From ac86a720b820448470fcc31290261626256aa27a Mon Sep 17 00:00:00 2001 From: Shao Yang Hong Date: Mon, 15 Feb 2021 02:19:33 +0800 Subject: [PATCH 001/243] MINOR: Update Serdes.java (#10117) Reviewers: Guozhang Wang --- .../kafka/common/serialization/Serdes.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/serialization/Serdes.java b/clients/src/main/java/org/apache/kafka/common/serialization/Serdes.java index 7b47dc6d29ba2..347bf8713ece8 100644 --- a/clients/src/main/java/org/apache/kafka/common/serialization/Serdes.java +++ b/clients/src/main/java/org/apache/kafka/common/serialization/Serdes.java @@ -189,77 +189,77 @@ static public Serde serdeFrom(final Serializer serializer, final Deser return new WrapperSerde<>(serializer, deserializer); } - /* + /** * A serde for nullable {@code Long} type. */ static public Serde Long() { return new LongSerde(); } - /* + /** * A serde for nullable {@code Integer} type. */ static public Serde Integer() { return new IntegerSerde(); } - /* + /** * A serde for nullable {@code Short} type. */ static public Serde Short() { return new ShortSerde(); } - /* + /** * A serde for nullable {@code Float} type. */ static public Serde Float() { return new FloatSerde(); } - /* + /** * A serde for nullable {@code Double} type. */ static public Serde Double() { return new DoubleSerde(); } - /* + /** * A serde for nullable {@code String} type. */ static public Serde String() { return new StringSerde(); } - /* + /** * A serde for nullable {@code ByteBuffer} type. */ static public Serde ByteBuffer() { return new ByteBufferSerde(); } - /* + /** * A serde for nullable {@code Bytes} type. */ static public Serde Bytes() { return new BytesSerde(); } - /* + /** * A serde for nullable {@code UUID} type */ static public Serde UUID() { return new UUIDSerde(); } - /* + /** * A serde for nullable {@code byte[]} type. */ static public Serde ByteArray() { return new ByteArraySerde(); } - /* + /** * A serde for {@code Void} type. */ static public Serde Void() { From 987aafeddf69645b6089306dac8f9f32ac7cd9b3 Mon Sep 17 00:00:00 2001 From: Rajini Sivaram Date: Mon, 15 Feb 2021 12:21:01 +0000 Subject: [PATCH 002/243] MINOR: Remove unused LeaderAndIsrResponse.partitions() since it has been replaced with partitionErrors() (#10127) Reviewers: David Jacot --- .../kafka/common/requests/LeaderAndIsrResponse.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index 60ab3d58cda1e..490983f1b5d21 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -20,11 +20,9 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError; -import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.utils.FlattenedIterator; import java.nio.ByteBuffer; import java.util.Collections; @@ -53,14 +51,6 @@ public List topics() { return this.data.topics(); } - public Iterable partitions() { - if (version < 5) { - return data.partitionErrors(); - } - return () -> new FlattenedIterator<>(data.topics().iterator(), - topic -> topic.partitionErrors().iterator()); - } - public Errors error() { return Errors.forCode(data.errorCode()); } From fe3f2bec8e46b22650a4ee2552379d2abbaa6f40 Mon Sep 17 00:00:00 2001 From: dengziming Date: Tue, 16 Feb 2021 23:55:20 +0800 Subject: [PATCH 003/243] MINOR: remove duplicate code of serializing auto-generated data (#10128) Reviewers: Chia-Ping Tsai --- .../common/message/RecordsSerdeTest.java | 14 ++--------- .../message/SimpleExampleMessageTest.java | 25 ++++--------------- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/common/message/RecordsSerdeTest.java b/clients/src/test/java/org/apache/kafka/common/message/RecordsSerdeTest.java index dcf6e0fd16cbc..739bed9cc46d8 100644 --- a/clients/src/test/java/org/apache/kafka/common/message/RecordsSerdeTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/RecordsSerdeTest.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.message; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.common.protocol.MessageUtil; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.SimpleRecord; @@ -69,7 +69,7 @@ private void testAllRoundTrips(SimpleRecordsMessageData message) throws Exceptio } private void testRoundTrip(SimpleRecordsMessageData message, short version) { - ByteBuffer buf = serialize(message, version); + ByteBuffer buf = MessageUtil.toByteBuffer(message, version); SimpleRecordsMessageData message2 = deserialize(buf.duplicate(), version); assertEquals(message, message2); assertEquals(message.hashCode(), message2.hashCode()); @@ -80,14 +80,4 @@ private SimpleRecordsMessageData deserialize(ByteBuffer buffer, short version) { return new SimpleRecordsMessageData(readable, version); } - private ByteBuffer serialize(SimpleRecordsMessageData message, short version) { - ObjectSerializationCache cache = new ObjectSerializationCache(); - int totalMessageSize = message.size(cache, version); - ByteBuffer buffer = ByteBuffer.allocate(totalMessageSize); - ByteBufferAccessor writer = new ByteBufferAccessor(buffer); - message.write(writer, cache, version); - buffer.flip(); - return buffer; - } - } diff --git a/clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java index 5b5f33d770819..1cdafcd0fdc0e 100644 --- a/clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.MessageUtil; import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.utils.ByteUtils; import org.junit.jupiter.api.Test; @@ -79,10 +80,7 @@ public void shouldRoundTripFieldThroughBuffer() { out.setProcessId(uuid); out.setZeroCopyByteBuffer(buf); - ObjectSerializationCache cache = new ObjectSerializationCache(); - final ByteBuffer buffer = ByteBuffer.allocate(out.size(cache, (short) 1)); - out.write(new ByteBufferAccessor(buffer), cache, (short) 1); - buffer.rewind(); + final ByteBuffer buffer = MessageUtil.toByteBuffer(out, (short) 1); final SimpleExampleMessageData in = new SimpleExampleMessageData(); in.read(new ByteBufferAccessor(buffer), (short) 1); @@ -104,10 +102,7 @@ public void shouldRoundTripFieldThroughBufferWithNullable() { out.setZeroCopyByteBuffer(buf1); out.setNullableZeroCopyByteBuffer(buf2); - ObjectSerializationCache cache = new ObjectSerializationCache(); - final ByteBuffer buffer = ByteBuffer.allocate(out.size(cache, (short) 1)); - out.write(new ByteBufferAccessor(buffer), cache, (short) 1); - buffer.rewind(); + final ByteBuffer buffer = MessageUtil.toByteBuffer(out, (short) 1); final SimpleExampleMessageData in = new SimpleExampleMessageData(); in.read(new ByteBufferAccessor(buffer), (short) 1); @@ -248,7 +243,7 @@ public void testTaggedLong() { testRoundTrip(new SimpleExampleMessageData(). setMyString("blah"). - setMyTaggedIntArray(Arrays.asList(4)). + setMyTaggedIntArray(Collections.singletonList(4)). setTaggedLong(0x123443211234432L), message -> assertEquals(0x123443211234432L, message.taggedLong())); @@ -306,16 +301,6 @@ public void testCommonStruct() { testRoundTrip(message, (short) 2); } - private ByteBuffer serialize(SimpleExampleMessageData message, short version) { - ObjectSerializationCache cache = new ObjectSerializationCache(); - int size = message.size(cache, version); - ByteBuffer buf = ByteBuffer.allocate(size); - message.write(new ByteBufferAccessor(buf), cache, version); - buf.flip(); - assertEquals(size, buf.remaining()); - return buf; - } - private SimpleExampleMessageData deserialize(ByteBuffer buf, short version) { SimpleExampleMessageData message = new SimpleExampleMessageData(); message.read(new ByteBufferAccessor(buf.duplicate()), version); @@ -335,7 +320,7 @@ private void testRoundTrip(SimpleExampleMessageData message, Consumer validator, short version) { validator.accept(message); - ByteBuffer buf = serialize(message, version); + ByteBuffer buf = MessageUtil.toByteBuffer(message, version); SimpleExampleMessageData message2 = deserialize(buf.duplicate(), version); validator.accept(message2); From fb7da1a245ff24a3d538e011ab3ec4e96ef86d28 Mon Sep 17 00:00:00 2001 From: Justine Olshan Date: Tue, 16 Feb 2021 18:12:29 -0500 Subject: [PATCH 004/243] Fixed README and added clearer error message. (#10133) The script `test-raft-server-start.sh` requires the config to be specified with `--config`. I've included this in the README and added an error message for this specific case. Reviewers: Jason Gustafson --- core/src/main/scala/kafka/tools/TestRaftServer.scala | 4 ++++ raft/README.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/tools/TestRaftServer.scala b/core/src/main/scala/kafka/tools/TestRaftServer.scala index 0a83fd9453581..2d6dd6772fca5 100644 --- a/core/src/main/scala/kafka/tools/TestRaftServer.scala +++ b/core/src/main/scala/kafka/tools/TestRaftServer.scala @@ -26,6 +26,7 @@ import kafka.raft.{KafkaRaftManager, RaftManager} import kafka.security.CredentialProvider import kafka.server.{KafkaConfig, KafkaRequestHandlerPool, MetaProperties} import kafka.utils.{CommandDefaultOptions, CommandLineUtils, CoreUtils, Exit, Logging, ShutdownableThread} +import org.apache.kafka.common.errors.InvalidConfigurationException import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.metrics.stats.Percentiles.BucketSizing import org.apache.kafka.common.metrics.stats.{Meter, Percentile, Percentiles} @@ -419,6 +420,9 @@ object TestRaftServer extends Logging { "Standalone raft server for performance testing") val configFile = opts.options.valueOf(opts.configOpt) + if (configFile == null) { + throw new InvalidConfigurationException("Missing configuration file. Should specify with '--config'") + } val serverProps = Utils.loadProps(configFile) // KafkaConfig requires either `process.roles` or `zookeeper.connect`. Neither are diff --git a/raft/README.md b/raft/README.md index 4d513230f9bc6..35d3969081c4b 100644 --- a/raft/README.md +++ b/raft/README.md @@ -9,7 +9,7 @@ we have a standalone test server which can be used for performance testing. Below we describe the details to set this up. ### Run Single Quorum ### - bin/test-raft-server-start.sh config/raft.properties + bin/test-raft-server-start.sh --config config/raft.properties ### Run Multi Node Quorum ### Create 3 separate raft quorum properties as the following: From da58ce4065695b0106d545e79a156faafa5d139b Mon Sep 17 00:00:00 2001 From: Jim Galasyn Date: Tue, 16 Feb 2021 17:11:24 -0800 Subject: [PATCH 005/243] MINOR: Clarify config names for EOS versions 1 and 2 (#9670) Reviewers: Boyang Chen , Matthias J. Sax --- docs/streams/core-concepts.html | 2 +- docs/streams/developer-guide/config-streams.html | 6 +++--- docs/streams/upgrade-guide.html | 11 ++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/streams/core-concepts.html b/docs/streams/core-concepts.html index ddb37eac2c19e..7391c022f1ed0 100644 --- a/docs/streams/core-concepts.html +++ b/docs/streams/core-concepts.html @@ -300,7 +300,7 @@

Kafka Streams Configs section.

diff --git a/docs/streams/developer-guide/config-streams.html b/docs/streams/developer-guide/config-streams.html index 07b47c0353f02..fe350260c6463 100644 --- a/docs/streams/developer-guide/config-streams.html +++ b/docs/streams/developer-guide/config-streams.html @@ -294,7 +294,7 @@

bootstrap.serversprocessing.guarantee Medium The processing mode. Can be either "at_least_once" (default), - "exactly_once", or "exactly_once_beta". + "exactly_once" (for EOS version 1), or "exactly_once_beta" (for EOS version 2). See Processing Guarantee poll.ms @@ -668,8 +668,8 @@

probing.rebalance.interval.ms
The processing guarantee that should be used. Possible values are "at_least_once" (default), - "exactly_once", - and "exactly_once_beta". + "exactly_once" (for EOS version 1), + and "exactly_once_beta" (for EOS version 2). Using "exactly_once" requires broker version 0.11.0 or newer, while using "exactly_once_beta" requires broker version 2.5 or newer. diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html index 7bb29718148f8..2a6a7604f43e2 100644 --- a/docs/streams/upgrade-guide.html +++ b/docs/streams/upgrade-guide.html @@ -53,8 +53,9 @@

Upgrade Guide and API Changes

- Starting in Kafka Streams 2.6.x, a new processing mode "exactly_once_beta" (configurable via parameter - processing.guarantee) is available. + Starting in Kafka Streams 2.6.x, a new processing mode is available, named EOS version 2, which is configurable by setting + processing.guarantee to "exactly_once_beta". + NOTE: The "exactly_once_beta" processing mode is ready for production (i.e., it's not "beta" software). To use this new feature, your brokers must be on version 2.5.x or newer. A switch from "exactly_once" to "exactly_once_beta" (or the other way around) is only possible if the application is on version 2.6.x. @@ -162,7 +163,7 @@

Streams API

Streams API changes in 2.6.0

- We added a new processing mode that improves application scalability using exactly-once guarantees + We added a new processing mode, EOS version 2, that improves application scalability using exactly-once guarantees (via KIP-447). You can enable this new feature by setting the configuration parameter processing.guarantee to the new value "exactly_once_beta". @@ -894,9 +895,9 @@

Metrics using exactly-once semantics:

- If "exactly_once" processing is enabled via the processing.guarantee parameter, + If "exactly_once" processing (EOS version 1) is enabled via the processing.guarantee parameter, internally Streams switches from a producer-per-thread to a producer-per-task runtime model. - Using "exactly_once_beta" does use a producer-per-thread, so client.id doesn't change, + Using "exactly_once_beta" (EOS version 2) does use a producer-per-thread, so client.id doesn't change, compared with "at_least_once" for this case). In order to distinguish the different producers, the producer's client.id additionally encodes the task-ID for this case. Because the producer's client.id is used to report JMX metrics, it might be required to update tools that receive those metrics. From f5c2f608b05c8094a898c14a6c4a3238f6388df1 Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Wed, 17 Feb 2021 12:14:08 +0800 Subject: [PATCH 006/243] MINOR: use 'mapKey' to avoid unnecessary grouped data (#10082) 1. add 'mapKey=true' to DescribeLogDirsRequest 2. rename PartitionIndex to Partitions for DescribeLogDirsRequest 3. add 'mapKey=true' to ElectLeadersRequest 4. rename PartitionId to Partitions for ElectLeadersRequest 5. add 'mapKey=true' to ConsumerProtocolAssignment Reviewers: David Jacot , Ismael Juma --- .../kafka/clients/admin/KafkaAdminClient.java | 10 +++--- .../consumer/internals/ConsumerProtocol.java | 32 +++++++++---------- .../common/requests/ElectLeadersRequest.java | 15 +++++---- .../message/ConsumerProtocolAssignment.json | 2 +- .../message/ConsumerProtocolSubscription.json | 2 +- .../message/DescribeLogDirsRequest.json | 2 +- .../common/message/ElectLeadersRequest.json | 4 +-- core/src/main/scala/kafka/api/package.scala | 2 +- .../main/scala/kafka/server/KafkaApis.scala | 2 +- .../kafka/api/AuthorizerIntegrationTest.scala | 2 +- .../unit/kafka/server/RequestQuotaTest.scala | 2 +- 11 files changed, 39 insertions(+), 36 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 3cad32800b73d..3296fb8be5f68 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -2578,13 +2578,13 @@ public DescribeReplicaLogDirsResult describeReplicaLogDirs(Collection new DescribeLogDirsRequestData()); DescribableLogDirTopic describableLogDirTopic = requestData.topics().find(replica.topic()); if (describableLogDirTopic == null) { - List partitionIndex = new ArrayList<>(); - partitionIndex.add(replica.partition()); + List partitions = new ArrayList<>(); + partitions.add(replica.partition()); describableLogDirTopic = new DescribableLogDirTopic().setTopic(replica.topic()) - .setPartitionIndex(partitionIndex); + .setPartitions(partitions); requestData.topics().add(describableLogDirTopic); } else { - describableLogDirTopic.partitionIndex().add(replica.partition()); + describableLogDirTopic.partitions().add(replica.partition()); } } @@ -2594,7 +2594,7 @@ public DescribeReplicaLogDirsResult describeReplicaLogDirs(Collection replicaDirInfoByPartition = new HashMap<>(); for (DescribableLogDirTopic topicPartition: topicPartitions.topics()) { - for (Integer partitionId : topicPartition.partitionIndex()) { + for (Integer partitionId : topicPartition.partitions()) { replicaDirInfoByPartition.put(new TopicPartition(topicPartition.topic(), partitionId), new ReplicaLogDirInfo()); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java index a05e8715eae84..a9c7430142935 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java @@ -25,12 +25,10 @@ import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.MessageUtil; import org.apache.kafka.common.protocol.types.SchemaException; -import org.apache.kafka.common.utils.CollectionUtils; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import java.util.Map; /** * ConsumerProtocol contains the schemas for consumer subscriptions and assignments for use with @@ -74,13 +72,14 @@ public static ByteBuffer serializeSubscription(final Subscription subscription, ConsumerProtocolSubscription data = new ConsumerProtocolSubscription(); data.setTopics(subscription.topics()); data.setUserData(subscription.userData() != null ? subscription.userData().duplicate() : null); - Map> partitionsByTopic = CollectionUtils.groupPartitionsByTopic(subscription.ownedPartitions()); - for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) { - data.ownedPartitions().add(new ConsumerProtocolSubscription.TopicPartition() - .setTopic(topicEntry.getKey()) - .setPartitions(topicEntry.getValue())); - } - + subscription.ownedPartitions().forEach(tp -> { + ConsumerProtocolSubscription.TopicPartition partition = data.ownedPartitions().find(tp.topic()); + if (partition == null) { + partition = new ConsumerProtocolSubscription.TopicPartition().setTopic(tp.topic()); + data.ownedPartitions().add(partition); + } + partition.partitions().add(tp.partition()); + }); return MessageUtil.toVersionPrefixedByteBuffer(version, data); } @@ -120,13 +119,14 @@ public static ByteBuffer serializeAssignment(final Assignment assignment, short ConsumerProtocolAssignment data = new ConsumerProtocolAssignment(); data.setUserData(assignment.userData() != null ? assignment.userData().duplicate() : null); - Map> partitionsByTopic = CollectionUtils.groupPartitionsByTopic(assignment.partitions()); - for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) { - data.assignedPartitions().add(new ConsumerProtocolAssignment.TopicPartition() - .setTopic(topicEntry.getKey()) - .setPartitions(topicEntry.getValue())); - } - + assignment.partitions().forEach(tp -> { + ConsumerProtocolAssignment.TopicPartition partition = data.assignedPartitions().find(tp.topic()); + if (partition == null) { + partition = new ConsumerProtocolAssignment.TopicPartition().setTopic(tp.topic()); + data.assignedPartitions().add(partition); + } + partition.partitions().add(tp.partition()); + }); return MessageUtil.toVersionPrefixedByteBuffer(version, data); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java index 92f6b45eed59d..1bc888af6050c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Map; import org.apache.kafka.common.ElectionType; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.UnsupportedVersionException; @@ -32,7 +31,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.MessageUtil; -import org.apache.kafka.common.utils.CollectionUtils; public class ElectLeadersRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -70,9 +68,14 @@ private ElectLeadersRequestData toRequestData(short version) { .setTimeoutMs(timeoutMs); if (topicPartitions != null) { - for (Map.Entry> tp : CollectionUtils.groupPartitionsByTopic(topicPartitions).entrySet()) { - data.topicPartitions().add(new ElectLeadersRequestData.TopicPartitions().setTopic(tp.getKey()).setPartitionId(tp.getValue())); - } + topicPartitions.forEach(tp -> { + ElectLeadersRequestData.TopicPartitions tps = data.topicPartitions().find(tp.topic()); + if (tps == null) { + tps = new ElectLeadersRequestData.TopicPartitions().setTopic(tp.topic()); + data.topicPartitions().add(tps); + } + tps.partitions().add(tp.partition()); + }); } else { data.setTopicPartitions(null); } @@ -104,7 +107,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ReplicaElectionResult electionResult = new ReplicaElectionResult(); electionResult.setTopic(topic.topic()); - for (Integer partitionId : topic.partitionId()) { + for (Integer partitionId : topic.partitions()) { PartitionResult partitionResult = new PartitionResult(); partitionResult.setPartitionId(partitionId); partitionResult.setErrorCode(apiError.error().code()); diff --git a/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json b/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json index 2ad373a5659ec..544db20b47ef0 100644 --- a/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json +++ b/clients/src/main/resources/common/message/ConsumerProtocolAssignment.json @@ -25,7 +25,7 @@ "fields": [ { "name": "AssignedPartitions", "type": "[]TopicPartition", "versions": "0+", "fields": [ - { "name": "Topic", "type": "string", "versions": "0+" }, + { "name": "Topic", "type": "string", "mapKey": true, "versions": "0+" }, { "name": "Partitions", "type": "[]int32", "versions": "0+" } ] }, diff --git a/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json b/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json index fa4c371d70de7..207dac79fbca5 100644 --- a/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json +++ b/clients/src/main/resources/common/message/ConsumerProtocolSubscription.json @@ -28,7 +28,7 @@ "default": "null", "zeroCopy": true }, { "name": "OwnedPartitions", "type": "[]TopicPartition", "versions": "1+", "ignorable": true, "fields": [ - { "name": "Topic", "type": "string", "versions": "1+" }, + { "name": "Topic", "type": "string", "mapKey": true, "versions": "1+" }, { "name": "Partitions", "type": "[]int32", "versions": "1+"} ] } diff --git a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json index 577f2eb4cf6a8..c498e0f22238a 100644 --- a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json +++ b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json @@ -26,7 +26,7 @@ "about": "Each topic that we want to describe log directories for, or null for all topics.", "fields": [ { "name": "Topic", "type": "string", "versions": "0+", "entityType": "topicName", "mapKey": true, "about": "The topic name" }, - { "name": "PartitionIndex", "type": "[]int32", "versions": "0+", + { "name": "Partitions", "type": "[]int32", "versions": "0+", "about": "The partition indxes." } ]} ] diff --git a/clients/src/main/resources/common/message/ElectLeadersRequest.json b/clients/src/main/resources/common/message/ElectLeadersRequest.json index 5b86c96b04d60..a2ba2bdca735c 100644 --- a/clients/src/main/resources/common/message/ElectLeadersRequest.json +++ b/clients/src/main/resources/common/message/ElectLeadersRequest.json @@ -28,9 +28,9 @@ { "name": "TopicPartitions", "type": "[]TopicPartitions", "versions": "0+", "nullableVersions": "0+", "about": "The topic partitions to elect leaders.", "fields": [ - { "name": "Topic", "type": "string", "versions": "0+", "entityType": "topicName", + { "name": "Topic", "type": "string", "versions": "0+", "entityType": "topicName", "mapKey": true, "about": "The name of a topic." }, - { "name": "PartitionId", "type": "[]int32", "versions": "0+", + { "name": "Partitions", "type": "[]int32", "versions": "0+", "about": "The partitions of this topic whose leader should be elected." } ] }, diff --git a/core/src/main/scala/kafka/api/package.scala b/core/src/main/scala/kafka/api/package.scala index 11a956d40b536..e0678f810ff4c 100644 --- a/core/src/main/scala/kafka/api/package.scala +++ b/core/src/main/scala/kafka/api/package.scala @@ -28,7 +28,7 @@ package object api { Set.empty } else { self.data.topicPartitions.asScala.iterator.flatMap { topicPartition => - topicPartition.partitionId.asScala.map { partitionId => + topicPartition.partitions.asScala.map { partitionId => new TopicPartition(topicPartition.topic, partitionId) } }.toSet diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 5d71c8e056b93..ffb1b8e7445a7 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -2772,7 +2772,7 @@ class KafkaApis(val requestChannel: RequestChannel, replicaManager.logManager.allLogs.map(_.topicPartition).toSet else describeLogDirsDirRequest.data.topics.asScala.flatMap( - logDirTopic => logDirTopic.partitionIndex.asScala.map(partitionIndex => + logDirTopic => logDirTopic.partitions.asScala.map(partitionIndex => new TopicPartition(logDirTopic.topic, partitionIndex))).toSet replicaManager.describeLogDirs(partitions) diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index fec75eb5ef94c..66f25eeaac1c9 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -609,7 +609,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { } private def describeLogDirsRequest = new DescribeLogDirsRequest.Builder(new DescribeLogDirsRequestData().setTopics(new DescribeLogDirsRequestData.DescribableLogDirTopicCollection(Collections.singleton( - new DescribeLogDirsRequestData.DescribableLogDirTopic().setTopic(tp.topic).setPartitionIndex(Collections.singletonList(tp.partition))).iterator()))).build() + new DescribeLogDirsRequestData.DescribableLogDirTopic().setTopic(tp.topic).setPartitions(Collections.singletonList(tp.partition))).iterator()))).build() private def addPartitionsToTxnRequest = new AddPartitionsToTxnRequest.Builder(transactionalId, 1, 1, Collections.singletonList(tp)).build() diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index 5b1363f6aec2e..1924034ffdf45 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -507,7 +507,7 @@ class RequestQuotaTest extends BaseRequestTest { val data = new DescribeLogDirsRequestData() data.topics.add(new DescribeLogDirsRequestData.DescribableLogDirTopic() .setTopic(tp.topic) - .setPartitionIndex(Collections.singletonList(tp.partition))) + .setPartitions(Collections.singletonList(tp.partition))) new DescribeLogDirsRequest.Builder(data) case ApiKeys.CREATE_PARTITIONS => From 0bdb469b56c63aa4dfae1ab24bb1868471cfa7bf Mon Sep 17 00:00:00 2001 From: runom Date: Wed, 17 Feb 2021 13:19:20 +0900 Subject: [PATCH 007/243] MINOR: Fix typo (thread -> threads) in MirrorMaker (#10130) Reviewers: Chia-Ping Tsai --- core/src/main/scala/kafka/tools/MirrorMaker.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala index c1146f2026503..70b70a8e3b32f 100755 --- a/core/src/main/scala/kafka/tools/MirrorMaker.scala +++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala @@ -44,7 +44,7 @@ import scala.util.{Failure, Success, Try} /** * The mirror maker has the following architecture: - * - There are N mirror maker thread, each of which is equipped with a separate KafkaConsumer instance. + * - There are N mirror maker threads, each of which is equipped with a separate KafkaConsumer instance. * - All the mirror maker threads share one producer. * - Each mirror maker thread periodically flushes the producer and then commits all offsets. * From 16a8f2c3c4e48e2ce84862dc60406ed6672d75b1 Mon Sep 17 00:00:00 2001 From: Justine Olshan Date: Wed, 17 Feb 2021 12:45:40 -0500 Subject: [PATCH 008/243] MINOR: Add note about topic IDs to upgrade doc (#10125) Reviewers: Jun Rao --- docs/upgrade.html | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/upgrade.html b/docs/upgrade.html index 9b43244beb5d5..9ce10371799e5 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -23,13 +23,17 @@

Notable changes in 2 - -
    +
  • + IBP 2.8 introduces topic IDs to topics as a part of + KIP-516. + When using ZooKeeper, this information is stored in the TopicZNode. If the cluster is downgraded to a previous IBP or version, + future topics will not get topic IDs and it is not guaranteed that topics will retain their topic IDs in ZooKeeper. + This means that upon upgrading again, some topics or all topics will be assigned new IDs. +
  • Kafka Streams introduce a type-safe split() operator as a substitution for deprecated KStream#branch() method (cf. KIP-418).
    • From 8c7c7494f9c2a0a6fc2d5b0b1fdee9da3d84b7a0 Mon Sep 17 00:00:00 2001 From: dengziming Date: Thu, 18 Feb 2021 11:08:29 +0800 Subject: [PATCH 009/243] MINOR: Import classes that is used in docs to fix warnings. (#10136) Reviewers: Chia-Ping Tsai --- .../apache/kafka/clients/admin/DescribeClientQuotasOptions.java | 1 + .../apache/kafka/clients/admin/DescribeClientQuotasResult.java | 1 + 2 files changed, 2 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java index 14e7e451219e6..c3bdc7b3ffb74 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java @@ -18,6 +18,7 @@ package org.apache.kafka.clients.admin; import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.quota.ClientQuotaFilter; /** * Options for {@link Admin#describeClientQuotas(ClientQuotaFilter, DescribeClientQuotasOptions)}. diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java index b4855901a03de..0e41bc7f78e8c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.common.quota.ClientQuotaEntity; +import org.apache.kafka.common.quota.ClientQuotaFilter; import java.util.Map; From 46690113cd0066e57f914539978cfdf69ebbef63 Mon Sep 17 00:00:00 2001 From: Geordie Date: Thu, 18 Feb 2021 11:15:56 +0800 Subject: [PATCH 010/243] =?UTF-8?q?KAFKA-10885=20Refactor=20MemoryRecordsB?= =?UTF-8?q?uilderTest/MemoryRecordsTest=20to=20avoid=20a=20lot=20of?= =?UTF-8?q?=E2=80=A6=20(#9906)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewers: Ismael Juma , Chia-Ping Tsai --- .../record/MemoryRecordsBuilderTest.java | 615 ++++++++---------- .../common/record/MemoryRecordsTest.java | 545 ++++++++-------- 2 files changed, 555 insertions(+), 605 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java index d6a3801f2e2b0..13eaa9d21cba1 100644 --- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java @@ -22,11 +22,13 @@ import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; +import org.junit.jupiter.params.provider.EnumSource; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -34,10 +36,14 @@ import java.util.Collections; import java.util.List; import java.util.Random; +import java.util.function.BiFunction; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Arrays.asList; +import static org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V0; +import static org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V1; import static org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V2; import static org.apache.kafka.common.utils.Utils.utf8; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -45,23 +51,24 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.junit.jupiter.api.Assumptions.assumeTrue; public class MemoryRecordsBuilderTest { private static class Args { final int bufferOffset; final CompressionType compressionType; + final byte magic; - public Args(int bufferOffset, CompressionType compressionType) { + public Args(int bufferOffset, CompressionType compressionType, byte magic) { this.bufferOffset = bufferOffset; this.compressionType = compressionType; + this.magic = magic; } @Override public String toString() { - return "bufferOffset=" + bufferOffset + + return "magic=" + magic + + ", bufferOffset=" + bufferOffset + ", compressionType=" + compressionType; } } @@ -71,260 +78,256 @@ private static class MemoryRecordsBuilderArgumentsProvider implements ArgumentsP public Stream provideArguments(ExtensionContext context) { List values = new ArrayList<>(); for (int bufferOffset : Arrays.asList(0, 15)) - for (CompressionType compressionType : CompressionType.values()) - values.add(Arguments.of(new Args(bufferOffset, compressionType))); + for (CompressionType type: CompressionType.values()) { + List magics = type == CompressionType.ZSTD + ? Collections.singletonList(RecordBatch.MAGIC_VALUE_V2) + : asList(RecordBatch.MAGIC_VALUE_V0, MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2); + for (byte magic : magics) + values.add(Arguments.of(new Args(bufferOffset, type, magic))); + } return values.stream(); } } private final Time time = Time.SYSTEM; + @Test + public void testUnsupportedCompress() { + BiFunction builderBiFunction = (magic, compressionType) -> + new MemoryRecordsBuilder(ByteBuffer.allocate(128), magic, compressionType, TimestampType.CREATE_TIME, 0L, 0L, + RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, + false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, 128); + + Arrays.asList(MAGIC_VALUE_V0, MAGIC_VALUE_V1).forEach(magic -> { + Exception e = assertThrows(IllegalArgumentException.class, () -> builderBiFunction.apply(magic, CompressionType.ZSTD)); + assertEquals(e.getMessage(), "ZStandard compression is not supported for magic " + magic); + }); + } + @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteEmptyRecordSet(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V0; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - + byte magic = args.magic; ByteBuffer buffer = allocateBuffer(128, args); - Supplier builderSupplier = () -> new MemoryRecordsBuilder(buffer, magic, + MemoryRecords records = new MemoryRecordsBuilder(buffer, magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, - false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()).build(); - if (args.compressionType != CompressionType.ZSTD) { - MemoryRecords records = builderSupplier.get().build(); - assertEquals(0, records.sizeInBytes()); - assertEquals(args.bufferOffset, buffer.position()); - } else { - Exception e = assertThrows(IllegalArgumentException.class, () -> builderSupplier.get().build()); - assertEquals(e.getMessage(), "ZStandard compression is not supported for magic " + magic); - } + assertEquals(0, records.sizeInBytes()); + assertEquals(args.bufferOffset, buffer.position()); } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteTransactionalRecordSet(Args args) { ByteBuffer buffer = allocateBuffer(128, args); - long pid = 9809; short epoch = 15; int sequence = 2342; - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, + Supplier supplier = () -> new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - builder.append(System.currentTimeMillis(), "foo".getBytes(), "bar".getBytes()); - MemoryRecords records = builder.build(); - - List batches = Utils.toList(records.batches().iterator()); - assertEquals(1, batches.size()); - assertTrue(batches.get(0).isTransactional()); - } - - @ParameterizedTest - @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) - public void testWriteTransactionalNotAllowedMagicV0(Args args) { - ByteBuffer buffer = allocateBuffer(128, args); - - long pid = 9809; - short epoch = 15; - int sequence = 2342; - - assertThrows(IllegalArgumentException.class, () -> new MemoryRecordsBuilder(buffer, RecordBatch.MAGIC_VALUE_V0, - args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, - true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity())); - } - - @ParameterizedTest - @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) - public void testWriteTransactionalNotAllowedMagicV1(Args args) { - ByteBuffer buffer = allocateBuffer(128, args); - - long pid = 9809; - short epoch = 15; - int sequence = 2342; - - assertThrows(IllegalArgumentException.class, () -> new MemoryRecordsBuilder(buffer, RecordBatch.MAGIC_VALUE_V1, - args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, - true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity())); - } - @ParameterizedTest - @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) - public void testWriteControlBatchNotAllowedMagicV0(Args args) { - ByteBuffer buffer = allocateBuffer(128, args); - - long pid = 9809; - short epoch = 15; - int sequence = 2342; - - assertThrows(IllegalArgumentException.class, () -> new MemoryRecordsBuilder(buffer, RecordBatch.MAGIC_VALUE_V0, - args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, - false, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity())); - } - - @ParameterizedTest - @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) - public void testWriteControlBatchNotAllowedMagicV1(Args args) { - ByteBuffer buffer = allocateBuffer(128, args); - - long pid = 9809; - short epoch = 15; - int sequence = 2342; + if (args.magic < MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + MemoryRecordsBuilder builder = supplier.get(); + builder.append(System.currentTimeMillis(), "foo".getBytes(), "bar".getBytes()); + MemoryRecords records = builder.build(); - assertThrows(IllegalArgumentException.class, () -> new MemoryRecordsBuilder(buffer, RecordBatch.MAGIC_VALUE_V1, - args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, - false, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity())); + List batches = Utils.toList(records.batches().iterator()); + assertEquals(1, batches.size()); + assertTrue(batches.get(0).isTransactional()); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteTransactionalWithInvalidPID(Args args) { ByteBuffer buffer = allocateBuffer(128, args); - long pid = RecordBatch.NO_PRODUCER_ID; short epoch = 15; int sequence = 2342; - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, TimestampType.CREATE_TIME, - 0L, 0L, pid, epoch, sequence, true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - assertThrows(IllegalArgumentException.class, builder::close); + Supplier supplier = () -> new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, + 0L, 0L, pid, epoch, sequence, true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + if (args.magic < MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + MemoryRecordsBuilder builder = supplier.get(); + assertThrows(IllegalArgumentException.class, builder::close); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteIdempotentWithInvalidEpoch(Args args) { ByteBuffer buffer = allocateBuffer(128, args); - long pid = 9809; short epoch = RecordBatch.NO_PRODUCER_EPOCH; int sequence = 2342; - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, TimestampType.CREATE_TIME, - 0L, 0L, pid, epoch, sequence, true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - assertThrows(IllegalArgumentException.class, builder::close); + Supplier supplier = () -> new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, + 0L, 0L, pid, epoch, sequence, true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + + if (args.magic < MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + MemoryRecordsBuilder builder = supplier.get(); + assertThrows(IllegalArgumentException.class, builder::close); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteIdempotentWithInvalidBaseSequence(Args args) { ByteBuffer buffer = allocateBuffer(128, args); - long pid = 9809; short epoch = 15; int sequence = RecordBatch.NO_SEQUENCE; - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, TimestampType.CREATE_TIME, - 0L, 0L, pid, epoch, sequence, true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - assertThrows(IllegalArgumentException.class, builder::close); + Supplier supplier = () -> new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, + 0L, 0L, pid, epoch, sequence, true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + + if (args.magic < MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + MemoryRecordsBuilder builder = supplier.get(); + assertThrows(IllegalArgumentException.class, builder::close); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteEndTxnMarkerNonTransactionalBatch(Args args) { ByteBuffer buffer = allocateBuffer(128, args); - long pid = 9809; short epoch = 15; int sequence = RecordBatch.NO_SEQUENCE; - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, TimestampType.CREATE_TIME, - 0L, 0L, pid, epoch, sequence, false, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - assertThrows(IllegalArgumentException.class, () -> builder.appendEndTxnMarker(RecordBatch.NO_TIMESTAMP, - new EndTransactionMarker(ControlRecordType.ABORT, 0))); + Supplier supplier = () -> new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, + TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, false, true, + RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + + if (args.magic < MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + MemoryRecordsBuilder builder = supplier.get(); + assertThrows(IllegalArgumentException.class, () -> builder.appendEndTxnMarker(RecordBatch.NO_TIMESTAMP, + new EndTransactionMarker(ControlRecordType.ABORT, 0))); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteEndTxnMarkerNonControlBatch(Args args) { ByteBuffer buffer = allocateBuffer(128, args); - long pid = 9809; short epoch = 15; int sequence = RecordBatch.NO_SEQUENCE; - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, TimestampType.CREATE_TIME, + Supplier supplier = () -> new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, pid, epoch, sequence, true, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - assertThrows(IllegalArgumentException.class, () -> builder.appendEndTxnMarker(RecordBatch.NO_TIMESTAMP, - new EndTransactionMarker(ControlRecordType.ABORT, 0))); + + if (args.magic < MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + MemoryRecordsBuilder builder = supplier.get(); + assertThrows(IllegalArgumentException.class, () -> builder.appendEndTxnMarker(RecordBatch.NO_TIMESTAMP, + new EndTransactionMarker(ControlRecordType.ABORT, 0))); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteLeaderChangeControlBatchWithoutLeaderEpoch(Args args) { ByteBuffer buffer = allocateBuffer(128, args); + Supplier supplier = () -> new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, + TimestampType.CREATE_TIME, 0L, 0L, + RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, + false, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - final int leaderId = 1; - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, TimestampType.CREATE_TIME, - 0L, 0L, - RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, - false, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - assertThrows(IllegalArgumentException.class, () -> builder.appendLeaderChangeMessage(RecordBatch.NO_TIMESTAMP, - new LeaderChangeMessage() - .setLeaderId(leaderId) - .setVoters(Collections.emptyList()))); + if (args.magic < MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + final int leaderId = 1; + MemoryRecordsBuilder builder = supplier.get(); + assertThrows(IllegalArgumentException.class, () -> builder.appendLeaderChangeMessage(RecordBatch.NO_TIMESTAMP, + new LeaderChangeMessage().setLeaderId(leaderId).setVoters(Collections.emptyList()))); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testWriteLeaderChangeControlBatch(Args args) { ByteBuffer buffer = allocateBuffer(128, args); - final int leaderId = 1; final int leaderEpoch = 5; final List voters = Arrays.asList(2, 3); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, TimestampType.CREATE_TIME, - 0L, 0L, - RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, - false, true, leaderEpoch, buffer.capacity()); - builder.appendLeaderChangeMessage(RecordBatch.NO_TIMESTAMP, - new LeaderChangeMessage() - .setLeaderId(leaderId) - .setVoters(voters.stream().map( - voterId -> new Voter().setVoterId(voterId)).collect(Collectors.toList()))); - - MemoryRecords built = builder.build(); - List records = TestUtils.toList(built.records()); - assertEquals(1, records.size()); - LeaderChangeMessage leaderChangeMessage = ControlRecordUtils.deserializeLeaderChangeMessage(records.get(0)); + Supplier supplier = () -> new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, + TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, + RecordBatch.NO_SEQUENCE, false, true, leaderEpoch, buffer.capacity()); + + if (args.magic < MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + MemoryRecordsBuilder builder = supplier.get(); + builder.appendLeaderChangeMessage(RecordBatch.NO_TIMESTAMP, + new LeaderChangeMessage() + .setLeaderId(leaderId) + .setVoters(voters.stream().map( + voterId -> new Voter().setVoterId(voterId)).collect(Collectors.toList()))); + + MemoryRecords built = builder.build(); + List records = TestUtils.toList(built.records()); + assertEquals(1, records.size()); + LeaderChangeMessage leaderChangeMessage = ControlRecordUtils.deserializeLeaderChangeMessage(records.get(0)); - assertEquals(leaderId, leaderChangeMessage.leaderId()); - assertEquals(voters, leaderChangeMessage.voters().stream().map(Voter::voterId).collect(Collectors.toList())); + assertEquals(leaderId, leaderChangeMessage.leaderId()); + assertEquals(voters, leaderChangeMessage.voters().stream().map(Voter::voterId).collect(Collectors.toList())); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) - public void testCompressionRateV0(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V0; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - + public void testLegacyCompressionRate(Args args) { + byte magic = args.magic; ByteBuffer buffer = allocateBuffer(1024, args); - LegacyRecord[] records = new LegacyRecord[] { + Supplier supplier = () -> new LegacyRecord[]{ LegacyRecord.create(magic, 0L, "a".getBytes(), "1".getBytes()), LegacyRecord.create(magic, 1L, "b".getBytes(), "2".getBytes()), LegacyRecord.create(magic, 2L, "c".getBytes(), "3".getBytes()), }; - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, - TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, - false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + if (magic >= MAGIC_VALUE_V2) { + assertThrows(IllegalArgumentException.class, supplier::get); + } else { + LegacyRecord[] records = supplier.get(); - int uncompressedSize = 0; - for (LegacyRecord record : records) { - uncompressedSize += record.sizeInBytes() + Records.LOG_OVERHEAD; - builder.append(record); - } + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, + TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, + false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - MemoryRecords built = builder.build(); - if (args.compressionType == CompressionType.NONE) { - assertEquals(1.0, builder.compressionRatio(), 0.00001); - } else { - int compressedSize = built.sizeInBytes() - Records.LOG_OVERHEAD - LegacyRecord.RECORD_OVERHEAD_V0; - double computedCompressionRate = (double) compressedSize / uncompressedSize; - assertEquals(computedCompressionRate, builder.compressionRatio(), 0.00001); + int uncompressedSize = 0; + for (LegacyRecord record : records) { + uncompressedSize += record.sizeInBytes() + Records.LOG_OVERHEAD; + builder.append(record); + } + + MemoryRecords built = builder.build(); + if (args.compressionType == CompressionType.NONE) { + assertEquals(1.0, builder.compressionRatio(), 0.00001); + } else { + int recordHeaad = magic == MAGIC_VALUE_V0 ? LegacyRecord.RECORD_OVERHEAD_V0 : LegacyRecord.RECORD_OVERHEAD_V1; + int compressedSize = built.sizeInBytes() - Records.LOG_OVERHEAD - recordHeaad; + double computedCompressionRate = (double) compressedSize / uncompressedSize; + assertEquals(computedCompressionRate, builder.compressionRatio(), 0.00001); + } } } @@ -333,7 +336,7 @@ public void testCompressionRateV0(Args args) { public void testEstimatedSizeInBytes(Args args) { ByteBuffer buffer = allocateBuffer(1024, args); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); @@ -352,49 +355,14 @@ public void testEstimatedSizeInBytes(Args args) { assertEquals(records.sizeInBytes(), bytesWrittenBeforeClose); } - @ParameterizedTest - @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) - public void testCompressionRateV1(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V1; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - - ByteBuffer buffer = allocateBuffer(1024, args); - - LegacyRecord[] records = new LegacyRecord[] { - LegacyRecord.create(magic, 0L, "a".getBytes(), "1".getBytes()), - LegacyRecord.create(magic, 1L, "b".getBytes(), "2".getBytes()), - LegacyRecord.create(magic, 2L, "c".getBytes(), "3".getBytes()), - }; - - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, - TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, - false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - - int uncompressedSize = 0; - for (LegacyRecord record : records) { - uncompressedSize += record.sizeInBytes() + Records.LOG_OVERHEAD; - builder.append(record); - } - - MemoryRecords built = builder.build(); - if (args.compressionType == CompressionType.NONE) { - assertEquals(1.0, builder.compressionRatio(), 0.00001); - } else { - int compressedSize = built.sizeInBytes() - Records.LOG_OVERHEAD - LegacyRecord.RECORD_OVERHEAD_V1; - double computedCompressionRate = (double) compressedSize / uncompressedSize; - assertEquals(computedCompressionRate, builder.compressionRatio(), 0.00001); - } - } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void buildUsingLogAppendTime(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V1; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - + byte magic = args.magic; ByteBuffer buffer = allocateBuffer(1024, args); - long logAppendTime = System.currentTimeMillis(); + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, TimestampType.LOG_APPEND_TIME, 0L, logAppendTime, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); @@ -406,24 +374,25 @@ public void buildUsingLogAppendTime(Args args) { MemoryRecordsBuilder.RecordsInfo info = builder.info(); assertEquals(logAppendTime, info.maxTimestamp); - if (args.compressionType != CompressionType.NONE) - assertEquals(2L, info.shallowOffsetOfMaxTimestamp); - else + if (args.compressionType == CompressionType.NONE && magic <= MAGIC_VALUE_V1) assertEquals(0L, info.shallowOffsetOfMaxTimestamp); + else + assertEquals(2L, info.shallowOffsetOfMaxTimestamp); for (RecordBatch batch : records.batches()) { - assertEquals(TimestampType.LOG_APPEND_TIME, batch.timestampType()); - for (Record record : batch) - assertEquals(logAppendTime, record.timestamp()); + if (magic == MAGIC_VALUE_V0) { + assertEquals(TimestampType.NO_TIMESTAMP_TYPE, batch.timestampType()); + } else { + assertEquals(TimestampType.LOG_APPEND_TIME, batch.timestampType()); + for (Record record : batch) + assertEquals(logAppendTime, record.timestamp()); + } } } - @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void buildUsingCreateTime(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V1; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - + byte magic = args.magic; ByteBuffer buffer = allocateBuffer(1024, args); long logAppendTime = System.currentTimeMillis(); @@ -436,9 +405,13 @@ public void buildUsingCreateTime(Args args) { MemoryRecords records = builder.build(); MemoryRecordsBuilder.RecordsInfo info = builder.info(); - assertEquals(2L, info.maxTimestamp); + if (magic == MAGIC_VALUE_V0) { + assertEquals(-1, info.maxTimestamp); + } else { + assertEquals(2L, info.maxTimestamp); + } - if (args.compressionType == CompressionType.NONE) + if (args.compressionType == CompressionType.NONE && magic == MAGIC_VALUE_V1) assertEquals(1L, info.shallowOffsetOfMaxTimestamp); else assertEquals(2L, info.shallowOffsetOfMaxTimestamp); @@ -446,30 +419,29 @@ public void buildUsingCreateTime(Args args) { int i = 0; long[] expectedTimestamps = new long[] {0L, 2L, 1L}; for (RecordBatch batch : records.batches()) { - assertEquals(TimestampType.CREATE_TIME, batch.timestampType()); - for (Record record : batch) - assertEquals(expectedTimestamps[i++], record.timestamp()); + if (magic == MAGIC_VALUE_V0) { + assertEquals(TimestampType.NO_TIMESTAMP_TYPE, batch.timestampType()); + } else { + assertEquals(TimestampType.CREATE_TIME, batch.timestampType()); + for (Record record : batch) + assertEquals(expectedTimestamps[i++], record.timestamp()); + } } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void testAppendedChecksumConsistency(Args args) { - assumeAtLeastV2OrNotZstd(RecordBatch.MAGIC_VALUE_V0, args.compressionType); - assumeAtLeastV2OrNotZstd(RecordBatch.MAGIC_VALUE_V1, args.compressionType); - ByteBuffer buffer = ByteBuffer.allocate(512); - for (byte magic : Arrays.asList(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2)) { - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, - TimestampType.CREATE_TIME, 0L, LegacyRecord.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, - RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, - RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); - Long checksumOrNull = builder.append(1L, "key".getBytes(), "value".getBytes()); - MemoryRecords memoryRecords = builder.build(); - List records = TestUtils.toList(memoryRecords.records()); - assertEquals(1, records.size()); - assertEquals(checksumOrNull, records.get(0).checksumOrNull()); - } + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, + TimestampType.CREATE_TIME, 0L, LegacyRecord.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, + RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); + Long checksumOrNull = builder.append(1L, "key".getBytes(), "value".getBytes()); + MemoryRecords memoryRecords = builder.build(); + List records = TestUtils.toList(memoryRecords.records()); + assertEquals(1, records.size()); + assertEquals(checksumOrNull, records.get(0).checksumOrNull()); } @ParameterizedTest @@ -481,7 +453,7 @@ public void testSmallWriteLimit(Args args) { byte[] value = "bar".getBytes(); int writeLimit = 0; ByteBuffer buffer = ByteBuffer.allocate(512); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, args.compressionType, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, LegacyRecord.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, writeLimit); @@ -504,9 +476,7 @@ public void testSmallWriteLimit(Args args) { @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void writePastLimit(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V1; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - + byte magic = args.magic; ByteBuffer buffer = allocateBuffer(64, args); long logAppendTime = System.currentTimeMillis(); @@ -522,14 +492,22 @@ public void writePastLimit(Args args) { MemoryRecords records = builder.build(); MemoryRecordsBuilder.RecordsInfo info = builder.info(); - assertEquals(2L, info.maxTimestamp); + if (magic == MAGIC_VALUE_V0) + assertEquals(-1, info.maxTimestamp); + else + assertEquals(2L, info.maxTimestamp); + assertEquals(2L, info.shallowOffsetOfMaxTimestamp); long i = 0L; for (RecordBatch batch : records.batches()) { - assertEquals(TimestampType.CREATE_TIME, batch.timestampType()); - for (Record record : batch) - assertEquals(i++, record.timestamp()); + if (magic == MAGIC_VALUE_V0) { + assertEquals(TimestampType.NO_TIMESTAMP_TYPE, batch.timestampType()); + } else { + assertEquals(TimestampType.CREATE_TIME, batch.timestampType()); + for (Record record : batch) + assertEquals(i++, record.timestamp()); + } } } @@ -539,7 +517,7 @@ public void testAppendAtInvalidOffset(Args args) { ByteBuffer buffer = allocateBuffer(1024, args); long logAppendTime = System.currentTimeMillis(); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.MAGIC_VALUE_V2, args.compressionType, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, logAppendTime, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); @@ -551,11 +529,11 @@ public void testAppendAtInvalidOffset(Args args) { } @ParameterizedTest - @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) - public void convertV2ToV1UsingMixedCreateAndLogAppendTime(Args args) { + @EnumSource(CompressionType.class) + public void convertV2ToV1UsingMixedCreateAndLogAppendTime(CompressionType compressionType) { ByteBuffer buffer = ByteBuffer.allocate(512); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, - args.compressionType, TimestampType.LOG_APPEND_TIME, 0L); + compressionType, TimestampType.LOG_APPEND_TIME, 0L); builder.append(10L, "1".getBytes(), "a".getBytes()); builder.close(); @@ -566,7 +544,7 @@ public void convertV2ToV1UsingMixedCreateAndLogAppendTime(Args args) { int position = buffer.position(); - builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, args.compressionType, + builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, compressionType, TimestampType.CREATE_TIME, 1L); builder.append(12L, "2".getBytes(), "b".getBytes()); builder.append(13L, "3".getBytes(), "c".getBytes()); @@ -580,18 +558,18 @@ public void convertV2ToV1UsingMixedCreateAndLogAppendTime(Args args) { buffer.flip(); Supplier> convertedRecordsSupplier = () -> - MemoryRecords.readableRecords(buffer).downConvert(RecordBatch.MAGIC_VALUE_V1, 0, time); + MemoryRecords.readableRecords(buffer).downConvert(MAGIC_VALUE_V1, 0, time); - if (args.compressionType != CompressionType.ZSTD) { + if (compressionType != CompressionType.ZSTD) { ConvertedRecords convertedRecords = convertedRecordsSupplier.get(); MemoryRecords records = convertedRecords.records(); // Transactional markers are skipped when down converting to V1, so exclude them from size - verifyRecordsProcessingStats(args.compressionType, convertedRecords.recordConversionStats(), + verifyRecordsProcessingStats(compressionType, convertedRecords.recordConversionStats(), 3, 3, records.sizeInBytes(), sizeExcludingTxnMarkers); List batches = Utils.toList(records.batches().iterator()); - if (args.compressionType != CompressionType.NONE) { + if (compressionType != CompressionType.NONE) { assertEquals(2, batches.size()); assertEquals(TimestampType.LOG_APPEND_TIME, batches.get(0).timestampType()); assertEquals(TimestampType.CREATE_TIME, batches.get(1).timestampType()); @@ -614,93 +592,93 @@ public void convertV2ToV1UsingMixedCreateAndLogAppendTime(Args args) { } @ParameterizedTest - @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) - public void convertToV1WithMixedV0AndV2Data(Args args) { - CompressionType compressionType = args.compressionType; - assumeAtLeastV2OrNotZstd(RecordBatch.MAGIC_VALUE_V0, compressionType); - assumeAtLeastV2OrNotZstd(RecordBatch.MAGIC_VALUE_V1, compressionType); - + @EnumSource(CompressionType.class) + public void convertToV1WithMixedV0AndV2Data(CompressionType compressionType) { ByteBuffer buffer = ByteBuffer.allocate(512); - MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V0, - compressionType, TimestampType.NO_TIMESTAMP_TYPE, 0L); - builder.append(RecordBatch.NO_TIMESTAMP, "1".getBytes(), "a".getBytes()); - builder.close(); - builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, compressionType, - TimestampType.CREATE_TIME, 1L); - builder.append(11L, "2".getBytes(), "b".getBytes()); - builder.append(12L, "3".getBytes(), "c".getBytes()); - builder.close(); - - buffer.flip(); + Supplier supplier = () -> MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V0, + compressionType, TimestampType.NO_TIMESTAMP_TYPE, 0L); - ConvertedRecords convertedRecords = MemoryRecords.readableRecords(buffer) - .downConvert(RecordBatch.MAGIC_VALUE_V1, 0, time); - MemoryRecords records = convertedRecords.records(); - verifyRecordsProcessingStats(compressionType, convertedRecords.recordConversionStats(), 3, 2, - records.sizeInBytes(), buffer.limit()); - - List batches = Utils.toList(records.batches().iterator()); - if (compressionType != CompressionType.NONE) { - assertEquals(2, batches.size()); - assertEquals(RecordBatch.MAGIC_VALUE_V0, batches.get(0).magic()); - assertEquals(0, batches.get(0).baseOffset()); - assertEquals(RecordBatch.MAGIC_VALUE_V1, batches.get(1).magic()); - assertEquals(1, batches.get(1).baseOffset()); + if (compressionType == CompressionType.ZSTD) { + assertThrows(IllegalArgumentException.class, supplier::get); } else { - assertEquals(3, batches.size()); - assertEquals(RecordBatch.MAGIC_VALUE_V0, batches.get(0).magic()); - assertEquals(0, batches.get(0).baseOffset()); - assertEquals(RecordBatch.MAGIC_VALUE_V1, batches.get(1).magic()); - assertEquals(1, batches.get(1).baseOffset()); - assertEquals(RecordBatch.MAGIC_VALUE_V1, batches.get(2).magic()); - assertEquals(2, batches.get(2).baseOffset()); - } + MemoryRecordsBuilder builder = supplier.get(); + builder.append(RecordBatch.NO_TIMESTAMP, "1".getBytes(), "a".getBytes()); + builder.close(); - List logRecords = Utils.toList(records.records().iterator()); - assertEquals("1", utf8(logRecords.get(0).key())); - assertEquals("2", utf8(logRecords.get(1).key())); - assertEquals("3", utf8(logRecords.get(2).key())); + builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, compressionType, + TimestampType.CREATE_TIME, 1L); + builder.append(11L, "2".getBytes(), "b".getBytes()); + builder.append(12L, "3".getBytes(), "c".getBytes()); + builder.close(); + + buffer.flip(); - convertedRecords = MemoryRecords.readableRecords(buffer).downConvert(RecordBatch.MAGIC_VALUE_V1, 2L, time); - records = convertedRecords.records(); + ConvertedRecords convertedRecords = MemoryRecords.readableRecords(buffer) + .downConvert(MAGIC_VALUE_V1, 0, time); + MemoryRecords records = convertedRecords.records(); + verifyRecordsProcessingStats(compressionType, convertedRecords.recordConversionStats(), 3, 2, + records.sizeInBytes(), buffer.limit()); - batches = Utils.toList(records.batches().iterator()); - logRecords = Utils.toList(records.records().iterator()); + List batches = Utils.toList(records.batches().iterator()); + if (compressionType != CompressionType.NONE) { + assertEquals(2, batches.size()); + assertEquals(RecordBatch.MAGIC_VALUE_V0, batches.get(0).magic()); + assertEquals(0, batches.get(0).baseOffset()); + assertEquals(MAGIC_VALUE_V1, batches.get(1).magic()); + assertEquals(1, batches.get(1).baseOffset()); + } else { + assertEquals(3, batches.size()); + assertEquals(RecordBatch.MAGIC_VALUE_V0, batches.get(0).magic()); + assertEquals(0, batches.get(0).baseOffset()); + assertEquals(MAGIC_VALUE_V1, batches.get(1).magic()); + assertEquals(1, batches.get(1).baseOffset()); + assertEquals(MAGIC_VALUE_V1, batches.get(2).magic()); + assertEquals(2, batches.get(2).baseOffset()); + } - if (compressionType != CompressionType.NONE) { - assertEquals(2, batches.size()); - assertEquals(RecordBatch.MAGIC_VALUE_V0, batches.get(0).magic()); - assertEquals(0, batches.get(0).baseOffset()); - assertEquals(RecordBatch.MAGIC_VALUE_V1, batches.get(1).magic()); - assertEquals(1, batches.get(1).baseOffset()); + List logRecords = Utils.toList(records.records().iterator()); assertEquals("1", utf8(logRecords.get(0).key())); assertEquals("2", utf8(logRecords.get(1).key())); assertEquals("3", utf8(logRecords.get(2).key())); - verifyRecordsProcessingStats(compressionType, convertedRecords.recordConversionStats(), 3, 2, - records.sizeInBytes(), buffer.limit()); - } else { - assertEquals(2, batches.size()); - assertEquals(RecordBatch.MAGIC_VALUE_V0, batches.get(0).magic()); - assertEquals(0, batches.get(0).baseOffset()); - assertEquals(RecordBatch.MAGIC_VALUE_V1, batches.get(1).magic()); - assertEquals(2, batches.get(1).baseOffset()); - assertEquals("1", utf8(logRecords.get(0).key())); - assertEquals("3", utf8(logRecords.get(1).key())); - verifyRecordsProcessingStats(compressionType, convertedRecords.recordConversionStats(), 3, 1, - records.sizeInBytes(), buffer.limit()); + + convertedRecords = MemoryRecords.readableRecords(buffer).downConvert(MAGIC_VALUE_V1, 2L, time); + records = convertedRecords.records(); + + batches = Utils.toList(records.batches().iterator()); + logRecords = Utils.toList(records.records().iterator()); + + if (compressionType != CompressionType.NONE) { + assertEquals(2, batches.size()); + assertEquals(RecordBatch.MAGIC_VALUE_V0, batches.get(0).magic()); + assertEquals(0, batches.get(0).baseOffset()); + assertEquals(MAGIC_VALUE_V1, batches.get(1).magic()); + assertEquals(1, batches.get(1).baseOffset()); + assertEquals("1", utf8(logRecords.get(0).key())); + assertEquals("2", utf8(logRecords.get(1).key())); + assertEquals("3", utf8(logRecords.get(2).key())); + verifyRecordsProcessingStats(compressionType, convertedRecords.recordConversionStats(), 3, 2, + records.sizeInBytes(), buffer.limit()); + } else { + assertEquals(2, batches.size()); + assertEquals(RecordBatch.MAGIC_VALUE_V0, batches.get(0).magic()); + assertEquals(0, batches.get(0).baseOffset()); + assertEquals(MAGIC_VALUE_V1, batches.get(1).magic()); + assertEquals(2, batches.get(1).baseOffset()); + assertEquals("1", utf8(logRecords.get(0).key())); + assertEquals("3", utf8(logRecords.get(1).key())); + verifyRecordsProcessingStats(compressionType, convertedRecords.recordConversionStats(), 3, 1, + records.sizeInBytes(), buffer.limit()); + } } } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void shouldThrowIllegalStateExceptionOnBuildWhenAborted(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V0; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - ByteBuffer buffer = allocateBuffer(128, args); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); builder.abort(); @@ -710,12 +688,9 @@ public void shouldThrowIllegalStateExceptionOnBuildWhenAborted(Args args) { @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void shouldResetBufferToInitialPositionOnAbort(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V0; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - ByteBuffer buffer = allocateBuffer(128, args); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); builder.append(0L, "a".getBytes(), "1".getBytes()); @@ -726,41 +701,25 @@ public void shouldResetBufferToInitialPositionOnAbort(Args args) { @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void shouldThrowIllegalStateExceptionOnCloseWhenAborted(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V0; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - ByteBuffer buffer = allocateBuffer(128, args); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); builder.abort(); - try { - builder.close(); - fail("Should have thrown IllegalStateException"); - } catch (IllegalStateException e) { - // ok - } + assertThrows(IllegalStateException.class, builder::close, "Should have thrown IllegalStateException"); } @ParameterizedTest @ArgumentsSource(MemoryRecordsBuilderArgumentsProvider.class) public void shouldThrowIllegalStateExceptionOnAppendWhenAborted(Args args) { - byte magic = RecordBatch.MAGIC_VALUE_V0; - assumeAtLeastV2OrNotZstd(magic, args.compressionType); - ByteBuffer buffer = allocateBuffer(128, args); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, args.compressionType, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity()); builder.abort(); - try { - builder.append(0L, "a".getBytes(), "1".getBytes()); - fail("Should have thrown IllegalStateException"); - } catch (IllegalStateException e) { - // ok - } + assertThrows(IllegalStateException.class, () -> builder.append(0L, "a".getBytes(), "1".getBytes()), "Should have thrown IllegalStateException"); } @ParameterizedTest @@ -778,11 +737,11 @@ public void testBuffersDereferencedOnClose(Args args) { int iterations = 0; while (iterations++ < 100) { buffer.rewind(); - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.MAGIC_VALUE_V2, args.compressionType, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, args.magic, args.compressionType, TimestampType.CREATE_TIME, 0L, 0L, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, false, RecordBatch.NO_PARTITION_LEADER_EPOCH, 0); - builder.append(1L, new byte[0], value); + builder.append(1L, key, value); builder.build(); builders.add(builder); @@ -822,10 +781,6 @@ else if (numRecordsConverted == numRecords) } } - private void assumeAtLeastV2OrNotZstd(byte magic, CompressionType compressionType) { - assumeTrue(compressionType != CompressionType.ZSTD || magic >= MAGIC_VALUE_V2); - } - private ByteBuffer allocateBuffer(int size, Args args) { ByteBuffer buffer = ByteBuffer.allocate(size); buffer.position(args.bufferOffset); diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java index ebac0bd818dfa..26b1c485d8d4d 100644 --- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java +++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.utils.BufferSupplier; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -36,10 +37,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.BiFunction; import java.util.function.Supplier; import java.util.stream.Stream; import static java.util.Arrays.asList; +import static org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V0; +import static org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V1; import static org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V2; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -47,7 +51,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; public class MemoryRecordsTest { @@ -87,21 +90,22 @@ private static class MemoryRecordsArgumentsProvider implements ArgumentsProvider public Stream provideArguments(ExtensionContext context) { List arguments = new ArrayList<>(); for (long firstOffset : asList(0L, 57L)) - for (byte magic : asList(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2)) - for (CompressionType type: CompressionType.values()) + for (CompressionType type: CompressionType.values()) { + List magics = type == CompressionType.ZSTD + ? Collections.singletonList(RecordBatch.MAGIC_VALUE_V2) + : asList(RecordBatch.MAGIC_VALUE_V0, RecordBatch.MAGIC_VALUE_V1, RecordBatch.MAGIC_VALUE_V2); + for (byte magic : magics) arguments.add(Arguments.of(new Args(magic, firstOffset, type))); + } return arguments.stream(); } } private final long logAppendTime = System.currentTimeMillis(); - private final int partitionLeaderEpoch = 998; @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testIterator(Args args) { - assumeAtLeastV2OrNotZstd(args); - CompressionType compression = args.compression; byte magic = args.magic; long pid = args.pid; @@ -110,6 +114,7 @@ public void testIterator(Args args) { long firstOffset = args.firstOffset; ByteBuffer buffer = ByteBuffer.allocate(1024); + int partitionLeaderEpoch = 998; MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, magic, compression, TimestampType.CREATE_TIME, firstOffset, logAppendTime, pid, epoch, firstSequence, false, false, partitionLeaderEpoch, buffer.limit()); @@ -192,7 +197,6 @@ public void testIterator(Args args) { @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testHasRoomForMethod(Args args) { - assumeAtLeastV2OrNotZstd(args); MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), args.magic, args.compression, TimestampType.CREATE_TIME, 0L); builder.append(0L, "a".getBytes(), "1".getBytes()); @@ -205,21 +209,16 @@ public void testHasRoomForMethod(Args args) { @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testHasRoomForMethodWithHeaders(Args args) { byte magic = args.magic; - if (magic >= RecordBatch.MAGIC_VALUE_V2) { - MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(100), magic, args.compression, - TimestampType.CREATE_TIME, 0L); - RecordHeaders headers = new RecordHeaders(); - headers.add("hello", "world.world".getBytes()); - headers.add("hello", "world.world".getBytes()); - headers.add("hello", "world.world".getBytes()); - headers.add("hello", "world.world".getBytes()); - headers.add("hello", "world.world".getBytes()); - builder.append(logAppendTime, "key".getBytes(), "value".getBytes()); - // Make sure that hasRoomFor accounts for header sizes by letting a record without headers pass, but stopping - // a record with a large number of headers. - assertTrue(builder.hasRoomFor(logAppendTime, "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS)); - assertFalse(builder.hasRoomFor(logAppendTime, "key".getBytes(), "value".getBytes(), headers.toArray())); - } + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(120), magic, args.compression, + TimestampType.CREATE_TIME, 0L); + builder.append(logAppendTime, "key".getBytes(), "value".getBytes()); + RecordHeaders headers = new RecordHeaders(); + for (int i = 0; i < 10; ++i) headers.add("hello", "world.world".getBytes()); + // Make sure that hasRoomFor accounts for header sizes by letting a record without headers pass, but stopping + // a record with a large number of headers. + assertTrue(builder.hasRoomFor(logAppendTime, "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS)); + if (magic < MAGIC_VALUE_V2) assertTrue(builder.hasRoomFor(logAppendTime, "key".getBytes(), "value".getBytes(), headers.toArray())); + else assertFalse(builder.hasRoomFor(logAppendTime, "key".getBytes(), "value".getBytes(), headers.toArray())); } /** @@ -265,102 +264,152 @@ public void testChecksum(Args args) { @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testFilterToPreservesPartitionLeaderEpoch(Args args) { byte magic = args.magic; - if (magic >= RecordBatch.MAGIC_VALUE_V2) { - int partitionLeaderEpoch = 67; + int partitionLeaderEpoch = 67; - ByteBuffer buffer = ByteBuffer.allocate(2048); - MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, args.compression, TimestampType.CREATE_TIME, - 0L, RecordBatch.NO_TIMESTAMP, partitionLeaderEpoch); - builder.append(10L, null, "a".getBytes()); - builder.append(11L, "1".getBytes(), "b".getBytes()); - builder.append(12L, null, "c".getBytes()); + ByteBuffer buffer = ByteBuffer.allocate(2048); + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, args.compression, TimestampType.CREATE_TIME, + 0L, RecordBatch.NO_TIMESTAMP, partitionLeaderEpoch); + builder.append(10L, null, "a".getBytes()); + builder.append(11L, "1".getBytes(), "b".getBytes()); + builder.append(12L, null, "c".getBytes()); - ByteBuffer filtered = ByteBuffer.allocate(2048); - builder.build().filterTo(new TopicPartition("foo", 0), new RetainNonNullKeysFilter(), filtered, - Integer.MAX_VALUE, BufferSupplier.NO_CACHING); + ByteBuffer filtered = ByteBuffer.allocate(2048); + builder.build().filterTo(new TopicPartition("foo", 0), new RetainNonNullKeysFilter(), filtered, + Integer.MAX_VALUE, BufferSupplier.NO_CACHING); - filtered.flip(); - MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered); + filtered.flip(); + MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered); - List batches = TestUtils.toList(filteredRecords.batches()); - assertEquals(1, batches.size()); + List batches = TestUtils.toList(filteredRecords.batches()); + assertEquals(1, batches.size()); - MutableRecordBatch firstBatch = batches.get(0); - assertEquals(partitionLeaderEpoch, firstBatch.partitionLeaderEpoch()); - } + MutableRecordBatch firstBatch = batches.get(0); + if (magic < MAGIC_VALUE_V2) assertEquals(RecordBatch.NO_PARTITION_LEADER_EPOCH, firstBatch.partitionLeaderEpoch()); + else assertEquals(partitionLeaderEpoch, firstBatch.partitionLeaderEpoch()); } @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testFilterToEmptyBatchRetention(Args args) { byte magic = args.magic; - if (magic >= RecordBatch.MAGIC_VALUE_V2) { - for (boolean isTransactional : Arrays.asList(true, false)) { - ByteBuffer buffer = ByteBuffer.allocate(2048); - long producerId = 23L; - short producerEpoch = 5; - long baseOffset = 3L; - int baseSequence = 10; - int partitionLeaderEpoch = 293; - int numRecords = 2; - - MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, args.compression, TimestampType.CREATE_TIME, - baseOffset, RecordBatch.NO_TIMESTAMP, producerId, producerEpoch, baseSequence, isTransactional, - partitionLeaderEpoch); + for (boolean isTransactional : Arrays.asList(true, false)) { + ByteBuffer buffer = ByteBuffer.allocate(2048); + long producerId = 23L; + short producerEpoch = 5; + long baseOffset = 3L; + int baseSequence = 10; + int partitionLeaderEpoch = 293; + int numRecords = 2; + + Supplier supplier = () -> MemoryRecords.builder(buffer, magic, args.compression, TimestampType.CREATE_TIME, + baseOffset, RecordBatch.NO_TIMESTAMP, producerId, producerEpoch, baseSequence, isTransactional, + partitionLeaderEpoch); + + if (isTransactional && magic < RecordBatch.MAGIC_VALUE_V2) assertThrows(IllegalArgumentException.class, supplier::get); + else { + MemoryRecordsBuilder builder = supplier.get(); builder.append(11L, "2".getBytes(), "b".getBytes()); builder.append(12L, "3".getBytes(), "c".getBytes()); - builder.close(); - MemoryRecords records = builder.build(); - - ByteBuffer filtered = ByteBuffer.allocate(2048); - MemoryRecords.FilterResult filterResult = records.filterTo(new TopicPartition("foo", 0), - new MemoryRecords.RecordFilter() { - @Override - protected BatchRetention checkBatchRetention(RecordBatch batch) { - // retain all batches - return BatchRetention.RETAIN_EMPTY; - } - - @Override - protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { - // delete the records - return false; - } - }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING); - - // Verify filter result - assertEquals(numRecords, filterResult.messagesRead()); - assertEquals(records.sizeInBytes(), filterResult.bytesRead()); - assertEquals(baseOffset + 1, filterResult.maxOffset()); - assertEquals(0, filterResult.messagesRetained()); - assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filterResult.bytesRetained()); - assertEquals(12, filterResult.maxTimestamp()); - assertEquals(baseOffset + 1, filterResult.shallowOffsetOfMaxTimestamp()); - - // Verify filtered records - filtered.flip(); - MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered); - - List batches = TestUtils.toList(filteredRecords.batches()); - assertEquals(1, batches.size()); - - MutableRecordBatch batch = batches.get(0); - assertEquals(0, batch.countOrNull().intValue()); - assertEquals(12L, batch.maxTimestamp()); - assertEquals(TimestampType.CREATE_TIME, batch.timestampType()); - assertEquals(baseOffset, batch.baseOffset()); - assertEquals(baseOffset + 1, batch.lastOffset()); - assertEquals(baseSequence, batch.baseSequence()); - assertEquals(baseSequence + 1, batch.lastSequence()); - assertEquals(isTransactional, batch.isTransactional()); + if (magic < MAGIC_VALUE_V2) assertThrows(IllegalArgumentException.class, builder::close); + else { + builder.close(); + MemoryRecords records = builder.build(); + ByteBuffer filtered = ByteBuffer.allocate(2048); + MemoryRecords.FilterResult filterResult = records.filterTo(new TopicPartition("foo", 0), + new MemoryRecords.RecordFilter() { + @Override + protected BatchRetention checkBatchRetention(RecordBatch batch) { + // retain all batches + return BatchRetention.RETAIN_EMPTY; + } + + @Override + protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { + // delete the records + return false; + } + }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING); + + // Verify filter result + assertEquals(numRecords, filterResult.messagesRead()); + assertEquals(records.sizeInBytes(), filterResult.bytesRead()); + assertEquals(baseOffset + 1, filterResult.maxOffset()); + assertEquals(0, filterResult.messagesRetained()); + assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filterResult.bytesRetained()); + assertEquals(12, filterResult.maxTimestamp()); + assertEquals(baseOffset + 1, filterResult.shallowOffsetOfMaxTimestamp()); + + // Verify filtered records + filtered.flip(); + MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered); + + List batches = TestUtils.toList(filteredRecords.batches()); + assertEquals(1, batches.size()); + + MutableRecordBatch batch = batches.get(0); + assertEquals(0, batch.countOrNull().intValue()); + assertEquals(12L, batch.maxTimestamp()); + assertEquals(TimestampType.CREATE_TIME, batch.timestampType()); + assertEquals(baseOffset, batch.baseOffset()); + assertEquals(baseOffset + 1, batch.lastOffset()); + assertEquals(baseSequence, batch.baseSequence()); + assertEquals(baseSequence + 1, batch.lastSequence()); + assertEquals(isTransactional, batch.isTransactional()); + } } } } - @ParameterizedTest - @ArgumentsSource(MemoryRecordsArgumentsProvider.class) - public void testEmptyBatchRetention(Args args) { - if (args.magic >= RecordBatch.MAGIC_VALUE_V2) { + @Test + public void testEmptyBatchRetention() { + ByteBuffer buffer = ByteBuffer.allocate(DefaultRecordBatch.RECORD_BATCH_OVERHEAD); + long producerId = 23L; + short producerEpoch = 5; + long baseOffset = 3L; + int baseSequence = 10; + int partitionLeaderEpoch = 293; + long timestamp = System.currentTimeMillis(); + + DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.MAGIC_VALUE_V2, producerId, producerEpoch, + baseSequence, baseOffset, baseOffset, partitionLeaderEpoch, TimestampType.CREATE_TIME, + timestamp, false, false); + buffer.flip(); + + ByteBuffer filtered = ByteBuffer.allocate(2048); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + MemoryRecords.FilterResult filterResult = records.filterTo(new TopicPartition("foo", 0), + new MemoryRecords.RecordFilter() { + @Override + protected BatchRetention checkBatchRetention(RecordBatch batch) { + // retain all batches + return BatchRetention.RETAIN_EMPTY; + } + + @Override + protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { + return false; + } + }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING); + + // Verify filter result + assertEquals(0, filterResult.messagesRead()); + assertEquals(records.sizeInBytes(), filterResult.bytesRead()); + assertEquals(baseOffset, filterResult.maxOffset()); + assertEquals(0, filterResult.messagesRetained()); + assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filterResult.bytesRetained()); + assertEquals(timestamp, filterResult.maxTimestamp()); + assertEquals(baseOffset, filterResult.shallowOffsetOfMaxTimestamp()); + assertTrue(filterResult.outputBuffer().position() > 0); + + // Verify filtered records + filtered.flip(); + MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered); + assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filteredRecords.sizeInBytes()); + } + + @Test + public void testEmptyBatchDeletion() { + for (final BatchRetention deleteRetention : Arrays.asList(BatchRetention.DELETE, BatchRetention.DELETE_EMPTY)) { ByteBuffer buffer = ByteBuffer.allocate(DefaultRecordBatch.RECORD_BATCH_OVERHEAD); long producerId = 23L; short producerEpoch = 5; @@ -380,8 +429,7 @@ public void testEmptyBatchRetention(Args args) { new MemoryRecords.RecordFilter() { @Override protected BatchRetention checkBatchRetention(RecordBatch batch) { - // retain all batches - return BatchRetention.RETAIN_EMPTY; + return deleteRetention; } @Override @@ -391,150 +439,90 @@ protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING); // Verify filter result - assertEquals(0, filterResult.messagesRead()); - assertEquals(records.sizeInBytes(), filterResult.bytesRead()); - assertEquals(baseOffset, filterResult.maxOffset()); - assertEquals(0, filterResult.messagesRetained()); - assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filterResult.bytesRetained()); - assertEquals(timestamp, filterResult.maxTimestamp()); - assertEquals(baseOffset, filterResult.shallowOffsetOfMaxTimestamp()); - assertTrue(filterResult.outputBuffer().position() > 0); + assertEquals(0, filterResult.outputBuffer().position()); // Verify filtered records filtered.flip(); MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered); - assertEquals(DefaultRecordBatch.RECORD_BATCH_OVERHEAD, filteredRecords.sizeInBytes()); + assertEquals(0, filteredRecords.sizeInBytes()); } } - @ParameterizedTest - @ArgumentsSource(MemoryRecordsArgumentsProvider.class) - public void testEmptyBatchDeletion(Args args) { - if (args.magic >= RecordBatch.MAGIC_VALUE_V2) { - for (final BatchRetention deleteRetention : Arrays.asList(BatchRetention.DELETE, BatchRetention.DELETE_EMPTY)) { - ByteBuffer buffer = ByteBuffer.allocate(DefaultRecordBatch.RECORD_BATCH_OVERHEAD); - long producerId = 23L; - short producerEpoch = 5; - long baseOffset = 3L; - int baseSequence = 10; - int partitionLeaderEpoch = 293; - long timestamp = System.currentTimeMillis(); - - DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.MAGIC_VALUE_V2, producerId, producerEpoch, - baseSequence, baseOffset, baseOffset, partitionLeaderEpoch, TimestampType.CREATE_TIME, - timestamp, false, false); - buffer.flip(); - - ByteBuffer filtered = ByteBuffer.allocate(2048); - MemoryRecords records = MemoryRecords.readableRecords(buffer); - MemoryRecords.FilterResult filterResult = records.filterTo(new TopicPartition("foo", 0), - new MemoryRecords.RecordFilter() { - @Override - protected BatchRetention checkBatchRetention(RecordBatch batch) { - return deleteRetention; - } - - @Override - protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { - return false; - } - }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING); - - // Verify filter result - assertEquals(0, filterResult.outputBuffer().position()); - - // Verify filtered records - filtered.flip(); - MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered); - assertEquals(0, filteredRecords.sizeInBytes()); - } - } - } + @Test + public void testBuildEndTxnMarker() { + long producerId = 73; + short producerEpoch = 13; + long initialOffset = 983L; + int coordinatorEpoch = 347; + int partitionLeaderEpoch = 29; + + EndTransactionMarker marker = new EndTransactionMarker(ControlRecordType.COMMIT, coordinatorEpoch); + MemoryRecords records = MemoryRecords.withEndTransactionMarker(initialOffset, System.currentTimeMillis(), + partitionLeaderEpoch, producerId, producerEpoch, marker); + // verify that buffer allocation was precise + assertEquals(records.buffer().remaining(), records.buffer().capacity()); + + List batches = TestUtils.toList(records.batches()); + assertEquals(1, batches.size()); - @ParameterizedTest - @ArgumentsSource(MemoryRecordsArgumentsProvider.class) - public void testBuildEndTxnMarker(Args args) { - if (args.magic >= RecordBatch.MAGIC_VALUE_V2) { - long producerId = 73; - short producerEpoch = 13; - long initialOffset = 983L; - int coordinatorEpoch = 347; - int partitionLeaderEpoch = 29; - - EndTransactionMarker marker = new EndTransactionMarker(ControlRecordType.COMMIT, coordinatorEpoch); - MemoryRecords records = MemoryRecords.withEndTransactionMarker(initialOffset, System.currentTimeMillis(), - partitionLeaderEpoch, producerId, producerEpoch, marker); - // verify that buffer allocation was precise - assertEquals(records.buffer().remaining(), records.buffer().capacity()); - - List batches = TestUtils.toList(records.batches()); - assertEquals(1, batches.size()); - - RecordBatch batch = batches.get(0); - assertTrue(batch.isControlBatch()); - assertEquals(producerId, batch.producerId()); - assertEquals(producerEpoch, batch.producerEpoch()); - assertEquals(initialOffset, batch.baseOffset()); - assertEquals(partitionLeaderEpoch, batch.partitionLeaderEpoch()); - assertTrue(batch.isValid()); - - List createdRecords = TestUtils.toList(batch); - assertEquals(1, createdRecords.size()); - - Record record = createdRecords.get(0); - assertTrue(record.isValid()); - EndTransactionMarker deserializedMarker = EndTransactionMarker.deserialize(record); - assertEquals(ControlRecordType.COMMIT, deserializedMarker.controlType()); - assertEquals(coordinatorEpoch, deserializedMarker.coordinatorEpoch()); - } + RecordBatch batch = batches.get(0); + assertTrue(batch.isControlBatch()); + assertEquals(producerId, batch.producerId()); + assertEquals(producerEpoch, batch.producerEpoch()); + assertEquals(initialOffset, batch.baseOffset()); + assertEquals(partitionLeaderEpoch, batch.partitionLeaderEpoch()); + assertTrue(batch.isValid()); + + List createdRecords = TestUtils.toList(batch); + assertEquals(1, createdRecords.size()); + + Record record = createdRecords.get(0); + assertTrue(record.isValid()); + EndTransactionMarker deserializedMarker = EndTransactionMarker.deserialize(record); + assertEquals(ControlRecordType.COMMIT, deserializedMarker.controlType()); + assertEquals(coordinatorEpoch, deserializedMarker.coordinatorEpoch()); } - @ParameterizedTest - @ArgumentsSource(MemoryRecordsArgumentsProvider.class) - public void testBuildLeaderChangeMessage(Args args) { - if (args.magic >= RecordBatch.MAGIC_VALUE_V2) { - - final int leaderId = 5; - final int leaderEpoch = 20; - final int voterId = 6; - - LeaderChangeMessage leaderChangeMessage = new LeaderChangeMessage() - .setLeaderId(leaderId) - .setVoters(Collections.singletonList( - new Voter().setVoterId(voterId))); - MemoryRecords records = MemoryRecords.withLeaderChangeMessage(System.currentTimeMillis(), - leaderEpoch, leaderChangeMessage); - - List batches = TestUtils.toList(records.batches()); - assertEquals(1, batches.size()); - - RecordBatch batch = batches.get(0); - assertTrue(batch.isControlBatch()); - assertEquals(0, batch.baseOffset()); - assertEquals(leaderEpoch, batch.partitionLeaderEpoch()); - assertTrue(batch.isValid()); - - List createdRecords = TestUtils.toList(batch); - assertEquals(1, createdRecords.size()); - - Record record = createdRecords.get(0); - assertTrue(record.isValid()); - assertEquals(ControlRecordType.LEADER_CHANGE, ControlRecordType.parse(record.key())); - - LeaderChangeMessage deserializedMessage = ControlRecordUtils.deserializeLeaderChangeMessage(record); - assertEquals(leaderId, deserializedMessage.leaderId()); - assertEquals(1, deserializedMessage.voters().size()); - assertEquals(voterId, deserializedMessage.voters().get(0).voterId()); - } + @Test + public void testBuildLeaderChangeMessage() { + final int leaderId = 5; + final int leaderEpoch = 20; + final int voterId = 6; + + LeaderChangeMessage leaderChangeMessage = new LeaderChangeMessage() + .setLeaderId(leaderId) + .setVoters(Collections.singletonList( + new Voter().setVoterId(voterId))); + MemoryRecords records = MemoryRecords.withLeaderChangeMessage(System.currentTimeMillis(), + leaderEpoch, leaderChangeMessage); + + List batches = TestUtils.toList(records.batches()); + assertEquals(1, batches.size()); + + RecordBatch batch = batches.get(0); + assertTrue(batch.isControlBatch()); + assertEquals(0, batch.baseOffset()); + assertEquals(leaderEpoch, batch.partitionLeaderEpoch()); + assertTrue(batch.isValid()); + + List createdRecords = TestUtils.toList(batch); + assertEquals(1, createdRecords.size()); + + Record record = createdRecords.get(0); + assertTrue(record.isValid()); + assertEquals(ControlRecordType.LEADER_CHANGE, ControlRecordType.parse(record.key())); + + LeaderChangeMessage deserializedMessage = ControlRecordUtils.deserializeLeaderChangeMessage(record); + assertEquals(leaderId, deserializedMessage.leaderId()); + assertEquals(1, deserializedMessage.voters().size()); + assertEquals(voterId, deserializedMessage.voters().get(0).voterId()); } @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testFilterToBatchDiscard(Args args) { - assumeAtLeastV2OrNotZstd(args); CompressionType compression = args.compression; byte magic = args.magic; - assumeTrue(compression != CompressionType.NONE || magic >= MAGIC_VALUE_V2); ByteBuffer buffer = ByteBuffer.allocate(2048); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 0L); @@ -578,15 +566,20 @@ protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered); List batches = TestUtils.toList(filteredRecords.batches()); - assertEquals(2, batches.size()); - assertEquals(0L, batches.get(0).lastOffset()); - assertEquals(5L, batches.get(1).lastOffset()); + if (compression != CompressionType.NONE || magic >= MAGIC_VALUE_V2) { + assertEquals(2, batches.size()); + assertEquals(0, batches.get(0).lastOffset()); + assertEquals(5, batches.get(1).lastOffset()); + } else { + assertEquals(5, batches.size()); + assertEquals(0, batches.get(0).lastOffset()); + assertEquals(1, batches.get(1).lastOffset()); + } } @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testFilterToAlreadyCompactedLog(Args args) { - assumeAtLeastV2OrNotZstd(args); byte magic = args.magic; CompressionType compression = args.compression; @@ -638,34 +631,39 @@ public void testFilterToAlreadyCompactedLog(Args args) { public void testFilterToPreservesProducerInfo(Args args) { byte magic = args.magic; CompressionType compression = args.compression; - if (magic >= RecordBatch.MAGIC_VALUE_V2) { - ByteBuffer buffer = ByteBuffer.allocate(2048); - - // non-idempotent, non-transactional - MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 0L); - builder.append(10L, null, "a".getBytes()); - builder.append(11L, "1".getBytes(), "b".getBytes()); - builder.append(12L, null, "c".getBytes()); + ByteBuffer buffer = ByteBuffer.allocate(2048); - builder.close(); + // non-idempotent, non-transactional + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 0L); + builder.append(10L, null, "a".getBytes()); + builder.append(11L, "1".getBytes(), "b".getBytes()); + builder.append(12L, null, "c".getBytes()); - // idempotent - long pid1 = 23L; - short epoch1 = 5; - int baseSequence1 = 10; - builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 3L, - RecordBatch.NO_TIMESTAMP, pid1, epoch1, baseSequence1); - builder.append(13L, null, "d".getBytes()); - builder.append(14L, "4".getBytes(), "e".getBytes()); - builder.append(15L, "5".getBytes(), "f".getBytes()); - builder.close(); + builder.close(); - // transactional - long pid2 = 99384L; - short epoch2 = 234; - int baseSequence2 = 15; - builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 3L, - RecordBatch.NO_TIMESTAMP, pid2, epoch2, baseSequence2, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); + // idempotent + long pid1 = 23L; + short epoch1 = 5; + int baseSequence1 = 10; + MemoryRecordsBuilder idempotentBuilder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 3L, + RecordBatch.NO_TIMESTAMP, pid1, epoch1, baseSequence1); + idempotentBuilder.append(13L, null, "d".getBytes()); + idempotentBuilder.append(14L, "4".getBytes(), "e".getBytes()); + idempotentBuilder.append(15L, "5".getBytes(), "f".getBytes()); + if (magic < MAGIC_VALUE_V2) assertThrows(IllegalArgumentException.class, idempotentBuilder::close); + else idempotentBuilder.close(); + + + // transactional + long pid2 = 99384L; + short epoch2 = 234; + int baseSequence2 = 15; + Supplier transactionSupplier = () -> MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 3L, + RecordBatch.NO_TIMESTAMP, pid2, epoch2, baseSequence2, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); + + if (magic < MAGIC_VALUE_V2) assertThrows(IllegalArgumentException.class, transactionSupplier::get); + else { + builder = transactionSupplier.get(); builder.append(16L, "6".getBytes(), "g".getBytes()); builder.append(17L, "7".getBytes(), "h".getBytes()); builder.append(18L, null, "i".getBytes()); @@ -734,7 +732,6 @@ public void testFilterToPreservesProducerInfo(Args args) { @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testFilterToWithUndersizedBuffer(Args args) { - assumeAtLeastV2OrNotZstd(args); byte magic = args.magic; CompressionType compression = args.compression; @@ -789,7 +786,6 @@ public void testFilterToWithUndersizedBuffer(Args args) { @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testFilterTo(Args args) { - assumeAtLeastV2OrNotZstd(args); byte magic = args.magic; CompressionType compression = args.compression; @@ -907,7 +903,6 @@ public void testFilterTo(Args args) { @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testFilterToPreservesLogAppendTime(Args args) { - assumeAtLeastV2OrNotZstd(args); byte magic = args.magic; CompressionType compression = args.compression; long pid = args.pid; @@ -958,8 +953,6 @@ public void testFilterToPreservesLogAppendTime(Args args) { @ParameterizedTest @ArgumentsSource(MemoryRecordsArgumentsProvider.class) public void testNextBatchSize(Args args) { - assumeAtLeastV2OrNotZstd(args); - ByteBuffer buffer = ByteBuffer.allocate(2048); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, args.magic, args.compression, TimestampType.LOG_APPEND_TIME, 0L, logAppendTime, args.pid, args.epoch, args.firstSequence); @@ -994,19 +987,21 @@ public void testNextBatchSize(Args args) { public void testWithRecords(Args args) { CompressionType compression = args.compression; byte magic = args.magic; - Supplier recordsSupplier = () -> MemoryRecords.withRecords(magic, compression, - new SimpleRecord(10L, "key1".getBytes(), "value1".getBytes())); - if (compression != CompressionType.ZSTD || magic >= MAGIC_VALUE_V2) { - MemoryRecords memoryRecords = recordsSupplier.get(); - String key = Utils.utf8(memoryRecords.batches().iterator().next().iterator().next().key()); - assertEquals("key1", key); - } else { - assertThrows(IllegalArgumentException.class, recordsSupplier::get); - } + MemoryRecords memoryRecords = MemoryRecords.withRecords(magic, compression, + new SimpleRecord(10L, "key1".getBytes(), "value1".getBytes())); + String key = Utils.utf8(memoryRecords.batches().iterator().next().iterator().next().key()); + assertEquals("key1", key); } - private void assumeAtLeastV2OrNotZstd(Args args) { - assumeTrue(args.compression != CompressionType.ZSTD || args.magic >= MAGIC_VALUE_V2); + @Test + public void testUnsupportedCompress() { + BiFunction builderBiFunction = (magic, compressionType) -> + MemoryRecords.withRecords(magic, compressionType, new SimpleRecord(10L, "key1".getBytes(), "value1".getBytes())); + + Arrays.asList(MAGIC_VALUE_V0, MAGIC_VALUE_V1).forEach(magic -> { + Exception e = assertThrows(IllegalArgumentException.class, () -> builderBiFunction.apply(magic, CompressionType.ZSTD)); + assertEquals(e.getMessage(), "ZStandard compression is not supported for magic " + magic); + }); } private static class RetainNonNullKeysFilter extends MemoryRecords.RecordFilter { From a30f92bf59cf31fc1dd36eadc2b9e554e0aef900 Mon Sep 17 00:00:00 2001 From: Ron Dagostino Date: Thu, 18 Feb 2021 00:35:13 -0500 Subject: [PATCH 011/243] MINOR: Add KIP-500 BrokerServer and ControllerServer (#10113) This PR adds the KIP-500 BrokerServer and ControllerServer classes and makes some related changes to get them working. Note that the ControllerServer does not instantiate a QuorumController object yet, since that will be added in PR #10070. * Add BrokerServer and ControllerServer * Change ApiVersions#computeMaxUsableProduceMagic so that it can handle endpoints which do not support PRODUCE (such as KIP-500 controller nodes) * KafkaAdminClientTest: fix some lingering references to decommissionBroker that should be references to unregisterBroker. * Make some changes to allow SocketServer to be used by ControllerServer as we as by the broker. * We now return a random active Broker ID as the Controller ID in MetadataResponse for the Raft-based case as per KIP-590. * Add the RaftControllerNodeProvider * Add EnvelopeUtils * Add MetaLogRaftShim * In ducktape, in config_property.py: use a KIP-500 compatible cluster ID. Reviewers: Colin P. McCabe , David Arthur --- checkstyle/import-control.xml | 1 + .../org/apache/kafka/clients/ApiVersions.java | 13 +- .../apache/kafka/clients/ApiVersionsTest.java | 17 + .../clients/admin/KafkaAdminClientTest.java | 4 +- .../src/main/scala/kafka/cluster/Broker.scala | 2 +- core/src/main/scala/kafka/log/LogConfig.scala | 2 +- .../scala/kafka/network/SocketServer.scala | 37 +- .../kafka/raft/KafkaNetworkChannel.scala | 3 +- .../main/scala/kafka/raft/RaftManager.scala | 2 + .../scala/kafka/server/AlterIsrManager.scala | 5 +- .../server/AutoTopicCreationManager.scala | 15 +- .../scala/kafka/server/BrokerServer.scala | 468 +++++++++++++++++- .../BrokerToControllerChannelManager.scala | 36 ++ .../scala/kafka/server/ControllerApis.scala | 453 +++++++++++++++++ .../scala/kafka/server/ControllerServer.scala | 198 +++++++- .../scala/kafka/server/EnvelopeUtils.scala | 137 +++++ .../main/scala/kafka/server/KafkaApis.scala | 103 +--- .../main/scala/kafka/server/KafkaBroker.scala | 8 + .../main/scala/kafka/server/KafkaConfig.scala | 32 +- .../scala/kafka/server/KafkaRaftServer.scala | 33 +- .../main/scala/kafka/server/KafkaServer.scala | 11 +- .../scala/kafka/server/MetadataSupport.scala | 20 +- core/src/main/scala/kafka/server/Server.scala | 12 + .../scala/kafka/tools/TestRaftServer.scala | 2 +- .../server/AutoTopicCreationManagerTest.scala | 12 +- .../kafka/server/ControllerApisTest.scala | 143 ++++++ .../unit/kafka/server/KafkaApisTest.scala | 40 +- .../unit/kafka/server/KafkaConfigTest.scala | 114 ++++- .../kafka/server/KafkaRaftServerTest.scala | 2 +- .../metadata/MetadataRequestBenchmark.java | 5 +- .../apache/kafka/raft/KafkaRaftClient.java | 21 +- .../org/apache/kafka/raft/NetworkChannel.java | 5 + .../org/apache/kafka/raft/RaftClient.java | 11 +- .../org/apache/kafka/raft/RaftConfig.java | 13 + .../apache/kafka/raft/ReplicatedCounter.java | 2 +- .../kafka/raft/metadata/MetaLogRaftShim.java | 119 +++++ .../apache/kafka/raft/MockNetworkChannel.java | 5 + .../kafka/raft/RaftClientTestContext.java | 2 +- .../services/kafka/config_property.py | 2 +- 39 files changed, 1908 insertions(+), 202 deletions(-) create mode 100644 core/src/main/scala/kafka/server/ControllerApis.scala create mode 100644 core/src/main/scala/kafka/server/EnvelopeUtils.scala create mode 100644 core/src/test/scala/unit/kafka/server/ControllerApisTest.scala create mode 100644 raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index bc0491e2c9c1b..9ec16b9b7c096 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -385,6 +385,7 @@ + diff --git a/clients/src/main/java/org/apache/kafka/clients/ApiVersions.java b/clients/src/main/java/org/apache/kafka/clients/ApiVersions.java index 8001f1c0f97f8..a09d58166b369 100644 --- a/clients/src/main/java/org/apache/kafka/clients/ApiVersions.java +++ b/clients/src/main/java/org/apache/kafka/clients/ApiVersions.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; /** * Maintains node api versions for access outside of NetworkClient (which is where the information is derived). @@ -51,12 +52,12 @@ public synchronized NodeApiVersions get(String nodeId) { private byte computeMaxUsableProduceMagic() { // use a magic version which is supported by all brokers to reduce the chance that // we will need to convert the messages when they are ready to be sent. - byte maxUsableMagic = RecordBatch.CURRENT_MAGIC_VALUE; - for (NodeApiVersions versions : this.nodeApiVersions.values()) { - byte nodeMaxUsableMagic = ProduceRequest.requiredMagicForVersion(versions.latestUsableVersion(ApiKeys.PRODUCE)); - maxUsableMagic = (byte) Math.min(nodeMaxUsableMagic, maxUsableMagic); - } - return maxUsableMagic; + Optional knownBrokerNodesMinRequiredMagicForProduce = this.nodeApiVersions.values().stream() + .filter(versions -> versions.apiVersion(ApiKeys.PRODUCE) != null) // filter out Raft controller nodes + .map(versions -> ProduceRequest.requiredMagicForVersion(versions.latestUsableVersion(ApiKeys.PRODUCE))) + .min(Byte::compare); + return (byte) Math.min(RecordBatch.CURRENT_MAGIC_VALUE, + knownBrokerNodesMinRequiredMagicForProduce.orElse(RecordBatch.CURRENT_MAGIC_VALUE)); } public synchronized byte maxUsableProduceMagic() { diff --git a/clients/src/test/java/org/apache/kafka/clients/ApiVersionsTest.java b/clients/src/test/java/org/apache/kafka/clients/ApiVersionsTest.java index 4a5c98d9d0a35..206e95e4d3074 100644 --- a/clients/src/test/java/org/apache/kafka/clients/ApiVersionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/ApiVersionsTest.java @@ -16,10 +16,13 @@ */ package org.apache.kafka.clients; +import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.record.RecordBatch; import org.junit.jupiter.api.Test; +import java.util.Collections; + import static org.junit.jupiter.api.Assertions.assertEquals; public class ApiVersionsTest { @@ -38,4 +41,18 @@ public void testMaxUsableProduceMagic() { apiVersions.remove("1"); assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, apiVersions.maxUsableProduceMagic()); } + + @Test + public void testMaxUsableProduceMagicWithRaftController() { + ApiVersions apiVersions = new ApiVersions(); + assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, apiVersions.maxUsableProduceMagic()); + + // something that doesn't support PRODUCE, which is the case with Raft-based controllers + apiVersions.update("2", new NodeApiVersions(Collections.singleton( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(ApiKeys.FETCH.id) + .setMinVersion((short) 0) + .setMaxVersion((short) 2)))); + assertEquals(RecordBatch.CURRENT_MAGIC_VALUE, apiVersions.maxUsableProduceMagic()); + } } diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index a5296dae04f5b..ec05d2c002fd1 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -5234,7 +5234,7 @@ public void testUnregisterBrokerTimeoutAndFailureRetry() { } @Test - public void testDecommissionBrokerTimeoutMaxRetry() { + public void testUnregisterBrokerTimeoutMaxRetry() { int nodeId = 1; try (final AdminClientUnitTestEnv env = mockClientEnv(Time.SYSTEM, AdminClientConfig.RETRIES_CONFIG, "1")) { env.kafkaClient().setNodeApiVersions( @@ -5251,7 +5251,7 @@ public void testDecommissionBrokerTimeoutMaxRetry() { } @Test - public void testDecommissionBrokerTimeoutMaxWait() { + public void testUnregisterBrokerTimeoutMaxWait() { int nodeId = 1; try (final AdminClientUnitTestEnv env = mockClientEnv()) { env.kafkaClient().setNodeApiVersions( diff --git a/core/src/main/scala/kafka/cluster/Broker.scala b/core/src/main/scala/kafka/cluster/Broker.scala index 46483d044d95d..657d89b8fe719 100755 --- a/core/src/main/scala/kafka/cluster/Broker.scala +++ b/core/src/main/scala/kafka/cluster/Broker.scala @@ -32,7 +32,7 @@ import scala.collection.Seq import scala.jdk.CollectionConverters._ object Broker { - private[cluster] case class ServerInfo(clusterResource: ClusterResource, + private[kafka] case class ServerInfo(clusterResource: ClusterResource, brokerId: Int, endpoints: util.List[Endpoint], interBrokerEndpoint: Endpoint) extends AuthorizerServerInfo diff --git a/core/src/main/scala/kafka/log/LogConfig.scala b/core/src/main/scala/kafka/log/LogConfig.scala index 4299534bc6638..c2ab1d843fab2 100755 --- a/core/src/main/scala/kafka/log/LogConfig.scala +++ b/core/src/main/scala/kafka/log/LogConfig.scala @@ -228,7 +228,7 @@ object LogConfig { } // Package private for testing, return a copy since it's a mutable global variable - private[log] def configDefCopy: LogConfigDef = new LogConfigDef(configDef) + private[kafka] def configDefCopy: LogConfigDef = new LogConfigDef(configDef) private val configDef: LogConfigDef = { import org.apache.kafka.common.config.ConfigDef.Importance._ diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 905c556a00be5..72c5141445f2a 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -78,12 +78,16 @@ class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time, val credentialProvider: CredentialProvider, - val allowControllerOnlyApis: Boolean = false) + allowControllerOnlyApis: Boolean = false, + controllerSocketServer: Boolean = false) extends Logging with KafkaMetricsGroup with BrokerReconfigurable { private val maxQueuedRequests = config.queuedMaxRequests - private val logContext = new LogContext(s"[SocketServer brokerId=${config.brokerId}] ") + private val nodeId = config.brokerId + + private val logContext = new LogContext(s"[SocketServer ${if (controllerSocketServer) "controller" else "broker"}Id=${nodeId}] ") + this.logIdent = logContext.logPrefix private val memoryPoolSensor = metrics.sensor("MemoryPoolUtilization") @@ -117,11 +121,15 @@ class SocketServer(val config: KafkaConfig, * when processors start up and invoke [[org.apache.kafka.common.network.Selector#poll]]. * * @param startProcessingRequests Flag indicating whether `Processor`s must be started. + * @param controlPlaneListener The control plane listener, or None if there is none. + * @param dataPlaneListeners The data plane listeners. */ - def startup(startProcessingRequests: Boolean = true): Unit = { + def startup(startProcessingRequests: Boolean = true, + controlPlaneListener: Option[EndPoint] = config.controlPlaneListener, + dataPlaneListeners: Seq[EndPoint] = config.dataPlaneListeners): Unit = { this.synchronized { - createControlPlaneAcceptorAndProcessor(config.controlPlaneListener) - createDataPlaneAcceptorsAndProcessors(config.numNetworkThreads, config.dataPlaneListeners) + createControlPlaneAcceptorAndProcessor(controlPlaneListener) + createDataPlaneAcceptorsAndProcessors(config.numNetworkThreads, dataPlaneListeners) if (startProcessingRequests) { this.startProcessingRequests() } @@ -224,9 +232,11 @@ class SocketServer(val config: KafkaConfig, private def startDataPlaneProcessorsAndAcceptors(authorizerFutures: Map[Endpoint, CompletableFuture[Void]]): Unit = { val interBrokerListener = dataPlaneAcceptors.asScala.keySet .find(_.listenerName == config.interBrokerListenerName) - .getOrElse(throw new IllegalStateException(s"Inter-broker listener ${config.interBrokerListenerName} not found, endpoints=${dataPlaneAcceptors.keySet}")) - val orderedAcceptors = List(dataPlaneAcceptors.get(interBrokerListener)) ++ - dataPlaneAcceptors.asScala.filter { case (k, _) => k != interBrokerListener }.values + val orderedAcceptors = interBrokerListener match { + case Some(interBrokerListener) => List(dataPlaneAcceptors.get(interBrokerListener)) ++ + dataPlaneAcceptors.asScala.filter { case (k, _) => k != interBrokerListener }.values + case None => dataPlaneAcceptors.asScala.values + } orderedAcceptors.foreach { acceptor => val endpoint = acceptor.endPoint startAcceptorAndProcessors(DataPlaneThreadPrefix, endpoint, acceptor, authorizerFutures) @@ -276,8 +286,7 @@ class SocketServer(val config: KafkaConfig, private def createAcceptor(endPoint: EndPoint, metricPrefix: String) : Acceptor = { val sendBufferSize = config.socketSendBufferBytes val recvBufferSize = config.socketReceiveBufferBytes - val brokerId = config.brokerId - new Acceptor(endPoint, sendBufferSize, recvBufferSize, brokerId, connectionQuotas, metricPrefix, time) + new Acceptor(endPoint, sendBufferSize, recvBufferSize, nodeId, connectionQuotas, metricPrefix, time) } private def addDataPlaneProcessors(acceptor: Acceptor, endpoint: EndPoint, newProcessorsPerListener: Int): Unit = { @@ -540,11 +549,13 @@ private[kafka] abstract class AbstractServerThread(connectionQuotas: ConnectionQ private[kafka] class Acceptor(val endPoint: EndPoint, val sendBufferSize: Int, val recvBufferSize: Int, - brokerId: Int, + nodeId: Int, connectionQuotas: ConnectionQuotas, metricPrefix: String, - time: Time) extends AbstractServerThread(connectionQuotas) with KafkaMetricsGroup { + time: Time, + logPrefix: String = "") extends AbstractServerThread(connectionQuotas) with KafkaMetricsGroup { + this.logIdent = logPrefix private val nioSelector = NSelector.open() val serverChannel = openServerSocket(endPoint.host, endPoint.port) private val processors = new ArrayBuffer[Processor]() @@ -573,7 +584,7 @@ private[kafka] class Acceptor(val endPoint: EndPoint, private def startProcessors(processors: Seq[Processor], processorThreadPrefix: String): Unit = synchronized { processors.foreach { processor => KafkaThread.nonDaemon( - s"${processorThreadPrefix}-kafka-network-thread-$brokerId-${endPoint.listenerName}-${endPoint.securityProtocol}-${processor.id}", + s"${processorThreadPrefix}-kafka-network-thread-$nodeId-${endPoint.listenerName}-${endPoint.securityProtocol}-${processor.id}", processor ).start() } diff --git a/core/src/main/scala/kafka/raft/KafkaNetworkChannel.scala b/core/src/main/scala/kafka/raft/KafkaNetworkChannel.scala index f3b7f11b012f7..68f7b4a87fb72 100644 --- a/core/src/main/scala/kafka/raft/KafkaNetworkChannel.scala +++ b/core/src/main/scala/kafka/raft/KafkaNetworkChannel.scala @@ -165,7 +165,7 @@ class KafkaNetworkChannel( RaftUtil.errorResponse(apiKey, error) } - def updateEndpoint(id: Int, spec: InetAddressSpec): Unit = { + override def updateEndpoint(id: Int, spec: InetAddressSpec): Unit = { val node = new Node(id, spec.address.getHostString, spec.address.getPort) endpoints.put(id, node) } @@ -181,5 +181,4 @@ class KafkaNetworkChannel( override def close(): Unit = { requestThread.shutdown() } - } diff --git a/core/src/main/scala/kafka/raft/RaftManager.scala b/core/src/main/scala/kafka/raft/RaftManager.scala index b9a77b702db9b..6a74c27bf06c2 100644 --- a/core/src/main/scala/kafka/raft/RaftManager.scala +++ b/core/src/main/scala/kafka/raft/RaftManager.scala @@ -121,6 +121,8 @@ class KafkaRaftManager[T]( private val raftClient = buildRaftClient() private val raftIoThread = new RaftIoThread(raftClient, threadNamePrefix) + def kafkaRaftClient: KafkaRaftClient[T] = raftClient + def startup(): Unit = { // Update the voter endpoints (if valid) with what's in RaftConfig val voterAddresses: util.Map[Integer, AddressSpec] = raftConfig.quorumVoterConnections diff --git a/core/src/main/scala/kafka/server/AlterIsrManager.scala b/core/src/main/scala/kafka/server/AlterIsrManager.scala index 70c0fc2df06bf..b58ca89da4045 100644 --- a/core/src/main/scala/kafka/server/AlterIsrManager.scala +++ b/core/src/main/scala/kafka/server/AlterIsrManager.scala @@ -70,7 +70,8 @@ object AlterIsrManager { time: Time, metrics: Metrics, threadNamePrefix: Option[String], - brokerEpochSupplier: () => Long + brokerEpochSupplier: () => Long, + brokerId: Int ): AlterIsrManager = { val nodeProvider = MetadataCacheControllerNodeProvider(config, metadataCache) @@ -87,7 +88,7 @@ object AlterIsrManager { controllerChannelManager = channelManager, scheduler = scheduler, time = time, - brokerId = config.brokerId, + brokerId = brokerId, brokerEpochSupplier = brokerEpochSupplier ) } diff --git a/core/src/main/scala/kafka/server/AutoTopicCreationManager.scala b/core/src/main/scala/kafka/server/AutoTopicCreationManager.scala index ec7f2df3e6cdc..01dabedf75428 100644 --- a/core/src/main/scala/kafka/server/AutoTopicCreationManager.scala +++ b/core/src/main/scala/kafka/server/AutoTopicCreationManager.scala @@ -61,8 +61,8 @@ object AutoTopicCreationManager { time: Time, metrics: Metrics, threadNamePrefix: Option[String], - adminManager: ZkAdminManager, - controller: KafkaController, + adminManager: Option[ZkAdminManager], + controller: Option[KafkaController], groupCoordinator: GroupCoordinator, txnCoordinator: TransactionCoordinator, enableForwarding: Boolean @@ -91,11 +91,14 @@ class DefaultAutoTopicCreationManager( config: KafkaConfig, metadataCache: MetadataCache, channelManager: Option[BrokerToControllerChannelManager], - adminManager: ZkAdminManager, - controller: KafkaController, + adminManager: Option[ZkAdminManager], + controller: Option[KafkaController], groupCoordinator: GroupCoordinator, txnCoordinator: TransactionCoordinator ) extends AutoTopicCreationManager with Logging { + if (controller.isEmpty && channelManager.isEmpty) { + throw new IllegalArgumentException("Must supply a channel manager if not supplying a controller") + } private val inflightTopics = Collections.newSetFromMap(new ConcurrentHashMap[String, java.lang.Boolean]()) @@ -116,7 +119,7 @@ class DefaultAutoTopicCreationManager( val creatableTopicResponses = if (creatableTopics.isEmpty) { Seq.empty - } else if (!controller.isActive && channelManager.isDefined) { + } else if (controller.isEmpty || !controller.get.isActive && channelManager.isDefined) { sendCreateTopicRequest(creatableTopics) } else { createTopicsInZk(creatableTopics, controllerMutationQuota) @@ -133,7 +136,7 @@ class DefaultAutoTopicCreationManager( try { // Note that we use timeout = 0 since we do not need to wait for metadata propagation // and we want to get the response error immediately. - adminManager.createTopics( + adminManager.get.createTopics( timeout = 0, validateOnly = false, creatableTopics, diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index 90f95ed2c4301..57ceb46202fbc 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -1,10 +1,10 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -14,14 +14,464 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package kafka.server +import java.util +import java.util.concurrent.{CompletableFuture, TimeUnit, TimeoutException} +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.locks.ReentrantLock + +import kafka.cluster.Broker.ServerInfo +import kafka.coordinator.group.GroupCoordinator +import kafka.coordinator.transaction.{ProducerIdGenerator, TransactionCoordinator} +import kafka.log.LogManager +import kafka.metrics.KafkaYammerMetrics +import kafka.network.SocketServer +import kafka.security.CredentialProvider +import kafka.server.KafkaBroker.metricsPrefix +import kafka.server.metadata.{BrokerMetadataListener, CachedConfigRepository, ClientQuotaCache, ClientQuotaMetadataManager, RaftMetadataCache} +import kafka.utils.{CoreUtils, KafkaScheduler} +import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.message.BrokerRegistrationRequestData.{Listener, ListenerCollection} +import org.apache.kafka.common.metrics.Metrics +import org.apache.kafka.common.network.ListenerName +import org.apache.kafka.common.security.auth.SecurityProtocol +import org.apache.kafka.common.security.scram.internals.ScramMechanism +import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache +import org.apache.kafka.common.utils.{AppInfoParser, LogContext, Time} +import org.apache.kafka.common.{ClusterResource, Endpoint, KafkaException} +import org.apache.kafka.metadata.{BrokerState, VersionRange} +import org.apache.kafka.metalog.MetaLogManager +import org.apache.kafka.raft.RaftConfig +import org.apache.kafka.server.authorizer.Authorizer + +import scala.collection.{Map, Seq} +import scala.jdk.CollectionConverters._ + /** - * Stubbed implementation of the KIP-500 broker which processes state - * from the `@metadata` topic which is replicated through Raft. + * A KIP-500 Kafka broker. */ -class BrokerServer { - def startup(): Unit = ??? - def shutdown(): Unit = ??? - def awaitShutdown(): Unit = ??? +class BrokerServer( + val config: KafkaConfig, + val metaProps: MetaProperties, + val metaLogManager: MetaLogManager, + val time: Time, + val metrics: Metrics, + val threadNamePrefix: Option[String], + val initialOfflineDirs: Seq[String], + val controllerQuorumVotersFuture: CompletableFuture[util.List[String]], + val supportedFeatures: util.Map[String, VersionRange] + ) extends KafkaBroker { + + import kafka.server.Server._ + + private val logContext: LogContext = new LogContext(s"[BrokerServer id=${config.nodeId}] ") + + this.logIdent = logContext.logPrefix + + val lifecycleManager: BrokerLifecycleManager = + new BrokerLifecycleManager(config, time, threadNamePrefix) + + private val isShuttingDown = new AtomicBoolean(false) + + val lock = new ReentrantLock() + val awaitShutdownCond = lock.newCondition() + var status: ProcessStatus = SHUTDOWN + + var dataPlaneRequestProcessor: KafkaApis = null + var controlPlaneRequestProcessor: KafkaApis = null + + var authorizer: Option[Authorizer] = None + var socketServer: SocketServer = null + var dataPlaneRequestHandlerPool: KafkaRequestHandlerPool = null + var controlPlaneRequestHandlerPool: KafkaRequestHandlerPool = null + + var logDirFailureChannel: LogDirFailureChannel = null + var logManager: LogManager = null + + var tokenManager: DelegationTokenManager = null + + var replicaManager: RaftReplicaManager = null + + var credentialProvider: CredentialProvider = null + var tokenCache: DelegationTokenCache = null + + var groupCoordinator: GroupCoordinator = null + + var transactionCoordinator: TransactionCoordinator = null + + var forwardingManager: ForwardingManager = null + + var alterIsrManager: AlterIsrManager = null + + var autoTopicCreationManager: AutoTopicCreationManager = null + + var kafkaScheduler: KafkaScheduler = null + + var metadataCache: RaftMetadataCache = null + + var quotaManagers: QuotaFactory.QuotaManagers = null + var quotaCache: ClientQuotaCache = null + + private var _brokerTopicStats: BrokerTopicStats = null + + val brokerFeatures: BrokerFeatures = BrokerFeatures.createDefault() + + val featureCache: FinalizedFeatureCache = new FinalizedFeatureCache(brokerFeatures) + + val clusterId: String = metaProps.clusterId.toString + + val configRepository = new CachedConfigRepository() + + var brokerMetadataListener: BrokerMetadataListener = null + + def kafkaYammerMetrics: kafka.metrics.KafkaYammerMetrics = KafkaYammerMetrics.INSTANCE + + private[kafka] def brokerTopicStats = _brokerTopicStats + + private def maybeChangeStatus(from: ProcessStatus, to: ProcessStatus): Boolean = { + lock.lock() + try { + if (status != from) return false + status = to + if (to == SHUTTING_DOWN) { + isShuttingDown.set(true) + } else if (to == SHUTDOWN) { + isShuttingDown.set(false) + awaitShutdownCond.signalAll() + } + } finally { + lock.unlock() + } + true + } + + def startup(): Unit = { + if (!maybeChangeStatus(SHUTDOWN, STARTING)) return + try { + info("Starting broker") + + /* start scheduler */ + kafkaScheduler = new KafkaScheduler(config.backgroundThreads) + kafkaScheduler.startup() + + /* register broker metrics */ + _brokerTopicStats = new BrokerTopicStats + + quotaManagers = QuotaFactory.instantiate(config, metrics, time, threadNamePrefix.getOrElse("")) + quotaCache = new ClientQuotaCache() + + logDirFailureChannel = new LogDirFailureChannel(config.logDirs.size) + + // Create log manager, but don't start it because we need to delay any potential unclean shutdown log recovery + // until we catch up on the metadata log and have up-to-date topic and broker configs. + logManager = LogManager(config, initialOfflineDirs, configRepository, kafkaScheduler, time, + brokerTopicStats, logDirFailureChannel, true) + + metadataCache = MetadataCache.raftMetadataCache(config.nodeId) + // Enable delegation token cache for all SCRAM mechanisms to simplify dynamic update. + // This keeps the cache up-to-date if new SCRAM mechanisms are enabled dynamically. + tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) + credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) + + // Create and start the socket server acceptor threads so that the bound port is known. + // Delay starting processors until the end of the initialization sequence to ensure + // that credentials have been loaded before processing authentications. + socketServer = new SocketServer(config, metrics, time, credentialProvider, allowControllerOnlyApis = false) + socketServer.startup(startProcessingRequests = false) + + val controllerNodes = + RaftConfig.quorumVoterStringsToNodes(controllerQuorumVotersFuture.get()).asScala + val controllerNodeProvider = RaftControllerNodeProvider(metaLogManager, config, controllerNodes) + val alterIsrChannelManager = BrokerToControllerChannelManager(controllerNodeProvider, + time, metrics, config, "alterisr", threadNamePrefix, 60000) + alterIsrManager = new DefaultAlterIsrManager( + controllerChannelManager = alterIsrChannelManager, + scheduler = kafkaScheduler, + time = time, + brokerId = config.nodeId, + brokerEpochSupplier = () => lifecycleManager.brokerEpoch() + ) + alterIsrManager.start() + + this.replicaManager = new RaftReplicaManager(config, metrics, time, + kafkaScheduler, logManager, isShuttingDown, quotaManagers, + brokerTopicStats, metadataCache, logDirFailureChannel, alterIsrManager, + configRepository, threadNamePrefix) + + val forwardingChannelManager = BrokerToControllerChannelManager(controllerNodeProvider, + time, metrics, config, "forwarding", threadNamePrefix, 60000) + forwardingManager = new ForwardingManagerImpl(forwardingChannelManager) + forwardingManager.start() + + /* start token manager */ + if (config.tokenAuthEnabled) { + throw new UnsupportedOperationException("Delegation tokens are not supported") + } + tokenManager = new DelegationTokenManager(config, tokenCache, time , null) + tokenManager.startup() // does nothing, we just need a token manager in order to compile right now... + + // Create group coordinator, but don't start it until we've started replica manager. + // Hardcode Time.SYSTEM for now as some Streams tests fail otherwise, it would be good to fix the underlying issue + groupCoordinator = GroupCoordinator(config, replicaManager, Time.SYSTEM, metrics) + + // Create transaction coordinator, but don't start it until we've started replica manager. + // Hardcode Time.SYSTEM for now as some Streams tests fail otherwise, it would be good to fix the underlying issue + transactionCoordinator = TransactionCoordinator(config, replicaManager, + new KafkaScheduler(threads = 1, threadNamePrefix = "transaction-log-manager-"), + createTemporaryProducerIdManager, metrics, metadataCache, Time.SYSTEM) + + val autoTopicCreationChannelManager = BrokerToControllerChannelManager(controllerNodeProvider, + time, metrics, config, "autocreate", threadNamePrefix, 60000) + autoTopicCreationManager = new DefaultAutoTopicCreationManager( + config, metadataCache, Some(autoTopicCreationChannelManager), None, None, + groupCoordinator, transactionCoordinator) + autoTopicCreationManager.start() + + /* Add all reconfigurables for config change notification before starting the metadata listener */ + config.dynamicConfig.addReconfigurables(this) + + val clientQuotaMetadataManager = new ClientQuotaMetadataManager( + quotaManagers, socketServer.connectionQuotas, quotaCache) + + brokerMetadataListener = new BrokerMetadataListener( + config.nodeId, + time, + metadataCache, + configRepository, + groupCoordinator, + replicaManager, + transactionCoordinator, + logManager, + threadNamePrefix, + clientQuotaMetadataManager) + + val networkListeners = new ListenerCollection() + config.advertisedListeners.foreach { ep => + networkListeners.add(new Listener(). + setHost(ep.host). + setName(ep.listenerName.value()). + setPort(socketServer.boundPort(ep.listenerName)). + setSecurityProtocol(ep.securityProtocol.id)) + } + lifecycleManager.start(() => brokerMetadataListener.highestMetadataOffset(), + BrokerToControllerChannelManager(controllerNodeProvider, time, metrics, config, + "heartbeat", threadNamePrefix, config.brokerSessionTimeoutMs.toLong), + metaProps.clusterId, networkListeners, supportedFeatures) + + // Register a listener with the Raft layer to receive metadata event notifications + metaLogManager.register(brokerMetadataListener) + + val endpoints = new util.ArrayList[Endpoint](networkListeners.size()) + var interBrokerListener: Endpoint = null + networkListeners.iterator().forEachRemaining(listener => { + val endPoint = new Endpoint(listener.name(), + SecurityProtocol.forId(listener.securityProtocol()), + listener.host(), listener.port()) + endpoints.add(endPoint) + if (listener.name().equals(config.interBrokerListenerName.value())) { + interBrokerListener = endPoint + } + }) + if (interBrokerListener == null) { + throw new RuntimeException("Unable to find inter-broker listener " + + config.interBrokerListenerName.value() + ". Found listener(s): " + + endpoints.asScala.map(ep => ep.listenerName().orElse("(none)")).mkString(", ")) + } + val authorizerInfo = ServerInfo(new ClusterResource(clusterId), + config.nodeId, endpoints, interBrokerListener) + + /* Get the authorizer and initialize it if one is specified.*/ + authorizer = config.authorizer + authorizer.foreach(_.configure(config.originals)) + val authorizerFutures: Map[Endpoint, CompletableFuture[Void]] = authorizer match { + case Some(authZ) => + authZ.start(authorizerInfo).asScala.map { case (ep, cs) => + ep -> cs.toCompletableFuture + } + case None => + authorizerInfo.endpoints.asScala.map { ep => + ep -> CompletableFuture.completedFuture[Void](null) + }.toMap + } + + val fetchManager = new FetchManager(Time.SYSTEM, + new FetchSessionCache(config.maxIncrementalFetchSessionCacheSlots, + KafkaServer.MIN_INCREMENTAL_FETCH_SESSION_EVICTION_MS)) + + // Start processing requests once we've caught up on the metadata log, recovered logs if necessary, + // and started all services that we previously delayed starting. + val raftSupport = RaftSupport(forwardingManager, metadataCache) + dataPlaneRequestProcessor = new KafkaApis(socketServer.dataPlaneRequestChannel, raftSupport, + replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, + config.nodeId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, + fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + + dataPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.nodeId, socketServer.dataPlaneRequestChannel, dataPlaneRequestProcessor, time, + config.numIoThreads, s"${SocketServer.DataPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.DataPlaneThreadPrefix) + + socketServer.controlPlaneRequestChannelOpt.foreach { controlPlaneRequestChannel => + controlPlaneRequestProcessor = new KafkaApis(controlPlaneRequestChannel, raftSupport, + replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, + config.nodeId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, + fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + + controlPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.nodeId, socketServer.controlPlaneRequestChannelOpt.get, controlPlaneRequestProcessor, time, + 1, s"${SocketServer.ControlPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.ControlPlaneThreadPrefix) + } + + // Block until we've caught up on the metadata log + lifecycleManager.initialCatchUpFuture.get() + // Start log manager, which will perform (potentially lengthy) recovery-from-unclean-shutdown if required. + logManager.startup(metadataCache.getAllTopics()) + // Start other services that we've delayed starting, in the appropriate order. + replicaManager.endMetadataChangeDeferral( + RequestHandlerHelper.onLeadershipChange(groupCoordinator, transactionCoordinator, _, _)) + replicaManager.startup() + replicaManager.startHighWatermarkCheckPointThread() + groupCoordinator.startup(() => metadataCache.numPartitions(Topic.GROUP_METADATA_TOPIC_NAME). + getOrElse(config.offsetsTopicPartitions)) + transactionCoordinator.startup(() => metadataCache.numPartitions(Topic.TRANSACTION_STATE_TOPIC_NAME). + getOrElse(config.transactionTopicPartitions)) + + socketServer.startProcessingRequests(authorizerFutures) + + // We're now ready to unfence the broker. + lifecycleManager.setReadyToUnfence() + + maybeChangeStatus(STARTING, STARTED) + } catch { + case e: Throwable => + maybeChangeStatus(STARTING, STARTED) + fatal("Fatal error during broker startup. Prepare to shutdown", e) + shutdown() + throw e + } + } + + class TemporaryProducerIdManager() extends ProducerIdGenerator { + val maxProducerIdsPerBrokerEpoch = 1000000 + var currentOffset = -1 + override def generateProducerId(): Long = { + currentOffset = currentOffset + 1 + if (currentOffset >= maxProducerIdsPerBrokerEpoch) { + fatal(s"Exhausted all demo/temporary producerIds as the next one will has extend past the block size of $maxProducerIdsPerBrokerEpoch") + throw new KafkaException("Have exhausted all demo/temporary producerIds.") + } + lifecycleManager.initialCatchUpFuture.get() + lifecycleManager.brokerEpoch() * maxProducerIdsPerBrokerEpoch + currentOffset + } + } + + def createTemporaryProducerIdManager(): ProducerIdGenerator = { + new TemporaryProducerIdManager() + } + + def shutdown(): Unit = { + if (!maybeChangeStatus(STARTED, SHUTTING_DOWN)) return + try { + info("shutting down") + + if (config.controlledShutdownEnable) { + lifecycleManager.beginControlledShutdown() + try { + lifecycleManager.controlledShutdownFuture.get(5L, TimeUnit.MINUTES) + } catch { + case _: TimeoutException => + error("Timed out waiting for the controller to approve controlled shutdown") + case e: Throwable => + error("Got unexpected exception waiting for controlled shutdown future", e) + } + } + lifecycleManager.beginShutdown() + + // Stop socket server to stop accepting any more connections and requests. + // Socket server will be shutdown towards the end of the sequence. + if (socketServer != null) { + CoreUtils.swallow(socketServer.stopProcessingRequests(), this) + } + if (dataPlaneRequestHandlerPool != null) + CoreUtils.swallow(dataPlaneRequestHandlerPool.shutdown(), this) + if (controlPlaneRequestHandlerPool != null) + CoreUtils.swallow(controlPlaneRequestHandlerPool.shutdown(), this) + if (kafkaScheduler != null) + CoreUtils.swallow(kafkaScheduler.shutdown(), this) + + if (dataPlaneRequestProcessor != null) + CoreUtils.swallow(dataPlaneRequestProcessor.close(), this) + if (controlPlaneRequestProcessor != null) + CoreUtils.swallow(controlPlaneRequestProcessor.close(), this) + CoreUtils.swallow(authorizer.foreach(_.close()), this) + + if (brokerMetadataListener != null) { + CoreUtils.swallow(brokerMetadataListener.close(), this) + } + if (transactionCoordinator != null) + CoreUtils.swallow(transactionCoordinator.shutdown(), this) + if (groupCoordinator != null) + CoreUtils.swallow(groupCoordinator.shutdown(), this) + + if (tokenManager != null) + CoreUtils.swallow(tokenManager.shutdown(), this) + + if (replicaManager != null) + CoreUtils.swallow(replicaManager.shutdown(), this) + + if (alterIsrManager != null) + CoreUtils.swallow(alterIsrManager.shutdown(), this) + + if (forwardingManager != null) + CoreUtils.swallow(forwardingManager.shutdown(), this) + + if (autoTopicCreationManager != null) + CoreUtils.swallow(autoTopicCreationManager.shutdown(), this) + + if (logManager != null) + CoreUtils.swallow(logManager.shutdown(), this) + + if (quotaManagers != null) + CoreUtils.swallow(quotaManagers.shutdown(), this) + + if (socketServer != null) + CoreUtils.swallow(socketServer.shutdown(), this) + if (metrics != null) + CoreUtils.swallow(metrics.close(), this) + if (brokerTopicStats != null) + CoreUtils.swallow(brokerTopicStats.close(), this) + + // Clear all reconfigurable instances stored in DynamicBrokerConfig + config.dynamicConfig.clear() + + isShuttingDown.set(false) + + CoreUtils.swallow(lifecycleManager.close(), this) + + CoreUtils.swallow(AppInfoParser.unregisterAppInfo(metricsPrefix, config.nodeId.toString, metrics), this) + info("shut down completed") + } catch { + case e: Throwable => + fatal("Fatal error during broker shutdown.", e) + throw e + } finally { + maybeChangeStatus(SHUTTING_DOWN, SHUTDOWN) + } + } + + def awaitShutdown(): Unit = { + lock.lock() + try { + while (true) { + if (status == SHUTDOWN) return + awaitShutdownCond.awaitUninterruptibly() + } + } finally { + lock.unlock() + } + } + + def boundPort(listenerName: ListenerName): Int = socketServer.boundPort(listenerName) + + def currentState(): BrokerState = lifecycleManager.state() + } diff --git a/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala b/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala index 1e7af76aeb457..3b535220994dc 100644 --- a/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala +++ b/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala @@ -31,7 +31,9 @@ import org.apache.kafka.common.requests.AbstractRequest import org.apache.kafka.common.security.JaasContext import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.{LogContext, Time} +import org.apache.kafka.metalog.MetaLogManager +import scala.collection.Seq import scala.jdk.CollectionConverters._ trait ControllerNodeProvider { @@ -71,6 +73,40 @@ class MetadataCacheControllerNodeProvider( } } +object RaftControllerNodeProvider { + def apply(metaLogManager: MetaLogManager, + config: KafkaConfig, + controllerQuorumVoterNodes: Seq[Node]): RaftControllerNodeProvider = { + + val listenerName = new ListenerName(config.controllerListenerNames.head) + val securityProtocol = config.listenerSecurityProtocolMap.getOrElse(listenerName, SecurityProtocol.forName(listenerName.value())) + new RaftControllerNodeProvider(metaLogManager, controllerQuorumVoterNodes, listenerName, securityProtocol) + } +} + +/** + * Finds the controller node by checking the metadata log manager. + * This provider is used when we are using a Raft-based metadata quorum. + */ +class RaftControllerNodeProvider(val metaLogManager: MetaLogManager, + controllerQuorumVoterNodes: Seq[Node], + val listenerName: ListenerName, + val securityProtocol: SecurityProtocol + ) extends ControllerNodeProvider with Logging { + val idToNode = controllerQuorumVoterNodes.map(node => node.id() -> node).toMap + + override def get(): Option[Node] = { + val leader = metaLogManager.leader() + if (leader == null) { + None + } else if (leader.nodeId() < 0) { + None + } else { + idToNode.get(leader.nodeId()) + } + } +} + object BrokerToControllerChannelManager { def apply( controllerNodeProvider: ControllerNodeProvider, diff --git a/core/src/main/scala/kafka/server/ControllerApis.scala b/core/src/main/scala/kafka/server/ControllerApis.scala new file mode 100644 index 0000000000000..2386da5d0b48c --- /dev/null +++ b/core/src/main/scala/kafka/server/ControllerApis.scala @@ -0,0 +1,453 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server + +import java.util + +import kafka.network.RequestChannel +import kafka.raft.RaftManager +import kafka.server.QuotaFactory.QuotaManagers +import kafka.utils.Logging +import org.apache.kafka.clients.admin.AlterConfigOp +import org.apache.kafka.common.acl.AclOperation.{ALTER, ALTER_CONFIGS, CLUSTER_ACTION, CREATE, DESCRIBE} +import org.apache.kafka.common.config.ConfigResource +import org.apache.kafka.common.errors.ApiException +import org.apache.kafka.common.internals.FatalExitError +import org.apache.kafka.common.message.ApiVersionsResponseData.{ApiVersion, SupportedFeatureKey} +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection +import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseBroker +import org.apache.kafka.common.message.{ApiVersionsResponseData, BeginQuorumEpochResponseData, BrokerHeartbeatResponseData, BrokerRegistrationResponseData, CreateTopicsResponseData, DescribeQuorumResponseData, EndQuorumEpochResponseData, FetchResponseData, MetadataResponseData, UnregisterBrokerResponseData, VoteResponseData} +import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage, Errors} +import org.apache.kafka.common.record.BaseRecords +import org.apache.kafka.common.requests._ +import org.apache.kafka.common.resource.Resource +import org.apache.kafka.common.resource.Resource.CLUSTER_NAME +import org.apache.kafka.common.resource.ResourceType.{CLUSTER, TOPIC} +import org.apache.kafka.common.utils.Time +import org.apache.kafka.common.Node +import org.apache.kafka.controller.Controller +import org.apache.kafka.metadata.{ApiMessageAndVersion, BrokerHeartbeatReply, BrokerRegistrationReply, FeatureMap, FeatureMapAndEpoch, VersionRange} +import org.apache.kafka.server.authorizer.Authorizer + +import scala.collection.mutable +import scala.jdk.CollectionConverters._ + +/** + * Request handler for Controller APIs + */ +class ControllerApis(val requestChannel: RequestChannel, + val authorizer: Option[Authorizer], + val quotas: QuotaManagers, + val time: Time, + val supportedFeatures: Map[String, VersionRange], + val controller: Controller, + val raftManager: RaftManager[ApiMessageAndVersion], + val config: KafkaConfig, + val metaProperties: MetaProperties, + val controllerNodes: Seq[Node]) extends ApiRequestHandler with Logging { + + val authHelper = new AuthHelper(authorizer) + val requestHelper = new RequestHandlerHelper(requestChannel, quotas, time, s"[ControllerApis id=${config.nodeId}] ") + + var supportedApiKeys = Set( + ApiKeys.FETCH, + ApiKeys.METADATA, + //ApiKeys.SASL_HANDSHAKE + ApiKeys.API_VERSIONS, + ApiKeys.CREATE_TOPICS, + //ApiKeys.DELETE_TOPICS, + //ApiKeys.DESCRIBE_ACLS, + //ApiKeys.CREATE_ACLS, + //ApiKeys.DELETE_ACLS, + //ApiKeys.DESCRIBE_CONFIGS, + //ApiKeys.ALTER_CONFIGS, + //ApiKeys.SASL_AUTHENTICATE, + //ApiKeys.CREATE_PARTITIONS, + //ApiKeys.CREATE_DELEGATION_TOKEN + //ApiKeys.RENEW_DELEGATION_TOKEN + //ApiKeys.EXPIRE_DELEGATION_TOKEN + //ApiKeys.DESCRIBE_DELEGATION_TOKEN + //ApiKeys.ELECT_LEADERS + ApiKeys.INCREMENTAL_ALTER_CONFIGS, + //ApiKeys.ALTER_PARTITION_REASSIGNMENTS + //ApiKeys.LIST_PARTITION_REASSIGNMENTS + ApiKeys.ALTER_CLIENT_QUOTAS, + //ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS + //ApiKeys.ALTER_USER_SCRAM_CREDENTIALS + //ApiKeys.UPDATE_FEATURES + ApiKeys.ENVELOPE, + ApiKeys.VOTE, + ApiKeys.BEGIN_QUORUM_EPOCH, + ApiKeys.END_QUORUM_EPOCH, + ApiKeys.DESCRIBE_QUORUM, + ApiKeys.ALTER_ISR, + ApiKeys.BROKER_REGISTRATION, + ApiKeys.BROKER_HEARTBEAT, + ApiKeys.UNREGISTER_BROKER, + ) + + private def maybeHandleInvalidEnvelope( + envelope: RequestChannel.Request, + forwardedApiKey: ApiKeys + ): Boolean = { + def sendEnvelopeError(error: Errors): Unit = { + requestHelper.sendErrorResponseMaybeThrottle(envelope, error.exception) + } + + if (!authHelper.authorize(envelope.context, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) { + // Forwarding request must have CLUSTER_ACTION authorization to reduce the risk of impersonation. + sendEnvelopeError(Errors.CLUSTER_AUTHORIZATION_FAILED) + true + } else if (!forwardedApiKey.forwardable) { + sendEnvelopeError(Errors.INVALID_REQUEST) + true + } else { + false + } + } + + override def handle(request: RequestChannel.Request): Unit = { + try { + val handled = request.envelope.exists(envelope => { + maybeHandleInvalidEnvelope(envelope, request.header.apiKey) + }) + + if (handled) + return + + request.header.apiKey match { + case ApiKeys.FETCH => handleFetch(request) + case ApiKeys.METADATA => handleMetadataRequest(request) + case ApiKeys.CREATE_TOPICS => handleCreateTopics(request) + case ApiKeys.API_VERSIONS => handleApiVersionsRequest(request) + case ApiKeys.VOTE => handleVote(request) + case ApiKeys.BEGIN_QUORUM_EPOCH => handleBeginQuorumEpoch(request) + case ApiKeys.END_QUORUM_EPOCH => handleEndQuorumEpoch(request) + case ApiKeys.DESCRIBE_QUORUM => handleDescribeQuorum(request) + case ApiKeys.ALTER_ISR => handleAlterIsrRequest(request) + case ApiKeys.BROKER_REGISTRATION => handleBrokerRegistration(request) + case ApiKeys.BROKER_HEARTBEAT => handleBrokerHeartBeatRequest(request) + case ApiKeys.UNREGISTER_BROKER => handleUnregisterBroker(request) + case ApiKeys.ALTER_CLIENT_QUOTAS => handleAlterClientQuotas(request) + case ApiKeys.INCREMENTAL_ALTER_CONFIGS => handleIncrementalAlterConfigs(request) + case ApiKeys.ENVELOPE => EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle) + case _ => throw new ApiException(s"Unsupported ApiKey ${request.context.header.apiKey()}") + } + } catch { + case e: FatalExitError => throw e + case e: Throwable => requestHelper.handleError(request, e) + } + } + + private def handleFetch(request: RequestChannel.Request): Unit = { + authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) + handleRaftRequest(request, response => new FetchResponse[BaseRecords](response.asInstanceOf[FetchResponseData])) + } + + def handleMetadataRequest(request: RequestChannel.Request): Unit = { + val metadataRequest = request.body[MetadataRequest] + def createResponseCallback(requestThrottleMs: Int): MetadataResponse = { + val metadataResponseData = new MetadataResponseData() + metadataResponseData.setThrottleTimeMs(requestThrottleMs) + controllerNodes.foreach { node => + metadataResponseData.brokers().add(new MetadataResponseBroker() + .setHost(node.host) + .setNodeId(node.id) + .setPort(node.port) + .setRack(node.rack)) + } + metadataResponseData.setClusterId(metaProperties.clusterId.toString) + if (controller.curClaimEpoch() > 0) { + metadataResponseData.setControllerId(config.nodeId) + } else { + metadataResponseData.setControllerId(MetadataResponse.NO_CONTROLLER_ID) + } + val clusterAuthorizedOperations = if (metadataRequest.data.includeClusterAuthorizedOperations) { + if (authHelper.authorize(request.context, DESCRIBE, CLUSTER, CLUSTER_NAME)) { + authHelper.authorizedOperations(request, Resource.CLUSTER) + } else { + 0 + } + } else { + Int.MinValue + } + // TODO: fill in information about the metadata topic + metadataResponseData.setClusterAuthorizedOperations(clusterAuthorizedOperations) + new MetadataResponse(metadataResponseData, request.header.apiVersion) + } + requestHelper.sendResponseMaybeThrottle(request, + requestThrottleMs => createResponseCallback(requestThrottleMs)) + } + + def handleCreateTopics(request: RequestChannel.Request): Unit = { + val createTopicRequest = request.body[CreateTopicsRequest] + val (authorizedCreateRequest, unauthorizedTopics) = + if (authHelper.authorize(request.context, CREATE, CLUSTER, CLUSTER_NAME)) { + (createTopicRequest.data, Seq.empty) + } else { + val duplicate = createTopicRequest.data.duplicate() + val authorizedTopics = new CreatableTopicCollection() + val unauthorizedTopics = mutable.Buffer.empty[String] + + createTopicRequest.data.topics.asScala.foreach { topicData => + if (authHelper.authorize(request.context, CREATE, TOPIC, topicData.name)) { + authorizedTopics.add(topicData) + } else { + unauthorizedTopics += topicData.name + } + } + (duplicate.setTopics(authorizedTopics), unauthorizedTopics) + } + + def sendResponse(response: CreateTopicsResponseData): Unit = { + unauthorizedTopics.foreach { topic => + val result = new CreatableTopicResult() + .setName(topic) + .setErrorCode(Errors.TOPIC_AUTHORIZATION_FAILED.code) + response.topics.add(result) + } + + requestHelper.sendResponseMaybeThrottle(request, throttleTimeMs => { + response.setThrottleTimeMs(throttleTimeMs) + new CreateTopicsResponse(response) + }) + } + + if (authorizedCreateRequest.topics.isEmpty) { + sendResponse(new CreateTopicsResponseData()) + } else { + val future = controller.createTopics(authorizedCreateRequest) + future.whenComplete((responseData, exception) => { + val response = if (exception != null) { + createTopicRequest.getErrorResponse(exception).asInstanceOf[CreateTopicsResponse].data + } else { + responseData + } + sendResponse(response) + }) + } + } + + def handleApiVersionsRequest(request: RequestChannel.Request): Unit = { + // Note that broker returns its full list of supported ApiKeys and versions regardless of current + // authentication state (e.g., before SASL authentication on an SASL listener, do note that no + // Kafka protocol requests may take place on an SSL listener before the SSL handshake is finished). + // If this is considered to leak information about the broker version a workaround is to use SSL + // with client authentication which is performed at an earlier stage of the connection where the + // ApiVersionRequest is not available. + def createResponseCallback(features: FeatureMapAndEpoch, + requestThrottleMs: Int): ApiVersionsResponse = { + val apiVersionRequest = request.body[ApiVersionsRequest] + if (apiVersionRequest.hasUnsupportedRequestVersion) + apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.UNSUPPORTED_VERSION.exception) + else if (!apiVersionRequest.isValid) + apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.INVALID_REQUEST.exception) + else { + val data = new ApiVersionsResponseData(). + setErrorCode(0.toShort). + setThrottleTimeMs(requestThrottleMs). + setFinalizedFeaturesEpoch(features.epoch()) + supportedFeatures.foreach { + case (k, v) => data.supportedFeatures().add(new SupportedFeatureKey(). + setName(k).setMaxVersion(v.max()).setMinVersion(v.min())) + } + // features.finalizedFeatures().asScala.foreach { + // case (k, v) => data.finalizedFeatures().add(new FinalizedFeatureKey(). + // setName(k).setMaxVersionLevel(v.max()).setMinVersionLevel(v.min())) + // } + ApiKeys.values().foreach { + key => + if (supportedApiKeys.contains(key)) { + data.apiKeys().add(new ApiVersion(). + setApiKey(key.id). + setMaxVersion(key.latestVersion()). + setMinVersion(key.oldestVersion())) + } + } + new ApiVersionsResponse(data) + } + } + // FutureConverters.toScala(controller.finalizedFeatures()).onComplete { + // case Success(features) => + requestHelper.sendResponseMaybeThrottle(request, + requestThrottleMs => createResponseCallback(new FeatureMapAndEpoch( + new FeatureMap(new util.HashMap()), 0), requestThrottleMs)) + // case Failure(e) => requestHelper.handleError(request, e) + // } + } + + private def handleVote(request: RequestChannel.Request): Unit = { + authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) + handleRaftRequest(request, response => new VoteResponse(response.asInstanceOf[VoteResponseData])) + } + + private def handleBeginQuorumEpoch(request: RequestChannel.Request): Unit = { + authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) + handleRaftRequest(request, response => new BeginQuorumEpochResponse(response.asInstanceOf[BeginQuorumEpochResponseData])) + } + + private def handleEndQuorumEpoch(request: RequestChannel.Request): Unit = { + authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) + handleRaftRequest(request, response => new EndQuorumEpochResponse(response.asInstanceOf[EndQuorumEpochResponseData])) + } + + private def handleDescribeQuorum(request: RequestChannel.Request): Unit = { + authHelper.authorizeClusterOperation(request, DESCRIBE) + handleRaftRequest(request, response => new DescribeQuorumResponse(response.asInstanceOf[DescribeQuorumResponseData])) + } + + def handleAlterIsrRequest(request: RequestChannel.Request): Unit = { + authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) + val alterIsrRequest = request.body[AlterIsrRequest] + val future = controller.alterIsr(alterIsrRequest.data()) + future.whenComplete((result, exception) => { + val response = if (exception != null) { + alterIsrRequest.getErrorResponse(exception) + } else { + new AlterIsrResponse(result) + } + requestHelper.sendResponseExemptThrottle(request, response) + }) + } + + def handleBrokerHeartBeatRequest(request: RequestChannel.Request): Unit = { + val heartbeatRequest = request.body[BrokerHeartbeatRequest] + authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) + + controller.processBrokerHeartbeat(heartbeatRequest.data).handle[Unit]((reply, e) => { + def createResponseCallback(requestThrottleMs: Int, + reply: BrokerHeartbeatReply, + e: Throwable): BrokerHeartbeatResponse = { + if (e != null) { + new BrokerHeartbeatResponse(new BrokerHeartbeatResponseData(). + setThrottleTimeMs(requestThrottleMs). + setErrorCode(Errors.forException(e).code())) + } else { + new BrokerHeartbeatResponse(new BrokerHeartbeatResponseData(). + setThrottleTimeMs(requestThrottleMs). + setErrorCode(Errors.NONE.code()). + setIsCaughtUp(reply.isCaughtUp()). + setIsFenced(reply.isFenced()). + setShouldShutDown(reply.shouldShutDown())) + } + } + requestHelper.sendResponseMaybeThrottle(request, + requestThrottleMs => createResponseCallback(requestThrottleMs, reply, e)) + }) + } + + def handleUnregisterBroker(request: RequestChannel.Request): Unit = { + val decommissionRequest = request.body[UnregisterBrokerRequest] + authHelper.authorizeClusterOperation(request, ALTER) + + controller.unregisterBroker(decommissionRequest.data().brokerId()).handle[Unit]((_, e) => { + def createResponseCallback(requestThrottleMs: Int, + e: Throwable): UnregisterBrokerResponse = { + if (e != null) { + new UnregisterBrokerResponse(new UnregisterBrokerResponseData(). + setThrottleTimeMs(requestThrottleMs). + setErrorCode(Errors.forException(e).code())) + } else { + new UnregisterBrokerResponse(new UnregisterBrokerResponseData(). + setThrottleTimeMs(requestThrottleMs)) + } + } + requestHelper.sendResponseMaybeThrottle(request, + requestThrottleMs => createResponseCallback(requestThrottleMs, e)) + }) + } + + def handleBrokerRegistration(request: RequestChannel.Request): Unit = { + val registrationRequest = request.body[BrokerRegistrationRequest] + authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) + + controller.registerBroker(registrationRequest.data).handle[Unit]((reply, e) => { + def createResponseCallback(requestThrottleMs: Int, + reply: BrokerRegistrationReply, + e: Throwable): BrokerRegistrationResponse = { + if (e != null) { + new BrokerRegistrationResponse(new BrokerRegistrationResponseData(). + setThrottleTimeMs(requestThrottleMs). + setErrorCode(Errors.forException(e).code())) + } else { + new BrokerRegistrationResponse(new BrokerRegistrationResponseData(). + setThrottleTimeMs(requestThrottleMs). + setErrorCode(Errors.NONE.code()). + setBrokerEpoch(reply.epoch)) + } + } + requestHelper.sendResponseMaybeThrottle(request, + requestThrottleMs => createResponseCallback(requestThrottleMs, reply, e)) + }) + } + + private def handleRaftRequest(request: RequestChannel.Request, + buildResponse: ApiMessage => AbstractResponse): Unit = { + val requestBody = request.body[AbstractRequest] + val future = raftManager.handleRequest(request.header, requestBody.data, time.milliseconds()) + + future.whenComplete((responseData, exception) => { + val response = if (exception != null) { + requestBody.getErrorResponse(exception) + } else { + buildResponse(responseData) + } + requestHelper.sendResponseExemptThrottle(request, response) + }) + } + + def handleAlterClientQuotas(request: RequestChannel.Request): Unit = { + val quotaRequest = request.body[AlterClientQuotasRequest] + authHelper.authorize(request.context, ALTER_CONFIGS, CLUSTER, CLUSTER_NAME) + + controller.alterClientQuotas(quotaRequest.entries(), quotaRequest.validateOnly()) + .whenComplete((results, exception) => { + if (exception != null) { + requestHelper.handleError(request, exception) + } else { + requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => + AlterClientQuotasResponse.fromQuotaEntities(results, requestThrottleMs)) + } + }) + } + + def handleIncrementalAlterConfigs(request: RequestChannel.Request): Unit = { + val alterConfigsRequest = request.body[IncrementalAlterConfigsRequest] + authHelper.authorize(request.context, ALTER_CONFIGS, CLUSTER, CLUSTER_NAME) + val configChanges = new util.HashMap[ConfigResource, util.Map[String, util.Map.Entry[AlterConfigOp.OpType, String]]]() + alterConfigsRequest.data.resources.forEach { resource => + val configResource = new ConfigResource(ConfigResource.Type.forId(resource.resourceType()), resource.resourceName()) + val altersByName = new util.HashMap[String, util.Map.Entry[AlterConfigOp.OpType, String]]() + resource.configs.forEach { config => + altersByName.put(config.name(), new util.AbstractMap.SimpleEntry[AlterConfigOp.OpType, String]( + AlterConfigOp.OpType.forId(config.configOperation()), config.value())) + } + configChanges.put(configResource, altersByName) + } + controller.incrementalAlterConfigs(configChanges, alterConfigsRequest.data().validateOnly()) + .whenComplete((results, exception) => { + if (exception != null) { + requestHelper.handleError(request, exception) + } else { + requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => + new IncrementalAlterConfigsResponse(requestThrottleMs, results)) + } + }) + } +} diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index b648e77328470..efcebb491c3df 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -1,10 +1,10 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -14,14 +14,194 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package kafka.server +import java.util.concurrent.CompletableFuture +import java.util +import java.util.concurrent.locks.ReentrantLock + +import kafka.cluster.Broker.ServerInfo +import kafka.metrics.{KafkaMetricsGroup, KafkaYammerMetrics, LinuxIoMetricsCollector} +import kafka.network.SocketServer +import kafka.raft.RaftManager +import kafka.security.CredentialProvider +import kafka.server.QuotaFactory.QuotaManagers +import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.common.metrics.Metrics +import org.apache.kafka.common.security.scram.internals.ScramMechanism +import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache +import org.apache.kafka.common.utils.{LogContext, Time} +import org.apache.kafka.common.{ClusterResource, Endpoint} +import org.apache.kafka.controller.Controller +import org.apache.kafka.metadata.{ApiMessageAndVersion, VersionRange} +import org.apache.kafka.metalog.MetaLogManager +import org.apache.kafka.raft.RaftConfig +import org.apache.kafka.server.authorizer.Authorizer + +import scala.jdk.CollectionConverters._ + /** - * Stubbed implementation of the KIP-500 controller which is responsible - * for managing the `@metadata` topic which is replicated through Raft. + * A KIP-500 Kafka controller. */ -class ControllerServer { - def startup(): Unit = ??? - def shutdown(): Unit = ??? - def awaitShutdown(): Unit = ??? +class ControllerServer( + val metaProperties: MetaProperties, + val config: KafkaConfig, + val metaLogManager: MetaLogManager, + val raftManager: RaftManager[ApiMessageAndVersion], + val time: Time, + val metrics: Metrics, + val threadNamePrefix: Option[String], + val controllerQuorumVotersFuture: CompletableFuture[util.List[String]] + ) extends Logging with KafkaMetricsGroup { + import kafka.server.Server._ + + val lock = new ReentrantLock() + val awaitShutdownCond = lock.newCondition() + var status: ProcessStatus = SHUTDOWN + + var linuxIoMetricsCollector: LinuxIoMetricsCollector = null + var authorizer: Option[Authorizer] = null + var tokenCache: DelegationTokenCache = null + var credentialProvider: CredentialProvider = null + var socketServer: SocketServer = null + val socketServerFirstBoundPortFuture = new CompletableFuture[Integer]() + var controller: Controller = null + val supportedFeatures: Map[String, VersionRange] = Map() + var quotaManagers: QuotaManagers = null + var controllerApis: ControllerApis = null + var controllerApisHandlerPool: KafkaRequestHandlerPool = null + + private def maybeChangeStatus(from: ProcessStatus, to: ProcessStatus): Boolean = { + lock.lock() + try { + if (status != from) return false + status = to + if (to == SHUTDOWN) awaitShutdownCond.signalAll() + } finally { + lock.unlock() + } + true + } + + def clusterId: String = metaProperties.clusterId.toString + + def startup(): Unit = { + if (!maybeChangeStatus(SHUTDOWN, STARTING)) return + try { + info("Starting controller") + + maybeChangeStatus(STARTING, STARTED) + // TODO: initialize the log dir(s) + this.logIdent = new LogContext(s"[ControllerServer id=${config.nodeId}] ").logPrefix() + + newGauge("ClusterId", () => clusterId) + newGauge("yammer-metrics-count", () => KafkaYammerMetrics.defaultRegistry.allMetrics.size) + + linuxIoMetricsCollector = new LinuxIoMetricsCollector("/proc", time, logger.underlying) + if (linuxIoMetricsCollector.usable()) { + newGauge("linux-disk-read-bytes", () => linuxIoMetricsCollector.readBytes()) + newGauge("linux-disk-write-bytes", () => linuxIoMetricsCollector.writeBytes()) + } + + val javaListeners = config.controllerListeners.map(_.toJava).asJava + authorizer = config.authorizer + authorizer.foreach(_.configure(config.originals)) + + val authorizerFutures: Map[Endpoint, CompletableFuture[Void]] = authorizer match { + case Some(authZ) => + // It would be nice to remove some of the broker-specific assumptions from + // AuthorizerServerInfo, such as the assumption that there is an inter-broker + // listener, or that ID is named brokerId. + val controllerAuthorizerInfo = ServerInfo( + new ClusterResource(clusterId), config.nodeId, javaListeners, javaListeners.get(0)) + authZ.start(controllerAuthorizerInfo).asScala.map { case (ep, cs) => + ep -> cs.toCompletableFuture + }.toMap + case None => + javaListeners.asScala.map { + ep => ep -> CompletableFuture.completedFuture[Void](null) + }.toMap + } + + tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) + credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) + socketServer = new SocketServer(config, + metrics, + time, + credentialProvider, + allowControllerOnlyApis = true, + controllerSocketServer = true) + socketServer.startup(false, None, config.controllerListeners) + socketServerFirstBoundPortFuture.complete(socketServer.boundPort( + config.controllerListeners.head.listenerName)) + + controller = null + quotaManagers = QuotaFactory.instantiate(config, metrics, time, threadNamePrefix.getOrElse("")) + val controllerNodes = + RaftConfig.quorumVoterStringsToNodes(controllerQuorumVotersFuture.get()).asScala + controllerApis = new ControllerApis(socketServer.dataPlaneRequestChannel, + authorizer, + quotaManagers, + time, + supportedFeatures, + controller, + raftManager, + config, + metaProperties, + controllerNodes.toSeq) + controllerApisHandlerPool = new KafkaRequestHandlerPool(config.nodeId, + socketServer.dataPlaneRequestChannel, + controllerApis, + time, + config.numIoThreads, + s"${SocketServer.DataPlaneMetricPrefix}RequestHandlerAvgIdlePercent", + SocketServer.DataPlaneThreadPrefix) + socketServer.startProcessingRequests(authorizerFutures) + } catch { + case e: Throwable => + maybeChangeStatus(STARTING, STARTED) + fatal("Fatal error during controller startup. Prepare to shutdown", e) + shutdown() + throw e + } + } + + def shutdown(): Unit = { + if (!maybeChangeStatus(STARTED, SHUTTING_DOWN)) return + try { + info("shutting down") + if (socketServer != null) + CoreUtils.swallow(socketServer.stopProcessingRequests(), this) + if (controller != null) + controller.beginShutdown() + if (socketServer != null) + CoreUtils.swallow(socketServer.shutdown(), this) + if (controllerApisHandlerPool != null) + CoreUtils.swallow(controllerApisHandlerPool.shutdown(), this) + if (quotaManagers != null) + CoreUtils.swallow(quotaManagers.shutdown(), this) + if (controller != null) + controller.close() + socketServerFirstBoundPortFuture.completeExceptionally(new RuntimeException("shutting down")) + } catch { + case e: Throwable => + fatal("Fatal error during controller shutdown.", e) + throw e + } finally { + maybeChangeStatus(SHUTTING_DOWN, SHUTDOWN) + } + } + + def awaitShutdown(): Unit = { + lock.lock() + try { + while (true) { + if (status == SHUTDOWN) return + awaitShutdownCond.awaitUninterruptibly() + } + } finally { + lock.unlock() + } + } } diff --git a/core/src/main/scala/kafka/server/EnvelopeUtils.scala b/core/src/main/scala/kafka/server/EnvelopeUtils.scala new file mode 100644 index 0000000000000..ec8871f3822ef --- /dev/null +++ b/core/src/main/scala/kafka/server/EnvelopeUtils.scala @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server + +import java.net.{InetAddress, UnknownHostException} +import java.nio.ByteBuffer + +import kafka.network.RequestChannel +import org.apache.kafka.common.errors.{InvalidRequestException, PrincipalDeserializationException, UnsupportedVersionException} +import org.apache.kafka.common.network.ClientInformation +import org.apache.kafka.common.requests.{EnvelopeRequest, RequestContext, RequestHeader} +import org.apache.kafka.common.security.auth.KafkaPrincipal + +import scala.compat.java8.OptionConverters._ + +object EnvelopeUtils { + def handleEnvelopeRequest( + request: RequestChannel.Request, + requestChannelMetrics: RequestChannel.Metrics, + handler: RequestChannel.Request => Unit): Unit = { + val envelope = request.body[EnvelopeRequest] + val forwardedPrincipal = parseForwardedPrincipal(request.context, envelope.requestPrincipal) + val forwardedClientAddress = parseForwardedClientAddress(envelope.clientAddress) + + val forwardedRequestBuffer = envelope.requestData.duplicate() + val forwardedRequestHeader = parseForwardedRequestHeader(forwardedRequestBuffer) + + val forwardedApi = forwardedRequestHeader.apiKey + if (!forwardedApi.forwardable) { + throw new InvalidRequestException(s"API $forwardedApi is not enabled or is not eligible for forwarding") + } + + val forwardedContext = new RequestContext( + forwardedRequestHeader, + request.context.connectionId, + forwardedClientAddress, + forwardedPrincipal, + request.context.listenerName, + request.context.securityProtocol, + ClientInformation.EMPTY, + request.context.fromPrivilegedListener + ) + + val forwardedRequest = parseForwardedRequest( + request, + forwardedContext, + forwardedRequestBuffer, + requestChannelMetrics + ) + handler(forwardedRequest) + } + + private def parseForwardedClientAddress( + address: Array[Byte] + ): InetAddress = { + try { + InetAddress.getByAddress(address) + } catch { + case e: UnknownHostException => + throw new InvalidRequestException("Failed to parse client address from envelope", e) + } + } + + private def parseForwardedRequest( + envelope: RequestChannel.Request, + forwardedContext: RequestContext, + buffer: ByteBuffer, + requestChannelMetrics: RequestChannel.Metrics + ): RequestChannel.Request = { + try { + new RequestChannel.Request( + processor = envelope.processor, + context = forwardedContext, + startTimeNanos = envelope.startTimeNanos, + envelope.memoryPool, + buffer, + requestChannelMetrics, + Some(envelope) + ) + } catch { + case e: InvalidRequestException => + // We use UNSUPPORTED_VERSION if the embedded request cannot be parsed. + // The purpose is to disambiguate structural errors in the envelope request + // itself, such as an invalid client address. + throw new UnsupportedVersionException(s"Failed to parse forwarded request " + + s"with header ${forwardedContext.header}", e) + } + } + + private def parseForwardedRequestHeader( + buffer: ByteBuffer + ): RequestHeader = { + try { + RequestHeader.parse(buffer) + } catch { + case e: InvalidRequestException => + // We use UNSUPPORTED_VERSION if the embedded request cannot be parsed. + // The purpose is to disambiguate structural errors in the envelope request + // itself, such as an invalid client address. + throw new UnsupportedVersionException("Failed to parse request header from envelope", e) + } + } + + private def parseForwardedPrincipal( + envelopeContext: RequestContext, + principalBytes: Array[Byte] + ): KafkaPrincipal = { + envelopeContext.principalSerde.asScala match { + case Some(serde) => + try { + serde.deserialize(principalBytes) + } catch { + case e: Exception => + throw new PrincipalDeserializationException("Failed to deserialize client principal from envelope", e) + } + + case None => + throw new PrincipalDeserializationException("Could not deserialize principal since " + + "no `KafkaPrincipalSerde` has been defined") + } + } +} diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index ffb1b8e7445a7..5e8340e6b6163 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -18,7 +18,6 @@ package kafka.server import java.lang.{Long => JLong} -import java.net.{InetAddress, UnknownHostException} import java.nio.ByteBuffer import java.util import java.util.concurrent.ConcurrentHashMap @@ -64,7 +63,7 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetFor import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult, OffsetForLeaderTopicResultCollection} import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, ApiVersionsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.network.{ClientInformation, ListenerName, Send} +import org.apache.kafka.common.network.{ListenerName, Send} import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record._ import org.apache.kafka.common.replica.ClientMetadata @@ -1224,7 +1223,7 @@ class KafkaApis(val requestChannel: RequestChannel, requestThrottleMs, brokers.flatMap(_.endpoints.get(request.context.listenerName.value())).toList.asJava, clusterId, - metadataCache.getControllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID), + metadataSupport.controllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID), completeTopicMetadata.asJava, clusterAuthorizedOperations )) @@ -3210,7 +3209,7 @@ class KafkaApis(val requestChannel: RequestChannel, } val brokers = metadataCache.getAliveBrokers - val controllerId = metadataCache.getControllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID) + val controllerId = metadataSupport.controllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID) requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => { val data = new DescribeClusterResponseData() @@ -3234,7 +3233,6 @@ class KafkaApis(val requestChannel: RequestChannel, def handleEnvelope(request: RequestChannel.Request): Unit = { val zkSupport = metadataSupport.requireZkOrThrow(KafkaApis.shouldNeverReceive(request)) - val envelope = request.body[EnvelopeRequest] // If forwarding is not yet enabled or this request has been received on an invalid endpoint, // then we treat the request as unparsable and close the connection. @@ -3258,101 +3256,8 @@ class KafkaApis(val requestChannel: RequestChannel, s"Broker $brokerId is not the active controller")) return } - - val forwardedPrincipal = parseForwardedPrincipal(request.context, envelope.requestPrincipal) - val forwardedClientAddress = parseForwardedClientAddress(envelope.clientAddress) - - val forwardedRequestBuffer = envelope.requestData.duplicate() - val forwardedRequestHeader = parseForwardedRequestHeader(forwardedRequestBuffer) - - val forwardedApi = forwardedRequestHeader.apiKey - if (!forwardedApi.forwardable) { - throw new InvalidRequestException(s"API $forwardedApi is not enabled or is not eligible for forwarding") - } - - val forwardedContext = new RequestContext( - forwardedRequestHeader, - request.context.connectionId, - forwardedClientAddress, - forwardedPrincipal, - request.context.listenerName, - request.context.securityProtocol, - ClientInformation.EMPTY, - request.context.fromPrivilegedListener - ) - - val forwardedRequest = parseForwardedRequest(request, forwardedContext, forwardedRequestBuffer) - handle(forwardedRequest) - } - - private def parseForwardedClientAddress( - address: Array[Byte] - ): InetAddress = { - try { - InetAddress.getByAddress(address) - } catch { - case e: UnknownHostException => - throw new InvalidRequestException("Failed to parse client address from envelope", e) - } - } - - private def parseForwardedRequest( - envelope: RequestChannel.Request, - forwardedContext: RequestContext, - buffer: ByteBuffer - ): RequestChannel.Request = { - try { - new RequestChannel.Request( - processor = envelope.processor, - context = forwardedContext, - startTimeNanos = envelope.startTimeNanos, - envelope.memoryPool, - buffer, - requestChannel.metrics, - Some(envelope) - ) - } catch { - case e: InvalidRequestException => - // We use UNSUPPORTED_VERSION if the embedded request cannot be parsed. - // The purpose is to disambiguate structural errors in the envelope request - // itself, such as an invalid client address. - throw new UnsupportedVersionException(s"Failed to parse forwarded request " + - s"with header ${forwardedContext.header}", e) - } - } - - private def parseForwardedRequestHeader( - buffer: ByteBuffer - ): RequestHeader = { - try { - RequestHeader.parse(buffer) - } catch { - case e: InvalidRequestException => - // We use UNSUPPORTED_VERSION if the embedded request cannot be parsed. - // The purpose is to disambiguate structural errors in the envelope request - // itself, such as an invalid client address. - throw new UnsupportedVersionException("Failed to parse request header from envelope", e) - } - } - - private def parseForwardedPrincipal( - envelopeContext: RequestContext, - principalBytes: Array[Byte] - ): KafkaPrincipal = { - envelopeContext.principalSerde.asScala match { - case Some(serde) => - try { - serde.deserialize(principalBytes) - } catch { - case e: Exception => - throw new PrincipalDeserializationException("Failed to deserialize client principal from envelope", e) - } - - case None => - throw new PrincipalDeserializationException("Could not deserialize principal since " + - "no `KafkaPrincipalSerde` has been defined") + EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle) } - } def handleDescribeProducersRequest(request: RequestChannel.Request): Unit = { val describeProducersRequest = request.body[DescribeProducersRequest] diff --git a/core/src/main/scala/kafka/server/KafkaBroker.scala b/core/src/main/scala/kafka/server/KafkaBroker.scala index 3613076c88bae..490c6166fe266 100644 --- a/core/src/main/scala/kafka/server/KafkaBroker.scala +++ b/core/src/main/scala/kafka/server/KafkaBroker.scala @@ -67,6 +67,14 @@ object KafkaBroker { case _ => //do nothing } } + + /** + * The log message that we print when the broker has been successfully started. + * The ducktape system tests look for a line matching the regex 'Kafka\s*Server.*started' + * to know when the broker is started, so it is best not to change this message -- but if + * you do change it, be sure to make it match that regex or the system tests will fail. + */ + val STARTED_MESSAGE = "Kafka Server started" } trait KafkaBroker extends KafkaMetricsGroup { diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 2fd04ae501b33..e01bf60babb0f 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -1025,7 +1025,7 @@ object KafkaConfig { val PasswordEncoderKeyLengthDoc = "The key length used for encoding dynamically configured passwords." val PasswordEncoderIterationsDoc = "The iteration count used for encoding dynamically configured passwords." - private val configDef = { + private[server] val configDef = { import ConfigDef.Importance._ import ConfigDef.Range._ import ConfigDef.Type._ @@ -1893,14 +1893,25 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO validateValues() private def validateValues(): Unit = { - if(brokerIdGenerationEnable) { - require(brokerId >= -1 && brokerId <= maxReservedBrokerId, "broker.id must be equal or greater than -1 and not greater than reserved.broker.max.id") + if (requiresZookeeper) { + if (zkConnect == null) { + throw new ConfigException(s"Missing required configuration `${KafkaConfig.ZkConnectProp}` which has no default value.") + } + if (brokerIdGenerationEnable) { + require(brokerId >= -1 && brokerId <= maxReservedBrokerId, "broker.id must be greater than or equal to -1 and not greater than reserved.broker.max.id") + } else { + require(brokerId >= 0, "broker.id must be greater than or equal to 0") + } } else { - require(brokerId >= 0, "broker.id must be equal or greater than 0") + // Raft-based metadata quorum + if (nodeId < 0) { + throw new ConfigException(s"Missing configuration `${KafkaConfig.NodeIdProp}` which is required " + + s"when `process.roles` is defined (i.e. when using the self-managed quorum).") + } } - require(logRollTimeMillis >= 1, "log.roll.ms must be equal or greater than 1") - require(logRollTimeJitterMillis >= 0, "log.roll.jitter.ms must be equal or greater than 0") - require(logRetentionTimeMillis >= 1 || logRetentionTimeMillis == -1, "log.retention.ms must be unlimited (-1) or, equal or greater than 1") + require(logRollTimeMillis >= 1, "log.roll.ms must be greater than or equal to 1") + require(logRollTimeJitterMillis >= 0, "log.roll.jitter.ms must be greater than or equal to 0") + require(logRetentionTimeMillis >= 1 || logRetentionTimeMillis == -1, "log.retention.ms must be unlimited (-1) or, greater than or equal to 1") require(logDirs.nonEmpty, "At least one log directory must be defined via log.dirs or log.dir.") require(logCleanerDedupeBufferSize / logCleanerThreads > 1024 * 1024, "log.cleaner.dedupe.buffer.size must be at least 1MB per cleaner thread.") require(replicaFetchWaitMaxMs <= replicaSocketTimeoutMs, "replica.socket.timeout.ms should always be at least replica.fetch.wait.max.ms" + @@ -1975,12 +1986,5 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO s"${KafkaConfig.FailedAuthenticationDelayMsProp}=$failedAuthenticationDelayMs should always be less than" + s" ${KafkaConfig.ConnectionsMaxIdleMsProp}=$connectionsMaxIdleMs to prevent failed" + s" authentication responses from timing out") - - if (requiresZookeeper && zkConnect == null) { - throw new ConfigException(s"Missing required configuration `${KafkaConfig.ZkConnectProp}` which has no default value.") - } else if (usesSelfManagedQuorum && nodeId < 0) { - throw new ConfigException(s"Missing required configuration `${KafkaConfig.NodeIdProp}` which is required " + - s"when `process.roles` is defined (i.e. when using the self-managed quorum).") - } } } diff --git a/core/src/main/scala/kafka/server/KafkaRaftServer.scala b/core/src/main/scala/kafka/server/KafkaRaftServer.scala index 1a072c3cb3c78..dc3fd16fed8a6 100644 --- a/core/src/main/scala/kafka/server/KafkaRaftServer.scala +++ b/core/src/main/scala/kafka/server/KafkaRaftServer.scala @@ -17,6 +17,7 @@ package kafka.server import java.io.File +import java.util.concurrent.CompletableFuture import kafka.common.{InconsistentNodeIdException, KafkaException} import kafka.log.Log @@ -26,7 +27,10 @@ import kafka.server.KafkaRaftServer.{BrokerRole, ControllerRole} import kafka.utils.{CoreUtils, Logging, Mx4jLoader, VerifiableProperties} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.utils.{AppInfoParser, Time} -import org.apache.kafka.raft.internals.StringSerde +import org.apache.kafka.metadata.ApiMessageAndVersion +import org.apache.kafka.raft.metadata.{MetaLogRaftShim, MetadataRecordSerde} + +import scala.collection.Seq /** * This class implements the KIP-500 server which relies on a self-managed @@ -47,7 +51,7 @@ class KafkaRaftServer( KafkaMetricsReporter.startReporters(VerifiableProperties(config.originals)) KafkaYammerMetrics.INSTANCE.configure(config.originals) - private val (metaProps, _) = KafkaRaftServer.initializeLogDirs(config) + private val (metaProps, offlineDirs) = KafkaRaftServer.initializeLogDirs(config) private val metrics = Server.initializeMetrics( config, @@ -55,24 +59,38 @@ class KafkaRaftServer( metaProps.clusterId.toString ) - private val raftManager = new KafkaRaftManager( + private val controllerQuorumVotersFuture = CompletableFuture.completedFuture(config.quorumVoters) + + private val raftManager = new KafkaRaftManager[ApiMessageAndVersion]( metaProps, config, - new StringSerde, + new MetadataRecordSerde, KafkaRaftServer.MetadataPartition, time, metrics, threadNamePrefix ) + private val metaLogShim = new MetaLogRaftShim(raftManager.kafkaRaftClient, config.nodeId) + private val broker: Option[BrokerServer] = if (config.processRoles.contains(BrokerRole)) { - Some(new BrokerServer()) + Some(new BrokerServer(config, metaProps, metaLogShim, time, metrics, threadNamePrefix, + offlineDirs, controllerQuorumVotersFuture, Server.SUPPORTED_FEATURES)) } else { None } private val controller: Option[ControllerServer] = if (config.processRoles.contains(ControllerRole)) { - Some(new ControllerServer()) + Some(new ControllerServer( + metaProps, + config, + metaLogShim, + raftManager, + time, + metrics, + threadNamePrefix, + CompletableFuture.completedFuture(config.quorumVoters) + )) } else { None } @@ -83,6 +101,7 @@ class KafkaRaftServer( controller.foreach(_.startup()) broker.foreach(_.startup()) AppInfoParser.registerAppInfo(Server.MetricsPrefix, config.brokerId.toString, metrics, time.milliseconds()) + info(KafkaBroker.STARTED_MESSAGE) } override def shutdown(): Unit = { @@ -118,7 +137,7 @@ object KafkaRaftServer { * be consistent across all log dirs) and the offline directories */ def initializeLogDirs(config: KafkaConfig): (MetaProperties, Seq[String]) = { - val logDirs = config.logDirs :+ config.metadataLogDir + val logDirs = (config.logDirs.toSet + config.metadataLogDir).toSeq val (rawMetaProperties, offlineDirs) = BrokerMetadataCheckpoint. getBrokerMetadataAndOfflineDirs(logDirs, ignoreMissing = false) diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 5b7f26f620806..3ad36874385e0 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -138,7 +138,7 @@ class KafkaServer( var kafkaScheduler: KafkaScheduler = null - var metadataCache: MetadataCache = null + var metadataCache: ZkMetadataCache = null var quotaManagers: QuotaFactory.QuotaManagers = null val zkClientConfig: ZKClientConfig = KafkaServer.zkClientConfigFromKafkaConfig(config).getOrElse(new ZKClientConfig()) @@ -275,7 +275,8 @@ class KafkaServer( time = time, metrics = metrics, threadNamePrefix = threadNamePrefix, - brokerEpochSupplier = () => kafkaController.brokerEpoch + brokerEpochSupplier = () => kafkaController.brokerEpoch, + config.brokerId ) } else { AlterIsrManager(kafkaScheduler, time, zkClient) @@ -332,8 +333,8 @@ class KafkaServer( time, metrics, threadNamePrefix, - adminManager, - kafkaController, + Some(adminManager), + Some(kafkaController), groupCoordinator, transactionCoordinator, enableForwarding @@ -359,7 +360,7 @@ class KafkaServer( KafkaServer.MIN_INCREMENTAL_FETCH_SESSION_EVICTION_MS)) /* start processing requests */ - val zkSupport = ZkSupport(adminManager, kafkaController, zkClient, forwardingManager) + val zkSupport = ZkSupport(adminManager, kafkaController, zkClient, forwardingManager, metadataCache) dataPlaneRequestProcessor = new KafkaApis(socketServer.dataPlaneRequestChannel, zkSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.brokerId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) diff --git a/core/src/main/scala/kafka/server/MetadataSupport.scala b/core/src/main/scala/kafka/server/MetadataSupport.scala index 00b029f582247..86390eaa094c7 100644 --- a/core/src/main/scala/kafka/server/MetadataSupport.scala +++ b/core/src/main/scala/kafka/server/MetadataSupport.scala @@ -19,6 +19,7 @@ package kafka.server import kafka.controller.KafkaController import kafka.network.RequestChannel +import kafka.server.metadata.RaftMetadataCache import kafka.zk.{AdminZkClient, KafkaZkClient} import org.apache.kafka.common.requests.AbstractResponse @@ -58,12 +59,15 @@ sealed trait MetadataSupport { def maybeForward(request: RequestChannel.Request, handler: RequestChannel.Request => Unit, responseCallback: Option[AbstractResponse] => Unit): Unit + + def controllerId: Option[Int] } case class ZkSupport(adminManager: ZkAdminManager, controller: KafkaController, zkClient: KafkaZkClient, - forwardingManager: Option[ForwardingManager]) extends MetadataSupport { + forwardingManager: Option[ForwardingManager], + metadataCache: ZkMetadataCache) extends MetadataSupport { val adminZkClient = new AdminZkClient(zkClient) override def requireZkOrThrow(createException: => Exception): ZkSupport = this @@ -83,9 +87,11 @@ case class ZkSupport(adminManager: ZkAdminManager, case _ => handler(request) } } + + override def controllerId: Option[Int] = metadataCache.getControllerId } -case class RaftSupport(fwdMgr: ForwardingManager) extends MetadataSupport { +case class RaftSupport(fwdMgr: ForwardingManager, metadataCache: RaftMetadataCache) extends MetadataSupport { override val forwardingManager: Option[ForwardingManager] = Some(fwdMgr) override def requireZkOrThrow(createException: => Exception): ZkSupport = throw createException override def requireRaftOrThrow(createException: => Exception): RaftSupport = this @@ -105,4 +111,14 @@ case class RaftSupport(fwdMgr: ForwardingManager) extends MetadataSupport { handler(request) // will reject } } + + override def controllerId: Option[Int] = { + // We send back a random controller ID when running with a Raft-based metadata quorum. + // Raft-based controllers are not directly accessible to clients; rather, clients can send + // requests destined for the controller to any broker node, and the receiving broker will + // automatically forward the request on the client's behalf to the active Raft-based + // controller as per KIP-590. + metadataCache.currentImage().brokers.randomAliveBrokerId() + } + } diff --git a/core/src/main/scala/kafka/server/Server.scala b/core/src/main/scala/kafka/server/Server.scala index 9126114a683dd..1b5aa598cdbe8 100644 --- a/core/src/main/scala/kafka/server/Server.scala +++ b/core/src/main/scala/kafka/server/Server.scala @@ -16,11 +16,15 @@ */ package kafka.server +import java.util.Collections import java.util.concurrent.TimeUnit import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.common.metrics.{JmxReporter, KafkaMetricsContext, MetricConfig, Metrics, MetricsReporter, Sensor} import org.apache.kafka.common.utils.Time +import org.apache.kafka.metadata.VersionRange + +import scala.jdk.CollectionConverters._ trait Server { def startup(): Unit @@ -91,4 +95,12 @@ object Server { reporters } + sealed trait ProcessStatus + case object SHUTDOWN extends ProcessStatus + case object STARTING extends ProcessStatus + case object STARTED extends ProcessStatus + case object SHUTTING_DOWN extends ProcessStatus + + val SUPPORTED_FEATURES = Collections. + unmodifiableMap[String, VersionRange](Map[String, VersionRange]().asJava) } diff --git a/core/src/main/scala/kafka/tools/TestRaftServer.scala b/core/src/main/scala/kafka/tools/TestRaftServer.scala index 2d6dd6772fca5..2391ca4c380ed 100644 --- a/core/src/main/scala/kafka/tools/TestRaftServer.scala +++ b/core/src/main/scala/kafka/tools/TestRaftServer.scala @@ -161,7 +161,7 @@ class TestRaftServer( eventQueue.offer(HandleClaim(epoch)) } - override def handleResign(): Unit = { + override def handleResign(epoch: Int): Unit = { eventQueue.offer(HandleResign) } diff --git a/core/src/test/scala/unit/kafka/server/AutoTopicCreationManagerTest.scala b/core/src/test/scala/unit/kafka/server/AutoTopicCreationManagerTest.scala index dc4dd06ef6115..9f9749bba66dc 100644 --- a/core/src/test/scala/unit/kafka/server/AutoTopicCreationManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/AutoTopicCreationManagerTest.scala @@ -96,8 +96,8 @@ class AutoTopicCreationManagerTest { config, metadataCache, Some(brokerToController), - adminManager, - controller, + Some(adminManager), + Some(controller), groupCoordinator, transactionCoordinator) @@ -125,8 +125,8 @@ class AutoTopicCreationManagerTest { config, metadataCache, None, - adminManager, - controller, + Some(adminManager), + Some(controller), groupCoordinator, transactionCoordinator) @@ -155,8 +155,8 @@ class AutoTopicCreationManagerTest { config, metadataCache, Some(brokerToController), - adminManager, - controller, + Some(adminManager), + Some(controller), groupCoordinator, transactionCoordinator) diff --git a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala new file mode 100644 index 0000000000000..fc0a38b6cd252 --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package unit.kafka.server + +import java.net.InetAddress +import java.util.Properties + +import kafka.network.RequestChannel +import kafka.raft.RaftManager +import kafka.server.QuotaFactory.QuotaManagers +import kafka.server.{ClientQuotaManager, ClientRequestQuotaManager, ControllerApis, ControllerMutationQuotaManager, KafkaConfig, MetaProperties, ReplicationQuotaManager} +import kafka.utils.MockTime +import org.apache.kafka.common.Uuid +import org.apache.kafka.common.errors.ClusterAuthorizationException +import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.BrokerRegistrationRequestData +import org.apache.kafka.common.network.{ClientInformation, ListenerName} +import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.requests.{AbstractRequest, BrokerRegistrationRequest, RequestContext, RequestHeader, RequestTestUtils} +import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} +import org.apache.kafka.controller.Controller +import org.apache.kafka.metadata.{ApiMessageAndVersion, VersionRange} +import org.apache.kafka.server.authorizer.{AuthorizableRequestContext, AuthorizationResult, Authorizer} +import org.easymock.{Capture, EasyMock, IAnswer} +import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.api.{AfterEach, Test} + +class ControllerApisTest { + // Mocks + private val nodeId = 1 + private val brokerRack = "Rack1" + private val clientID = "Client1" + private val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) + private val requestChannel: RequestChannel = EasyMock.createNiceMock(classOf[RequestChannel]) + private val time = new MockTime + private val clientQuotaManager: ClientQuotaManager = EasyMock.createNiceMock(classOf[ClientQuotaManager]) + private val clientRequestQuotaManager: ClientRequestQuotaManager = EasyMock.createNiceMock(classOf[ClientRequestQuotaManager]) + private val clientControllerQuotaManager: ControllerMutationQuotaManager = EasyMock.createNiceMock(classOf[ControllerMutationQuotaManager]) + private val replicaQuotaManager: ReplicationQuotaManager = EasyMock.createNiceMock(classOf[ReplicationQuotaManager]) + private val raftManager: RaftManager[ApiMessageAndVersion] = EasyMock.createNiceMock(classOf[RaftManager[ApiMessageAndVersion]]) + private val quotas = QuotaManagers( + clientQuotaManager, + clientQuotaManager, + clientRequestQuotaManager, + clientControllerQuotaManager, + replicaQuotaManager, + replicaQuotaManager, + replicaQuotaManager, + None) + private val controller: Controller = EasyMock.createNiceMock(classOf[Controller]) + + private def createControllerApis(authorizer: Option[Authorizer], + supportedFeatures: Map[String, VersionRange] = Map.empty): ControllerApis = { + val props = new Properties() + props.put(KafkaConfig.NodeIdProp, nodeId) + props.put(KafkaConfig.ProcessRolesProp, "controller") + new ControllerApis( + requestChannel, + authorizer, + quotas, + time, + supportedFeatures, + controller, + raftManager, + new KafkaConfig(props), + + // FIXME: Would make more sense to set controllerId here + MetaProperties(Uuid.fromString("JgxuGe9URy-E-ceaL04lEw"), nodeId = nodeId), + Seq.empty + ) + } + + /** + * Build a RequestChannel.Request from the AbstractRequest + * + * @param request - AbstractRequest + * @param listenerName - Default listener for the RequestChannel + * @tparam T - Type of AbstractRequest + * @return + */ + private def buildRequest[T <: AbstractRequest](request: AbstractRequest, + listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): RequestChannel.Request = { + val buffer = RequestTestUtils.serializeRequestWithHeader( + new RequestHeader(request.apiKey, request.version, clientID, 0), request) + + // read the header from the buffer first so that the body can be read next from the Request constructor + val header = RequestHeader.parse(buffer) + val context = new RequestContext(header, "1", InetAddress.getLocalHost, KafkaPrincipal.ANONYMOUS, + listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, false) + new RequestChannel.Request(processor = 1, context = context, startTimeNanos = 0, MemoryPool.NONE, buffer, + requestChannelMetrics) + } + + @Test + def testBrokerRegistration(): Unit = { + val brokerRegistrationRequest = new BrokerRegistrationRequest.Builder( + new BrokerRegistrationRequestData() + .setBrokerId(nodeId) + .setRack(brokerRack) + ).build() + + val request = buildRequest(brokerRegistrationRequest) + + val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + + val authorizer = Some[Authorizer](EasyMock.createNiceMock(classOf[Authorizer])) + EasyMock.expect(authorizer.get.authorize(EasyMock.anyObject[AuthorizableRequestContext](), EasyMock.anyObject())).andAnswer( + new IAnswer[java.util.List[AuthorizationResult]]() { + override def answer(): java.util.List[AuthorizationResult] = { + new java.util.ArrayList[AuthorizationResult](){ + add(AuthorizationResult.DENIED) + } + } + } + ) + EasyMock.replay(requestChannel, authorizer.get) + + val assertion = assertThrows(classOf[ClusterAuthorizationException], + () => createControllerApis(authorizer = authorizer).handleBrokerRegistration(request)) + assert(Errors.forException(assertion) == Errors.CLUSTER_AUTHORIZATION_FAILED) + } + + @AfterEach + def tearDown(): Unit = { + quotas.shutdown() + } +} diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 88bf8ebc3affe..5138bf67655fb 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -34,7 +34,7 @@ import kafka.log.AppendOrigin import kafka.network.RequestChannel import kafka.network.RequestChannel.{CloseConnectionResponse, SendResponse} import kafka.server.QuotaFactory.QuotaManagers -import kafka.server.metadata.{ConfigRepository, CachedConfigRepository} +import kafka.server.metadata.{CachedConfigRepository, ConfigRepository, RaftMetadataCache} import kafka.utils.{MockTime, TestUtils} import kafka.zk.KafkaZkClient import org.apache.kafka.clients.NodeApiVersions @@ -148,8 +148,23 @@ class KafkaApisTest { else None + val metadataSupport = if (raftSupport) { + // it will be up to the test to replace the default ZkMetadataCache implementation + // with a RaftMetadataCache instance + metadataCache match { + case raftMetadataCache: RaftMetadataCache => + RaftSupport(forwardingManager, raftMetadataCache) + case _ => throw new IllegalStateException("Test must set an instance of RaftMetadataCache") + } + } else { + metadataCache match { + case zkMetadataCache: ZkMetadataCache => + ZkSupport(adminManager, controller, zkClient, forwardingManagerOpt, zkMetadataCache) + case _ => throw new IllegalStateException("Test must set an instance of ZkMetadataCache") + } + } new KafkaApis(requestChannel, - if (raftSupport) RaftSupport(forwardingManager) else ZkSupport(adminManager, controller, zkClient, forwardingManagerOpt), + metadataSupport, replicaManager, groupCoordinator, txnCoordinator, @@ -321,6 +336,7 @@ class KafkaApisTest { EasyMock.expect(controller.isActive).andReturn(true) + EasyMock.expect(requestChannel.metrics).andReturn(EasyMock.niceMock(classOf[RequestChannel.Metrics])) EasyMock.expect(requestChannel.updateErrorMetrics(ApiKeys.ENVELOPE, Map(Errors.INVALID_REQUEST -> 1))) val capturedResponse = expectNoThrottling() @@ -3460,101 +3476,121 @@ class KafkaApisTest { @Test def testRaftShouldNeverHandleLeaderAndIsrRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleLeaderAndIsrRequest) } @Test def testRaftShouldNeverHandleStopReplicaRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleStopReplicaRequest) } @Test def testRaftShouldNeverHandleUpdateMetadataRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleUpdateMetadataRequest) } @Test def testRaftShouldNeverHandleControlledShutdownRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleControlledShutdownRequest) } @Test def testRaftShouldNeverHandleAlterIsrRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleAlterIsrRequest) } @Test def testRaftShouldNeverHandleEnvelope(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldNeverHandle(createKafkaApis(raftSupport = true).handleEnvelope) } @Test def testRaftShouldAlwaysForwardCreateTopicsRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleCreateTopicsRequest) } @Test def testRaftShouldAlwaysForwardCreatePartitionsRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleCreatePartitionsRequest) } @Test def testRaftShouldAlwaysForwardDeleteTopicsRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleDeleteTopicsRequest) } @Test def testRaftShouldAlwaysForwardCreateAcls(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleCreateAcls) } @Test def testRaftShouldAlwaysForwardDeleteAcls(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleDeleteAcls) } @Test def testRaftShouldAlwaysForwardAlterConfigsRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleAlterConfigsRequest) } @Test def testRaftShouldAlwaysForwardAlterPartitionReassignmentsRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleAlterPartitionReassignmentsRequest) } @Test def testRaftShouldAlwaysForwardIncrementalAlterConfigsRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleIncrementalAlterConfigsRequest) } @Test def testRaftShouldAlwaysForwardCreateTokenRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleCreateTokenRequest) } @Test def testRaftShouldAlwaysForwardRenewTokenRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleRenewTokenRequest) } @Test def testRaftShouldAlwaysForwardExpireTokenRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleExpireTokenRequest) } @Test def testRaftShouldAlwaysForwardAlterClientQuotasRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleAlterClientQuotasRequest) } @Test def testRaftShouldAlwaysForwardAlterUserScramCredentialsRequest(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleAlterUserScramCredentialsRequest) } @Test def testRaftShouldAlwaysForwardUpdateFeatures(): Unit = { + metadataCache = MetadataCache.raftMetadataCache(brokerId) verifyShouldAlwaysForward(createKafkaApis(raftSupport = true).handleUpdateFeatures) } } diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala index d6c456b28889b..6271105c8ca35 100755 --- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala @@ -33,10 +33,13 @@ import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test import java.net.InetSocketAddress import java.util -import java.util.Properties +import java.util.{Collections, Properties} +import org.apache.kafka.common.Node import org.junit.jupiter.api.function.Executable +import scala.jdk.CollectionConverters._ + class KafkaConfigTest { @Test @@ -1034,7 +1037,17 @@ class KafkaConfigTest { } @Test - def testInvalidQuorumVotersConfig(): Unit = { + def testControllerQuorumVoterStringsToNodes(): Unit = { + assertThrows(classOf[ConfigException], () => RaftConfig.quorumVoterStringsToNodes(Collections.singletonList(""))) + assertEquals(Seq(new Node(3000, "example.com", 9093)), + RaftConfig.quorumVoterStringsToNodes(util.Arrays.asList("3000@example.com:9093")).asScala.toSeq) + assertEquals(Seq(new Node(3000, "example.com", 9093), + new Node(3001, "example.com", 9094)), + RaftConfig.quorumVoterStringsToNodes(util.Arrays.asList("3000@example.com:9093","3001@example.com:9094")).asScala.toSeq) + } + + @Test + def testInvalidQuorumVoterConfig(): Unit = { assertInvalidQuorumVoters("1") assertInvalidQuorumVoters("1@") assertInvalidQuorumVoters("1:") @@ -1046,6 +1059,7 @@ class KafkaConfigTest { assertInvalidQuorumVoters("1@kafka1:9092,2@") assertInvalidQuorumVoters("1@kafka1:9092,2@blah") assertInvalidQuorumVoters("1@kafka1:9092,2@blah,") + assertInvalidQuorumVoters("1@kafka1:9092:1@kafka2:9092") } private def assertInvalidQuorumVoters(value: String): Unit = { @@ -1080,6 +1094,102 @@ class KafkaConfigTest { assertEquals(expectedVoters, raftConfig.quorumVoterConnections()) } + @Test + def testAcceptsLargeNodeIdForRaftBasedCase(): Unit = { + // Generation of Broker IDs is not supported when using Raft-based controller quorums, + // so pick a broker ID greater than reserved.broker.max.id, which defaults to 1000, + // and make sure it is allowed despite broker.id.generation.enable=true (true is the default) + val largeBrokerId = 2000 + val props = new Properties() + props.put(KafkaConfig.ProcessRolesProp, "broker") + props.put(KafkaConfig.NodeIdProp, largeBrokerId.toString) + assertTrue(isValidKafkaConfig(props)) + } + + @Test + def testRejectsNegativeNodeIdForRaftBasedBrokerCaseWithAutoGenEnabled(): Unit = { + // -1 is the default for both node.id and broker.id + val props = new Properties() + props.put(KafkaConfig.ProcessRolesProp, "broker") + assertFalse(isValidKafkaConfig(props)) + } + + @Test + def testRejectsNegativeNodeIdForRaftBasedControllerCaseWithAutoGenEnabled(): Unit = { + // -1 is the default for both node.id and broker.id + val props = new Properties() + props.put(KafkaConfig.ProcessRolesProp, "controller") + assertFalse(isValidKafkaConfig(props)) + } + + @Test + def testRejectsNegativeNodeIdForRaftBasedCaseWithAutoGenDisabled(): Unit = { + // -1 is the default for both node.id and broker.id + val props = new Properties() + props.put(KafkaConfig.ProcessRolesProp, "broker") + props.put(KafkaConfig.BrokerIdGenerationEnableProp, "false") + assertFalse(isValidKafkaConfig(props)) + } + + @Test + def testRejectsLargeNodeIdForZkBasedCaseWithAutoGenEnabled(): Unit = { + // Generation of Broker IDs is supported when using ZooKeeper-based controllers, + // so pick a broker ID greater than reserved.broker.max.id, which defaults to 1000, + // and make sure it is not allowed with broker.id.generation.enable=true (true is the default) + val largeBrokerId = 2000 + val props = TestUtils.createBrokerConfig(largeBrokerId, TestUtils.MockZkConnect, port = TestUtils.MockZkPort) + val listeners = "PLAINTEXT://A:9092,SSL://B:9093,SASL_SSL://C:9094" + props.put(KafkaConfig.ListenersProp, listeners) + props.put(KafkaConfig.AdvertisedListenersProp, listeners) + assertFalse(isValidKafkaConfig(props)) + } + + @Test + def testAcceptsNegativeOneNodeIdForZkBasedCaseWithAutoGenEnabled(): Unit = { + // -1 is the default for both node.id and broker.id; it implies "auto-generate" and should succeed + val props = TestUtils.createBrokerConfig(-1, TestUtils.MockZkConnect, port = TestUtils.MockZkPort) + val listeners = "PLAINTEXT://A:9092,SSL://B:9093,SASL_SSL://C:9094" + props.put(KafkaConfig.ListenersProp, listeners) + props.put(KafkaConfig.AdvertisedListenersProp, listeners) + assertTrue(isValidKafkaConfig(props)) + } + + @Test + def testRejectsNegativeTwoNodeIdForZkBasedCaseWithAutoGenEnabled(): Unit = { + // -1 implies "auto-generate" and should succeed, but -2 does not and should fail + val negativeTwoNodeId = -2 + val props = TestUtils.createBrokerConfig(negativeTwoNodeId, TestUtils.MockZkConnect, port = TestUtils.MockZkPort) + val listeners = "PLAINTEXT://A:9092,SSL://B:9093,SASL_SSL://C:9094" + props.put(KafkaConfig.ListenersProp, listeners) + props.put(KafkaConfig.AdvertisedListenersProp, listeners) + props.put(KafkaConfig.NodeIdProp, negativeTwoNodeId.toString) + props.put(KafkaConfig.BrokerIdProp, negativeTwoNodeId.toString) + assertFalse(isValidKafkaConfig(props)) + } + + @Test + def testAcceptsLargeNodeIdForZkBasedCaseWithAutoGenDisabled(): Unit = { + // Ensure a broker ID greater than reserved.broker.max.id, which defaults to 1000, + // is allowed with broker.id.generation.enable=false + val largeBrokerId = 2000 + val props = TestUtils.createBrokerConfig(largeBrokerId, TestUtils.MockZkConnect, port = TestUtils.MockZkPort) + val listeners = "PLAINTEXT://A:9092,SSL://B:9093,SASL_SSL://C:9094" + props.put(KafkaConfig.ListenersProp, listeners) + props.put(KafkaConfig.AdvertisedListenersProp, listeners) + props.put(KafkaConfig.BrokerIdGenerationEnableProp, "false") + assertTrue(isValidKafkaConfig(props)) + } + + @Test + def testRejectsNegativeNodeIdForZkBasedCaseWithAutoGenDisabled(): Unit = { + // -1 is the default for both node.id and broker.id + val props = TestUtils.createBrokerConfig(-1, TestUtils.MockZkConnect, port = TestUtils.MockZkPort) + val listeners = "PLAINTEXT://A:9092,SSL://B:9093,SASL_SSL://C:9094" + props.put(KafkaConfig.ListenersProp, listeners) + props.put(KafkaConfig.BrokerIdGenerationEnableProp, "false") + assertFalse(isValidKafkaConfig(props)) + } + @Test def testZookeeperConnectRequiredIfEmptyProcessRoles(): Unit = { val props = new Properties() diff --git a/core/src/test/scala/unit/kafka/server/KafkaRaftServerTest.scala b/core/src/test/scala/unit/kafka/server/KafkaRaftServerTest.scala index 4cf7d1e28e514..6166d73decdf3 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaRaftServerTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaRaftServerTest.scala @@ -66,7 +66,7 @@ class KafkaRaftServerTest { private def invokeLoadMetaProperties( metaProperties: MetaProperties, configProperties: Properties - ): (MetaProperties, Seq[String]) = { + ): (MetaProperties, collection.Seq[String]) = { val tempLogDir = TestUtils.tempDirectory() try { writeMetaProperties(tempLogDir, metaProperties) diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java index 887d53dad0b0b..c71c05878cc5c 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java @@ -34,6 +34,7 @@ import kafka.server.KafkaConfig; import kafka.server.KafkaConfig$; import kafka.server.MetadataCache; +import kafka.server.ZkMetadataCache; import kafka.server.QuotaFactory; import kafka.server.ReplicaManager; import kafka.server.ReplicationQuotaManager; @@ -105,7 +106,7 @@ public class MetadataRequestBenchmark { private KafkaZkClient kafkaZkClient = Mockito.mock(KafkaZkClient.class); private Metrics metrics = new Metrics(); private int brokerId = 1; - private MetadataCache metadataCache = MetadataCache.zkMetadataCache(brokerId); + private ZkMetadataCache metadataCache = MetadataCache.zkMetadataCache(brokerId); private ClientQuotaManager clientQuotaManager = Mockito.mock(ClientQuotaManager.class); private ClientRequestQuotaManager clientRequestQuotaManager = Mockito.mock(ClientRequestQuotaManager.class); private ControllerMutationQuotaManager controllerMutationQuotaManager = Mockito.mock(ControllerMutationQuotaManager.class); @@ -173,7 +174,7 @@ private KafkaApis createKafkaApis() { kafkaProps.put(KafkaConfig$.MODULE$.BrokerIdProp(), brokerId + ""); BrokerFeatures brokerFeatures = BrokerFeatures.createDefault(); return new KafkaApis(requestChannel, - new ZkSupport(adminManager, kafkaController, kafkaZkClient, Option.empty()), + new ZkSupport(adminManager, kafkaController, kafkaZkClient, Option.empty(), metadataCache), replicaManager, groupCoordinator, transactionCoordinator, diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index b021e3a52a699..164a9214e24f8 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -250,6 +250,12 @@ public KafkaRaftClient( random); this.kafkaRaftMetrics = new KafkaRaftMetrics(metrics, "raft", quorum); kafkaRaftMetrics.updateNumUnknownVoterConnections(quorum.remoteVoters().size()); + + // Update the voter endpoints with what's in RaftConfig + Map voterAddresses = raftConfig.quorumVoterConnections(); + voterAddresses.entrySet().stream() + .filter(e -> e.getValue() instanceof RaftConfig.InetAddressSpec) + .forEach(e -> this.channel.updateEndpoint(e.getKey(), (RaftConfig.InetAddressSpec) e.getValue())); } private void updateFollowerHighWatermark( @@ -343,9 +349,9 @@ private void maybeFireHandleClaim(LeaderState state) { } } - private void fireHandleResign() { + private void fireHandleResign(int epoch) { for (ListenerContext listenerContext : listenerContexts) { - listenerContext.fireHandleResign(); + listenerContext.fireHandleResign(epoch); } } @@ -377,6 +383,11 @@ public void register(Listener listener) { wakeup(); } + @Override + public LeaderAndEpoch leaderAndEpoch() { + return quorum.leaderAndEpoch(); + } + private OffsetAndEpoch endOffset() { return new OffsetAndEpoch(log.endOffset().offset, log.lastFetchedEpoch()); } @@ -464,7 +475,7 @@ private void onBecomeCandidate(long currentTimeMs) throws IOException { private void maybeResignLeadership() { if (quorum.isLeader()) { - fireHandleResign(); + fireHandleResign(quorum.epoch()); } if (accumulator != null) { @@ -2364,8 +2375,8 @@ void maybeFireHandleClaim(int epoch, long epochStartOffset) { } } - void fireHandleResign() { - listener.handleResign(); + void fireHandleResign(int epoch) { + listener.handleResign(epoch); } public synchronized void onClose(BatchReader reader) { diff --git a/raft/src/main/java/org/apache/kafka/raft/NetworkChannel.java b/raft/src/main/java/org/apache/kafka/raft/NetworkChannel.java index f023955beba8e..e3482e567511b 100644 --- a/raft/src/main/java/org/apache/kafka/raft/NetworkChannel.java +++ b/raft/src/main/java/org/apache/kafka/raft/NetworkChannel.java @@ -36,6 +36,11 @@ public interface NetworkChannel extends Closeable { */ void send(RaftRequest.Outbound request); + /** + * Update connection information for the given id. + */ + void updateEndpoint(int id, RaftConfig.InetAddressSpec address); + default void close() {} } diff --git a/raft/src/main/java/org/apache/kafka/raft/RaftClient.java b/raft/src/main/java/org/apache/kafka/raft/RaftClient.java index 554ce6173df98..e2bec0ed4ee79 100644 --- a/raft/src/main/java/org/apache/kafka/raft/RaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/RaftClient.java @@ -57,15 +57,16 @@ default void handleClaim(int epoch) {} /** * Invoked after a leader has stepped down. This callback may or may not * fire before the next leader has been elected. + * + * @param epoch the epoch that the leader is resigning from */ - default void handleResign() {} + default void handleResign(int epoch) {} } /** * Initialize the client. * This should only be called once on startup. * - * @param raftConfig the Raft quorum configuration * @throws IOException For any IO errors during initialization */ void initialize() throws IOException; @@ -77,6 +78,12 @@ default void handleResign() {} */ void register(Listener listener); + /** + * Return the current {@link LeaderAndEpoch}. + * @return the current {@link LeaderAndEpoch} + */ + LeaderAndEpoch leaderAndEpoch(); + /** * Append a list of records to the log. The write will be scheduled for some time * in the future. There is no guarantee that appended records will be written to diff --git a/raft/src/main/java/org/apache/kafka/raft/RaftConfig.java b/raft/src/main/java/org/apache/kafka/raft/RaftConfig.java index de40b35079bfc..13dd8794b78d0 100644 --- a/raft/src/main/java/org/apache/kafka/raft/RaftConfig.java +++ b/raft/src/main/java/org/apache/kafka/raft/RaftConfig.java @@ -17,6 +17,7 @@ package org.apache.kafka.raft; import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.Node; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; @@ -28,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * RaftConfig encapsulates configuration specific to the Raft quorum voter nodes. @@ -233,6 +235,17 @@ public static Map parseVoterConnections(List voter return voterMap; } + public static List quorumVoterStringsToNodes(List voters) { + return parseVoterConnections(voters).entrySet().stream() + .filter(connection -> connection.getValue() instanceof InetAddressSpec) + .map(connection -> { + InetAddressSpec inetAddressSpec = InetAddressSpec.class.cast(connection.getValue()); + return new Node(connection.getKey(), inetAddressSpec.address.getHostName(), + inetAddressSpec.address.getPort()); + }) + .collect(Collectors.toList()); + } + public static class ControllerQuorumVotersValidator implements ConfigDef.Validator { @Override public void ensureValid(String name, Object value) { diff --git a/raft/src/main/java/org/apache/kafka/raft/ReplicatedCounter.java b/raft/src/main/java/org/apache/kafka/raft/ReplicatedCounter.java index 47dae5d8e1372..3db4d736a53f7 100644 --- a/raft/src/main/java/org/apache/kafka/raft/ReplicatedCounter.java +++ b/raft/src/main/java/org/apache/kafka/raft/ReplicatedCounter.java @@ -96,7 +96,7 @@ public synchronized void handleClaim(int epoch) { } @Override - public synchronized void handleResign() { + public synchronized void handleResign(int epoch) { log.debug("Counter uncommitted value reset after resigning leadership"); this.uncommitted = -1; this.claimedEpoch = Optional.empty(); diff --git a/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java b/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java new file mode 100644 index 0000000000000..bf88e7d8120a1 --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.raft.metadata; + +import org.apache.kafka.common.protocol.ApiMessage; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.metalog.MetaLogLeader; +import org.apache.kafka.metalog.MetaLogListener; +import org.apache.kafka.metalog.MetaLogManager; +import org.apache.kafka.raft.BatchReader; +import org.apache.kafka.raft.LeaderAndEpoch; +import org.apache.kafka.raft.RaftClient; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * For now, we rely on a shim to translate from `RaftClient` to `MetaLogManager`. + * Once we check in to trunk, we can drop `RaftClient` and implement `MetaLogManager` + * directly. + */ +public class MetaLogRaftShim implements MetaLogManager { + private final RaftClient client; + private final int nodeId; + + public MetaLogRaftShim(RaftClient client, int nodeId) { + this.client = client; + this.nodeId = nodeId; + } + + @Override + public void initialize() { + // NO-OP - The RaftClient is initialized externally + } + + @Override + public void register(MetaLogListener listener) { + client.register(new ListenerShim(listener)); + } + + @Override + public long scheduleWrite(long epoch, List batch) { + return client.scheduleAppend((int) epoch, batch); + } + + @Override + public void renounce(long epoch) { + throw new UnsupportedOperationException(); + } + + @Override + public MetaLogLeader leader() { + LeaderAndEpoch leaderAndEpoch = client.leaderAndEpoch(); + return new MetaLogLeader(leaderAndEpoch.leaderId.orElse(-1), leaderAndEpoch.epoch); + } + + @Override + public int nodeId() { + return nodeId; + } + + private class ListenerShim implements RaftClient.Listener { + private final MetaLogListener listener; + + private ListenerShim(MetaLogListener listener) { + this.listener = listener; + } + + @Override + public void handleCommit(BatchReader reader) { + try { + // TODO: The `BatchReader` might need to read from disk if this is + // not a leader. We want to move this IO to the state machine so that + // it does not block Raft replication + while (reader.hasNext()) { + BatchReader.Batch batch = reader.next(); + List records = batch.records().stream() + .map(ApiMessageAndVersion::message) + .collect(Collectors.toList()); + listener.handleCommits(batch.lastOffset(), records); + } + } finally { + reader.close(); + } + } + + @Override + public void handleClaim(int epoch) { + listener.handleNewLeader(new MetaLogLeader(nodeId, epoch)); + } + + @Override + public void handleResign(int epoch) { + listener.handleRenounce(epoch); + } + + @Override + public String toString() { + return "ListenerShim(" + + "listener=" + listener + + ')'; + } + } + +} diff --git a/raft/src/test/java/org/apache/kafka/raft/MockNetworkChannel.java b/raft/src/test/java/org/apache/kafka/raft/MockNetworkChannel.java index 7a5b385462e68..2a9793170f9bc 100644 --- a/raft/src/test/java/org/apache/kafka/raft/MockNetworkChannel.java +++ b/raft/src/test/java/org/apache/kafka/raft/MockNetworkChannel.java @@ -56,6 +56,11 @@ public void send(RaftRequest.Outbound request) { sendQueue.add(request); } + @Override + public void updateEndpoint(int id, RaftConfig.InetAddressSpec address) { + // empty + } + public List drainSendQueue() { return drainSentRequests(Optional.empty()); } diff --git a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java index efe7c95bfbcbf..9d19b8698257c 100644 --- a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java +++ b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java @@ -975,7 +975,7 @@ public void handleClaim(int epoch) { } @Override - public void handleResign() { + public void handleResign(int epoch) { this.currentClaimedEpoch = OptionalInt.empty(); } diff --git a/tests/kafkatest/services/kafka/config_property.py b/tests/kafkatest/services/kafka/config_property.py index 2222c16aa8fe5..42243cf30673e 100644 --- a/tests/kafkatest/services/kafka/config_property.py +++ b/tests/kafkatest/services/kafka/config_property.py @@ -22,7 +22,7 @@ FIRST_BROKER_PORT = 9092 FIRST_CONTROLLER_PORT = FIRST_BROKER_PORT + 500 FIRST_CONTROLLER_ID = 3001 -CLUSTER_ID = "6bd37820-6745-4790-ae98-620300e1f61b" +CLUSTER_ID = "I2eXt9rvSnyhct8BYmW6-w" PORT = "port" ADVERTISED_HOSTNAME = "advertised.host.name" From 38ec258c66b2e7762885c07f5b104349b69908ee Mon Sep 17 00:00:00 2001 From: iczellion <37164659+iczellion@users.noreply.github.com> Date: Thu, 18 Feb 2021 04:54:02 -0500 Subject: [PATCH 012/243] MINOR: Fix Typo in MirrorMaker README file (#10144) Fix Typo in metric name of MirrorMaker README file from 'replication-latecny-ms' to 'replication-latency-ms' Reviewers: Mickael Maison , Eric Beaudet --- connect/mirror/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connect/mirror/README.md b/connect/mirror/README.md index 9c7c259ae2998..3c8aebc635fe7 100644 --- a/connect/mirror/README.md +++ b/connect/mirror/README.md @@ -236,7 +236,7 @@ The following metrics are emitted: record-age-ms-min record-age-ms-max record-age-ms-avg - replication-latecny-ms # time it takes records to propagate source->target + replication-latency-ms # time it takes records to propagate source->target replication-latency-ms-min replication-latency-ms-max replication-latency-ms-avg From b58b944e70780029806a6004114bba43ce5b9483 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Thu, 18 Feb 2021 10:01:49 -0500 Subject: [PATCH 013/243] KAFKA-12303: Fix handling of null values by Flatten SMT (#10073) Reviewers: Mickael Maison , Greg Harris --- .../kafka/connect/transforms/Flatten.java | 2 +- .../kafka/connect/transforms/FlattenTest.java | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java index cad8d79e99f34..782cb3a27b9de 100644 --- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java +++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Flatten.java @@ -106,7 +106,7 @@ private void applySchemaless(Map originalRecord, String fieldNam Object value = entry.getValue(); if (value == null) { newRecord.put(fieldName(fieldNamePrefix, entry.getKey()), null); - return; + continue; } Schema.Type inferredType = ConnectSchema.schemaType(value.getClass()); diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/FlattenTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/FlattenTest.java index 541ca142abffa..af311bc4f3c58 100644 --- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/FlattenTest.java +++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/FlattenTest.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -326,4 +327,47 @@ public void tombstoneEventWithSchemaShouldPassThrough() { assertNull(transformedRecord.value()); assertEquals(simpleStructSchema, transformedRecord.valueSchema()); } + + @Test + public void testMapWithNullFields() { + xformValue.configure(Collections.emptyMap()); + + // Use a LinkedHashMap to ensure the SMT sees entries in a specific order + Map value = new LinkedHashMap<>(); + value.put("firstNull", null); + value.put("firstNonNull", "nonNull"); + value.put("secondNull", null); + value.put("secondNonNull", "alsoNonNull"); + value.put("thirdNonNull", null); + + final SourceRecord record = new SourceRecord(null, null, "test", 0, null, value); + final SourceRecord transformedRecord = xformValue.apply(record); + + assertEquals(value, transformedRecord.value()); + } + + @Test + public void testStructWithNullFields() { + xformValue.configure(Collections.emptyMap()); + + final Schema structSchema = SchemaBuilder.struct() + .field("firstNull", Schema.OPTIONAL_STRING_SCHEMA) + .field("firstNonNull", Schema.OPTIONAL_STRING_SCHEMA) + .field("secondNull", Schema.OPTIONAL_STRING_SCHEMA) + .field("secondNonNull", Schema.OPTIONAL_STRING_SCHEMA) + .field("thirdNonNull", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + final Struct value = new Struct(structSchema); + value.put("firstNull", null); + value.put("firstNonNull", "nonNull"); + value.put("secondNull", null); + value.put("secondNonNull", "alsoNonNull"); + value.put("thirdNonNull", null); + + final SourceRecord record = new SourceRecord(null, null, "test", 0, structSchema, value); + final SourceRecord transformedRecord = xformValue.apply(record); + + assertEquals(value, transformedRecord.value()); + } } From 97c9ae119ac12e9b3cf60d6639900a46ab30d062 Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Fri, 19 Feb 2021 05:47:14 +0800 Subject: [PATCH 014/243] HOTFIX: Fix build error caused by ControllerApisTest.scala (#10146) Introduced by #10113. The error was: ``` 14:01:06 > Task :core:compileTestScala 14:01:06 [Error] /home/jenkins/agent/workspace/LoopTest-Kafka/kafka/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala:70: the result type of an implicit conversion must be more specific than Object ``` Reviewers: Ismael Juma Date: Thu, 18 Feb 2021 16:02:25 -0800 Subject: [PATCH 015/243] TRIVIAL: fix JavaDocs formatting (#10134) Reviewers: Chia-Ping Tsai , Bill Bejeck --- .../org/apache/kafka/streams/processor/ProcessorSupplier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/ProcessorSupplier.java b/streams/src/main/java/org/apache/kafka/streams/processor/ProcessorSupplier.java index b81b8b93e2f36..a3e5f30c2c464 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/ProcessorSupplier.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/ProcessorSupplier.java @@ -26,7 +26,7 @@ * It is used in {@link Topology} for adding new processor operators, whose generated * topology can then be replicated (and thus creating one or more {@link Processor} instances) * and distributed to multiple stream threads. - * + *

    * The supplier should always generate a new instance each time {@link ProcessorSupplier#get()} gets called. Creating * a single {@link Processor} object and returning the same object reference in {@link ProcessorSupplier#get()} would be * a violation of the supplier pattern and leads to runtime exceptions. From 698319b8e2c1f6cb574f339eede6f2a5b1919b55 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 4 Feb 2021 10:04:17 -0800 Subject: [PATCH 016/243] KAFKA-12278; Ensure exposed api versions are consistent within listener (#10666) Previously all APIs were accessible on every listener exposed by the broker, but with KIP-500, that is no longer true. We now have more complex requirements for API accessibility. For example, the KIP-500 controller exposes some APIs which are not exposed by brokers, such as BrokerHeartbeatRequest, and does not expose most client APIs, such as JoinGroupRequest, etc. Similarly, the KIP-500 broker does not implement some APIs that the ZK-based broker does, such as LeaderAndIsrRequest and UpdateFeaturesRequest. All of this means that we need more sophistication in how we expose APIs and keep them consistent with the ApiVersions API. Up until now, we have been working around this using the controllerOnly flag inside ApiKeys, but this is not rich enough to support all of the cases listed above. This PR introduces a new "listeners" field to the request schema definitions. This field is an array of strings which indicate the listener types in which the API should be exposed. We currently support "zkBroker", "broker", and "controller". ("broker" indicates the KIP-500 broker, whereas zkBroker indicates the old broker). This PR also creates ApiVersionManager to encapsulate the creation of the ApiVersionsResponse based on the listener type. Additionally, it modifies SocketServer to check the listener type of received requests before forwarding them to the request handler. Finally, this PR also fixes a bug in the handling of the ApiVersionsResponse prior to authentication. Previously a static response was sent, which means that changes to features would not get reflected. This also meant that the logic to ensure that only the intersection of version ranges supported by the controller would get exposed did not work. I think this is important because some clients rely on the initial pre-authenticated ApiVersions response rather than doing a second round after authentication as the Java client does. One final cleanup note: I have removed the expectation that envelope requests are only allowed on "privileged" listeners. This made sense initially because we expected to use forwarding before the KIP-500 controller was available. That is not the case anymore and we expect the Envelope API to only be exposed on the controller listener. I have nevertheless preserved the existing workarounds to allow verification of the forwarding behavior in integration testing. Reviewers: Colin P. McCabe , Ismael Juma --- checkstyle/import-control.xml | 2 + .../apache/kafka/clients/NodeApiVersions.java | 10 +- .../kafka/common/network/ChannelBuilders.java | 16 +- .../common/network/SaslChannelBuilder.java | 12 +- .../apache/kafka/common/protocol/ApiKeys.java | 71 ++++--- .../kafka/common/protocol/Protocol.java | 2 +- .../common/requests/ApiVersionsResponse.java | 89 ++++++--- .../SaslServerAuthenticator.java | 13 +- .../message/AddOffsetsToTxnRequest.json | 1 + .../message/AddPartitionsToTxnRequest.json | 1 + .../message/AlterClientQuotasRequest.json | 1 + .../common/message/AlterConfigsRequest.json | 1 + .../common/message/AlterIsrRequest.json | 1 + .../AlterPartitionReassignmentsRequest.json | 1 + .../message/AlterReplicaLogDirsRequest.json | 1 + .../AlterUserScramCredentialsRequest.json | 1 + .../common/message/ApiVersionsRequest.json | 1 + .../message/BeginQuorumEpochRequest.json | 1 + .../message/BrokerHeartbeatRequest.json | 1 + .../message/BrokerRegistrationRequest.json | 1 + .../message/ControlledShutdownRequest.json | 1 + .../common/message/CreateAclsRequest.json | 1 + .../message/CreateDelegationTokenRequest.json | 1 + .../message/CreatePartitionsRequest.json | 1 + .../common/message/CreateTopicsRequest.json | 1 + .../common/message/DeleteAclsRequest.json | 1 + .../common/message/DeleteGroupsRequest.json | 1 + .../common/message/DeleteRecordsRequest.json | 1 + .../common/message/DeleteTopicsRequest.json | 1 + .../common/message/DescribeAclsRequest.json | 1 + .../message/DescribeClientQuotasRequest.json | 1 + .../message/DescribeClusterRequest.json | 1 + .../message/DescribeConfigsRequest.json | 1 + .../DescribeDelegationTokenRequest.json | 1 + .../common/message/DescribeGroupsRequest.json | 1 + .../message/DescribeLogDirsRequest.json | 1 + .../message/DescribeProducersRequest.json | 1 + .../common/message/DescribeQuorumRequest.json | 1 + .../DescribeUserScramCredentialsRequest.json | 1 + .../common/message/ElectLeadersRequest.json | 1 + .../common/message/EndQuorumEpochRequest.json | 1 + .../common/message/EndTxnRequest.json | 1 + .../common/message/EnvelopeRequest.json | 1 + .../message/ExpireDelegationTokenRequest.json | 1 + .../common/message/FetchRequest.json | 1 + .../common/message/FetchSnapshotRequest.json | 1 + .../message/FindCoordinatorRequest.json | 1 + .../common/message/HeartbeatRequest.json | 1 + .../IncrementalAlterConfigsRequest.json | 1 + .../common/message/InitProducerIdRequest.json | 1 + .../common/message/JoinGroupRequest.json | 1 + .../common/message/LeaderAndIsrRequest.json | 1 + .../common/message/LeaveGroupRequest.json | 1 + .../common/message/ListGroupsRequest.json | 1 + .../common/message/ListOffsetsRequest.json | 1 + .../ListPartitionReassignmentsRequest.json | 1 + .../common/message/MetadataRequest.json | 1 + .../common/message/OffsetCommitRequest.json | 1 + .../common/message/OffsetDeleteRequest.json | 1 + .../common/message/OffsetFetchRequest.json | 1 + .../message/OffsetForLeaderEpochRequest.json | 1 + .../common/message/ProduceRequest.json | 1 + .../message/RenewDelegationTokenRequest.json | 1 + .../message/SaslAuthenticateRequest.json | 1 + .../common/message/SaslHandshakeRequest.json | 1 + .../common/message/StopReplicaRequest.json | 1 + .../common/message/SyncGroupRequest.json | 1 + .../message/TxnOffsetCommitRequest.json | 1 + .../message/UnregisterBrokerRequest.json | 1 + .../common/message/UpdateFeaturesRequest.json | 1 + .../common/message/UpdateMetadataRequest.json | 1 + .../resources/common/message/VoteRequest.json | 1 + .../message/WriteTxnMarkersRequest.json | 1 + .../kafka/clients/NetworkClientTest.java | 17 +- .../kafka/clients/NodeApiVersionsTest.java | 36 ++-- .../clients/admin/KafkaAdminClientTest.java | 14 +- .../consumer/internals/FetcherTest.java | 6 +- .../producer/internals/SenderTest.java | 27 +-- .../kafka/common/network/NioEchoServer.java | 5 +- .../network/SaslChannelBuilderTest.java | 14 +- .../common/network/SslTransportLayerTest.java | 24 ++- .../kafka/common/protocol/ApiKeysTest.java | 22 ++- .../requests/ApiVersionsResponseTest.java | 51 ++--- .../common/requests/RequestResponseTest.java | 100 +++++----- .../authenticator/SaslAuthenticatorTest.java | 177 ++++++++---------- .../SaslServerAuthenticatorTest.java | 8 +- .../src/main/scala/kafka/api/ApiVersion.scala | 71 ++++--- .../scala/kafka/network/RequestChannel.scala | 17 +- .../scala/kafka/network/SocketServer.scala | 36 ++-- .../kafka/server/ApiVersionManager.scala | 126 +++++++++++++ .../scala/kafka/server/BrokerServer.scala | 51 +++-- .../scala/kafka/server/ControllerServer.scala | 8 +- .../main/scala/kafka/server/KafkaApis.scala | 58 ++---- .../scala/kafka/server/KafkaRaftServer.scala | 15 +- .../main/scala/kafka/server/KafkaServer.scala | 40 ++-- .../kafka/tools/TestRaftRequestHandler.scala | 8 +- .../scala/kafka/tools/TestRaftServer.scala | 9 +- .../admin/BrokerApiVersionsCommandTest.scala | 2 +- .../server/GssapiAuthenticationTest.scala | 5 +- .../scala/unit/kafka/api/ApiVersionTest.scala | 23 ++- .../unit/kafka/network/SocketServerTest.scala | 61 +++--- .../AbstractApiVersionsRequestTest.scala | 15 +- .../kafka/server/ApiVersionManagerTest.scala | 115 ++++++++++++ .../kafka/server/ApiVersionsRequestTest.scala | 8 +- .../kafka/server/ForwardingManagerTest.scala | 3 +- .../unit/kafka/server/KafkaApisTest.scala | 85 ++------- .../unit/kafka/server/RequestQuotaTest.scala | 6 +- .../server/SaslApiVersionsRequestTest.scala | 4 +- .../message/ApiMessageTypeGenerator.java | 69 ++++++- .../kafka/message/MessageGenerator.java | 6 +- .../org/apache/kafka/message/MessageSpec.java | 16 +- .../kafka/message/RequestListenerType.java | 30 +++ .../metadata/MetadataRequestBenchmark.java | 8 +- 113 files changed, 1091 insertions(+), 585 deletions(-) create mode 100644 core/src/main/scala/kafka/server/ApiVersionManager.scala create mode 100644 core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala create mode 100644 generator/src/main/java/org/apache/kafka/message/RequestListenerType.java diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 9ec16b9b7c096..aad58b0ddd8cd 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -50,6 +50,7 @@ + @@ -108,6 +109,7 @@ + diff --git a/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java b/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java index 658d481308ead..3c09f0eb4e781 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java +++ b/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java @@ -16,9 +16,6 @@ */ package org.apache.kafka.clients; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; @@ -27,7 +24,10 @@ import org.apache.kafka.common.utils.Utils; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.EnumMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -62,7 +62,7 @@ public static NodeApiVersions create() { */ public static NodeApiVersions create(Collection overrides) { List apiVersions = new LinkedList<>(overrides); - for (ApiKeys apiKey : ApiKeys.brokerApis()) { + for (ApiKeys apiKey : ApiKeys.zkBrokerApis()) { boolean exists = false; for (ApiVersion apiVersion : apiVersions) { if (apiVersion.apiKey() == apiKey.id) { @@ -170,7 +170,7 @@ public String toString(boolean lineBreaks) { // Also handle the case where some apiKey types are not specified at all in the given ApiVersions, // which may happen when the remote is too old. - for (ApiKeys apiKey : ApiKeys.brokerApis()) { + for (ApiKeys apiKey : ApiKeys.zkBrokerApis()) { if (!apiKeysText.containsKey(apiKey.id)) { StringBuilder bld = new StringBuilder(); bld.append(apiKey.name).append("("). diff --git a/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java b/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java index ee5ed75828920..b4a1ce87cf1cd 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.config.SslClientAuth; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.errors.InvalidConfigurationException; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.JaasContext; import org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.Supplier; public class ChannelBuilders { private static final Logger log = LoggerFactory.getLogger(ChannelBuilders.class); @@ -77,7 +79,7 @@ public static ChannelBuilder clientChannelBuilder( throw new IllegalArgumentException("`clientSaslMechanism` must be non-null in client mode if `securityProtocol` is `" + securityProtocol + "`"); } return create(securityProtocol, Mode.CLIENT, contextType, config, listenerName, false, clientSaslMechanism, - saslHandshakeRequestEnable, null, null, time, logContext); + saslHandshakeRequestEnable, null, null, time, logContext, null); } /** @@ -89,6 +91,7 @@ public static ChannelBuilder clientChannelBuilder( * @param tokenCache Delegation token cache * @param time the time instance * @param logContext the log context instance + * @param apiVersionSupplier supplier for ApiVersions responses sent prior to authentication * * @return the configured `ChannelBuilder` */ @@ -99,10 +102,11 @@ public static ChannelBuilder serverChannelBuilder(ListenerName listenerName, CredentialCache credentialCache, DelegationTokenCache tokenCache, Time time, - LogContext logContext) { + LogContext logContext, + Supplier apiVersionSupplier) { return create(securityProtocol, Mode.SERVER, JaasContext.Type.SERVER, config, listenerName, isInterBrokerListener, null, true, credentialCache, - tokenCache, time, logContext); + tokenCache, time, logContext, apiVersionSupplier); } private static ChannelBuilder create(SecurityProtocol securityProtocol, @@ -116,7 +120,8 @@ private static ChannelBuilder create(SecurityProtocol securityProtocol, CredentialCache credentialCache, DelegationTokenCache tokenCache, Time time, - LogContext logContext) { + LogContext logContext, + Supplier apiVersionSupplier) { Map configs = channelBuilderConfigs(config, listenerName); ChannelBuilder channelBuilder; @@ -174,7 +179,8 @@ private static ChannelBuilder create(SecurityProtocol securityProtocol, tokenCache, sslClientAuthOverride, time, - logContext); + logContext, + apiVersionSupplier); break; case PLAINTEXT: channelBuilder = new PlaintextChannelBuilder(listenerName); diff --git a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java index 900162614d998..17988db87a650 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.JaasContext; import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.Login; @@ -85,6 +86,7 @@ public class SaslChannelBuilder implements ChannelBuilder, ListenerReconfigurabl private final DelegationTokenCache tokenCache; private final Map loginManagers; private final Map subjects; + private final Supplier apiVersionSupplier; private SslFactory sslFactory; private Map configs; @@ -108,7 +110,8 @@ public SaslChannelBuilder(Mode mode, DelegationTokenCache tokenCache, String sslClientAuthOverride, Time time, - LogContext logContext) { + LogContext logContext, + Supplier apiVersionSupplier) { this.mode = mode; this.jaasContexts = jaasContexts; this.loginManagers = new HashMap<>(jaasContexts.size()); @@ -126,6 +129,11 @@ public SaslChannelBuilder(Mode mode, this.time = time; this.logContext = logContext; this.log = logContext.logger(getClass()); + this.apiVersionSupplier = apiVersionSupplier; + + if (mode == Mode.SERVER && apiVersionSupplier == null) { + throw new IllegalArgumentException("Server channel builder must provide an ApiVersionResponse supplier"); + } } @SuppressWarnings("unchecked") @@ -266,7 +274,7 @@ protected SaslServerAuthenticator buildServerAuthenticator(Map config ChannelMetadataRegistry metadataRegistry) { return new SaslServerAuthenticator(configs, callbackHandlers, id, subjects, kerberosShortNamer, listenerName, securityProtocol, transportLayer, - connectionsMaxReauthMsByMechanism, metadataRegistry, time); + connectionsMaxReauthMsByMechanism, metadataRegistry, time, apiVersionSupplier); } // Visible to override for testing diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 49a6130a30079..475fc84c7355c 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -21,7 +21,10 @@ import org.apache.kafka.common.protocol.types.Type; import org.apache.kafka.common.record.RecordBatch; +import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumMap; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -90,19 +93,28 @@ public enum ApiKeys { ALTER_CLIENT_QUOTAS(ApiMessageType.ALTER_CLIENT_QUOTAS, false, true), DESCRIBE_USER_SCRAM_CREDENTIALS(ApiMessageType.DESCRIBE_USER_SCRAM_CREDENTIALS), ALTER_USER_SCRAM_CREDENTIALS(ApiMessageType.ALTER_USER_SCRAM_CREDENTIALS, false, true), - VOTE(ApiMessageType.VOTE, true, RecordBatch.MAGIC_VALUE_V0, false, true), - BEGIN_QUORUM_EPOCH(ApiMessageType.BEGIN_QUORUM_EPOCH, true, RecordBatch.MAGIC_VALUE_V0, false, true), - END_QUORUM_EPOCH(ApiMessageType.END_QUORUM_EPOCH, true, RecordBatch.MAGIC_VALUE_V0, false, true), - DESCRIBE_QUORUM(ApiMessageType.DESCRIBE_QUORUM, true, RecordBatch.MAGIC_VALUE_V0, false, true), + VOTE(ApiMessageType.VOTE, true, RecordBatch.MAGIC_VALUE_V0, false), + BEGIN_QUORUM_EPOCH(ApiMessageType.BEGIN_QUORUM_EPOCH, true, RecordBatch.MAGIC_VALUE_V0, false), + END_QUORUM_EPOCH(ApiMessageType.END_QUORUM_EPOCH, true, RecordBatch.MAGIC_VALUE_V0, false), + DESCRIBE_QUORUM(ApiMessageType.DESCRIBE_QUORUM, true, RecordBatch.MAGIC_VALUE_V0, false), ALTER_ISR(ApiMessageType.ALTER_ISR, true), UPDATE_FEATURES(ApiMessageType.UPDATE_FEATURES, false, true), - ENVELOPE(ApiMessageType.ENVELOPE, true, RecordBatch.MAGIC_VALUE_V0, false, true), - FETCH_SNAPSHOT(ApiMessageType.FETCH_SNAPSHOT, false, RecordBatch.MAGIC_VALUE_V0, false, true), + ENVELOPE(ApiMessageType.ENVELOPE, true, RecordBatch.MAGIC_VALUE_V0, false), + FETCH_SNAPSHOT(ApiMessageType.FETCH_SNAPSHOT, false, RecordBatch.MAGIC_VALUE_V0, false), DESCRIBE_CLUSTER(ApiMessageType.DESCRIBE_CLUSTER), DESCRIBE_PRODUCERS(ApiMessageType.DESCRIBE_PRODUCERS), - BROKER_REGISTRATION(ApiMessageType.BROKER_REGISTRATION, true, RecordBatch.MAGIC_VALUE_V0, false, true), - BROKER_HEARTBEAT(ApiMessageType.BROKER_HEARTBEAT, true, RecordBatch.MAGIC_VALUE_V0, false, true), - UNREGISTER_BROKER(ApiMessageType.UNREGISTER_BROKER, false, RecordBatch.MAGIC_VALUE_V0, true, false); + BROKER_REGISTRATION(ApiMessageType.BROKER_REGISTRATION, true, RecordBatch.MAGIC_VALUE_V0, false), + BROKER_HEARTBEAT(ApiMessageType.BROKER_HEARTBEAT, true, RecordBatch.MAGIC_VALUE_V0, false), + UNREGISTER_BROKER(ApiMessageType.UNREGISTER_BROKER, false, RecordBatch.MAGIC_VALUE_V0, true); + + private static final Map> APIS_BY_LISTENER = + new EnumMap<>(ApiMessageType.ListenerType.class); + + static { + for (ApiMessageType.ListenerType listenerType : ApiMessageType.ListenerType.values()) { + APIS_BY_LISTENER.put(listenerType, filterApisForListener(listenerType)); + } + } // The generator ensures every `ApiMessageType` has a unique id private static final Map ID_TO_TYPE = Arrays.stream(ApiKeys.values()) @@ -120,9 +132,6 @@ public enum ApiKeys { /** indicates the minimum required inter broker magic required to support the API */ public final byte minRequiredInterBrokerMagic; - /** indicates whether this is an API which is only exposed by the KIP-500 controller **/ - public final boolean isControllerOnlyApi; - /** indicates whether the API is enabled for forwarding **/ public final boolean forwardable; @@ -142,24 +151,17 @@ public enum ApiKeys { this(messageType, clusterAction, RecordBatch.MAGIC_VALUE_V0, forwardable); } - ApiKeys(ApiMessageType messageType, boolean clusterAction, byte minRequiredInterBrokerMagic, boolean forwardable) { - this(messageType, clusterAction, minRequiredInterBrokerMagic, forwardable, false); - } - ApiKeys( ApiMessageType messageType, boolean clusterAction, byte minRequiredInterBrokerMagic, - boolean forwardable, - boolean isControllerOnlyApi + boolean forwardable ) { this.messageType = messageType; this.id = messageType.apiKey(); this.name = messageType.name; this.clusterAction = clusterAction; this.minRequiredInterBrokerMagic = minRequiredInterBrokerMagic; - this.isControllerOnlyApi = isControllerOnlyApi; - this.requiresDelayedAllocation = forwardable || shouldRetainsBufferReference(messageType.requestSchemas()); this.forwardable = forwardable; } @@ -195,6 +197,14 @@ public short oldestVersion() { return messageType.lowestSupportedVersion(); } + public List allVersions() { + List versions = new ArrayList<>(latestVersion() - oldestVersion() + 1); + for (short version = oldestVersion(); version < latestVersion(); version++) { + versions.add(version); + } + return versions; + } + public boolean isVersionSupported(short apiVersion) { return apiVersion >= oldestVersion() && apiVersion <= latestVersion(); } @@ -207,6 +217,10 @@ public short responseHeaderVersion(short apiVersion) { return messageType.responseHeaderVersion(apiVersion); } + public boolean inScope(ApiMessageType.ListenerType listener) { + return messageType.listeners().contains(listener); + } + private static String toHtml() { final StringBuilder b = new StringBuilder(); b.append("\n"); @@ -214,7 +228,7 @@ private static String toHtml() { b.append("\n"); b.append("\n"); b.append(""); - for (ApiKeys key : ApiKeys.brokerApis()) { + for (ApiKeys key : zkBrokerApis()) { b.append("\n"); b.append(" - - - - @@ -912,10 +908,6 @@

    Parameters controlled by Kafka StreamsProducer

    - - - - From f5e01f743be2276c1c0365b4e3e6639de32b985a Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Tue, 23 Feb 2021 10:54:14 +0800 Subject: [PATCH 043/243] =?UTF-8?q?KAFKA-12273=20InterBrokerSendThread#pol?= =?UTF-8?q?lOnce=20throws=20FatalExitError=20even=E2=80=A6=20(#10024)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewers: David Jacot --- .../kafka/common/InterBrokerSendThread.scala | 5 +-- .../common/InterBrokerSendThreadTest.scala | 31 ++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/kafka/common/InterBrokerSendThread.scala b/core/src/main/scala/kafka/common/InterBrokerSendThread.scala index d0ba2edd42d89..c2724e24f1d3b 100644 --- a/core/src/main/scala/kafka/common/InterBrokerSendThread.scala +++ b/core/src/main/scala/kafka/common/InterBrokerSendThread.scala @@ -18,11 +18,10 @@ package kafka.common import java.util.Map.Entry import java.util.{ArrayDeque, ArrayList, Collection, Collections, HashMap, Iterator} - import kafka.utils.ShutdownableThread import org.apache.kafka.clients.{ClientRequest, ClientResponse, KafkaClient, RequestCompletionHandler} import org.apache.kafka.common.Node -import org.apache.kafka.common.errors.AuthenticationException +import org.apache.kafka.common.errors.{AuthenticationException, DisconnectException} import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.requests.AbstractRequest import org.apache.kafka.common.utils.Time @@ -78,6 +77,8 @@ abstract class InterBrokerSendThread( failExpiredRequests(now) unsentRequests.clean() } catch { + case _: DisconnectException if !networkClient.active() => + // DisconnectException is expected when NetworkClient#initiateClose is called case e: FatalExitError => throw e case t: Throwable => error(s"unhandled exception caught in InterBrokerSendThread", t) diff --git a/core/src/test/scala/kafka/common/InterBrokerSendThreadTest.scala b/core/src/test/scala/kafka/common/InterBrokerSendThreadTest.scala index 53a6b909ad37f..f2d9fb9424811 100644 --- a/core/src/test/scala/kafka/common/InterBrokerSendThreadTest.scala +++ b/core/src/test/scala/kafka/common/InterBrokerSendThreadTest.scala @@ -16,18 +16,18 @@ */ package kafka.common -import java.util - import kafka.utils.MockTime import org.apache.kafka.clients.{ClientRequest, ClientResponse, NetworkClient, RequestCompletionHandler} import org.apache.kafka.common.Node -import org.apache.kafka.common.errors.AuthenticationException +import org.apache.kafka.common.errors.{AuthenticationException, DisconnectException} import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.requests.AbstractRequest import org.easymock.EasyMock import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test +import org.mockito.{ArgumentMatchers, Mockito} +import java.util import scala.collection.mutable class InterBrokerSendThreadTest { @@ -36,8 +36,9 @@ class InterBrokerSendThreadTest { private val completionHandler = new StubCompletionHandler private val requestTimeoutMs = 1000 - class TestInterBrokerSendThread( - ) extends InterBrokerSendThread("name", networkClient, requestTimeoutMs, time) { + class TestInterBrokerSendThread(networkClient: NetworkClient = networkClient, + exceptionCallback: Throwable => Unit = t => throw t) + extends InterBrokerSendThread("name", networkClient, requestTimeoutMs, time) { private val queue = mutable.Queue[RequestAndCompletionHandler]() def enqueue(request: RequestAndCompletionHandler): Unit = { @@ -51,6 +52,26 @@ class InterBrokerSendThreadTest { Some(queue.dequeue()) } } + override def pollOnce(maxTimeoutMs: Long): Unit = { + try super.pollOnce(maxTimeoutMs) + catch { + case e: Throwable => exceptionCallback(e) + } + } + + } + + @Test + def shutdownThreadShouldNotCauseException(): Unit = { + val networkClient = Mockito.mock(classOf[NetworkClient]) + // InterBrokerSendThread#shutdown calls NetworkClient#initiateClose first so NetworkClient#poll + // can throw DisconnectException when thread is running + Mockito.when(networkClient.poll(ArgumentMatchers.anyLong, ArgumentMatchers.anyLong)).thenThrow(new DisconnectException()) + var exception: Throwable = null + val thread = new TestInterBrokerSendThread(networkClient, e => exception = e) + thread.shutdown() + thread.pollOnce(100) + assertNull(exception) } @Test From 23b85417b3ed53bf092877baef42608f7b778843 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Mon, 22 Feb 2021 20:34:25 -0800 Subject: [PATCH 044/243] MINOR: Move `RequestChannel.Response` creation logic into `RequestChannel` (#9912) This patch moves some common response creation logic from `RequestHandlerHelper` and into `RequestChannel`. This refactor has the following benefits: - It allows us to get rid of some logic that was previously duplicated in both `RequestHandlerHelper` and `TestRaftRequestHandler`. - It ensures that we do not need to rely on the caller to ensure that `updateErrorMetrics` gets called since this is handled internally in `RequestChannel`. - It provides better encapsulation of the quota workflow which relies on custom `Response` objects. Previously it was quite confusing for `KafkaApis` to handle this directly through the `sendResponse` API. Reviewers: Ismael Juma --- .../kafka/common/requests/RequestContext.java | 15 + .../scala/kafka/network/RequestChannel.scala | 40 +- .../kafka/server/ClientQuotaManager.scala | 8 +- .../scala/kafka/server/ControllerApis.scala | 2 +- .../main/scala/kafka/server/KafkaApis.scala | 29 +- .../kafka/server/RequestHandlerHelper.scala | 106 ++-- .../scala/kafka/server/ThrottledChannel.scala | 22 +- .../kafka/tools/TestRaftRequestHandler.scala | 63 +-- .../unit/kafka/network/SocketServerTest.scala | 16 +- .../server/BaseClientQuotaManagerTest.scala | 16 +- .../kafka/server/ControllerApisTest.scala | 81 +-- .../unit/kafka/server/KafkaApisTest.scala | 508 ++++++++---------- .../ThrottledChannelExpirationTest.scala | 55 +- 13 files changed, 449 insertions(+), 512 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java index 225db375370d5..d7a6df19c7c02 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java @@ -175,4 +175,19 @@ public String clientId() { public int correlationId() { return header.correlationId(); } + + @Override + public String toString() { + return "RequestContext(" + + "header=" + header + + ", connectionId='" + connectionId + '\'' + + ", clientAddress=" + clientAddress + + ", principal=" + principal + + ", listenerName=" + listenerName + + ", securityProtocol=" + securityProtocol + + ", clientInformation=" + clientInformation + + ", fromPrivilegedListener=" + fromPrivilegedListener + + ", principalSerde=" + principalSerde + + ')'; + } } diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 48f723f4d3375..40d0bd62e622d 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.JsonNode import com.typesafe.scalalogging.Logger import com.yammer.metrics.core.Meter import kafka.metrics.KafkaMetricsGroup +import kafka.network import kafka.server.KafkaConfig import kafka.utils.{Logging, NotNothing, Pool} import kafka.utils.Implicits._ @@ -372,9 +373,44 @@ class RequestChannel(val queueSize: Int, requestQueue.put(request) } - /** Send a response back to the socket server to be sent over the network */ - def sendResponse(response: RequestChannel.Response): Unit = { + def closeConnection( + request: RequestChannel.Request, + errorCounts: java.util.Map[Errors, Integer] + ): Unit = { + // This case is used when the request handler has encountered an error, but the client + // does not expect a response (e.g. when produce request has acks set to 0) + updateErrorMetrics(request.header.apiKey, errorCounts.asScala) + sendResponse(new RequestChannel.CloseConnectionResponse(request)) + } + + def sendResponse( + request: RequestChannel.Request, + response: AbstractResponse, + onComplete: Option[Send => Unit] + ): Unit = { + updateErrorMetrics(request.header.apiKey, response.errorCounts.asScala) + sendResponse(new RequestChannel.SendResponse( + request, + request.buildResponseSend(response), + request.responseNode(response), + onComplete + )) + } + + def sendNoOpResponse(request: RequestChannel.Request): Unit = { + sendResponse(new network.RequestChannel.NoOpResponse(request)) + } + def startThrottling(request: RequestChannel.Request): Unit = { + sendResponse(new RequestChannel.StartThrottlingResponse(request)) + } + + def endThrottling(request: RequestChannel.Request): Unit = { + sendResponse(new EndThrottlingResponse(request)) + } + + /** Send a response back to the socket server to be sent over the network */ + private[network] def sendResponse(response: RequestChannel.Response): Unit = { if (isTraceEnabled) { val requestHeader = response.request.headerForLoggingOrThrottling() val message = response match { diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala index e32978cf1710e..1f5b752d614bc 100644 --- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala +++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala @@ -335,11 +335,15 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, * @param throttleTimeMs Duration in milliseconds for which the channel is to be muted. * @param channelThrottlingCallback Callback for channel throttling */ - def throttle(request: RequestChannel.Request, throttleTimeMs: Int, channelThrottlingCallback: Response => Unit): Unit = { + def throttle( + request: RequestChannel.Request, + throttleCallback: ThrottleCallback, + throttleTimeMs: Int + ): Unit = { if (throttleTimeMs > 0) { val clientSensors = getOrCreateQuotaSensors(request.session, request.headerForLoggingOrThrottling().clientId) clientSensors.throttleTimeSensor.record(throttleTimeMs) - val throttledChannel = new ThrottledChannel(request, time, throttleTimeMs, channelThrottlingCallback) + val throttledChannel = new ThrottledChannel(time, throttleTimeMs, throttleCallback) delayQueue.add(throttledChannel) delayQueueSensor.record() debug("Channel throttled for sensor (%s). Delay time: (%d)".format(clientSensors.quotaSensor.name(), throttleTimeMs)) diff --git a/core/src/main/scala/kafka/server/ControllerApis.scala b/core/src/main/scala/kafka/server/ControllerApis.scala index abd8506be67cb..336775c4933e8 100644 --- a/core/src/main/scala/kafka/server/ControllerApis.scala +++ b/core/src/main/scala/kafka/server/ControllerApis.scala @@ -63,7 +63,7 @@ class ControllerApis(val requestChannel: RequestChannel, val controllerNodes: Seq[Node]) extends ApiRequestHandler with Logging { val authHelper = new AuthHelper(authorizer) - val requestHelper = new RequestHandlerHelper(requestChannel, quotas, time, s"[ControllerApis id=${config.nodeId}] ") + val requestHelper = new RequestHandlerHelper(requestChannel, quotas, time) var supportedApiKeys = Set( ApiKeys.FETCH, diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 5a926d4ace403..1245fe74d2e0c 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -120,7 +120,7 @@ class KafkaApis(val requestChannel: RequestChannel, private val alterAclsPurgatory = new DelayedFuturePurgatory(purgatoryName = "AlterAcls", brokerId = config.brokerId) val authHelper = new AuthHelper(authorizer) - val requestHelper = new RequestHandlerHelper(requestChannel, quotas, time, logIdent) + val requestHelper = new RequestHandlerHelper(requestChannel, quotas, time) def close(): Unit = { alterAclsPurgatory.shutdown() @@ -142,7 +142,7 @@ class KafkaApis(val requestChannel: RequestChannel, info(s"The client connection will be closed due to controller responded " + s"unsupported version exception during $request forwarding. " + s"This could happen when the controller changed after the connection was established.") - requestHelper.closeConnection(request, Collections.emptyMap()) + requestChannel.closeConnection(request, Collections.emptyMap()) } } @@ -226,7 +226,10 @@ class KafkaApis(val requestChannel: RequestChannel, } } catch { case e: FatalExitError => throw e - case e: Throwable => requestHelper.handleError(request, e) + case e: Throwable => + error(s"Unexpected error handling request ${request.requestDesc(true)} " + + s"with context ${request.context}", e) + requestHelper.handleError(request, e) } finally { // try to complete delayed action. In order to avoid conflicting locking, the actions to complete delayed requests // are kept in a queue. We add the logic to check the ReplicaManager queue at the end of KafkaApis.handle() and the @@ -593,9 +596,9 @@ class KafkaApis(val requestChannel: RequestChannel, if (maxThrottleTimeMs > 0) { request.apiThrottleTimeMs = maxThrottleTimeMs if (bandwidthThrottleTimeMs > requestThrottleTimeMs) { - quotas.produce.throttle(request, bandwidthThrottleTimeMs, requestChannel.sendResponse) + requestHelper.throttle(quotas.produce, request, bandwidthThrottleTimeMs) } else { - quotas.request.throttle(request, requestThrottleTimeMs, requestChannel.sendResponse) + requestHelper.throttle(quotas.request, request, requestThrottleTimeMs) } } @@ -613,14 +616,14 @@ class KafkaApis(val requestChannel: RequestChannel, s"from client id ${request.header.clientId} with ack=0\n" + s"Topic and partition to exceptions: $exceptionsSummary" ) - requestHelper.closeConnection(request, new ProduceResponse(mergedResponseStatus.asJava).errorCounts) + requestChannel.closeConnection(request, new ProduceResponse(mergedResponseStatus.asJava).errorCounts) } else { // Note that although request throttling is exempt for acks == 0, the channel may be throttled due to // bandwidth quota violation. requestHelper.sendNoOpResponseExemptThrottle(request) } } else { - requestHelper.sendResponse(request, Some(new ProduceResponse(mergedResponseStatus.asJava, maxThrottleTimeMs)), None) + requestChannel.sendResponse(request, new ProduceResponse(mergedResponseStatus.asJava, maxThrottleTimeMs), None) } } @@ -872,9 +875,9 @@ class KafkaApis(val requestChannel: RequestChannel, // from the fetch quota because we are going to return an empty response. quotas.fetch.unrecordQuotaSensor(request, responseSize, timeMs) if (bandwidthThrottleTimeMs > requestThrottleTimeMs) { - quotas.fetch.throttle(request, bandwidthThrottleTimeMs, requestChannel.sendResponse) + requestHelper.throttle(quotas.fetch, request, bandwidthThrottleTimeMs) } else { - quotas.request.throttle(request, requestThrottleTimeMs, requestChannel.sendResponse) + requestHelper.throttle(quotas.request, request, requestThrottleTimeMs) } // If throttling is required, return an empty response. unconvertedFetchResponse = fetchContext.getThrottledResponse(maxThrottleTimeMs) @@ -885,7 +888,7 @@ class KafkaApis(val requestChannel: RequestChannel, } // Send the response immediately. - requestHelper.sendResponse(request, Some(createResponse(maxThrottleTimeMs)), Some(updateConversionStats)) + requestChannel.sendResponse(request, createResponse(maxThrottleTimeMs), Some(updateConversionStats)) } } @@ -3207,13 +3210,13 @@ class KafkaApis(val requestChannel: RequestChannel, if (!isForwardingEnabled(request)) { info(s"Closing connection ${request.context.connectionId} because it sent an `Envelope` " + "request even though forwarding has not been enabled") - requestHelper.closeConnection(request, Collections.emptyMap()) + requestChannel.closeConnection(request, Collections.emptyMap()) return } else if (!request.context.fromPrivilegedListener) { info(s"Closing connection ${request.context.connectionId} from listener ${request.context.listenerName} " + s"because it sent an `Envelope` request, which is only accepted on the inter-broker listener " + s"${config.interBrokerListenerName}.") - requestHelper.closeConnection(request, Collections.emptyMap()) + requestChannel.closeConnection(request, Collections.emptyMap()) return } else if (!authHelper.authorize(request.context, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) { requestHelper.sendErrorResponseMaybeThrottle(request, new ClusterAuthorizationException( @@ -3225,7 +3228,7 @@ class KafkaApis(val requestChannel: RequestChannel, return } EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle) - } + } def handleDescribeProducersRequest(request: RequestChannel.Request): Unit = { val describeProducersRequest = request.body[DescribeProducersRequest] diff --git a/core/src/main/scala/kafka/server/RequestHandlerHelper.scala b/core/src/main/scala/kafka/server/RequestHandlerHelper.scala index cf2b816763790..ef9ff8210b53d 100644 --- a/core/src/main/scala/kafka/server/RequestHandlerHelper.scala +++ b/core/src/main/scala/kafka/server/RequestHandlerHelper.scala @@ -22,17 +22,12 @@ import kafka.coordinator.group.GroupCoordinator import kafka.coordinator.transaction.TransactionCoordinator import kafka.network.RequestChannel import kafka.server.QuotaFactory.QuotaManagers -import kafka.utils.Logging import org.apache.kafka.common.errors.ClusterAuthorizationException import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.network.Send -import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse} import org.apache.kafka.common.utils.Time -import scala.jdk.CollectionConverters._ - - object RequestHandlerHelper { def onLeadershipChange(groupCoordinator: GroupCoordinator, @@ -56,38 +51,55 @@ object RequestHandlerHelper { txnCoordinator.onResignation(partition.partitionId, Some(partition.getLeaderEpoch)) } } -} - +} -class RequestHandlerHelper(requestChannel: RequestChannel, - quotas: QuotaManagers, - time: Time, - logPrefix: String) extends Logging { - - this.logIdent = logPrefix +class RequestHandlerHelper( + requestChannel: RequestChannel, + quotas: QuotaManagers, + time: Time +) { + + def throttle( + quotaManager: ClientQuotaManager, + request: RequestChannel.Request, + throttleTimeMs: Int + ): Unit = { + val callback = new ThrottleCallback { + override def startThrottling(): Unit = requestChannel.startThrottling(request) + override def endThrottling(): Unit = requestChannel.endThrottling(request) + } + quotaManager.throttle(request, callback, throttleTimeMs) + } def handleError(request: RequestChannel.Request, e: Throwable): Unit = { val mayThrottle = e.isInstanceOf[ClusterAuthorizationException] || !request.header.apiKey.clusterAction - error("Error when handling request: " + - s"clientId=${request.header.clientId}, " + - s"correlationId=${request.header.correlationId}, " + - s"api=${request.header.apiKey}, " + - s"version=${request.header.apiVersion}, " + - s"body=${request.body[AbstractRequest]}", e) if (mayThrottle) sendErrorResponseMaybeThrottle(request, e) else sendErrorResponseExemptThrottle(request, e) } + def sendErrorOrCloseConnection( + request: RequestChannel.Request, + error: Throwable, + throttleMs: Int + ): Unit = { + val requestBody = request.body[AbstractRequest] + val response = requestBody.getErrorResponse(throttleMs, error) + if (response == null) + requestChannel.closeConnection(request, requestBody.errorCounts(error)) + else + requestChannel.sendResponse(request, response, None) + } + def sendForwardedResponse(request: RequestChannel.Request, response: AbstractResponse): Unit = { // For forwarded requests, we take the throttle time from the broker that // the request was forwarded to val throttleTimeMs = response.throttleTimeMs() - quotas.request.throttle(request, throttleTimeMs, requestChannel.sendResponse) - sendResponse(request, Some(response), None) + throttle(quotas.request, request, throttleTimeMs) + requestChannel.sendResponse(request, response, None) } // Throttle the channel if the request quota is enabled but has been violated. Regardless of throttling, send the @@ -97,15 +109,15 @@ class RequestHandlerHelper(requestChannel: RequestChannel, val throttleTimeMs = maybeRecordAndGetThrottleTimeMs(request) // Only throttle non-forwarded requests if (!request.isForwarded) - quotas.request.throttle(request, throttleTimeMs, requestChannel.sendResponse) - sendResponse(request, Some(createResponse(throttleTimeMs)), None) + throttle(quotas.request, request, throttleTimeMs) + requestChannel.sendResponse(request, createResponse(throttleTimeMs), None) } def sendErrorResponseMaybeThrottle(request: RequestChannel.Request, error: Throwable): Unit = { val throttleTimeMs = maybeRecordAndGetThrottleTimeMs(request) // Only throttle non-forwarded requests or cluster authorization failures if (error.isInstanceOf[ClusterAuthorizationException] || !request.isForwarded) - quotas.request.throttle(request, throttleTimeMs, requestChannel.sendResponse) + throttle(quotas.request, request, throttleTimeMs) sendErrorOrCloseConnection(request, error, throttleTimeMs) } @@ -130,29 +142,20 @@ class RequestHandlerHelper(requestChannel: RequestChannel, if (maxThrottleTimeMs > 0 && !request.isForwarded) { request.apiThrottleTimeMs = maxThrottleTimeMs if (controllerThrottleTimeMs > requestThrottleTimeMs) { - quotas.controllerMutation.throttle(request, controllerThrottleTimeMs, requestChannel.sendResponse) + throttle(quotas.controllerMutation, request, controllerThrottleTimeMs) } else { - quotas.request.throttle(request, requestThrottleTimeMs, requestChannel.sendResponse) + throttle(quotas.request, request, requestThrottleTimeMs) } } - sendResponse(request, Some(createResponse(maxThrottleTimeMs)), None) + requestChannel.sendResponse(request, createResponse(maxThrottleTimeMs), None) } def sendResponseExemptThrottle(request: RequestChannel.Request, response: AbstractResponse, onComplete: Option[Send => Unit] = None): Unit = { quotas.request.maybeRecordExempt(request) - sendResponse(request, Some(response), onComplete) - } - - def sendErrorOrCloseConnection(request: RequestChannel.Request, error: Throwable, throttleMs: Int): Unit = { - val requestBody = request.body[AbstractRequest] - val response = requestBody.getErrorResponse(throttleMs, error) - if (response == null) - closeConnection(request, requestBody.errorCounts(error)) - else - sendResponse(request, Some(response), None) + requestChannel.sendResponse(request, response, onComplete) } def sendErrorResponseExemptThrottle(request: RequestChannel.Request, error: Throwable): Unit = { @@ -162,34 +165,7 @@ class RequestHandlerHelper(requestChannel: RequestChannel, def sendNoOpResponseExemptThrottle(request: RequestChannel.Request): Unit = { quotas.request.maybeRecordExempt(request) - sendResponse(request, None, None) - } - - def closeConnection(request: RequestChannel.Request, errorCounts: java.util.Map[Errors, Integer]): Unit = { - // This case is used when the request handler has encountered an error, but the client - // does not expect a response (e.g. when produce request has acks set to 0) - requestChannel.updateErrorMetrics(request.header.apiKey, errorCounts.asScala) - requestChannel.sendResponse(new RequestChannel.CloseConnectionResponse(request)) + requestChannel.sendNoOpResponse(request) } - def sendResponse(request: RequestChannel.Request, - responseOpt: Option[AbstractResponse], - onComplete: Option[Send => Unit]): Unit = { - // Update error metrics for each error code in the response including Errors.NONE - responseOpt.foreach(response => requestChannel.updateErrorMetrics(request.header.apiKey, response.errorCounts.asScala)) - - val response = responseOpt match { - case Some(response) => - new RequestChannel.SendResponse( - request, - request.buildResponseSend(response), - request.responseNode(response), - onComplete - ) - case None => - new RequestChannel.NoOpResponse(request) - } - - requestChannel.sendResponse(response) - } } diff --git a/core/src/main/scala/kafka/server/ThrottledChannel.scala b/core/src/main/scala/kafka/server/ThrottledChannel.scala index 531ef5da43199..809167857e56d 100644 --- a/core/src/main/scala/kafka/server/ThrottledChannel.scala +++ b/core/src/main/scala/kafka/server/ThrottledChannel.scala @@ -19,33 +19,35 @@ package kafka.server import java.util.concurrent.{Delayed, TimeUnit} -import kafka.network -import kafka.network.RequestChannel -import kafka.network.RequestChannel.Response import kafka.utils.Logging import org.apache.kafka.common.utils.Time +trait ThrottleCallback { + def startThrottling(): Unit + def endThrottling(): Unit +} /** * Represents a request whose response has been delayed. - * @param request The request that has been delayed * @param time Time instance to use * @param throttleTimeMs Delay associated with this request - * @param channelThrottlingCallback Callback for channel throttling + * @param callback Callback for channel throttling */ -class ThrottledChannel(val request: RequestChannel.Request, val time: Time, val throttleTimeMs: Int, - channelThrottlingCallback: Response => Unit) - extends Delayed with Logging { +class ThrottledChannel( + val time: Time, + val throttleTimeMs: Int, + val callback: ThrottleCallback +) extends Delayed with Logging { private val endTimeNanos = time.nanoseconds() + TimeUnit.MILLISECONDS.toNanos(throttleTimeMs) // Notify the socket server that throttling has started for this channel. - channelThrottlingCallback(new RequestChannel.StartThrottlingResponse(request)) + callback.startThrottling() // Notify the socket server that throttling has been done for this channel. def notifyThrottlingDone(): Unit = { trace(s"Channel throttled for: $throttleTimeMs ms") - channelThrottlingCallback(new network.RequestChannel.EndThrottlingResponse(request)) + callback.endThrottling() } override def getDelay(unit: TimeUnit): Long = { diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index e4dec2ee66af3..db825ff7d91d2 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -18,19 +18,16 @@ package kafka.tools import kafka.network.RequestChannel -import kafka.network.RequestConvertToJson import kafka.raft.RaftManager import kafka.server.{ApiRequestHandler, ApiVersionManager} import kafka.utils.Logging import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, EndQuorumEpochResponseData, FetchResponseData, FetchSnapshotResponseData, VoteResponseData} -import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage, Errors} +import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage} import org.apache.kafka.common.record.BaseRecords import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, BeginQuorumEpochResponse, EndQuorumEpochResponse, FetchResponse, FetchSnapshotResponse, VoteResponse} import org.apache.kafka.common.utils.Time -import scala.jdk.CollectionConverters._ - /** * Simple request handler implementation for use by [[TestRaftServer]]. */ @@ -43,8 +40,7 @@ class TestRaftRequestHandler( override def handle(request: RequestChannel.Request): Unit = { try { - trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" + - s"securityProtocol:${request.context.securityProtocol},principal:${request.context.principal}") + trace(s"Handling request:${request.requestDesc(true)} with context ${request.context}") request.header.apiKey match { case ApiKeys.API_VERSIONS => handleApiVersions(request) case ApiKeys.VOTE => handleVote(request) @@ -56,7 +52,11 @@ class TestRaftRequestHandler( } } catch { case e: FatalExitError => throw e - case e: Throwable => handleError(request, e) + case e: Throwable => + error(s"Unexpected error handling request ${request.requestDesc(true)} " + + s"with context ${request.context}", e) + val errorResponse = request.body[AbstractRequest].getErrorResponse(e) + requestChannel.sendResponse(request, errorResponse, None) } finally { // The local completion time may be set while processing the request. Only record it if it's unset. if (request.apiLocalCompleteTimeNanos < 0) @@ -65,7 +65,7 @@ class TestRaftRequestHandler( } private def handleApiVersions(request: RequestChannel.Request): Unit = { - sendResponse(request, Some(apiVersionManager.apiVersionResponse(throttleTimeMs = 0))) + requestChannel.sendResponse(request, apiVersionManager.apiVersionResponse(throttleTimeMs = 0), None) } private def handleVote(request: RequestChannel.Request): Unit = { @@ -106,53 +106,8 @@ class TestRaftRequestHandler( } else { buildResponse(response) } - sendResponse(request, Some(res)) + requestChannel.sendResponse(request, res, None) }) } - private def handleError(request: RequestChannel.Request, err: Throwable): Unit = { - error("Error when handling request: " + - s"clientId=${request.header.clientId}, " + - s"correlationId=${request.header.correlationId}, " + - s"api=${request.header.apiKey}, " + - s"version=${request.header.apiVersion}, " + - s"body=${request.body[AbstractRequest]}", err) - - val requestBody = request.body[AbstractRequest] - val response = requestBody.getErrorResponse(0, err) - if (response == null) - closeConnection(request, requestBody.errorCounts(err)) - else - sendResponse(request, Some(response)) - } - - private def closeConnection(request: RequestChannel.Request, errorCounts: java.util.Map[Errors, Integer]): Unit = { - // This case is used when the request handler has encountered an error, but the client - // does not expect a response (e.g. when produce request has acks set to 0) - requestChannel.updateErrorMetrics(request.header.apiKey, errorCounts.asScala) - requestChannel.sendResponse(new RequestChannel.CloseConnectionResponse(request)) - } - - private def sendResponse(request: RequestChannel.Request, - responseOpt: Option[AbstractResponse]): Unit = { - // Update error metrics for each error code in the response including Errors.NONE - responseOpt.foreach(response => requestChannel.updateErrorMetrics(request.header.apiKey, response.errorCounts.asScala)) - - val response = responseOpt match { - case Some(response) => - val responseSend = request.context.buildResponseSend(response) - val responseString = - if (RequestChannel.isRequestLoggingEnabled) Some(RequestConvertToJson.response(response, request.context.apiVersion)) - else None - new RequestChannel.SendResponse(request, responseSend, responseString, None) - case None => - new RequestChannel.NoOpResponse(request) - } - sendResponse(response) - } - - private def sendResponse(response: RequestChannel.Response): Unit = { - requestChannel.sendResponse(response) - } - } diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 293614432cb76..d3230308cc13b 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -31,7 +31,7 @@ import com.yammer.metrics.core.{Gauge, Meter} import javax.net.ssl._ import kafka.metrics.KafkaYammerMetrics import kafka.security.CredentialProvider -import kafka.server.{KafkaConfig, SimpleApiVersionManager, ThrottledChannel} +import kafka.server.{KafkaConfig, SimpleApiVersionManager, ThrottleCallback, ThrottledChannel} import kafka.utils.Implicits._ import kafka.utils.TestUtils import org.apache.kafka.common.memory.MemoryPool @@ -147,7 +147,7 @@ class SocketServerTest { } def processRequestNoOpResponse(channel: RequestChannel, request: RequestChannel.Request): Unit = { - channel.sendResponse(new RequestChannel.NoOpResponse(request)) + channel.sendNoOpResponse(request) } def connect(s: SocketServer = server, @@ -247,7 +247,7 @@ class SocketServerTest { assertEquals(ClientInformation.UNKNOWN_NAME_OR_VERSION, receivedReq.context.clientInformation.softwareName) assertEquals(ClientInformation.UNKNOWN_NAME_OR_VERSION, receivedReq.context.clientInformation.softwareVersion) - server.dataPlaneRequestChannel.sendResponse(new RequestChannel.NoOpResponse(receivedReq)) + server.dataPlaneRequestChannel.sendNoOpResponse(receivedReq) // Send ProduceRequest - client info expected sendRequest(plainSocket, producerRequestBytes()) @@ -256,7 +256,7 @@ class SocketServerTest { assertEquals(expectedClientSoftwareName, receivedReq.context.clientInformation.softwareName) assertEquals(expectedClientSoftwareVersion, receivedReq.context.clientInformation.softwareVersion) - server.dataPlaneRequestChannel.sendResponse(new RequestChannel.NoOpResponse(receivedReq)) + server.dataPlaneRequestChannel.sendNoOpResponse(receivedReq) // Close the socket plainSocket.setSoLinger(true, 0) @@ -678,10 +678,12 @@ class SocketServerTest { val request = receiveRequest(server.dataPlaneRequestChannel) val byteBuffer = RequestTestUtils.serializeRequestWithHeader(request.header, request.body[AbstractRequest]) val send = new NetworkSend(request.context.connectionId, ByteBufferSend.sizePrefixed(byteBuffer)) - def channelThrottlingCallback(response: RequestChannel.Response): Unit = { - server.dataPlaneRequestChannel.sendResponse(response) + + val channelThrottlingCallback = new ThrottleCallback { + override def startThrottling(): Unit = server.dataPlaneRequestChannel.startThrottling(request) + override def endThrottling(): Unit = server.dataPlaneRequestChannel.endThrottling(request) } - val throttledChannel = new ThrottledChannel(request, new MockTime(), 100, channelThrottlingCallback) + val throttledChannel = new ThrottledChannel(new MockTime(), 100, channelThrottlingCallback) val headerLog = RequestConvertToJson.requestHeaderNode(request.header) val response = if (!noOpResponse) diff --git a/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala index 76b3d3eb56723..48379ca08aebf 100644 --- a/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala @@ -20,9 +20,7 @@ import java.net.InetAddress import java.util import java.util.Collections import kafka.network.RequestChannel -import kafka.network.RequestChannel.EndThrottlingResponse import kafka.network.RequestChannel.Session -import kafka.network.RequestChannel.StartThrottlingResponse import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.memory.MemoryPool import org.apache.kafka.common.metrics.MetricConfig @@ -47,11 +45,11 @@ class BaseClientQuotaManagerTest { metrics.close() } - protected def callback(response: RequestChannel.Response): Unit = { - // Count how many times this callback is called for notifyThrottlingDone(). - (response: @unchecked) match { - case _: StartThrottlingResponse => - case _: EndThrottlingResponse => numCallbacks += 1 + protected def callback: ThrottleCallback = new ThrottleCallback { + override def startThrottling(): Unit = {} + override def endThrottling(): Unit = { + // Count how many times this callback is called for notifyThrottlingDone(). + numCallbacks += 1 } } @@ -82,8 +80,8 @@ class BaseClientQuotaManagerTest { } protected def throttle(quotaManager: ClientQuotaManager, user: String, clientId: String, throttleTimeMs: Int, - channelThrottlingCallback: RequestChannel.Response => Unit): Unit = { + channelThrottlingCallback: ThrottleCallback): Unit = { val (_, request) = buildRequest(FetchRequest.Builder.forConsumer(0, 1000, new util.HashMap[TopicPartition, PartitionData])) - quotaManager.throttle(request, throttleTimeMs, channelThrottlingCallback) + quotaManager.throttle(request, channelThrottlingCallback, throttleTimeMs) } } diff --git a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala index 3533fcfdee2cd..0ca44f3cc17f2 100644 --- a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala @@ -26,33 +26,36 @@ import kafka.server.QuotaFactory.QuotaManagers import kafka.server.{ClientQuotaManager, ClientRequestQuotaManager, ControllerApis, ControllerMutationQuotaManager, KafkaConfig, MetaProperties, ReplicationQuotaManager} import kafka.utils.MockTime import org.apache.kafka.common.Uuid -import org.apache.kafka.common.errors.ClusterAuthorizationException import org.apache.kafka.common.memory.MemoryPool import org.apache.kafka.common.message.BrokerRegistrationRequestData import org.apache.kafka.common.network.{ClientInformation, ListenerName} import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.requests.{AbstractRequest, BrokerRegistrationRequest, RequestContext, RequestHeader, RequestTestUtils} +import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, BrokerRegistrationRequest, BrokerRegistrationResponse, RequestContext, RequestHeader, RequestTestUtils} import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.controller.Controller -import org.apache.kafka.metadata.{ApiMessageAndVersion, VersionRange} -import org.apache.kafka.server.authorizer.{AuthorizableRequestContext, AuthorizationResult, Authorizer} -import org.easymock.{Capture, EasyMock, IAnswer} +import org.apache.kafka.metadata.ApiMessageAndVersion +import org.apache.kafka.server.authorizer.{Action, AuthorizableRequestContext, AuthorizationResult, Authorizer} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} +import org.mockito.ArgumentMatchers._ +import org.mockito.Mockito._ +import org.mockito.{ArgumentCaptor, ArgumentMatchers} +import scala.jdk.CollectionConverters._ class ControllerApisTest { - // Mocks private val nodeId = 1 private val brokerRack = "Rack1" private val clientID = "Client1" - private val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) - private val requestChannel: RequestChannel = EasyMock.createNiceMock(classOf[RequestChannel]) + private val requestChannelMetrics: RequestChannel.Metrics = mock(classOf[RequestChannel.Metrics]) + private val requestChannel: RequestChannel = mock(classOf[RequestChannel]) private val time = new MockTime - private val clientQuotaManager: ClientQuotaManager = EasyMock.createNiceMock(classOf[ClientQuotaManager]) - private val clientRequestQuotaManager: ClientRequestQuotaManager = EasyMock.createNiceMock(classOf[ClientRequestQuotaManager]) - private val clientControllerQuotaManager: ControllerMutationQuotaManager = EasyMock.createNiceMock(classOf[ControllerMutationQuotaManager]) - private val replicaQuotaManager: ReplicationQuotaManager = EasyMock.createNiceMock(classOf[ReplicationQuotaManager]) - private val raftManager: RaftManager[ApiMessageAndVersion] = EasyMock.createNiceMock(classOf[RaftManager[ApiMessageAndVersion]]) + private val clientQuotaManager: ClientQuotaManager = mock(classOf[ClientQuotaManager]) + private val clientRequestQuotaManager: ClientRequestQuotaManager = mock(classOf[ClientRequestQuotaManager]) + private val clientControllerQuotaManager: ControllerMutationQuotaManager = mock(classOf[ControllerMutationQuotaManager]) + private val replicaQuotaManager: ReplicationQuotaManager = mock(classOf[ReplicationQuotaManager]) + private val raftManager: RaftManager[ApiMessageAndVersion] = mock(classOf[RaftManager[ApiMessageAndVersion]]) + private val authorizer: Authorizer = mock(classOf[Authorizer]) + private val quotas = QuotaManagers( clientQuotaManager, clientQuotaManager, @@ -62,24 +65,21 @@ class ControllerApisTest { replicaQuotaManager, replicaQuotaManager, None) - private val controller: Controller = EasyMock.createNiceMock(classOf[Controller]) + private val controller: Controller = mock(classOf[Controller]) - private def createControllerApis(authorizer: Option[Authorizer], - supportedFeatures: Map[String, VersionRange] = Map.empty): ControllerApis = { + private def createControllerApis(): ControllerApis = { val props = new Properties() props.put(KafkaConfig.NodeIdProp, nodeId: java.lang.Integer) props.put(KafkaConfig.ProcessRolesProp, "controller") new ControllerApis( requestChannel, - authorizer, + Some(authorizer), quotas, time, - supportedFeatures, + Map.empty, controller, raftManager, new KafkaConfig(props), - - // FIXME: Would make more sense to set controllerId here MetaProperties(Uuid.fromString("JgxuGe9URy-E-ceaL04lEw"), nodeId = nodeId), Seq.empty ) @@ -93,8 +93,10 @@ class ControllerApisTest { * @tparam T - Type of AbstractRequest * @return */ - private def buildRequest[T <: AbstractRequest](request: AbstractRequest, - listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): RequestChannel.Request = { + private def buildRequest[T <: AbstractRequest]( + request: AbstractRequest, + listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT) + ): RequestChannel.Request = { val buffer = RequestTestUtils.serializeRequestWithHeader( new RequestHeader(request.apiKey, request.version, clientID, 0), request) @@ -107,7 +109,7 @@ class ControllerApisTest { } @Test - def testBrokerRegistration(): Unit = { + def testUnauthorizedBrokerRegistration(): Unit = { val brokerRegistrationRequest = new BrokerRegistrationRequest.Builder( new BrokerRegistrationRequestData() .setBrokerId(nodeId) @@ -115,25 +117,26 @@ class ControllerApisTest { ).build() val request = buildRequest(brokerRegistrationRequest) + val capturedResponse: ArgumentCaptor[AbstractResponse] = ArgumentCaptor.forClass(classOf[AbstractResponse]) - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) - - val authorizer = Some[Authorizer](EasyMock.createNiceMock(classOf[Authorizer])) - EasyMock.expect(authorizer.get.authorize(EasyMock.anyObject[AuthorizableRequestContext](), EasyMock.anyObject())).andAnswer( - new IAnswer[java.util.List[AuthorizationResult]]() { - override def answer(): java.util.List[AuthorizationResult] = { - new java.util.ArrayList[AuthorizationResult](){ - add(AuthorizationResult.DENIED) - } - } - } + when(authorizer.authorize( + any(classOf[AuthorizableRequestContext]), + any(classOf[java.util.List[Action]]) + )).thenReturn( + java.util.Collections.singletonList(AuthorizationResult.DENIED) ) - EasyMock.replay(requestChannel, authorizer.get) - val assertion = assertThrows(classOf[ClusterAuthorizationException], - () => createControllerApis(authorizer = authorizer).handleBrokerRegistration(request)) - assert(Errors.forException(assertion) == Errors.CLUSTER_AUTHORIZATION_FAILED) + createControllerApis().handle(request) + verify(requestChannel).sendResponse( + ArgumentMatchers.eq(request), + capturedResponse.capture(), + ArgumentMatchers.eq(None)) + + assertNotNull(capturedResponse.getValue) + + val brokerRegistrationResponse = capturedResponse.getValue.asInstanceOf[BrokerRegistrationResponse] + assertEquals(Map(Errors.CLUSTER_AUTHORIZATION_FAILED -> 1), + brokerRegistrationResponse.errorCounts().asScala) } @AfterEach diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index e80c6eb3a736c..43b9ca597194f 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -32,7 +32,6 @@ import kafka.coordinator.group._ import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator} import kafka.log.AppendOrigin import kafka.network.RequestChannel -import kafka.network.RequestChannel.{CloseConnectionResponse, SendResponse} import kafka.server.QuotaFactory.QuotaManagers import kafka.server.metadata.{CachedConfigRepository, ConfigRepository, RaftMetadataCache} import kafka.utils.{MockTime, TestUtils} @@ -94,7 +93,6 @@ class KafkaApisTest { private val forwardingManager: ForwardingManager = EasyMock.createNiceMock(classOf[ForwardingManager]) private val autoTopicCreationManager: AutoTopicCreationManager = EasyMock.createNiceMock(classOf[AutoTopicCreationManager]) - private val hostAddress: Array[Byte] = InetAddress.getByName("192.168.1.1").getAddress private val kafkaPrincipalSerde = new KafkaPrincipalSerde { override def serialize(principal: KafkaPrincipal): Array[Byte] = Utils.utf8(principal.toString) override def deserialize(bytes: Array[Byte]): KafkaPrincipal = SecurityUtils.parseKafkaPrincipal(Utils.utf8(bytes)) @@ -211,8 +209,6 @@ class KafkaApisTest { .andReturn(Seq(AuthorizationResult.ALLOWED).asJava) .once() - val capturedResponse = expectNoThrottling() - val configRepository: ConfigRepository = EasyMock.strictMock(classOf[ConfigRepository]) val topicConfigs = new Properties() val propName = "min.insync.replicas" @@ -229,8 +225,6 @@ class KafkaApisTest { expect(metadataCache.contains(resourceName)).andReturn(true) - EasyMock.replay(metadataCache, replicaManager, clientRequestQuotaManager, requestChannel, authorizer, configRepository, adminManager) - val describeConfigsRequest = new DescribeConfigsRequest.Builder(new DescribeConfigsRequestData() .setIncludeSynonyms(true) .setResources(List(new DescribeConfigsRequestData.DescribeConfigsResource() @@ -239,12 +233,16 @@ class KafkaApisTest { .build(requestHeader.apiVersion) val request = buildRequest(describeConfigsRequest, requestHeader = Option(requestHeader)) - createKafkaApis(authorizer = Some(authorizer), configRepository = configRepository).handleDescribeConfigsRequest(request) + val capturedResponse = expectNoThrottling(request) + + EasyMock.replay(metadataCache, replicaManager, clientRequestQuotaManager, requestChannel, + authorizer, configRepository, adminManager) + createKafkaApis(authorizer = Some(authorizer), configRepository = configRepository) + .handleDescribeConfigsRequest(request) verify(authorizer, replicaManager) - val response = readResponse(describeConfigsRequest, capturedResponse) - .asInstanceOf[DescribeConfigsResponse] + val response = capturedResponse.getValue.asInstanceOf[DescribeConfigsResponse] val results = response.data().results() assertEquals(1, results.size()) val describeConfigsResult: DescribeConfigsResult = results.get(0) @@ -252,7 +250,7 @@ class KafkaApisTest { assertEquals(resourceName, describeConfigsResult.resourceName()) val configs = describeConfigsResult.configs().asScala.filter(_.name() == propName) assertEquals(1, configs.length) - val describeConfigsResponseData = configs(0) + val describeConfigsResponseData = configs.head assertEquals(propName, describeConfigsResponseData.name()) assertEquals(propValue, describeConfigsResponseData.value()) } @@ -290,37 +288,33 @@ class KafkaApisTest { authorizeResource(authorizer, operation, ResourceType.TOPIC, resourceName, AuthorizationResult.ALLOWED) - val capturedResponse = expectNoThrottling() - val configResource = new ConfigResource(ConfigResource.Type.TOPIC, resourceName) EasyMock.expect(adminManager.alterConfigs(anyObject(), EasyMock.eq(false))) .andAnswer(() => { Map(configResource -> alterConfigHandler.apply()) }) - EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, authorizer, - adminManager, controller) - val configs = Map( configResource -> new AlterConfigsRequest.Config( Seq(new AlterConfigsRequest.ConfigEntry("foo", "bar")).asJava)) val alterConfigsRequest = new AlterConfigsRequest.Builder(configs.asJava, false).build(requestHeader.apiVersion) val request = buildRequestWithEnvelope(alterConfigsRequest, fromPrivilegedListener = true) + val capturedResponse = EasyMock.newCapture[AbstractResponse]() + val capturedRequest = EasyMock.newCapture[RequestChannel.Request]() - createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handle(request) - - val envelopeRequest = request.body[EnvelopeRequest] - val response = readResponse(envelopeRequest, capturedResponse) - .asInstanceOf[EnvelopeResponse] - - assertEquals(Errors.NONE, response.error) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.capture(capturedRequest), + EasyMock.capture(capturedResponse), + EasyMock.anyObject() + )) - val innerResponse = AbstractResponse.parseResponse( - response.responseData(), - requestHeader - ).asInstanceOf[AlterConfigsResponse] + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, authorizer, + adminManager, controller) + createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handle(request) + assertEquals(Some(request), capturedRequest.getValue.envelope) + val innerResponse = capturedResponse.getValue.asInstanceOf[AlterConfigsResponse] val responseMap = innerResponse.data.responses().asScala.map { resourceResponse => resourceResponse.resourceName() -> Errors.forCode(resourceResponse.errorCode) }.toMap @@ -336,29 +330,16 @@ class KafkaApisTest { clientId, 0) val leaveGroupRequest = new LeaveGroupRequest.Builder("group", Collections.singletonList(new MemberIdentity())).build(requestHeader.apiVersion) - val serializedRequestData = RequestTestUtils.serializeRequestWithHeader(requestHeader, leaveGroupRequest) - - resetToStrict(requestChannel) EasyMock.expect(controller.isActive).andReturn(true) - EasyMock.expect(requestChannel.metrics).andReturn(EasyMock.niceMock(classOf[RequestChannel.Metrics])) - EasyMock.expect(requestChannel.updateErrorMetrics(ApiKeys.ENVELOPE, Map(Errors.INVALID_REQUEST -> 1))) - val capturedResponse = expectNoThrottling() - - EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, controller) - - val envelopeHeader = new RequestHeader(ApiKeys.ENVELOPE, ApiKeys.ENVELOPE.latestVersion, - clientId, 0) - - val envelopeRequest = new EnvelopeRequest.Builder(serializedRequestData, new Array[Byte](0), hostAddress) - .build(envelopeHeader.apiVersion) val request = buildRequestWithEnvelope(leaveGroupRequest, fromPrivilegedListener = true) + val capturedResponse = expectNoThrottling(request) + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, controller) createKafkaApis(enableForwarding = true).handle(request) - val response = readResponse(envelopeRequest, capturedResponse) - .asInstanceOf[EnvelopeResponse] + val response = capturedResponse.getValue.asInstanceOf[EnvelopeResponse] assertEquals(Errors.INVALID_REQUEST, response.error()) } @@ -397,13 +378,8 @@ class KafkaApisTest { EasyMock.expect(controller.isActive).andReturn(isActiveController) - val capturedResponse = expectNoThrottling() - val configResource = new ConfigResource(ConfigResource.Type.TOPIC, resourceName) - EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, authorizer, - adminManager, controller) - val configs = Map( configResource -> new AlterConfigsRequest.Config( Seq(new AlterConfigsRequest.ConfigEntry("foo", "bar")).asJava)) @@ -412,19 +388,31 @@ class KafkaApisTest { val request = buildRequestWithEnvelope(alterConfigsRequest, fromPrivilegedListener = fromPrivilegedListener) - createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handle(request) + val capturedResponse = EasyMock.newCapture[AbstractResponse]() if (shouldCloseConnection) { - assertTrue(capturedResponse.getValue.isInstanceOf[CloseConnectionResponse]) + EasyMock.expect(requestChannel.closeConnection( + EasyMock.eq(request), + EasyMock.eq(java.util.Collections.emptyMap()) + )) } else { - val envelopeRequest = request.body[EnvelopeRequest] - val response = readResponse(envelopeRequest, capturedResponse) - .asInstanceOf[EnvelopeResponse] + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) + } - assertEquals(expectedError, response.error()) + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, authorizer, + adminManager, controller) + createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handle(request) - verify(authorizer, adminManager) + if (!shouldCloseConnection) { + val response = capturedResponse.getValue.asInstanceOf[EnvelopeResponse] + assertEquals(expectedError, response.error) } + + verify(authorizer, adminManager, requestChannel) } @Test @@ -452,7 +440,7 @@ class KafkaApisTest { EasyMock.expect(controller.isActive).andReturn(false) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.expect(adminManager.alterConfigs(anyObject(), EasyMock.eq(false))) .andReturn(Map(authorizedResource -> ApiError.NONE)) @@ -483,7 +471,7 @@ class KafkaApisTest { EasyMock.expect(controller.isActive).andReturn(false) - expectNoThrottling() + expectNoThrottling(request) EasyMock.expect(forwardingManager.forwardRequest( EasyMock.eq(request), @@ -519,10 +507,9 @@ class KafkaApisTest { } private def verifyAlterConfigResult(alterConfigsRequest: AlterConfigsRequest, - capturedResponse: Capture[RequestChannel.Response], + capturedResponse: Capture[AbstractResponse], expectedResults: Map[String, Errors]): Unit = { - val response = readResponse(alterConfigsRequest, capturedResponse) - .asInstanceOf[AlterConfigsResponse] + val response = capturedResponse.getValue.asInstanceOf[AlterConfigsResponse] val responseMap = response.data.responses().asScala.map { resourceResponse => resourceResponse.resourceName() -> Errors.forCode(resourceResponse.errorCode) }.toMap @@ -559,19 +546,19 @@ class KafkaApisTest { EasyMock.expect(controller.isActive).andReturn(true) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.expect(adminManager.incrementalAlterConfigs(anyObject(), EasyMock.eq(false))) .andReturn(Map(authorizedResource -> ApiError.NONE)) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, authorizer, adminManager, controller) - createKafkaApis(authorizer = Some(authorizer)).handleIncrementalAlterConfigsRequest(request) - verifyIncrementalAlterConfigResult(incrementalAlterConfigsRequest, - capturedResponse, Map(authorizedTopic -> Errors.NONE, - unauthorizedTopic -> Errors.TOPIC_AUTHORIZATION_FAILED)) + verifyIncrementalAlterConfigResult(capturedResponse, Map( + authorizedTopic -> Errors.NONE, + unauthorizedTopic -> Errors.TOPIC_AUTHORIZATION_FAILED + )) verify(authorizer, adminManager) } @@ -593,15 +580,12 @@ class KafkaApisTest { new IncrementalAlterConfigsRequest.Builder(resourceMap, false) } - private def verifyIncrementalAlterConfigResult(incrementalAlterConfigsRequest: IncrementalAlterConfigsRequest, - capturedResponse: Capture[RequestChannel.Response], + private def verifyIncrementalAlterConfigResult(capturedResponse: Capture[AbstractResponse], expectedResults: Map[String, Errors]): Unit = { - val response = readResponse(incrementalAlterConfigsRequest, capturedResponse) - .asInstanceOf[IncrementalAlterConfigsResponse] + val response = capturedResponse.getValue.asInstanceOf[IncrementalAlterConfigsResponse] val responseMap = response.data.responses().asScala.map { resourceResponse => resourceResponse.resourceName() -> Errors.forCode(resourceResponse.errorCode) }.toMap - assertEquals(expectedResults, responseMap) } @@ -624,15 +608,13 @@ class KafkaApisTest { EasyMock.expect(controller.isActive).andReturn(true) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, authorizer, adminManager, controller) - createKafkaApis(authorizer = Some(authorizer)).handleAlterClientQuotasRequest(request) - verifyAlterClientQuotaResult(alterClientQuotasRequest, - capturedResponse, Map(quotaEntity -> Errors.CLUSTER_AUTHORIZATION_FAILED)) + verifyAlterClientQuotaResult(capturedResponse, Map(quotaEntity -> Errors.CLUSTER_AUTHORIZATION_FAILED)) verify(authorizer, adminManager) } @@ -643,11 +625,9 @@ class KafkaApisTest { testForwardableAPI(ApiKeys.ALTER_CLIENT_QUOTAS, requestBuilder) } - private def verifyAlterClientQuotaResult(alterClientQuotasRequest: AlterClientQuotasRequest, - capturedResponse: Capture[RequestChannel.Response], + private def verifyAlterClientQuotaResult(capturedResponse: Capture[AbstractResponse], expected: Map[ClientQuotaEntity, Errors]): Unit = { - val response = readResponse(alterClientQuotasRequest, capturedResponse) - .asInstanceOf[AlterClientQuotasResponse] + val response = capturedResponse.getValue.asInstanceOf[AlterClientQuotasResponse] val futures = expected.keys.map(quotaEntity => quotaEntity -> new KafkaFutureImpl[Void]()).toMap response.complete(futures.asJava) futures.foreach { @@ -678,8 +658,6 @@ class KafkaApisTest { EasyMock.expect(controller.isActive).andReturn(true) - val capturedResponse = expectNoThrottling() - val topics = new CreateTopicsRequestData.CreatableTopicCollection(2) val topicToCreate = new CreateTopicsRequestData.CreatableTopic() .setName(authorizedTopic) @@ -699,6 +677,8 @@ class KafkaApisTest { val request = buildRequest(createTopicsRequest, fromPrivilegedListener = true, requestHeader = Option(requestHeader)) + val capturedResponse = expectNoThrottling(request) + EasyMock.expect(clientControllerQuotaManager.newQuotaFor( EasyMock.eq(request), EasyMock.eq(6))).andReturn(UnboundedControllerMutationQuota) @@ -776,10 +756,9 @@ class KafkaApisTest { } private def verifyCreateTopicsResult(createTopicsRequest: CreateTopicsRequest, - capturedResponse: Capture[RequestChannel.Response], + capturedResponse: Capture[AbstractResponse], expectedResults: Map[String, Errors]): Unit = { - val response = readResponse(createTopicsRequest, capturedResponse) - .asInstanceOf[CreateTopicsResponse] + val response = capturedResponse.getValue.asInstanceOf[CreateTopicsResponse] val responseMap = response.data.topics().asScala.map { topicResponse => topicResponse.name() -> Errors.forCode(topicResponse.errorCode) }.toMap @@ -910,7 +889,7 @@ class KafkaApisTest { ).build(requestHeader.apiVersion) val request = buildRequest(findCoordinatorRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) verifyTopicCreation(topicName, true, true, request) @@ -920,8 +899,7 @@ class KafkaApisTest { createKafkaApis(authorizer = Some(authorizer), overrideProperties = topicConfigOverride).handleFindCoordinatorRequest(request) - val response = readResponse(findCoordinatorRequest, capturedResponse) - .asInstanceOf[FindCoordinatorResponse] + val response = capturedResponse.getValue.asInstanceOf[FindCoordinatorResponse] assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, response.error()) verify(authorizer, autoTopicCreationManager) @@ -1012,7 +990,7 @@ class KafkaApisTest { ).build(requestHeader.apiVersion) val request = buildRequest(metadataRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) verifyTopicCreation(topicName, enableAutoTopicCreation, isInternal, request) @@ -1022,9 +1000,7 @@ class KafkaApisTest { createKafkaApis(authorizer = Some(authorizer), enableForwarding = enableAutoTopicCreation, overrideProperties = topicConfigOverride).handleTopicMetadataRequest(request) - val response = readResponse(metadataRequest, capturedResponse) - .asInstanceOf[MetadataResponse] - + val response = capturedResponse.getValue.asInstanceOf[MetadataResponse] val expectedMetadataResponse = util.Collections.singletonList(new TopicMetadata( expectedError, topicName, @@ -1088,12 +1064,11 @@ class KafkaApisTest { ))).build() val request = buildRequest(offsetCommitRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) createKafkaApis().handleOffsetCommitRequest(request) - val response = readResponse(offsetCommitRequest, capturedResponse) - .asInstanceOf[OffsetCommitResponse] + val response = capturedResponse.getValue.asInstanceOf[OffsetCommitResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.forCode(response.data().topics().get(0).partitions().get(0).errorCode())) } @@ -1122,12 +1097,11 @@ class KafkaApisTest { ).build() val request = buildRequest(offsetCommitRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) createKafkaApis().handleTxnOffsetCommitRequest(request) - val response = readResponse(offsetCommitRequest, capturedResponse) - .asInstanceOf[TxnOffsetCommitResponse] + val response = capturedResponse.getValue.asInstanceOf[TxnOffsetCommitResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response.errors().get(invalidTopicPartition)) } @@ -1144,7 +1118,7 @@ class KafkaApisTest { EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel, groupCoordinator) val topicPartition = new TopicPartition(topic, 1) - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val responseCallback: Capture[Map[TopicPartition, Errors] => Unit] = EasyMock.newCapture() val partitionOffsetCommitData = new TxnOffsetCommitRequest.CommittedOffset(15L, "", Optional.empty()) @@ -1175,14 +1149,17 @@ class KafkaApisTest { )).andAnswer( () => responseCallback.getValue.apply(Map(topicPartition -> Errors.COORDINATOR_LOAD_IN_PROGRESS))) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, groupCoordinator) createKafkaApis().handleTxnOffsetCommitRequest(request) - val response = readResponse(offsetCommitRequest, capturedResponse) - .asInstanceOf[TxnOffsetCommitResponse] + val response = capturedResponse.getValue.asInstanceOf[TxnOffsetCommitResponse] if (version < 2) { assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, response.errors().get(topicPartition)) @@ -1201,7 +1178,7 @@ class KafkaApisTest { EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val responseCallback: Capture[InitProducerIdResult => Unit] = EasyMock.newCapture() val transactionalId = "txnId" @@ -1240,14 +1217,17 @@ class KafkaApisTest { )).andAnswer( () => responseCallback.getValue.apply(InitProducerIdResult(producerId, epoch, Errors.PRODUCER_FENCED))) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) createKafkaApis().handleInitProducerIdRequest(request) - val response = readResponse(initProducerIdRequest, capturedResponse) - .asInstanceOf[InitProducerIdResponse] + val response = capturedResponse.getValue.asInstanceOf[InitProducerIdResponse] if (version < 4) { assertEquals(Errors.INVALID_PRODUCER_EPOCH.code, response.data.errorCode) @@ -1266,7 +1246,7 @@ class KafkaApisTest { EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel, groupCoordinator, txnCoordinator) - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val responseCallback: Capture[Errors => Unit] = EasyMock.newCapture() val groupId = "groupId" @@ -1297,14 +1277,17 @@ class KafkaApisTest { )).andAnswer( () => responseCallback.getValue.apply(Errors.PRODUCER_FENCED)) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator, groupCoordinator) createKafkaApis().handleAddOffsetsToTxnRequest(request) - val response = readResponse(addOffsetsToTxnRequest, capturedResponse) - .asInstanceOf[AddOffsetsToTxnResponse] + val response = capturedResponse.getValue.asInstanceOf[AddOffsetsToTxnResponse] if (version < 2) { assertEquals(Errors.INVALID_PRODUCER_EPOCH.code, response.data.errorCode) @@ -1323,7 +1306,7 @@ class KafkaApisTest { EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val responseCallback: Capture[Errors => Unit] = EasyMock.newCapture() val transactionalId = "txnId" @@ -1351,14 +1334,17 @@ class KafkaApisTest { )).andAnswer( () => responseCallback.getValue.apply(Errors.PRODUCER_FENCED)) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) createKafkaApis().handleAddPartitionToTxnRequest(request) - val response = readResponse(addPartitionsToTxnRequest, capturedResponse) - .asInstanceOf[AddPartitionsToTxnResponse] + val response = capturedResponse.getValue.asInstanceOf[AddPartitionsToTxnResponse] if (version < 2) { assertEquals(Collections.singletonMap(topicPartition, Errors.INVALID_PRODUCER_EPOCH), response.errors()) @@ -1374,10 +1360,9 @@ class KafkaApisTest { addTopicToMetadataCache(topic, numPartitions = 2) for (version <- ApiKeys.END_TXN.oldestVersion to ApiKeys.END_TXN.latestVersion) { - EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val responseCallback: Capture[Errors => Unit] = EasyMock.newCapture() val transactionalId = "txnId" @@ -1402,14 +1387,16 @@ class KafkaApisTest { )).andAnswer( () => responseCallback.getValue.apply(Errors.PRODUCER_FENCED)) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - createKafkaApis().handleEndTxnRequest(request) - val response = readResponse(endTxnRequest, capturedResponse) - .asInstanceOf[EndTxnResponse] + val response = capturedResponse.getValue.asInstanceOf[EndTxnResponse] if (version < 2) { assertEquals(Errors.INVALID_PRODUCER_EPOCH.code, response.data.errorCode) @@ -1456,7 +1443,7 @@ class KafkaApisTest { EasyMock.anyObject()) ).andAnswer(() => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.INVALID_PRODUCER_EPOCH)))) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.expect(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( anyObject[RequestChannel.Request](), anyDouble, anyLong)).andReturn(0) @@ -1464,8 +1451,7 @@ class KafkaApisTest { createKafkaApis().handleProduceRequest(request) - val response = readResponse(produceRequest, capturedResponse) - .asInstanceOf[ProduceResponse] + val response = capturedResponse.getValue.asInstanceOf[ProduceResponse] assertEquals(1, response.responses().size()) for (partitionResponse <- response.responses().asScala) { @@ -1488,12 +1474,11 @@ class KafkaApisTest { ).build() val request = buildRequest(addPartitionsToTxnRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) createKafkaApis().handleAddPartitionToTxnRequest(request) - val response = readResponse(addPartitionsToTxnRequest, capturedResponse) - .asInstanceOf[AddPartitionsToTxnResponse] + val response = capturedResponse.getValue.asInstanceOf[AddPartitionsToTxnResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, response.errors().get(invalidTopicPartition)) } @@ -1531,17 +1516,20 @@ class KafkaApisTest { val topicPartition = new TopicPartition("t", 0) val (writeTxnMarkersRequest, request) = createWriteTxnMarkersRequest(asList(topicPartition)) val expectedErrors = Map(topicPartition -> Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT).asJava - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() EasyMock.expect(replicaManager.getMagic(topicPartition)) .andReturn(Some(RecordBatch.MAGIC_VALUE_V1)) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, replicaQuotaManager, requestChannel) createKafkaApis().handleWriteTxnMarkersRequest(request) - val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) - .asInstanceOf[WriteTxnMarkersResponse] + val markersResponse = capturedResponse.getValue.asInstanceOf[WriteTxnMarkersResponse] assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) } @@ -1550,17 +1538,20 @@ class KafkaApisTest { val topicPartition = new TopicPartition("t", 0) val (writeTxnMarkersRequest, request) = createWriteTxnMarkersRequest(asList(topicPartition)) val expectedErrors = Map(topicPartition -> Errors.UNKNOWN_TOPIC_OR_PARTITION).asJava - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() EasyMock.expect(replicaManager.getMagic(topicPartition)) .andReturn(None) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, replicaQuotaManager, requestChannel) createKafkaApis().handleWriteTxnMarkersRequest(request) - val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) - .asInstanceOf[WriteTxnMarkersResponse] + val markersResponse = capturedResponse.getValue.asInstanceOf[WriteTxnMarkersResponse] assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) } @@ -1571,7 +1562,7 @@ class KafkaApisTest { val (writeTxnMarkersRequest, request) = createWriteTxnMarkersRequest(asList(tp1, tp2)) val expectedErrors = Map(tp1 -> Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, tp2 -> Errors.NONE).asJava - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val responseCallback: Capture[Map[TopicPartition, PartitionResponse] => Unit] = EasyMock.newCapture() EasyMock.expect(replicaManager.getMagic(tp1)) @@ -1589,13 +1580,16 @@ class KafkaApisTest { EasyMock.anyObject()) ).andAnswer(() => responseCallback.getValue.apply(Map(tp2 -> new PartitionResponse(Errors.NONE)))) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, replicaQuotaManager, requestChannel) createKafkaApis().handleWriteTxnMarkersRequest(request) - val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) - .asInstanceOf[WriteTxnMarkersResponse] + val markersResponse = capturedResponse.getValue.asInstanceOf[WriteTxnMarkersResponse] assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) EasyMock.verify(replicaManager) } @@ -1708,7 +1702,7 @@ class KafkaApisTest { val (writeTxnMarkersRequest, request) = createWriteTxnMarkersRequest(asList(tp1, tp2)) val expectedErrors = Map(tp1 -> Errors.UNKNOWN_TOPIC_OR_PARTITION, tp2 -> Errors.NONE).asJava - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val responseCallback: Capture[Map[TopicPartition, PartitionResponse] => Unit] = EasyMock.newCapture() EasyMock.expect(replicaManager.getMagic(tp1)) @@ -1726,13 +1720,16 @@ class KafkaApisTest { EasyMock.anyObject()) ).andAnswer(() => responseCallback.getValue.apply(Map(tp2 -> new PartitionResponse(Errors.NONE)))) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, replicaQuotaManager, requestChannel) createKafkaApis().handleWriteTxnMarkersRequest(request) - val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) - .asInstanceOf[WriteTxnMarkersResponse] + val markersResponse = capturedResponse.getValue.asInstanceOf[WriteTxnMarkersResponse] assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) EasyMock.verify(replicaManager) } @@ -1798,15 +1795,14 @@ class KafkaApisTest { ).build() val request = buildRequest(describeGroupsRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.expect(groupCoordinator.handleDescribeGroup(EasyMock.eq(groupId))) .andReturn((Errors.NONE, groupSummary)) EasyMock.replay(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel) createKafkaApis().handleDescribeGroupRequest(request) - val response = readResponse(describeGroupsRequest, capturedResponse) - .asInstanceOf[DescribeGroupsResponse] + val response = capturedResponse.getValue.asInstanceOf[DescribeGroupsResponse] val group = response.data().groups().get(0) assertEquals(Errors.NONE, Errors.forCode(group.errorCode())) @@ -1852,7 +1848,7 @@ class KafkaApisTest { ).build() val request = buildRequest(offsetDeleteRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.expect(groupCoordinator.handleDeleteOffsets( EasyMock.eq(group), EasyMock.eq(Seq( @@ -1872,8 +1868,7 @@ class KafkaApisTest { createKafkaApis().handleOffsetDeleteRequest(request) - val response = readResponse(offsetDeleteRequest, capturedResponse) - .asInstanceOf[OffsetDeleteResponse] + val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] def errorForPartition(topic: String, partition: Int): Errors = { Errors.forCode(response.data.topics.find(topic).partitions.find(partition).errorCode()) @@ -1906,16 +1901,15 @@ class KafkaApisTest { .setTopics(topics) ).build() val request = buildRequest(offsetDeleteRequest) + val capturedResponse = expectNoThrottling(request) - val capturedResponse = expectNoThrottling() EasyMock.expect(groupCoordinator.handleDeleteOffsets(EasyMock.eq(group), EasyMock.eq(Seq.empty))) .andReturn((Errors.NONE, Map.empty)) EasyMock.replay(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel) createKafkaApis().handleOffsetDeleteRequest(request) - val response = readResponse(offsetDeleteRequest, capturedResponse) - .asInstanceOf[OffsetDeleteResponse] + val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.forCode(response.data.topics.find(topic).partitions.find(invalidPartitionId).errorCode())) @@ -1937,15 +1931,14 @@ class KafkaApisTest { ).build() val request = buildRequest(offsetDeleteRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.expect(groupCoordinator.handleDeleteOffsets(EasyMock.eq(group), EasyMock.eq(Seq.empty))) .andReturn((Errors.GROUP_ID_NOT_FOUND, Map.empty)) EasyMock.replay(groupCoordinator, replicaManager, clientRequestQuotaManager, requestChannel) createKafkaApis().handleOffsetDeleteRequest(request) - val response = readResponse(offsetDeleteRequest, capturedResponse) - .asInstanceOf[OffsetDeleteResponse] + val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] assertEquals(Errors.GROUP_ID_NOT_FOUND, Errors.forCode(response.data.errorCode())) } @@ -1963,9 +1956,6 @@ class KafkaApisTest { fetchOnlyFromLeader = EasyMock.eq(true)) ).andThrow(error.exception) - val capturedResponse = expectNoThrottling() - EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) - val targetTimes = List(new ListOffsetsTopic() .setName(tp.topic) .setPartitions(List(new ListOffsetsPartition() @@ -1975,10 +1965,12 @@ class KafkaApisTest { val listOffsetRequest = ListOffsetsRequest.Builder.forConsumer(true, isolationLevel) .setTargetTimes(targetTimes).build() val request = buildRequest(listOffsetRequest) + val capturedResponse = expectNoThrottling(request) + + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) createKafkaApis().handleListOffsetRequest(request) - val response = readResponse(listOffsetRequest, capturedResponse) - .asInstanceOf[ListOffsetsResponse] + val response = capturedResponse.getValue.asInstanceOf[ListOffsetsResponse] val partitionDataOptional = response.topics.asScala.find(_.name == tp.topic).get .partitions.asScala.find(_.partitionIndex == tp.partition) assertTrue(partitionDataOptional.isDefined) @@ -2125,19 +2117,18 @@ class KafkaApisTest { anyObject[util.List[TopicPartition]], anyBoolean)).andReturn(fetchContext) - val capturedResponse = expectNoThrottling() EasyMock.expect(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( anyObject[RequestChannel.Request](), anyDouble, anyLong)).andReturn(0) - EasyMock.replay(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, fetchManager) - val fetchRequest = new FetchRequest.Builder(9, 9, -1, 100, 0, fetchData) .build() val request = buildRequest(fetchRequest) + val capturedResponse = expectNoThrottling(request) + + EasyMock.replay(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, fetchManager) createKafkaApis().handleFetchRequest(request) - val response = readResponse(fetchRequest, capturedResponse) - .asInstanceOf[FetchResponse[BaseRecords]] + val response = capturedResponse.getValue.asInstanceOf[FetchResponse[BaseRecords]] assertTrue(response.responseData.containsKey(tp)) val partitionData = response.responseData.get(tp) @@ -2216,8 +2207,6 @@ class KafkaApisTest { def testJoinGroupWhenAnErrorOccurs(version: Short): Unit = { EasyMock.reset(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - val capturedResponse = expectNoThrottling() - val groupId = "group" val memberId = "member1" val protocolType = "consumer" @@ -2250,17 +2239,16 @@ class KafkaApisTest { ).build(version) val requestChannelRequest = buildRequest(joinGroupRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - createKafkaApis().handleJoinGroupRequest(requestChannelRequest) EasyMock.verify(groupCoordinator) capturedCallback.getValue.apply(JoinGroupResult(memberId, Errors.INCONSISTENT_GROUP_PROTOCOL)) - val response = readResponse(joinGroupRequest, capturedResponse) - .asInstanceOf[JoinGroupResponse] + val response = capturedResponse.getValue.asInstanceOf[JoinGroupResponse] assertEquals(Errors.INCONSISTENT_GROUP_PROTOCOL, response.error) assertEquals(0, response.data.members.size) @@ -2288,8 +2276,6 @@ class KafkaApisTest { def testJoinGroupProtocolType(version: Short): Unit = { EasyMock.reset(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - val capturedResponse = expectNoThrottling() - val groupId = "group" val memberId = "member1" val protocolType = "consumer" @@ -2323,9 +2309,9 @@ class KafkaApisTest { ).build(version) val requestChannelRequest = buildRequest(joinGroupRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - createKafkaApis().handleJoinGroupRequest(requestChannelRequest) EasyMock.verify(groupCoordinator) @@ -2340,8 +2326,7 @@ class KafkaApisTest { error = Errors.NONE )) - val response = readResponse(joinGroupRequest, capturedResponse) - .asInstanceOf[JoinGroupResponse] + val response = capturedResponse.getValue.asInstanceOf[JoinGroupResponse] assertEquals(Errors.NONE, response.error) assertEquals(0, response.data.members.size) @@ -2349,12 +2334,7 @@ class KafkaApisTest { assertEquals(0, response.data.generationId) assertEquals(memberId, response.data.leader) assertEquals(protocolName, response.data.protocolName) - - if (version >= 7) { - assertEquals(protocolType, response.data.protocolType) - } else { - assertNull(response.data.protocolType) - } + assertEquals(protocolType, response.data.protocolType) EasyMock.verify(clientRequestQuotaManager, requestChannel) } @@ -2369,8 +2349,6 @@ class KafkaApisTest { def testSyncGroupProtocolTypeAndName(version: Short): Unit = { EasyMock.reset(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - val capturedResponse = expectNoThrottling() - val groupId = "group" val memberId = "member1" val protocolType = "consumer" @@ -2399,9 +2377,9 @@ class KafkaApisTest { ).build(version) val requestChannelRequest = buildRequest(syncGroupRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - createKafkaApis().handleSyncGroupRequest(requestChannelRequest) EasyMock.verify(groupCoordinator) @@ -2413,17 +2391,11 @@ class KafkaApisTest { error = Errors.NONE )) - val response = readResponse(syncGroupRequest, capturedResponse) - .asInstanceOf[SyncGroupResponse] + val response = capturedResponse.getValue.asInstanceOf[SyncGroupResponse] assertEquals(Errors.NONE, response.error) assertArrayEquals(Array.empty[Byte], response.data.assignment) - - if (version >= 5) { - assertEquals(protocolType, response.data.protocolType) - } else { - assertNull(response.data.protocolType) - } + assertEquals(protocolType, response.data.protocolType) EasyMock.verify(clientRequestQuotaManager, requestChannel) } @@ -2438,8 +2410,6 @@ class KafkaApisTest { def testSyncGroupProtocolTypeAndNameAreMandatorySinceV5(version: Short): Unit = { EasyMock.reset(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - val capturedResponse = expectNoThrottling() - val groupId = "group" val memberId = "member1" val protocolType = "consumer" @@ -2468,9 +2438,9 @@ class KafkaApisTest { ).build(version) val requestChannelRequest = buildRequest(syncGroupRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) EasyMock.replay(groupCoordinator, clientRequestQuotaManager, requestChannel, replicaManager) - createKafkaApis().handleSyncGroupRequest(requestChannelRequest) EasyMock.verify(groupCoordinator) @@ -2484,8 +2454,7 @@ class KafkaApisTest { )) } - val response = readResponse(syncGroupRequest, capturedResponse) - .asInstanceOf[SyncGroupResponse] + val response = capturedResponse.getValue.asInstanceOf[SyncGroupResponse] if (version < 5) { assertEquals(Errors.NONE, response.error) @@ -2498,9 +2467,6 @@ class KafkaApisTest { @Test def rejectJoinGroupRequestWhenStaticMembershipNotSupported(): Unit = { - val capturedResponse = expectNoThrottling() - EasyMock.replay(clientRequestQuotaManager, requestChannel) - val joinGroupRequest = new JoinGroupRequest.Builder( new JoinGroupRequestData() .setGroupId("test") @@ -2511,18 +2477,18 @@ class KafkaApisTest { ).build() val requestChannelRequest = buildRequest(joinGroupRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) + + EasyMock.replay(clientRequestQuotaManager, requestChannel) createKafkaApis(KAFKA_2_2_IV1).handleJoinGroupRequest(requestChannelRequest) - val response = readResponse(joinGroupRequest, capturedResponse).asInstanceOf[JoinGroupResponse] + val response = capturedResponse.getValue.asInstanceOf[JoinGroupResponse] assertEquals(Errors.UNSUPPORTED_VERSION, response.error()) EasyMock.replay(groupCoordinator) } @Test def rejectSyncGroupRequestWhenStaticMembershipNotSupported(): Unit = { - val capturedResponse = expectNoThrottling() - EasyMock.replay(clientRequestQuotaManager, requestChannel) - val syncGroupRequest = new SyncGroupRequest.Builder( new SyncGroupRequestData() .setGroupId("test") @@ -2532,18 +2498,18 @@ class KafkaApisTest { ).build() val requestChannelRequest = buildRequest(syncGroupRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) + + EasyMock.replay(clientRequestQuotaManager, requestChannel) createKafkaApis(KAFKA_2_2_IV1).handleSyncGroupRequest(requestChannelRequest) - val response = readResponse(syncGroupRequest, capturedResponse).asInstanceOf[SyncGroupResponse] + val response = capturedResponse.getValue.asInstanceOf[SyncGroupResponse] assertEquals(Errors.UNSUPPORTED_VERSION, response.error) EasyMock.replay(groupCoordinator) } @Test def rejectHeartbeatRequestWhenStaticMembershipNotSupported(): Unit = { - val capturedResponse = expectNoThrottling() - EasyMock.replay(clientRequestQuotaManager, requestChannel) - val heartbeatRequest = new HeartbeatRequest.Builder( new HeartbeatRequestData() .setGroupId("test") @@ -2552,18 +2518,18 @@ class KafkaApisTest { .setGenerationId(1) ).build() val requestChannelRequest = buildRequest(heartbeatRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) + + EasyMock.replay(clientRequestQuotaManager, requestChannel) createKafkaApis(KAFKA_2_2_IV1).handleHeartbeatRequest(requestChannelRequest) - val response = readResponse(heartbeatRequest, capturedResponse).asInstanceOf[HeartbeatResponse] + val response = capturedResponse.getValue.asInstanceOf[HeartbeatResponse] assertEquals(Errors.UNSUPPORTED_VERSION, response.error()) EasyMock.replay(groupCoordinator) } @Test def rejectOffsetCommitRequestWhenStaticMembershipNotSupported(): Unit = { - val capturedResponse = expectNoThrottling() - EasyMock.replay(clientRequestQuotaManager, requestChannel) - val offsetCommitRequest = new OffsetCommitRequest.Builder( new OffsetCommitRequestData() .setGroupId("test") @@ -2584,6 +2550,9 @@ class KafkaApisTest { ).build() val requestChannelRequest = buildRequest(offsetCommitRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) + + EasyMock.replay(clientRequestQuotaManager, requestChannel) createKafkaApis(KAFKA_2_2_IV1).handleOffsetCommitRequest(requestChannelRequest) val expectedTopicErrors = Collections.singletonList( @@ -2595,7 +2564,7 @@ class KafkaApisTest { .setErrorCode(Errors.UNSUPPORTED_VERSION.code()) )) ) - val response = readResponse(offsetCommitRequest, capturedResponse).asInstanceOf[OffsetCommitResponse] + val response = capturedResponse.getValue.asInstanceOf[OffsetCommitResponse] assertEquals(expectedTopicErrors, response.data.topics()) EasyMock.replay(groupCoordinator) } @@ -2723,9 +2692,6 @@ class KafkaApisTest { @Test def rejectInitProducerIdWhenIdButNotEpochProvided(): Unit = { - val capturedResponse = expectNoThrottling() - EasyMock.replay(clientRequestQuotaManager, requestChannel) - val initProducerIdRequest = new InitProducerIdRequest.Builder( new InitProducerIdRequestData() .setTransactionalId("known") @@ -2735,18 +2701,17 @@ class KafkaApisTest { ).build() val requestChannelRequest = buildRequest(initProducerIdRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) + + EasyMock.replay(clientRequestQuotaManager, requestChannel) createKafkaApis(KAFKA_2_2_IV1).handleInitProducerIdRequest(requestChannelRequest) - val response = readResponse(initProducerIdRequest, capturedResponse) - .asInstanceOf[InitProducerIdResponse] + val response = capturedResponse.getValue.asInstanceOf[InitProducerIdResponse] assertEquals(Errors.INVALID_REQUEST, response.error) } @Test def rejectInitProducerIdWhenEpochButNotIdProvided(): Unit = { - val capturedResponse = expectNoThrottling() - EasyMock.replay(clientRequestQuotaManager, requestChannel) - val initProducerIdRequest = new InitProducerIdRequest.Builder( new InitProducerIdRequestData() .setTransactionalId("known") @@ -2755,9 +2720,12 @@ class KafkaApisTest { .setProducerEpoch(2) ).build() val requestChannelRequest = buildRequest(initProducerIdRequest) + val capturedResponse = expectNoThrottling(requestChannelRequest) + + EasyMock.replay(clientRequestQuotaManager, requestChannel) createKafkaApis(KAFKA_2_2_IV1).handleInitProducerIdRequest(requestChannelRequest) - val response = readResponse(initProducerIdRequest, capturedResponse).asInstanceOf[InitProducerIdResponse] + val response = capturedResponse.getValue.asInstanceOf[InitProducerIdResponse] assertEquals(Errors.INVALID_REQUEST, response.error) } @@ -2783,7 +2751,7 @@ class KafkaApisTest { val updateMetadataRequest = createBasicMetadataRequest("topicA", 1, brokerEpochInRequest, 1) val request = buildRequest(updateMetadataRequest) - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() EasyMock.expect(controller.brokerEpoch).andStubReturn(currentBrokerEpoch) EasyMock.expect(replicaManager.maybeUpdateMetadataCache( @@ -2793,12 +2761,15 @@ class KafkaApisTest { Seq() ) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, controller, requestChannel) createKafkaApis().handleUpdateMetadataRequest(request) - val updateMetadataResponse = readResponse(updateMetadataRequest, capturedResponse) - .asInstanceOf[UpdateMetadataResponse] + val updateMetadataResponse = capturedResponse.getValue.asInstanceOf[UpdateMetadataResponse] assertEquals(expectedError, updateMetadataResponse.error()) EasyMock.verify(replicaManager) } @@ -2824,7 +2795,7 @@ class KafkaApisTest { def testLeaderAndIsrRequest(currentBrokerEpoch: Long, brokerEpochInRequest: Long, expectedError: Errors): Unit = { val controllerId = 2 val controllerEpoch = 6 - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val partitionStates = Seq( new LeaderAndIsrRequestData.LeaderAndIsrPartitionState() .setTopicName("topicW") @@ -2860,12 +2831,15 @@ class KafkaApisTest { response ) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(replicaManager, controller, requestChannel) createKafkaApis().handleLeaderAndIsrRequest(request) - val leaderAndIsrResponse = readResponse(leaderAndIsrRequest, capturedResponse) - .asInstanceOf[LeaderAndIsrResponse] + val leaderAndIsrResponse = capturedResponse.getValue.asInstanceOf[LeaderAndIsrResponse] assertEquals(expectedError, leaderAndIsrResponse.error()) EasyMock.verify(replicaManager) } @@ -2891,7 +2865,7 @@ class KafkaApisTest { def testStopReplicaRequest(currentBrokerEpoch: Long, brokerEpochInRequest: Long, expectedError: Errors): Unit = { val controllerId = 0 val controllerEpoch = 5 - val capturedResponse: Capture[RequestChannel.Response] = EasyMock.newCapture() + val capturedResponse: Capture[AbstractResponse] = EasyMock.newCapture() val fooPartition = new TopicPartition("foo", 0) val topicStates = Seq( new StopReplicaTopicState() @@ -2923,13 +2897,16 @@ class KafkaApisTest { fooPartition -> Errors.NONE ), Errors.NONE) ) - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.eq(None) + )) EasyMock.replay(controller, replicaManager, requestChannel) createKafkaApis().handleStopReplicaRequest(request) - val stopReplicaResponse = readResponse(stopReplicaRequest, capturedResponse) - .asInstanceOf[StopReplicaResponse] + val stopReplicaResponse = capturedResponse.getValue.asInstanceOf[StopReplicaResponse] assertEquals(expectedError, stopReplicaResponse.error()) EasyMock.verify(replicaManager) } @@ -2965,7 +2942,7 @@ class KafkaApisTest { val listGroupsRequest = new ListGroupsRequest.Builder(data).build() val requestChannelRequest = buildRequest(listGroupsRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(requestChannelRequest) val expectedStates: Set[String] = if (state.isDefined) Set(state.get) else Set() EasyMock.expect(groupCoordinator.handleListGroups(expectedStates)) .andReturn((Errors.NONE, overviews)) @@ -2973,7 +2950,7 @@ class KafkaApisTest { createKafkaApis().handleListGroupsRequest(requestChannelRequest) - val response = readResponse(listGroupsRequest, capturedResponse).asInstanceOf[ListGroupsResponse] + val response = capturedResponse.getValue.asInstanceOf[ListGroupsResponse] assertEquals(Errors.NONE.code, response.data.errorCode) response } @@ -3006,17 +2983,16 @@ class KafkaApisTest { 0, 0, Seq.empty[UpdateMetadataPartitionState].asJava, brokers.asJava, Collections.emptyMap()).build() metadataCache.updateMetadata(correlationId = 0, updateMetadataRequest) - val capturedResponse = expectNoThrottling() - EasyMock.replay(clientRequestQuotaManager, requestChannel) - val describeClusterRequest = new DescribeClusterRequest.Builder(new DescribeClusterRequestData() .setIncludeClusterAuthorizedOperations(true)).build() val request = buildRequest(describeClusterRequest, plaintextListener) + val capturedResponse = expectNoThrottling(request) + + EasyMock.replay(clientRequestQuotaManager, requestChannel) createKafkaApis().handleDescribeCluster(request) - val describeClusterResponse = readResponse(describeClusterRequest, capturedResponse) - .asInstanceOf[DescribeClusterResponse] + val describeClusterResponse = capturedResponse.getValue.asInstanceOf[DescribeClusterResponse] assertEquals(metadataCache.getControllerId.get, describeClusterResponse.data.controllerId) assertEquals(clusterId, describeClusterResponse.data.clusterId) @@ -3064,14 +3040,14 @@ class KafkaApisTest { } private def sendMetadataRequestWithInconsistentListeners(requestListener: ListenerName): MetadataResponse = { - val capturedResponse = expectNoThrottling() - EasyMock.replay(clientRequestQuotaManager, requestChannel) - val metadataRequest = MetadataRequest.Builder.allTopics.build() val requestChannelRequest = buildRequest(metadataRequest, requestListener) + val capturedResponse = expectNoThrottling(requestChannelRequest) + EasyMock.replay(clientRequestQuotaManager, requestChannel) + createKafkaApis().handleTopicMetadataRequest(requestChannelRequest) - readResponse(metadataRequest, capturedResponse).asInstanceOf[MetadataResponse] + capturedResponse.getValue.asInstanceOf[MetadataResponse] } private def testConsumerListOffsetLatest(isolationLevel: IsolationLevel): Unit = { @@ -3087,9 +3063,6 @@ class KafkaApisTest { fetchOnlyFromLeader = EasyMock.eq(true)) ).andReturn(Some(new TimestampAndOffset(ListOffsetsResponse.UNKNOWN_TIMESTAMP, latestOffset, currentLeaderEpoch))) - val capturedResponse = expectNoThrottling() - EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) - val targetTimes = List(new ListOffsetsTopic() .setName(tp.topic) .setPartitions(List(new ListOffsetsPartition() @@ -3098,9 +3071,12 @@ class KafkaApisTest { val listOffsetRequest = ListOffsetsRequest.Builder.forConsumer(true, isolationLevel) .setTargetTimes(targetTimes).build() val request = buildRequest(listOffsetRequest) + val capturedResponse = expectNoThrottling(request) + + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel) createKafkaApis().handleListOffsetRequest(request) - val response = readResponse(listOffsetRequest, capturedResponse).asInstanceOf[ListOffsetsResponse] + val response = capturedResponse.getValue.asInstanceOf[ListOffsetsResponse] val partitionDataOptional = response.topics.asScala.find(_.name == tp.topic).get .partitions.asScala.find(_.partitionIndex == tp.partition) assertTrue(partitionDataOptional.isDefined) @@ -3164,28 +3140,22 @@ class KafkaApisTest { requestChannelMetrics, envelope = None) } - private def readResponse(request: AbstractRequest, capturedResponse: Capture[RequestChannel.Response]) = { - val api = request.apiKey - val response = capturedResponse.getValue - assertTrue(response.isInstanceOf[SendResponse], s"Unexpected response type: ${response.getClass}") - val sendResponse = response.asInstanceOf[SendResponse] - val send = sendResponse.responseSend - val channel = new ByteBufferChannel(send.size) - send.writeTo(channel) - channel.close() - channel.buffer.getInt() // read the size - ResponseHeader.parse(channel.buffer, api.responseHeaderVersion(request.version)) - AbstractResponse.parseResponse(api, channel.buffer, request.version) - } - - private def expectNoThrottling(): Capture[RequestChannel.Response] = { + private def expectNoThrottling(request: RequestChannel.Request): Capture[AbstractResponse] = { EasyMock.expect(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(EasyMock.anyObject[RequestChannel.Request](), EasyMock.anyObject[Long])).andReturn(0) - EasyMock.expect(clientRequestQuotaManager.throttle(EasyMock.anyObject[RequestChannel.Request](), EasyMock.eq(0), - EasyMock.anyObject[RequestChannel.Response => Unit]())) - val capturedResponse = EasyMock.newCapture[RequestChannel.Response]() - EasyMock.expect(requestChannel.sendResponse(EasyMock.capture(capturedResponse))) + EasyMock.expect(clientRequestQuotaManager.throttle( + EasyMock.eq(request), + EasyMock.anyObject[ThrottleCallback](), + EasyMock.eq(0))) + + val capturedResponse = EasyMock.newCapture[AbstractResponse]() + EasyMock.expect(requestChannel.sendResponse( + EasyMock.eq(request), + EasyMock.capture(capturedResponse), + EasyMock.anyObject() + )) + capturedResponse } @@ -3244,7 +3214,7 @@ class KafkaApisTest { EasyMock.reset(replicaManager, clientRequestQuotaManager, requestChannel) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) val t0p0 = new TopicPartition("t0", 0) val t0p1 = new TopicPartition("t0", 1) val t0p2 = new TopicPartition("t0", 2) @@ -3261,8 +3231,7 @@ class KafkaApisTest { createKafkaApis().handleAlterReplicaLogDirsRequest(request) - val response = readResponse(alterReplicaLogDirsRequest, capturedResponse) - .asInstanceOf[AlterReplicaLogDirsResponse] + val response = capturedResponse.getValue.asInstanceOf[AlterReplicaLogDirsResponse] assertEquals(partitionResults, response.data.results.asScala.flatMap { tr => tr.partitions().asScala.map { pr => new TopicPartition(tr.topicName, pr.partitionIndex) -> Errors.forCode(pr.errorCode) @@ -3358,13 +3327,12 @@ class KafkaApisTest { val describeProducersRequest = new DescribeProducersRequest.Builder(data).build() val request = buildRequest(describeProducersRequest) - val capturedResponse = expectNoThrottling() + val capturedResponse = expectNoThrottling(request) EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator, authorizer) createKafkaApis(authorizer = Some(authorizer)).handleDescribeProducersRequest(request) - val response = readResponse(describeProducersRequest, capturedResponse) - .asInstanceOf[DescribeProducersResponse] + val response = capturedResponse.getValue.asInstanceOf[DescribeProducersResponse] assertEquals(3, response.data.topics.size()) assertEquals(Set("foo", "bar", "baz"), response.data.topics.asScala.map(_.name).toSet) diff --git a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala index d7dec5b58729f..15ad22d97258f 100644 --- a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala +++ b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala @@ -18,22 +18,11 @@ package kafka.server -import java.net.InetAddress -import java.util import java.util.Collections import java.util.concurrent.{DelayQueue, TimeUnit} -import kafka.network.RequestChannel -import kafka.network.RequestChannel.{EndThrottlingResponse, Response, StartThrottlingResponse} -import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.memory.MemoryPool + import org.apache.kafka.common.metrics.MetricConfig -import org.apache.kafka.common.network.ClientInformation -import org.apache.kafka.common.network.ListenerName -import org.apache.kafka.common.requests.FetchRequest.PartitionData -import org.apache.kafka.common.requests.{AbstractRequest, FetchRequest, RequestContext, RequestHeader, RequestTestUtils} -import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.utils.MockTime -import org.easymock.EasyMock import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{BeforeEach, Test} @@ -44,28 +33,13 @@ class ThrottledChannelExpirationTest { private val metrics = new org.apache.kafka.common.metrics.Metrics(new MetricConfig(), Collections.emptyList(), time) - private val request = buildRequest(FetchRequest.Builder.forConsumer(0, 1000, new util.HashMap[TopicPartition, PartitionData]))._2 - - private def buildRequest[T <: AbstractRequest](builder: AbstractRequest.Builder[T], - listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { - - val request = builder.build() - val buffer = RequestTestUtils.serializeRequestWithHeader( - new RequestHeader(builder.apiKey, request.version, "", 0), request) - val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) - - // read the header from the buffer first so that the body can be read next from the Request constructor - val header = RequestHeader.parse(buffer) - val context = new RequestContext(header, "1", InetAddress.getLocalHost, KafkaPrincipal.ANONYMOUS, - listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, false) - (request, new RequestChannel.Request(processor = 1, context = context, startTimeNanos = 0, MemoryPool.NONE, buffer, - requestChannelMetrics)) - } + private val callback = new ThrottleCallback { + override def startThrottling(): Unit = { + numCallbacksForStartThrottling += 1 + } - def callback(response: Response): Unit = { - (response: @unchecked) match { - case _: StartThrottlingResponse => numCallbacksForStartThrottling += 1 - case _: EndThrottlingResponse => numCallbacksForEndThrottling += 1 + override def endThrottling(): Unit = { + numCallbacksForEndThrottling += 1 } } @@ -83,10 +57,10 @@ class ThrottledChannelExpirationTest { val reaper = new clientMetrics.ThrottledChannelReaper(delayQueue, "") try { // Add 4 elements to the queue out of order. Add 2 elements with the same expire timestamp. - val channel1 = new ThrottledChannel(request, time, 10, callback) - val channel2 = new ThrottledChannel(request, time, 30, callback) - val channel3 = new ThrottledChannel(request, time, 30, callback) - val channel4 = new ThrottledChannel(request, time, 20, callback) + val channel1 = new ThrottledChannel(time, 10, callback) + val channel2 = new ThrottledChannel(time, 30, callback) + val channel3 = new ThrottledChannel(time, 30, callback) + val channel4 = new ThrottledChannel(time, 20, callback) delayQueue.add(channel1) delayQueue.add(channel2) delayQueue.add(channel3) @@ -110,9 +84,9 @@ class ThrottledChannelExpirationTest { @Test def testThrottledChannelDelay(): Unit = { - val t1: ThrottledChannel = new ThrottledChannel(request, time, 10, callback) - val t2: ThrottledChannel = new ThrottledChannel(request, time, 20, callback) - val t3: ThrottledChannel = new ThrottledChannel(request, time, 20, callback) + val t1: ThrottledChannel = new ThrottledChannel(time, 10, callback) + val t2: ThrottledChannel = new ThrottledChannel(time, 20, callback) + val t3: ThrottledChannel = new ThrottledChannel(time, 20, callback) assertEquals(10, t1.throttleTimeMs) assertEquals(20, t2.throttleTimeMs) assertEquals(20, t3.throttleTimeMs) @@ -124,4 +98,5 @@ class ThrottledChannelExpirationTest { time.sleep(10) } } + } From da1d9da7ea6a5ae515c94c68aefd4768b388a3d1 Mon Sep 17 00:00:00 2001 From: Luke Chen <43372967+showuon@users.noreply.github.com> Date: Tue, 23 Feb 2021 12:51:06 +0800 Subject: [PATCH 045/243] MINOR: update the memberMetadata#toString output (#10166) Reviewers: Chia-Ping Tsai --- .../src/main/scala/kafka/coordinator/group/MemberMetadata.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala index e0ae0ec2e34d6..73ec0e14ec36e 100644 --- a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala +++ b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala @@ -148,7 +148,7 @@ private[group] class MemberMetadata(var memberId: String, s"clientHost=$clientHost, " + s"sessionTimeoutMs=$sessionTimeoutMs, " + s"rebalanceTimeoutMs=$rebalanceTimeoutMs, " + - s"supportedProtocols=${supportedProtocols.map(_._1)}, " + + s"supportedProtocols=${supportedProtocols.map(_._1)}" + ")" } } From b5265e98ebc48c44f6d9b836d4ab9d1c9cd371af Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Mon, 22 Feb 2021 21:11:17 -0800 Subject: [PATCH 046/243] KAFKA-12357: Do not inline methods from the scala package by default (#10174) As mentioned in #9548, users currently use the kafka jar (`core` module) for integration testing and the current inlining behavior causes problems when the user's classpath contains a different Scala version than the one that was used for compilation (e.g. 2.13.4 versus 2.13.3). An example error: `java.lang.NoClassDefFoundError: scala/math/Ordering$$anon$7` We now disable inlining of the `scala` package by default, but make it easy to enable it for those who so desire (a good option if you can ensure the scala library version matches the one used for compilation). While at it, we make it possible to disable scala compiler optimizations (`none`) or to use only method local optimizations (`method`). This can be useful if optimizing for compilation time during development. Verified behavior by running gradlew with `--debug` and checking the output after `[zinc] The Scala compiler is invoked with:` Reviewers: Chia-Ping Tsai --- README.md | 7 +++++++ build.gradle | 32 ++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 34e13b6ea7da7..3c3f013662344 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,13 @@ The following options should be set with a `-P` switch, for example `./gradlew - * `enableTestCoverage`: enables test coverage plugins and tasks, including bytecode enhancement of classes required to track said coverage. Note that this introduces some overhead when running tests and hence why it's disabled by default (the overhead varies, but 15-20% is a reasonable estimate). +* `scalaOptimizerMode`: configures the optimizing behavior of the scala compiler, the value should be one of `none`, `method`, `inline-kafka` or +`inline-scala` (the default is `inline-kafka`). `none` is the scala compiler default, which only eliminates unreachable code. `method` also +includes method-local optimizations. `inline-kafka` adds inlining of methods within the kafka packages. Finally, `inline-scala` also +includes inlining of methods within the scala library (which avoids lambda allocations for methods like `Option.exists`). `inline-scala` is +only safe if the Scala library version is the same at compile time and runtime. Since we cannot guarantee this for all cases (for example, users +may depend on the kafka jar for integration tests where they may include a scala library with a different version), we don't enable it by +default. See https://www.lightbend.com/blog/scala-inliner-optimizer for more details. ### Dependency Analysis ### diff --git a/build.gradle b/build.gradle index bf92f9645eb7c..e9b25123b44f8 100644 --- a/build.gradle +++ b/build.gradle @@ -134,6 +134,12 @@ ext { userEnableTestCoverage = project.hasProperty("enableTestCoverage") ? enableTestCoverage : false + // See README.md for details on this option and the reasoning for the default + userScalaOptimizerMode = project.hasProperty("scalaOptimizerMode") ? scalaOptimizerMode : "inline-kafka" + def scalaOptimizerValues = ["none", "method", "inline-kafka", "inline-scala"] + if (!scalaOptimizerValues.contains(userScalaOptimizerMode)) + throw new GradleException("Unexpected value for scalaOptimizerMode property. Expected one of $scalaOptimizerValues), but received: $userScalaOptimizerMode") + generatedDocsDir = new File("${project.rootDir}/docs/generated") commitId = project.hasProperty('commitId') ? commitId : null @@ -515,21 +521,19 @@ subprojects { "-Xlint:unused" ] - // Inline more aggressively when compiling the `core` jar since it's not meant to be used as a library. - // More specifically, inline classes from the Scala library so that we can inline methods like `Option.exists` - // and avoid lambda allocations. This is only safe if the Scala library version is the same at compile time - // and runtime. We cannot guarantee this for libraries like kafka streams, so only inline classes from the - // Kafka project in that case. - List inlineFrom - if (project.name.equals('core')) - inlineFrom = ["-opt-inline-from:scala.**", "-opt-inline-from:kafka.**", "-opt-inline-from:org.apache.kafka.**"] - else - inlineFrom = ["-opt-inline-from:org.apache.kafka.**"] + // See README.md for details on this option and the meaning of each value + if (userScalaOptimizerMode.equals("method")) + scalaCompileOptions.additionalParameters += ["-opt:l:method"] + else if (userScalaOptimizerMode.startsWith("inline-")) { + List inlineFrom = ["-opt-inline-from:org.apache.kafka.**"] + if (project.name.equals('core')) + inlineFrom.add("-opt-inline-from:kafka.**") + if (userScalaOptimizerMode.equals("inline-scala")) + inlineFrom.add("-opt-inline-from:scala.**") - // Somewhat confusingly, `-opt:l:inline` enables all optimizations. `inlineFrom` configures what can be inlined. - // See https://www.lightbend.com/blog/scala-inliner-optimizer for more information about the optimizer. - scalaCompileOptions.additionalParameters += ["-opt:l:inline"] - scalaCompileOptions.additionalParameters += inlineFrom + scalaCompileOptions.additionalParameters += ["-opt:l:inline"] + scalaCompileOptions.additionalParameters += inlineFrom + } if (versions.baseScala != '2.12') { scalaCompileOptions.additionalParameters += ["-opt-warnings", "-Xlint:strict-unsealed-patmat"] From dd34e40f3e8ea097374654e8afea7dabc050272d Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Mon, 22 Feb 2021 22:02:38 -0800 Subject: [PATCH 047/243] MINOR: Update Scala to 2.13.5 (#10169) This includes a fix from Chia-Ping that removes tuple allocations when `Map.forKeyValue` is used (https://github.com/scala/scala/pull/9425) and support for JDK 16. Release notes: https://github.com/scala/scala/releases/tag/v2.13.5 Reviewers: Chia-Ping Tsai --- bin/kafka-run-class.sh | 2 +- bin/windows/kafka-run-class.bat | 2 +- gradle.properties | 2 +- gradle/dependencies.gradle | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index f485f5b04459b..0d322854a504e 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -48,7 +48,7 @@ should_include_file() { base_dir=$(dirname $0)/.. if [ -z "$SCALA_VERSION" ]; then - SCALA_VERSION=2.13.4 + SCALA_VERSION=2.13.5 if [[ -f "$base_dir/gradle.properties" ]]; then SCALA_VERSION=`grep "^scalaVersion=" "$base_dir/gradle.properties" | cut -d= -f 2` fi diff --git a/bin/windows/kafka-run-class.bat b/bin/windows/kafka-run-class.bat index 3490588e37ded..5c69c102c2286 100755 --- a/bin/windows/kafka-run-class.bat +++ b/bin/windows/kafka-run-class.bat @@ -27,7 +27,7 @@ set BASE_DIR=%CD% popd IF ["%SCALA_VERSION%"] EQU [""] ( - set SCALA_VERSION=2.13.4 + set SCALA_VERSION=2.13.5 ) IF ["%SCALA_BINARY_VERSION%"] EQU [""] ( diff --git a/gradle.properties b/gradle.properties index 9ea539c4d0726..c3de576dcdceb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,7 +21,7 @@ group=org.apache.kafka # - tests/kafkatest/version.py (variable DEV_VERSION) # - kafka-merge-pr.py version=2.9.0-SNAPSHOT -scalaVersion=2.13.4 +scalaVersion=2.13.5 task=build org.gradle.jvmargs=-Xmx2g -Xss4m -XX:+UseParallelGC org.gradle.parallel=true diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 2606ea4c2b79f..a81a05c4f17e9 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -28,7 +28,7 @@ ext { // Add Scala version def defaultScala212Version = '2.12.13' -def defaultScala213Version = '2.13.4' +def defaultScala213Version = '2.13.5' if (hasProperty('scalaVersion')) { if (scalaVersion == '2.12') { versions["scala"] = defaultScala212Version From 837208009fa994751958b5d748dc422c9058993e Mon Sep 17 00:00:00 2001 From: Luke Chen <43372967+showuon@users.noreply.github.com> Date: Tue, 23 Feb 2021 17:29:11 +0800 Subject: [PATCH 048/243] MINOR: add toString to Subscription classes (#10172) Reviewers: Chia-Ping Tsai --- .../consumer/ConsumerPartitionAssignor.java | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java index 8708ea4f7e343..9fd7673a025df 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java @@ -132,6 +132,16 @@ public void setGroupInstanceId(Optional groupInstanceId) { public Optional groupInstanceId() { return groupInstanceId; } + + @Override + public String toString() { + return "Subscription(" + + "topics=" + topics + + (userData == null ? "" : ", userDataSize=" + userData.remaining()) + + ", ownedPartitions=" + ownedPartitions + + ", groupInstanceId=" + (groupInstanceId.map(String::toString).orElse("null")) + + ")"; + } } final class Assignment { @@ -158,9 +168,9 @@ public ByteBuffer userData() { @Override public String toString() { return "Assignment(" + - "partitions=" + partitions + - (userData == null ? "" : ", userDataSize=" + userData.remaining()) + - ')'; + "partitions=" + partitions + + (userData == null ? "" : ", userDataSize=" + userData.remaining()) + + ')'; } } @@ -174,6 +184,13 @@ public GroupSubscription(Map subscriptions) { public Map groupSubscription() { return subscriptions; } + + @Override + public String toString() { + return "GroupSubscription(" + + "subscriptions=" + subscriptions + + ")"; + } } final class GroupAssignment { @@ -186,6 +203,13 @@ public GroupAssignment(Map assignments) { public Map groupAssignment() { return assignments; } + + @Override + public String toString() { + return "GroupAssignment(" + + "assignments=" + assignments + + ")"; + } } /** From a74b5eb0df7ecfa46601ee572e44877373af0cd6 Mon Sep 17 00:00:00 2001 From: Manikumar Reddy Date: Tue, 23 Feb 2021 15:10:24 +0530 Subject: [PATCH 049/243] MINOR: Update HttpClient to "4.5.13" Update HttpClient to recent bug fix version 4.5.13. Author: Manikumar Reddy Reviewers: Chia-Ping Tsai Closes #10188 from omkreddy/http-client --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index a81a05c4f17e9..b5a143c734962 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -64,7 +64,7 @@ versions += [ gradle: "6.8.1", gradleVersionsPlugin: "0.36.0", grgit: "4.1.0", - httpclient: "4.5.12", + httpclient: "4.5.13", easymock: "4.2", jackson: "2.10.5", jacksonDatabind: "2.10.5.1", From 0605b690e94ae2fa442551b2741472c8d5cac485 Mon Sep 17 00:00:00 2001 From: "A. Sophie Blee-Goldman" Date: Tue, 23 Feb 2021 08:39:19 -0800 Subject: [PATCH 050/243] MINOR: document restriction against running multiple Streams apps on same state.dir (#10187) Reviewers: Leah Thomas , Almog Gavra , Guozhang Wang --- docs/streams/upgrade-guide.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html index 38138e134e69c..5b8d58161e670 100644 --- a/docs/streams/upgrade-guide.html +++ b/docs/streams/upgrade-guide.html @@ -87,6 +87,11 @@

    Upgrade Guide and API Changes

    More details about the new config StreamsConfig#TOPOLOGY_OPTIMIZATION can be found in     KIP-295.

    +

    + Note: Kafka Streams does not support running multiple instances of the same application as different processes on the same physical state directory. Starting in 2.8.0 (as well as 2.7.1 and 2.6.2), + this restriction will be enforced. If you wish to run more than one instance of Kafka Streams, you must configure them with different values for state.dir. +

    +

    Streams API changes in 2.8.0

    We extended StreamJoined to include the options withLoggingEnabled() and withLoggingDisabled() in From 6f2cb60ef6a6512d48a7dbfd0c9adc0f1cc3fa11 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Tue, 23 Feb 2021 10:47:53 -0800 Subject: [PATCH 051/243] KAFKA-12341: Ensure consistent versions for javassist (#10191) And update to 3.27.0-GA. Reviewers: Chia-Ping Tsai --- build.gradle | 13 +++++-------- gradle/dependencies.gradle | 2 ++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index e9b25123b44f8..05f77e054f04a 100644 --- a/build.gradle +++ b/build.gradle @@ -80,17 +80,14 @@ allprojects { if (name != "zinc") { resolutionStrategy { force( - // ensure we have a single version of scala jars in the classpath, we enable inlining - // in the scala compiler for the `core` module so binary compatibility is only - // guaranteed if the exact same version of the scala jars is used for compilation - // and at runtime + // be explicit about the javassist dependency version instead of relying on the transitive version + libs.javassist, + // ensure we have a single version in the classpath despite transitive dependencies libs.scalaLibrary, libs.scalaReflect, - // ensures we have a single version of jackson-annotations in the classpath even if - // some modules only have a transitive reference to an older version libs.jacksonAnnotations, - // be explicit about the Netty dependency version instead of relying on the version - // set by ZooKeeper (potentially older and containing CVEs) + // be explicit about the Netty dependency version instead of relying on the version set by + // ZooKeeper (potentially older and containing CVEs) libs.nettyHandler, libs.nettyTransportNativeEpoll ) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index b5a143c734962..709ef013cc1f5 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -69,6 +69,7 @@ versions += [ jackson: "2.10.5", jacksonDatabind: "2.10.5.1", jacoco: "0.8.5", + javassist: "3.27.0-GA", jetty: "9.4.36.v20210114", jersey: "2.31", jline: "3.12.1", @@ -144,6 +145,7 @@ libs += [ jacksonJaxrsJsonProvider: "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:$versions.jackson", jaxbApi: "javax.xml.bind:jaxb-api:$versions.jaxb", jaxrsApi: "javax.ws.rs:javax.ws.rs-api:$versions.jaxrs", + javassist: "org.javassist:javassist:$versions.javassist", jettyServer: "org.eclipse.jetty:jetty-server:$versions.jetty", jettyClient: "org.eclipse.jetty:jetty-client:$versions.jetty", jettyServlet: "org.eclipse.jetty:jetty-servlet:$versions.jetty", From 16d1439fc413f42ff97f62ebb9fd102cfbf2e063 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Tue, 23 Feb 2021 15:25:08 -0800 Subject: [PATCH 052/243] MINOR: Reduce log level of spammy message in `LeaderEpochFileCache` (#10198) Since we do epoch validation as part of the fetch handling these days, the log message `LeaderEpochFileCache.endOffsetFor` has become very noisy when DEBUG is enabled (which is often the default for system tests). This patch reduces the level to TRACE. Reviewers: David Jacot --- .../main/scala/kafka/server/epoch/LeaderEpochFileCache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala b/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala index 03775439f00a6..92f9393522ee3 100644 --- a/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala +++ b/core/src/main/scala/kafka/server/epoch/LeaderEpochFileCache.scala @@ -221,7 +221,7 @@ class LeaderEpochFileCache(topicPartition: TopicPartition, } } } - debug(s"Processed end offset request for epoch $requestedEpoch and returning epoch ${epochAndOffset._1} " + + trace(s"Processed end offset request for epoch $requestedEpoch and returning epoch ${epochAndOffset._1} " + s"with end offset ${epochAndOffset._2} from epoch cache of size ${epochs.size}") epochAndOffset } From bc04c335fc6c39a507f14d0cda71c8a2780a53cf Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Tue, 23 Feb 2021 21:32:42 -0500 Subject: [PATCH 053/243] KAFKA-12361; Use default request.timeout.ms value for Connect producers (#10178) Connect uses a high request timeout as a holdover from the days prior to KIP-91 when this was required to guarantee records would not get timed out in the accumulator. Having a high request timeout makes it harder for the producer to gracefully handle unclean connection terminations, which might happen in the case of sudden broker death. Reducing that value to the default of 30 seconds should address that issue, without compromising the existing delivery guarantees of the Connect framework. Since the delivery timeout is still set to a very-high value, this change shouldn't make it more likely for `Producer::send` to throw an exception and fail the task. Reviewers: Jason Gustafson --- .../src/main/java/org/apache/kafka/connect/runtime/Worker.java | 1 - .../test/java/org/apache/kafka/connect/runtime/WorkerTest.java | 1 - .../kafka/connect/runtime/WorkerWithTopicCreationTest.java | 1 - 3 files changed, 3 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java index 45a93dfc4650b..1f6a8d151511c 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java @@ -660,7 +660,6 @@ static Map producerConfigs(ConnectorTaskId id, producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); // These settings will execute infinite retries on retriable exceptions. They *may* be overridden via configs passed to the worker, // but this may compromise the delivery guarantees of Kafka Connect. - producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, Integer.toString(Integer.MAX_VALUE)); producerProps.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, Long.toString(Long.MAX_VALUE)); producerProps.put(ProducerConfig.ACKS_CONFIG, "all"); producerProps.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1"); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java index cbf0802cd4b28..1ac262589bdd3 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java @@ -182,7 +182,6 @@ public void setup() { ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); defaultProducerConfigs.put( ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - defaultProducerConfigs.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, Integer.toString(Integer.MAX_VALUE)); defaultProducerConfigs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, Long.toString(Long.MAX_VALUE)); defaultProducerConfigs.put(ProducerConfig.ACKS_CONFIG, "all"); defaultProducerConfigs.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1"); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerWithTopicCreationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerWithTopicCreationTest.java index 3bf61088ed80b..912dfacdd9912 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerWithTopicCreationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerWithTopicCreationTest.java @@ -175,7 +175,6 @@ public void setup() { ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); defaultProducerConfigs.put( ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - defaultProducerConfigs.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, Integer.toString(Integer.MAX_VALUE)); defaultProducerConfigs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, Long.toString(Long.MAX_VALUE)); defaultProducerConfigs.put(ProducerConfig.ACKS_CONFIG, "all"); defaultProducerConfigs.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1"); From f75efb96fae99a22eb54b5d0ef4e23b28fe8cd2d Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 23 Feb 2021 20:41:02 -0800 Subject: [PATCH 054/243] KAFKA-12323: Set timestamp in record context when punctuate (#10170) We need to preserve the timestamp when punctuating so that downstream operators would retain it via context. Reviewers: Matthias J. Sax --- .../processor/internals/StreamTask.java | 29 ++++--- .../processor/internals/StreamThreadTest.java | 85 ++++++++++++++++++- 2 files changed, 99 insertions(+), 15 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java index 36dc02ac0e886..aefa3ac95a308 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; @@ -688,17 +689,14 @@ record = partitionGroup.nextRecord(recordInfo, wallClockTime); log.trace("Start processing one record [{}]", record); - updateProcessorContext( - currNode, - wallClockTime, - new ProcessorRecordContext( - record.timestamp, - record.offset(), - record.partition(), - record.topic(), - record.headers() - ) + final ProcessorRecordContext recordContext = new ProcessorRecordContext( + record.timestamp, + record.offset(), + record.partition(), + record.topic(), + record.headers() ); + updateProcessorContext(currNode, wallClockTime, recordContext); maybeRecordE2ELatency(record.timestamp, wallClockTime, currNode.name()); final Record toProcess = new Record<>( @@ -792,7 +790,16 @@ public void punctuate(final ProcessorNode node, throw new IllegalStateException(String.format("%sCurrent node is not null", logPrefix)); } - updateProcessorContext(node, time.milliseconds(), null); + // when punctuating, we need to preserve the timestamp (this can be either system time or event time) + // while other record context are set as dummy: null topic, -1 partition, -1 offset and empty header + final ProcessorRecordContext recordContext = new ProcessorRecordContext( + timestamp, + -1L, + -1, + null, + new RecordHeaders() + ); + updateProcessorContext(node, time.milliseconds(), recordContext); if (log.isTraceEnabled()) { log.trace("Punctuating processor {} with timestamp {} and punctuation type {}", node.name(), timestamp, type); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java index bfc32d5340eee..e2b1549b905c0 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java @@ -48,6 +48,7 @@ import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.errors.LogAndContinueExceptionHandler; import org.apache.kafka.streams.errors.StreamsException; @@ -1805,7 +1806,7 @@ public void shouldPunctuateActiveTask() { final List punctuatedStreamTime = new ArrayList<>(); final List punctuatedWallClockTime = new ArrayList<>(); final org.apache.kafka.streams.processor.ProcessorSupplier punctuateProcessor = - () -> new org.apache.kafka.streams.processor.Processor() { + () -> new org.apache.kafka.streams.processor.AbstractProcessor() { @Override public void init(final org.apache.kafka.streams.processor.ProcessorContext context) { context.schedule(Duration.ofMillis(100L), PunctuationType.STREAM_TIME, punctuatedStreamTime::add); @@ -1814,9 +1815,6 @@ public void init(final org.apache.kafka.streams.processor.ProcessorContext conte @Override public void process(final Object key, final Object value) {} - - @Override - public void close() {} }; internalStreamsBuilder.stream(Collections.singleton(topic1), consumed).process(punctuateProcessor); @@ -1874,6 +1872,85 @@ public void close() {} assertEquals(2, punctuatedWallClockTime.size()); } + @Test + public void shouldPunctuateWithTimestampPreservedInProcessorContext() { + final org.apache.kafka.streams.kstream.TransformerSupplier> punctuateProcessor = + () -> new org.apache.kafka.streams.kstream.Transformer>() { + @Override + public void init(final org.apache.kafka.streams.processor.ProcessorContext context) { + context.schedule(Duration.ofMillis(100L), PunctuationType.WALL_CLOCK_TIME, timestamp -> context.forward("key", "value")); + context.schedule(Duration.ofMillis(100L), PunctuationType.STREAM_TIME, timestamp -> context.forward("key", "value")); + } + + @Override + public KeyValue transform(final Object key, final Object value) { + return null; + } + + @Override + public void close() {} + }; + + final List peekedContextTime = new ArrayList<>(); + final org.apache.kafka.streams.processor.ProcessorSupplier peekProcessor = + () -> new org.apache.kafka.streams.processor.AbstractProcessor() { + @Override + public void process(final Object key, final Object value) { + peekedContextTime.add(context.timestamp()); + } + }; + + internalStreamsBuilder.stream(Collections.singleton(topic1), consumed) + .transform(punctuateProcessor) + .process(peekProcessor); + internalStreamsBuilder.buildAndOptimizeTopology(); + + final long currTime = mockTime.milliseconds(); + final StreamThread thread = createStreamThread(CLIENT_ID, config, false); + + thread.setState(StreamThread.State.STARTING); + thread.rebalanceListener().onPartitionsRevoked(Collections.emptySet()); + final List assignedPartitions = new ArrayList<>(); + + final Map> activeTasks = new HashMap<>(); + + // assign single partition + assignedPartitions.add(t1p1); + activeTasks.put(task1, Collections.singleton(t1p1)); + + thread.taskManager().handleAssignment(activeTasks, emptyMap()); + + clientSupplier.consumer.assign(assignedPartitions); + clientSupplier.consumer.updateBeginningOffsets(Collections.singletonMap(t1p1, 0L)); + thread.rebalanceListener().onPartitionsAssigned(assignedPartitions); + + thread.runOnce(); + assertEquals(0, peekedContextTime.size()); + + mockTime.sleep(100L); + thread.runOnce(); + + assertEquals(1, peekedContextTime.size()); + assertEquals(currTime + 100L, peekedContextTime.get(0).longValue()); + + clientSupplier.consumer.addRecord(new ConsumerRecord<>( + topic1, + 1, + 0L, + 100L, + TimestampType.CREATE_TIME, + ConsumerRecord.NULL_CHECKSUM, + "K".getBytes().length, + "V".getBytes().length, + "K".getBytes(), + "V".getBytes())); + + thread.runOnce(); + + assertEquals(2, peekedContextTime.size()); + assertEquals(0L, peekedContextTime.get(1).longValue()); + } + @Test public void shouldAlwaysUpdateTasksMetadataAfterChangingState() { final StreamThread thread = createStreamThread(CLIENT_ID, config, false); From 5a9435afd18e0f9fdc34b4567bd51442e599053d Mon Sep 17 00:00:00 2001 From: Luke Chen <43372967+showuon@users.noreply.github.com> Date: Thu, 25 Feb 2021 01:52:34 +0800 Subject: [PATCH 055/243] KAFKA-12350: Correct the default value in doc (#10165) Reviewers: Chia-Ping Tsai --- docs/ops.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/ops.html b/docs/ops.html index 6762b6df63dad..1bef09aed1d90 100644 --- a/docs/ops.html +++ b/docs/ops.html @@ -787,18 +787,18 @@

    Streams API into the constructor, it is no longer required to set mandatory configuration parameters (cf. KIP-680).

    +

    + We added the prefixScan() method to interface ReadOnlyKeyValueStore. + The new prefixScan() allows fetching all values whose keys start with a given prefix. + See KIP-614 for more details. +

    Kafka Streams is now handling TimeoutException thrown by the consumer, producer, and admin client. If a timeout occurs on a task, Kafka Streams moves to the next task and retries to make progress on the failed From 059c9b3fcf427697be6e06269a4c0ace271e729a Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 24 Feb 2021 12:23:24 -0800 Subject: [PATCH 057/243] MINOR: Fix the generation extraction util (#10204) Reviewers: Matthias J. Sax , Anna Sophie Blee-Goldman , Chia-Ping Tsai --- tests/kafkatest/tests/streams/utils/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kafkatest/tests/streams/utils/util.py b/tests/kafkatest/tests/streams/utils/util.py index 7bec20cd2a2e1..6871e08af2774 100644 --- a/tests/kafkatest/tests/streams/utils/util.py +++ b/tests/kafkatest/tests/streams/utils/util.py @@ -35,7 +35,7 @@ def stop_processors(processors, stopped_message): verify_stopped(processor, stopped_message) def extract_generation_from_logs(processor): - return list(processor.node.account.ssh_capture("grep \"Successfully joined group with generation\" %s| awk \'{for(i=1;i<=NF;i++) {if ($i == \"generation\") beginning=i+1; if($i== \"(org.apache.kafka.clients.consumer.internals.AbstractCoordinator)\") ending=i }; for (j=beginning;j Date: Wed, 24 Feb 2021 12:50:18 -0800 Subject: [PATCH 058/243] KAFKA-12267; Implement `DescribeTransactions` API (#10183) This patch implements the `DescribeTransactions` API as documented in KIP-664: https://cwiki.apache.org/confluence/display/KAFKA/KIP-664%3A+Provide+tooling+to+detect+and+abort+hanging+transactions. This is only the server-side implementation and does not contain the `Admin` API. Reviewers: Chia-Ping Tsai --- .../TransactionalIdNotFoundException.java | 24 ++++ .../apache/kafka/common/protocol/ApiKeys.java | 3 +- .../apache/kafka/common/protocol/Errors.java | 4 +- .../common/requests/AbstractRequest.java | 2 + .../common/requests/AbstractResponse.java | 2 + .../requests/DescribeTransactionsRequest.java | 85 ++++++++++++ .../DescribeTransactionsResponse.java | 67 ++++++++++ .../message/DescribeProducersResponse.json | 2 +- .../message/DescribeTransactionsRequest.json | 27 ++++ .../message/DescribeTransactionsResponse.json | 42 ++++++ .../common/requests/RequestResponseTest.java | 52 ++++++++ .../transaction/TransactionCoordinator.scala | 54 +++++++- .../transaction/TransactionMetadata.scala | 49 +++++-- .../kafka/network/RequestConvertToJson.scala | 2 + .../main/scala/kafka/server/KafkaApis.scala | 31 ++++- .../kafka/api/AuthorizerIntegrationTest.scala | 49 ++++++- .../TransactionCoordinatorTest.scala | 51 ++++++++ .../unit/kafka/server/KafkaApisTest.scala | 121 ++++++++++++++++++ .../unit/kafka/server/RequestQuotaTest.scala | 4 + .../scala/unit/kafka/utils/TestUtils.scala | 27 +++- 20 files changed, 669 insertions(+), 29 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/TransactionalIdNotFoundException.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/DescribeTransactionsRequest.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/DescribeTransactionsResponse.java create mode 100644 clients/src/main/resources/common/message/DescribeTransactionsRequest.json create mode 100644 clients/src/main/resources/common/message/DescribeTransactionsResponse.json diff --git a/clients/src/main/java/org/apache/kafka/common/errors/TransactionalIdNotFoundException.java b/clients/src/main/java/org/apache/kafka/common/errors/TransactionalIdNotFoundException.java new file mode 100644 index 0000000000000..240eaa33a8474 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/errors/TransactionalIdNotFoundException.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.errors; + +public class TransactionalIdNotFoundException extends ApiException { + + public TransactionalIdNotFoundException(String message) { + super(message); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 475fc84c7355c..07f1a62aec1f8 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -105,7 +105,8 @@ public enum ApiKeys { DESCRIBE_PRODUCERS(ApiMessageType.DESCRIBE_PRODUCERS), BROKER_REGISTRATION(ApiMessageType.BROKER_REGISTRATION, true, RecordBatch.MAGIC_VALUE_V0, false), BROKER_HEARTBEAT(ApiMessageType.BROKER_HEARTBEAT, true, RecordBatch.MAGIC_VALUE_V0, false), - UNREGISTER_BROKER(ApiMessageType.UNREGISTER_BROKER, false, RecordBatch.MAGIC_VALUE_V0, true); + UNREGISTER_BROKER(ApiMessageType.UNREGISTER_BROKER, false, RecordBatch.MAGIC_VALUE_V0, true), + DESCRIBE_TRANSACTIONS(ApiMessageType.DESCRIBE_TRANSACTIONS); private static final Map> APIS_BY_LISTENER = new EnumMap<>(ApiMessageType.ListenerType.class); diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java index ca246c13e728f..5a758d9fd660a 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java @@ -110,6 +110,7 @@ import org.apache.kafka.common.errors.TopicExistsException; import org.apache.kafka.common.errors.TransactionCoordinatorFencedException; import org.apache.kafka.common.errors.TransactionalIdAuthorizationException; +import org.apache.kafka.common.errors.TransactionalIdNotFoundException; import org.apache.kafka.common.errors.UnacceptableCredentialException; import org.apache.kafka.common.errors.UnknownLeaderEpochException; import org.apache.kafka.common.errors.UnknownMemberIdException; @@ -360,7 +361,8 @@ public enum Errors { DUPLICATE_BROKER_REGISTRATION(101, "This broker ID is already in use.", DuplicateBrokerRegistrationException::new), BROKER_ID_NOT_REGISTERED(102, "The given broker ID was not registered.", BrokerIdNotRegisteredException::new), INCONSISTENT_TOPIC_ID(103, "The log's topic ID did not match the topic ID in the request", InconsistentTopicIdException::new), - INCONSISTENT_CLUSTER_ID(104, "The clusterId in the request does not match that found on the server", InconsistentClusterIdException::new); + INCONSISTENT_CLUSTER_ID(104, "The clusterId in the request does not match that found on the server", InconsistentClusterIdException::new), + TRANSACTIONAL_ID_NOT_FOUND(105, "The transactionalId could not be found", TransactionalIdNotFoundException::new); private static final Logger log = LoggerFactory.getLogger(Errors.class); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 64befeba62d31..2b754e056ce0a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -284,6 +284,8 @@ private static AbstractRequest doParseRequest(ApiKeys apiKey, short apiVersion, return BrokerHeartbeatRequest.parse(buffer, apiVersion); case UNREGISTER_BROKER: return UnregisterBrokerRequest.parse(buffer, apiVersion); + case DESCRIBE_TRANSACTIONS: + return DescribeTransactionsRequest.parse(buffer, apiVersion); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index c4dd7d85f4a22..e35589f68ec4d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -241,6 +241,8 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, ByteBuffer response return BrokerHeartbeatResponse.parse(responseBuffer, version); case UNREGISTER_BROKER: return UnregisterBrokerResponse.parse(responseBuffer, version); + case DESCRIBE_TRANSACTIONS: + return DescribeTransactionsResponse.parse(responseBuffer, version); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeTransactionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeTransactionsRequest.java new file mode 100644 index 0000000000000..a6e44fa5f6ddb --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeTransactionsRequest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.message.DescribeTransactionsRequestData; +import org.apache.kafka.common.message.DescribeTransactionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; + +import java.nio.ByteBuffer; + +public class DescribeTransactionsRequest extends AbstractRequest { + public static class Builder extends AbstractRequest.Builder { + public final DescribeTransactionsRequestData data; + + public Builder(DescribeTransactionsRequestData data) { + super(ApiKeys.DESCRIBE_TRANSACTIONS); + this.data = data; + } + + @Override + public DescribeTransactionsRequest build(short version) { + return new DescribeTransactionsRequest(data, version); + } + + @Override + public String toString() { + return data.toString(); + } + } + + private final DescribeTransactionsRequestData data; + + private DescribeTransactionsRequest(DescribeTransactionsRequestData data, short version) { + super(ApiKeys.DESCRIBE_TRANSACTIONS, version); + this.data = data; + } + + @Override + public DescribeTransactionsRequestData data() { + return data; + } + + @Override + public DescribeTransactionsResponse getErrorResponse(int throttleTimeMs, Throwable e) { + Errors error = Errors.forException(e); + DescribeTransactionsResponseData response = new DescribeTransactionsResponseData() + .setThrottleTimeMs(throttleTimeMs); + + for (String transactionalId : data.transactionalIds()) { + DescribeTransactionsResponseData.TransactionState transactionState = + new DescribeTransactionsResponseData.TransactionState() + .setTransactionalId(transactionalId) + .setErrorCode(error.code()); + response.transactionStates().add(transactionState); + } + return new DescribeTransactionsResponse(response); + } + + public static DescribeTransactionsRequest parse(ByteBuffer buffer, short version) { + return new DescribeTransactionsRequest(new DescribeTransactionsRequestData( + new ByteBufferAccessor(buffer), version), version); + } + + @Override + public String toString(boolean verbose) { + return data.toString(); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeTransactionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeTransactionsResponse.java new file mode 100644 index 0000000000000..cf151b35bba35 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeTransactionsResponse.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.message.DescribeTransactionsResponseData; +import org.apache.kafka.common.message.DescribeTransactionsResponseData.TransactionState; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +public class DescribeTransactionsResponse extends AbstractResponse { + private final DescribeTransactionsResponseData data; + + public DescribeTransactionsResponse(DescribeTransactionsResponseData data) { + super(ApiKeys.DESCRIBE_TRANSACTIONS); + this.data = data; + } + + @Override + public DescribeTransactionsResponseData data() { + return data; + } + + @Override + public Map errorCounts() { + Map errorCounts = new HashMap<>(); + for (TransactionState transactionState : data.transactionStates()) { + Errors error = Errors.forCode(transactionState.errorCode()); + updateErrorCounts(errorCounts, error); + } + return errorCounts; + } + + public static DescribeTransactionsResponse parse(ByteBuffer buffer, short version) { + return new DescribeTransactionsResponse(new DescribeTransactionsResponseData( + new ByteBufferAccessor(buffer), version)); + } + + @Override + public String toString() { + return data.toString(); + } + + @Override + public int throttleTimeMs() { + return data.throttleTimeMs(); + } + +} diff --git a/clients/src/main/resources/common/message/DescribeProducersResponse.json b/clients/src/main/resources/common/message/DescribeProducersResponse.json index 2ac977cf95c5f..c456ee4fb985f 100644 --- a/clients/src/main/resources/common/message/DescribeProducersResponse.json +++ b/clients/src/main/resources/common/message/DescribeProducersResponse.json @@ -35,7 +35,7 @@ { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", "default": "null", "about": "The partition error message, which may be null if no additional details are available" }, { "name": "ActiveProducers", "type": "[]ProducerState", "versions": "0+", "fields": [ - { "name": "ProducerId", "type": "int64", "versions": "0+" }, + { "name": "ProducerId", "type": "int64", "versions": "0+", "entityType": "producerId" }, { "name": "ProducerEpoch", "type": "int32", "versions": "0+" }, { "name": "LastSequence", "type": "int32", "versions": "0+", "default": "-1" }, { "name": "LastTimestamp", "type": "int64", "versions": "0+", "default": "-1" }, diff --git a/clients/src/main/resources/common/message/DescribeTransactionsRequest.json b/clients/src/main/resources/common/message/DescribeTransactionsRequest.json new file mode 100644 index 0000000000000..442f11f8b0b47 --- /dev/null +++ b/clients/src/main/resources/common/message/DescribeTransactionsRequest.json @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 65, + "type": "request", + "listeners": ["zkBroker", "broker"], + "name": "DescribeTransactionsRequest", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "TransactionalIds", "entityType": "transactionalId", "type": "[]string", "versions": "0+", + "about": "Array of transactionalIds to include in describe results. If empty, then no results will be returned." } + ] +} diff --git a/clients/src/main/resources/common/message/DescribeTransactionsResponse.json b/clients/src/main/resources/common/message/DescribeTransactionsResponse.json new file mode 100644 index 0000000000000..affc5aa4f09a8 --- /dev/null +++ b/clients/src/main/resources/common/message/DescribeTransactionsResponse.json @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 65, + "type": "response", + "name": "DescribeTransactionsResponse", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "TransactionStates", "type": "[]TransactionState", "versions": "0+", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+" }, + { "name": "TransactionalId", "type": "string", "versions": "0+", "entityType": "transactionalId" }, + { "name": "TransactionState", "type": "string", "versions": "0+" }, + { "name": "TransactionTimeoutMs", "type": "int32", "versions": "0+" }, + { "name": "TransactionStartTimeMs", "type": "int64", "versions": "0+" }, + { "name": "ProducerId", "type": "int64", "versions": "0+", "entityType": "producerId" }, + { "name": "ProducerEpoch", "type": "int16", "versions": "0+" }, + { "name": "Topics", "type": "[]TopicData", "versions": "0+", + "about": "The set of partitions included in the current transaction (if active). When a transaction is preparing to commit or abort, this will include only partitions which do not have markers.", + "fields": [ + { "name": "Topic", "type": "string", "versions": "0+", "entityType": "topicName", "mapKey": true }, + { "name": "Partitions", "type": "[]int32", "versions": "0+" } + ] + } + ]} + ] +} diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 920a9511e40c4..219c5c344a005 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -101,6 +101,8 @@ import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup; import org.apache.kafka.common.message.DescribeProducersRequestData; import org.apache.kafka.common.message.DescribeProducersResponseData; +import org.apache.kafka.common.message.DescribeTransactionsRequestData; +import org.apache.kafka.common.message.DescribeTransactionsResponseData; import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult; import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; import org.apache.kafka.common.message.EndTxnRequestData; @@ -549,6 +551,15 @@ public void testDescribeProducersSerialization() { } } + @Test + public void testDescribeTransactionsSerialization() { + for (short v : ApiKeys.DESCRIBE_TRANSACTIONS.allVersions()) { + checkRequest(createDescribeTransactionsRequest(v), true); + checkErrorResponse(createDescribeTransactionsRequest(v), unknownServerException, true); + checkResponse(createDescribeTransactionsResponse(), v, true); + } + } + @Test public void testDescribeClusterSerialization() { for (short v : ApiKeys.DESCRIBE_CLUSTER.allVersions()) { @@ -2754,4 +2765,45 @@ public void testErrorCountsIncludesNone() { assertEquals(Integer.valueOf(1), createUpdateMetadataResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createWriteTxnMarkersResponse().errorCounts().get(Errors.NONE)); } + + private DescribeTransactionsRequest createDescribeTransactionsRequest(short version) { + DescribeTransactionsRequestData data = new DescribeTransactionsRequestData() + .setTransactionalIds(asList("t1", "t2", "t3")); + return new DescribeTransactionsRequest.Builder(data).build(version); + } + + private DescribeTransactionsResponse createDescribeTransactionsResponse() { + DescribeTransactionsResponseData data = new DescribeTransactionsResponseData(); + data.setTransactionStates(asList( + new DescribeTransactionsResponseData.TransactionState() + .setErrorCode(Errors.NONE.code()) + .setTransactionalId("t1") + .setProducerId(12345L) + .setProducerEpoch((short) 15) + .setTransactionStartTimeMs(13490218304L) + .setTransactionState("Empty"), + new DescribeTransactionsResponseData.TransactionState() + .setErrorCode(Errors.NONE.code()) + .setTransactionalId("t2") + .setProducerId(98765L) + .setProducerEpoch((short) 30) + .setTransactionStartTimeMs(13490218304L) + .setTransactionState("Ongoing") + .setTopics(new DescribeTransactionsResponseData.TopicDataCollection( + asList( + new DescribeTransactionsResponseData.TopicData() + .setTopic("foo") + .setPartitions(asList(1, 3, 5, 7)), + new DescribeTransactionsResponseData.TopicData() + .setTopic("bar") + .setPartitions(asList(1, 3)) + ).iterator() + )), + new DescribeTransactionsResponseData.TransactionState() + .setErrorCode(Errors.NOT_COORDINATOR.code()) + .setTransactionalId("t3") + )); + return new DescribeTransactionsResponse(data); + } + } diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala index 0a2c45b100f8a..19f92435dcc42 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala @@ -23,6 +23,7 @@ import kafka.server.{KafkaConfig, MetadataCache, ReplicaManager} import kafka.utils.{Logging, Scheduler} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.message.DescribeTransactionsResponseData import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.record.RecordBatch @@ -187,10 +188,10 @@ class TransactionCoordinator(brokerId: Int, } private def prepareInitProducerIdTransit(transactionalId: String, - transactionTimeoutMs: Int, - coordinatorEpoch: Int, - txnMetadata: TransactionMetadata, - expectedProducerIdAndEpoch: Option[ProducerIdAndEpoch]): ApiResult[(Int, TxnTransitMetadata)] = { + transactionTimeoutMs: Int, + coordinatorEpoch: Int, + txnMetadata: TransactionMetadata, + expectedProducerIdAndEpoch: Option[ProducerIdAndEpoch]): ApiResult[(Int, TxnTransitMetadata)] = { def isValidProducerId(producerIdAndEpoch: ProducerIdAndEpoch): Boolean = { // If a producer ID and epoch are provided by the request, fence the producer unless one of the following is true: @@ -255,6 +256,51 @@ class TransactionCoordinator(brokerId: Int, } } + def handleDescribeTransactions( + transactionalId: String + ): DescribeTransactionsResponseData.TransactionState = { + if (transactionalId == null) { + throw new IllegalArgumentException("Invalid null transactionalId") + } + + val transactionState = new DescribeTransactionsResponseData.TransactionState() + .setTransactionalId(transactionalId) + + if (!isActive.get()) { + transactionState.setErrorCode(Errors.COORDINATOR_NOT_AVAILABLE.code) + } else if (transactionalId.isEmpty) { + transactionState.setErrorCode(Errors.INVALID_REQUEST.code) + } else { + txnManager.getTransactionState(transactionalId) match { + case Left(error) => + transactionState.setErrorCode(error.code) + case Right(None) => + transactionState.setErrorCode(Errors.TRANSACTIONAL_ID_NOT_FOUND.code) + case Right(Some(coordinatorEpochAndMetadata)) => + val txnMetadata = coordinatorEpochAndMetadata.transactionMetadata + txnMetadata.inLock { + txnMetadata.topicPartitions.foreach { topicPartition => + var topicData = transactionState.topics.find(topicPartition.topic) + if (topicData == null) { + topicData = new DescribeTransactionsResponseData.TopicData() + .setTopic(topicPartition.topic) + transactionState.topics.add(topicData) + } + topicData.partitions.add(topicPartition.partition) + } + + transactionState + .setErrorCode(Errors.NONE.code) + .setProducerId(txnMetadata.producerId) + .setProducerEpoch(txnMetadata.producerEpoch) + .setTransactionState(txnMetadata.state.name) + .setTransactionTimeoutMs(txnMetadata.txnTimeoutMs) + .setTransactionStartTimeMs(txnMetadata.txnStartTimestamp) + } + } + } + } + def handleAddPartitionsToTransaction(transactionalId: String, producerId: Long, producerEpoch: Short, diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMetadata.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMetadata.scala index e059b04a37374..5269f3e3349a2 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMetadata.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMetadata.scala @@ -25,7 +25,14 @@ import org.apache.kafka.common.record.RecordBatch import scala.collection.{immutable, mutable} -private[transaction] sealed trait TransactionState { def byte: Byte } +private[transaction] sealed trait TransactionState { + def byte: Byte + + /** + * Get the name of this state. This is exposed through the `DescribeTransactions` API. + */ + def name: String +} /** * Transaction has not existed yet @@ -33,7 +40,10 @@ private[transaction] sealed trait TransactionState { def byte: Byte } * transition: received AddPartitionsToTxnRequest => Ongoing * received AddOffsetsToTxnRequest => Ongoing */ -private[transaction] case object Empty extends TransactionState { val byte: Byte = 0 } +private[transaction] case object Empty extends TransactionState { + val byte: Byte = 0 + val name: String = "Empty" +} /** * Transaction has started and ongoing @@ -43,46 +53,67 @@ private[transaction] case object Empty extends TransactionState { val byte: Byte * received AddPartitionsToTxnRequest => Ongoing * received AddOffsetsToTxnRequest => Ongoing */ -private[transaction] case object Ongoing extends TransactionState { val byte: Byte = 1 } +private[transaction] case object Ongoing extends TransactionState { + val byte: Byte = 1 + val name: String = "Ongoing" +} /** * Group is preparing to commit * * transition: received acks from all partitions => CompleteCommit */ -private[transaction] case object PrepareCommit extends TransactionState { val byte: Byte = 2} +private[transaction] case object PrepareCommit extends TransactionState { + val byte: Byte = 2 + val name: String = "PrepareCommit" +} /** * Group is preparing to abort * * transition: received acks from all partitions => CompleteAbort */ -private[transaction] case object PrepareAbort extends TransactionState { val byte: Byte = 3 } +private[transaction] case object PrepareAbort extends TransactionState { + val byte: Byte = 3 + val name: String = "PrepareAbort" +} /** * Group has completed commit * * Will soon be removed from the ongoing transaction cache */ -private[transaction] case object CompleteCommit extends TransactionState { val byte: Byte = 4 } +private[transaction] case object CompleteCommit extends TransactionState { + val byte: Byte = 4 + val name: String = "CompleteCommit" +} /** * Group has completed abort * * Will soon be removed from the ongoing transaction cache */ -private[transaction] case object CompleteAbort extends TransactionState { val byte: Byte = 5 } +private[transaction] case object CompleteAbort extends TransactionState { + val byte: Byte = 5 + val name: String = "CompleteAbort" +} /** * TransactionalId has expired and is about to be removed from the transaction cache */ -private[transaction] case object Dead extends TransactionState { val byte: Byte = 6 } +private[transaction] case object Dead extends TransactionState { + val byte: Byte = 6 + val name: String = "Dead" +} /** * We are in the middle of bumping the epoch and fencing out older producers. */ -private[transaction] case object PrepareEpochFence extends TransactionState { val byte: Byte = 7} +private[transaction] case object PrepareEpochFence extends TransactionState { + val byte: Byte = 7 + val name: String = "PrepareEpochFence" +} private[transaction] object TransactionMetadata { def apply(transactionalId: String, producerId: Long, producerEpoch: Short, txnTimeoutMs: Int, timestamp: Long) = diff --git a/core/src/main/scala/kafka/network/RequestConvertToJson.scala b/core/src/main/scala/kafka/network/RequestConvertToJson.scala index 66ece9911307a..aacd24ec28524 100644 --- a/core/src/main/scala/kafka/network/RequestConvertToJson.scala +++ b/core/src/main/scala/kafka/network/RequestConvertToJson.scala @@ -92,6 +92,7 @@ object RequestConvertToJson { case req: FetchSnapshotRequest => FetchSnapshotRequestDataJsonConverter.write(req.data, request.version) case req: DescribeClusterRequest => DescribeClusterRequestDataJsonConverter.write(req.data, request.version) case req: DescribeProducersRequest => DescribeProducersRequestDataJsonConverter.write(req.data, request.version) + case req: DescribeTransactionsRequest => DescribeTransactionsRequestDataJsonConverter.write(req.data, request.version) case _ => throw new IllegalStateException(s"ApiKey ${request.apiKey} is not currently handled in `request`, the " + "code should be updated to do so."); } @@ -164,6 +165,7 @@ object RequestConvertToJson { case res: FetchSnapshotResponse => FetchSnapshotResponseDataJsonConverter.write(res.data, version) case res: DescribeClusterResponse => DescribeClusterResponseDataJsonConverter.write(res.data, version) case res: DescribeProducersResponse => DescribeProducersResponseDataJsonConverter.write(res.data, version) + case res: DescribeTransactionsResponse => DescribeTransactionsResponseDataJsonConverter.write(res.data, version) case _ => throw new IllegalStateException(s"ApiKey ${response.apiKey} is not currently handled in `response`, the " + "code should be updated to do so."); } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 1245fe74d2e0c..d4fd527d3d4e1 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -61,7 +61,7 @@ import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsParti import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic} import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult, OffsetForLeaderTopicResultCollection} -import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} +import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, DescribeTransactionsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.{ListenerName, Send} import org.apache.kafka.common.protocol.{ApiKeys, Errors} @@ -222,6 +222,7 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.DESCRIBE_CLUSTER => handleDescribeCluster(request) case ApiKeys.DESCRIBE_PRODUCERS => handleDescribeProducersRequest(request) case ApiKeys.UNREGISTER_BROKER => maybeForwardToController(request, handleUnregisterBrokerRequest) + case ApiKeys.DESCRIBE_TRANSACTIONS => handleDescribeTransactionsRequest(request) case _ => throw new IllegalStateException(s"No handler for request api key ${request.header.apiKey}") } } catch { @@ -3274,6 +3275,34 @@ class KafkaApis(val requestChannel: RequestChannel, "Apache ZooKeeper mode.") } + def handleDescribeTransactionsRequest(request: RequestChannel.Request): Unit = { + val describeTransactionsRequest = request.body[DescribeTransactionsRequest] + val response = new DescribeTransactionsResponseData() + + describeTransactionsRequest.data.transactionalIds.forEach { transactionalId => + val transactionState = if (!authHelper.authorize(request.context, DESCRIBE, TRANSACTIONAL_ID, transactionalId)) { + new DescribeTransactionsResponseData.TransactionState() + .setTransactionalId(transactionalId) + .setErrorCode(Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.code) + } else { + txnCoordinator.handleDescribeTransactions(transactionalId) + } + + // Include only partitions which the principal is authorized to describe + val topicIter = transactionState.topics.iterator() + while (topicIter.hasNext) { + val topic = topicIter.next().topic + if (!authHelper.authorize(request.context, DESCRIBE, TOPIC, topic)) { + topicIter.remove() + } + } + response.transactionStates.add(transactionState) + } + + requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => + new DescribeTransactionsResponse(response.setThrottleTimeMs(requestThrottleMs))) + } + private def updateRecordConversionStats(request: RequestChannel.Request, tp: TopicPartition, conversionStats: RecordConversionStats): Unit = { diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index 66f25eeaac1c9..b100962e35c9c 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -36,7 +36,6 @@ import org.apache.kafka.common.config.internals.BrokerSecurityConfigs import org.apache.kafka.common.config.{ConfigResource, LogLevelConfig} import org.apache.kafka.common.errors._ import org.apache.kafka.common.internals.Topic.GROUP_METADATA_TOPIC_NAME -import org.apache.kafka.common.message.{AddOffsetsToTxnRequestData, AlterPartitionReassignmentsRequestData, AlterReplicaLogDirsRequestData, ControlledShutdownRequestData, CreateAclsRequestData, CreatePartitionsRequestData, CreateTopicsRequestData, DeleteAclsRequestData, DeleteGroupsRequestData, DeleteRecordsRequestData, DeleteTopicsRequestData, DescribeClusterRequestData, DescribeConfigsRequestData, DescribeGroupsRequestData, DescribeLogDirsRequestData, DescribeProducersRequestData, FindCoordinatorRequestData, HeartbeatRequestData, IncrementalAlterConfigsRequestData, JoinGroupRequestData, ListPartitionReassignmentsRequestData, MetadataRequestData, OffsetCommitRequestData, ProduceRequestData, SyncGroupRequestData} import org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopic import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicCollection} import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData.{AlterConfigsResource, AlterableConfig, AlterableConfigCollection} @@ -44,11 +43,10 @@ import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProt import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity import org.apache.kafka.common.message.ListOffsetsRequestData.{ListOffsetsPartition, ListOffsetsTopic} -import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderPartition -import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic -import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopicCollection +import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.{OffsetForLeaderPartition, OffsetForLeaderTopic, OffsetForLeaderTopicCollection} import org.apache.kafka.common.message.StopReplicaRequestData.{StopReplicaPartitionState, StopReplicaTopicState} import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState} +import org.apache.kafka.common.message.{AddOffsetsToTxnRequestData, AlterPartitionReassignmentsRequestData, AlterReplicaLogDirsRequestData, ControlledShutdownRequestData, CreateAclsRequestData, CreatePartitionsRequestData, CreateTopicsRequestData, DeleteAclsRequestData, DeleteGroupsRequestData, DeleteRecordsRequestData, DeleteTopicsRequestData, DescribeClusterRequestData, DescribeConfigsRequestData, DescribeGroupsRequestData, DescribeLogDirsRequestData, DescribeProducersRequestData, DescribeTransactionsRequestData, FindCoordinatorRequestData, HeartbeatRequestData, IncrementalAlterConfigsRequestData, JoinGroupRequestData, ListPartitionReassignmentsRequestData, MetadataRequestData, OffsetCommitRequestData, ProduceRequestData, SyncGroupRequestData} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, Records, SimpleRecord} @@ -64,9 +62,9 @@ import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import scala.annotation.nowarn -import scala.jdk.CollectionConverters._ import scala.collection.mutable import scala.collection.mutable.Buffer +import scala.jdk.CollectionConverters._ object AuthorizerIntegrationTest { val BrokerPrincipal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "broker") @@ -237,6 +235,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest { .partitions.asScala.find(_.partitionIndex == part).get .errorCode ) + }), + ApiKeys.DESCRIBE_TRANSACTIONS -> ((resp: DescribeTransactionsResponse) => { + Errors.forCode( + resp.data + .transactionStates.asScala.find(_.transactionalId == transactionalId).get + .errorCode + ) }) ) @@ -285,7 +290,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> clusterAlterAcl, ApiKeys.LIST_PARTITION_REASSIGNMENTS -> clusterDescribeAcl, ApiKeys.OFFSET_DELETE -> groupReadAcl, - ApiKeys.DESCRIBE_PRODUCERS -> topicReadAcl + ApiKeys.DESCRIBE_PRODUCERS -> topicReadAcl, + ApiKeys.DESCRIBE_TRANSACTIONS -> transactionalIdDescribeAcl ) @BeforeEach @@ -636,6 +642,10 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ).asJava) ).build() + private def describeTransactionsRequest: DescribeTransactionsRequest = new DescribeTransactionsRequest.Builder( + new DescribeTransactionsRequestData().setTransactionalIds(List(transactionalId).asJava) + ).build() + private def alterPartitionReassignmentsRequest = new AlterPartitionReassignmentsRequest.Builder( new AlterPartitionReassignmentsRequestData().setTopics( List(new AlterPartitionReassignmentsRequestData.ReassignableTopic() @@ -712,6 +722,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> alterPartitionReassignmentsRequest, ApiKeys.LIST_PARTITION_REASSIGNMENTS -> listPartitionReassignmentsRequest, ApiKeys.DESCRIBE_PRODUCERS -> describeProducersRequest, + ApiKeys.DESCRIBE_TRANSACTIONS -> describeTransactionsRequest, // Inter-broker APIs use an invalid broker epoch, so does not affect the test case ApiKeys.UPDATE_METADATA -> createUpdateMetadataRequest, @@ -1790,6 +1801,28 @@ class AuthorizerIntegrationTest extends BaseRequestTest { assertThrows(classOf[TransactionalIdAuthorizationException], () => producer.commitTransaction()) } + @Test + def shouldNotIncludeUnauthorizedTopicsInDescribeTransactionsResponse(): Unit = { + createTopic(topic) + addAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, WRITE, ALLOW)), transactionalIdResource) + addAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, WRITE, ALLOW)), topicResource) + + // Start a transaction and write to a topic. + val producer = buildTransactionalProducer() + producer.initTransactions() + producer.beginTransaction() + producer.send(new ProducerRecord(tp.topic, tp.partition, "1".getBytes, "1".getBytes)).get + + // Remove only topic authorization so that we can verify that the + // topic does not get included in the response. + removeAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, WRITE, ALLOW)), topicResource) + val response = connectAndReceive[DescribeTransactionsResponse](describeTransactionsRequest) + assertEquals(1, response.data.transactionStates.size) + val transactionStateData = response.data.transactionStates.asScala.find(_.transactionalId == transactionalId).get + assertEquals("Ongoing", transactionStateData.transactionState) + assertEquals(List.empty, transactionStateData.topics.asScala.toList) + } + @Test def shouldSuccessfullyAbortTransactionAfterTopicAuthorizationException(): Unit = { createTopic(topic) @@ -2114,6 +2147,10 @@ class AuthorizerIntegrationTest extends BaseRequestTest { TestUtils.addAndVerifyAcls(servers.head, acls, resource) } + private def removeAndVerifyAcls(acls: Set[AccessControlEntry], resource: ResourcePattern): Unit = { + TestUtils.removeAndVerifyAcls(servers.head, acls, resource) + } + private def consumeRecords(consumer: Consumer[Array[Byte], Array[Byte]], numRecords: Int = 1, startingOffset: Int = 0, diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala index b5381e2defd7b..0cc52bf6e6d58 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionCoordinatorTest.scala @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test import scala.collection.mutable +import scala.jdk.CollectionConverters._ class TransactionCoordinatorTest { @@ -1078,6 +1079,56 @@ class TransactionCoordinatorTest { EasyMock.verify(transactionManager) } + @Test + def testDescribeTransactionsWithEmptyTransactionalId(): Unit = { + coordinator.startup(() => transactionStatePartitionCount, enableTransactionalIdExpiration = false) + val result = coordinator.handleDescribeTransactions("") + assertEquals("", result.transactionalId) + assertEquals(Errors.INVALID_REQUEST, Errors.forCode(result.errorCode)) + } + + @Test + def testDescribeTransactionsWhileCoordinatorLoading(): Unit = { + EasyMock.expect(transactionManager.getTransactionState(EasyMock.eq(transactionalId))) + .andReturn(Left(Errors.COORDINATOR_LOAD_IN_PROGRESS)) + + EasyMock.replay(transactionManager) + + coordinator.startup(() => transactionStatePartitionCount, enableTransactionalIdExpiration = false) + val result = coordinator.handleDescribeTransactions(transactionalId) + assertEquals(transactionalId, result.transactionalId) + assertEquals(Errors.COORDINATOR_LOAD_IN_PROGRESS, Errors.forCode(result.errorCode)) + + EasyMock.verify(transactionManager) + } + + @Test + def testDescribeTransactions(): Unit = { + val txnMetadata = new TransactionMetadata(transactionalId, producerId, producerId, producerEpoch, + RecordBatch.NO_PRODUCER_EPOCH, txnTimeoutMs, Ongoing, partitions, time.milliseconds(), time.milliseconds()) + + EasyMock.expect(transactionManager.getTransactionState(EasyMock.eq(transactionalId))) + .andReturn(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata)))) + + EasyMock.replay(transactionManager) + + coordinator.startup(() => transactionStatePartitionCount, enableTransactionalIdExpiration = false) + val result = coordinator.handleDescribeTransactions(transactionalId) + assertEquals(Errors.NONE, Errors.forCode(result.errorCode)) + assertEquals(transactionalId, result.transactionalId) + assertEquals(producerId, result.producerId) + assertEquals(producerEpoch, result.producerEpoch) + assertEquals(txnTimeoutMs, result.transactionTimeoutMs) + assertEquals(time.milliseconds(), result.transactionStartTimeMs) + + val addedPartitions = result.topics.asScala.flatMap { topicData => + topicData.partitions.asScala.map(partition => new TopicPartition(topicData.topic, partition)) + }.toSet + assertEquals(partitions, addedPartitions) + + EasyMock.verify(transactionManager) + } + private def validateRespondsWithConcurrentTransactionsOnInitPidWhenInPrepareState(state: TransactionState): Unit = { EasyMock.expect(transactionManager.validateTransactionTimeoutMs(EasyMock.anyInt())) .andReturn(true).anyTimes() diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 43b9ca597194f..1fec7892434a8 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -3355,7 +3355,128 @@ class KafkaApisTest { val bazTopic = response.data.topics.asScala.find(_.name == tp3.topic).get val bazPartition = bazTopic.partitions.asScala.find(_.partitionIndex == tp3.partition).get assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.forCode(bazPartition.errorCode)) + } + + @Test + def testDescribeTransactions(): Unit = { + val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) + val data = new DescribeTransactionsRequestData() + .setTransactionalIds(List("foo", "bar").asJava) + val describeTransactionsRequest = new DescribeTransactionsRequest.Builder(data).build() + val request = buildRequest(describeTransactionsRequest) + val capturedResponse = expectNoThrottling(request) + + def buildExpectedActions(transactionalId: String): util.List[Action] = { + val pattern = new ResourcePattern(ResourceType.TRANSACTIONAL_ID, transactionalId, PatternType.LITERAL) + val action = new Action(AclOperation.DESCRIBE, pattern, 1, true, true) + Collections.singletonList(action) + } + + EasyMock.expect(txnCoordinator.handleDescribeTransactions("foo")) + .andReturn(new DescribeTransactionsResponseData.TransactionState() + .setErrorCode(Errors.NONE.code) + .setTransactionalId("foo") + .setProducerId(12345L) + .setProducerEpoch(15) + .setTransactionStartTimeMs(time.milliseconds()) + .setTransactionState("CompleteCommit") + .setTransactionTimeoutMs(10000)) + + EasyMock.expect(authorizer.authorize(anyObject[RequestContext], EasyMock.eq(buildExpectedActions("foo")))) + .andReturn(Seq(AuthorizationResult.ALLOWED).asJava) + .once() + + EasyMock.expect(authorizer.authorize(anyObject[RequestContext], EasyMock.eq(buildExpectedActions("bar")))) + .andReturn(Seq(AuthorizationResult.DENIED).asJava) + .once() + + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator, authorizer) + createKafkaApis(authorizer = Some(authorizer)).handleDescribeTransactionsRequest(request) + val response = capturedResponse.getValue.asInstanceOf[DescribeTransactionsResponse] + assertEquals(2, response.data.transactionStates.size) + + val fooState = response.data.transactionStates.asScala.find(_.transactionalId == "foo").get + assertEquals(Errors.NONE.code, fooState.errorCode) + assertEquals(12345L, fooState.producerId) + assertEquals(15, fooState.producerEpoch) + assertEquals(time.milliseconds(), fooState.transactionStartTimeMs) + assertEquals("CompleteCommit", fooState.transactionState) + assertEquals(10000, fooState.transactionTimeoutMs) + assertEquals(List.empty, fooState.topics.asScala.toList) + + val barState = response.data.transactionStates.asScala.find(_.transactionalId == "bar").get + assertEquals(Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.code, barState.errorCode) + } + + @Test + def testDescribeTransactionsFiltersUnauthorizedTopics(): Unit = { + val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) + val transactionalId = "foo" + val data = new DescribeTransactionsRequestData() + .setTransactionalIds(List(transactionalId).asJava) + val describeTransactionsRequest = new DescribeTransactionsRequest.Builder(data).build() + val request = buildRequest(describeTransactionsRequest) + val capturedResponse = expectNoThrottling(request) + + def expectDescribe( + resourceType: ResourceType, + transactionalId: String, + result: AuthorizationResult + ): Unit = { + val pattern = new ResourcePattern(resourceType, transactionalId, PatternType.LITERAL) + val action = new Action(AclOperation.DESCRIBE, pattern, 1, true, true) + val actions = Collections.singletonList(action) + + EasyMock.expect(authorizer.authorize(anyObject[RequestContext], EasyMock.eq(actions))) + .andReturn(Seq(result).asJava) + .once() + } + + // Principal is authorized to one of the two topics. The second topic should be + // filtered from the result. + expectDescribe(ResourceType.TRANSACTIONAL_ID, transactionalId, AuthorizationResult.ALLOWED) + expectDescribe(ResourceType.TOPIC, "foo", AuthorizationResult.ALLOWED) + expectDescribe(ResourceType.TOPIC, "bar", AuthorizationResult.DENIED) + + def mkTopicData( + topic: String, + partitions: Seq[Int] + ): DescribeTransactionsResponseData.TopicData = { + new DescribeTransactionsResponseData.TopicData() + .setTopic(topic) + .setPartitions(partitions.map(Int.box).asJava) + } + + val describeTransactionsResponse = new DescribeTransactionsResponseData.TransactionState() + .setErrorCode(Errors.NONE.code) + .setTransactionalId(transactionalId) + .setProducerId(12345L) + .setProducerEpoch(15) + .setTransactionStartTimeMs(time.milliseconds()) + .setTransactionState("Ongoing") + .setTransactionTimeoutMs(10000) + + describeTransactionsResponse.topics.add(mkTopicData(topic = "foo", Seq(1, 2))) + describeTransactionsResponse.topics.add(mkTopicData(topic = "bar", Seq(3, 4))) + + EasyMock.expect(txnCoordinator.handleDescribeTransactions("foo")) + .andReturn(describeTransactionsResponse) + + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator, authorizer) + createKafkaApis(authorizer = Some(authorizer)).handleDescribeTransactionsRequest(request) + + val response = capturedResponse.getValue.asInstanceOf[DescribeTransactionsResponse] + assertEquals(1, response.data.transactionStates.size) + + val fooState = response.data.transactionStates.asScala.find(_.transactionalId == "foo").get + assertEquals(Errors.NONE.code, fooState.errorCode) + assertEquals(12345L, fooState.producerId) + assertEquals(15, fooState.producerEpoch) + assertEquals(time.milliseconds(), fooState.transactionStartTimeMs) + assertEquals("Ongoing", fooState.transactionState) + assertEquals(10000, fooState.transactionTimeoutMs) + assertEquals(List(mkTopicData(topic = "foo", Seq(1, 2))), fooState.topics.asScala.toList) } private def createMockRequest(): RequestChannel.Request = { diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index 7706c83cca9bf..a5d5078a761d8 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -635,6 +635,10 @@ class RequestQuotaTest extends BaseRequestTest { case ApiKeys.UNREGISTER_BROKER => new UnregisterBrokerRequest.Builder(new UnregisterBrokerRequestData()) + case ApiKeys.DESCRIBE_TRANSACTIONS => + new DescribeTransactionsRequest.Builder(new DescribeTransactionsRequestData() + .setTransactionalIds(List("test-transactional-id").asJava)) + case _ => throw new IllegalArgumentException("Unsupported API key " + apiKey) } diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index 43df2b97f4bd0..38a5a12289bfc 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -24,18 +24,19 @@ import java.nio.file.{Files, StandardOpenOption} import java.security.cert.X509Certificate import java.time.Duration import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} -import java.util.{Arrays, Collections, Properties} import java.util.concurrent.{Callable, ExecutionException, Executors, TimeUnit} +import java.util.{Arrays, Collections, Properties} + +import com.yammer.metrics.core.Meter import javax.net.ssl.X509TrustManager import kafka.api._ import kafka.cluster.{Broker, EndPoint, IsrChangeListener} +import kafka.controller.LeaderIsrAndControllerEpoch import kafka.log._ +import kafka.metrics.KafkaYammerMetrics import kafka.security.auth.{Acl, Resource, Authorizer => LegacyAuthorizer} import kafka.server._ import kafka.server.checkpoints.OffsetCheckpointFile -import com.yammer.metrics.core.Meter -import kafka.controller.LeaderIsrAndControllerEpoch -import kafka.metrics.KafkaYammerMetrics import kafka.server.metadata.{CachedConfigRepository, ConfigRepository, MetadataBroker} import kafka.utils.Implicits._ import kafka.zk._ @@ -57,8 +58,8 @@ import org.apache.kafka.common.record._ import org.apache.kafka.common.resource.ResourcePattern import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, Deserializer, IntegerSerializer, Serializer} -import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.utils.Utils._ +import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.{KafkaFuture, Node, TopicPartition} import org.apache.kafka.server.authorizer.{Authorizer => JAuthorizer} import org.apache.kafka.test.{TestSslUtils, TestUtils => JTestUtils} @@ -67,11 +68,11 @@ import org.apache.zookeeper.ZooDefs._ import org.apache.zookeeper.data.ACL import org.junit.jupiter.api.Assertions._ -import scala.jdk.CollectionConverters._ import scala.collection.mutable.{ArrayBuffer, ListBuffer} import scala.collection.{Map, Seq, mutable} import scala.concurrent.duration.FiniteDuration import scala.concurrent.{Await, ExecutionContext, Future} +import scala.jdk.CollectionConverters._ /** * Utility functions to help with testing @@ -1853,4 +1854,18 @@ object TestUtils extends Logging { authorizer, resource) } + def removeAndVerifyAcls(server: KafkaServer, acls: Set[AccessControlEntry], resource: ResourcePattern): Unit = { + val authorizer = server.dataPlaneRequestProcessor.authorizer.get + val aclBindingFilters = acls.map { acl => new AclBindingFilter(resource.toFilter, acl.toFilter) } + authorizer.deleteAcls(null, aclBindingFilters.toList.asJava).asScala + .map(_.toCompletableFuture.get) + .foreach { result => + result.exception.ifPresent { e => throw e } + } + val aclFilter = new AclBindingFilter(resource.toFilter, AccessControlEntryFilter.ANY) + waitAndVerifyAcls( + authorizer.acls(aclFilter).asScala.map(_.entry).toSet -- acls, + authorizer, resource) + } + } From e2a0d0c90e1916d77223a420e3595e8aba643001 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Wed, 24 Feb 2021 17:49:18 -0800 Subject: [PATCH 059/243] MINOR: bump release version to 3.0.0-SNAPSHOT (#10186) Reviewers: Konstantine Karantasis , Chia-Ping Tsai --- docs/js/templateData.js | 6 +++--- gradle.properties | 2 +- kafka-merge-pr.py | 2 +- streams/quickstart/java/pom.xml | 2 +- .../java/src/main/resources/archetype-resources/pom.xml | 2 +- streams/quickstart/pom.xml | 2 +- tests/kafkatest/__init__.py | 2 +- tests/kafkatest/version.py | 8 ++++---- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/js/templateData.js b/docs/js/templateData.js index 64fe9d8b07ecd..bd30fa9ccc2f3 100644 --- a/docs/js/templateData.js +++ b/docs/js/templateData.js @@ -17,8 +17,8 @@ limitations under the License. // Define variables for doc templates var context={ - "version": "29", - "dotVersion": "2.9", - "fullDotVersion": "2.9.0", + "version": "30", + "dotVersion": "3.0", + "fullDotVersion": "3.0.0", "scalaVersion": "2.13" }; diff --git a/gradle.properties b/gradle.properties index c3de576dcdceb..6c6fcd38f0969 100644 --- a/gradle.properties +++ b/gradle.properties @@ -20,7 +20,7 @@ group=org.apache.kafka # - tests/kafkatest/__init__.py # - tests/kafkatest/version.py (variable DEV_VERSION) # - kafka-merge-pr.py -version=2.9.0-SNAPSHOT +version=3.0.0-SNAPSHOT scalaVersion=2.13.5 task=build org.gradle.jvmargs=-Xmx2g -Xss4m -XX:+UseParallelGC diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py index 3ab8b58e7392f..673db4b377bf9 100755 --- a/kafka-merge-pr.py +++ b/kafka-merge-pr.py @@ -70,7 +70,7 @@ DEV_BRANCH_NAME = "trunk" -DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "2.9.0") +DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.0.0") def get_json(url): try: diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml index 74fe61d72e4c6..5eeb2d51e8193 100644 --- a/streams/quickstart/java/pom.xml +++ b/streams/quickstart/java/pom.xml @@ -26,7 +26,7 @@ org.apache.kafka streams-quickstart - 2.9.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml index 6fd14359ce073..4a81d7d555fa4 100644 --- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml +++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml @@ -29,7 +29,7 @@ UTF-8 - 2.9.0-SNAPSHOT + 3.0.0-SNAPSHOT 1.7.7 1.2.17 diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml index c1c36851cc9c7..8621858af4424 100644 --- a/streams/quickstart/pom.xml +++ b/streams/quickstart/pom.xml @@ -22,7 +22,7 @@ org.apache.kafka streams-quickstart pom - 2.9.0-SNAPSHOT + 3.0.0-SNAPSHOT Kafka Streams :: Quickstart diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py index 0e8c3f4af6344..cb42768337300 100644 --- a/tests/kafkatest/__init__.py +++ b/tests/kafkatest/__init__.py @@ -22,4 +22,4 @@ # Instead, in development branches, the version should have a suffix of the form ".devN" # # For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0" -__version__ = '2.9.0.dev0' +__version__ = '3.0.0.dev0' diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py index 566dba5fe30ad..672ba8647d5da 100644 --- a/tests/kafkatest/version.py +++ b/tests/kafkatest/version.py @@ -115,7 +115,7 @@ def get_version(node=None): return DEV_BRANCH DEV_BRANCH = KafkaVersion("dev") -DEV_VERSION = KafkaVersion("2.9.0-SNAPSHOT") +DEV_VERSION = KafkaVersion("3.0.0-SNAPSHOT") # 0.8.2.x versions V_0_8_2_1 = KafkaVersion("0.8.2.1") @@ -208,6 +208,6 @@ def get_version(node=None): V_2_8_0 = KafkaVersion("2.8.0") LATEST_2_8 = V_2_8_0 -# 2.9.x versions -V_2_9_0 = KafkaVersion("2.9.0") -LATEST_2_9 = V_2_9_0 +# 3.0.x versions +V_3_0_0 = KafkaVersion("3.0.0") +LATEST_3_0 = V_3_0_0 From bd04f7557a4962d5d5054dc8339f8727b8e428b6 Mon Sep 17 00:00:00 2001 From: Ron Dagostino Date: Thu, 25 Feb 2021 15:14:38 -0500 Subject: [PATCH 060/243] MINOR: fix syntax error in upgrade_test.py (#10210) Reviewers: Colin P. McCabe --- tests/kafkatest/tests/core/upgrade_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/kafkatest/tests/core/upgrade_test.py b/tests/kafkatest/tests/core/upgrade_test.py index 183e4900e8c1d..59db68f692b73 100644 --- a/tests/kafkatest/tests/core/upgrade_test.py +++ b/tests/kafkatest/tests/core/upgrade_test.py @@ -139,8 +139,9 @@ def test_upgrade(self, from_kafka_version, to_message_format_version, compressio - Finally, validate that every message acked by the producer was consumed by the consumer """ self.zk = ZookeeperService(self.test_context, num_nodes=1, version=KafkaVersion(from_kafka_version)) + fromKafkaVersion = KafkaVersion(from_kafka_version) self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, - version=KafkaVersion(from_kafka_version), + version=fromKafkaVersion, topics={self.topic: {"partitions": self.partitions, "replication-factor": self.replication_factor, 'configs': {"min.insync.replicas": 2}}}) @@ -171,7 +172,7 @@ def test_upgrade(self, from_kafka_version, to_message_format_version, compressio # after leader change. Tolerate limited data loss for this case to avoid transient test failures. self.may_truncate_acked_records = False if from_kafka_version >= V_0_11_0_0 else True - new_consumer = from_kafka_version.consumer_supports_bootstrap_server() + new_consumer = fromKafkaVersion.consumer_supports_bootstrap_server() # TODO - reduce the timeout self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, new_consumer=new_consumer, consumer_timeout_ms=30000, From a8eda3c785104615fa7144d6722a91223ecfea58 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 25 Feb 2021 14:17:19 -0800 Subject: [PATCH 061/243] KAFKA-12367; Ensure partition epoch is propagated to `Partition` state (#10200) This patch fixes two problem with the AlterIsr handling of the quorum controller: - Ensure that partition epoch is updated correctly after partition change records and is propagated to Partition - Ensure that AlterIsr response includes partitions that were successfully updated As part of this patch, I've renamed BrokersToIsrs.TopicPartition to BrokersToIsrs.TopicIdPartition to avoid confusion with the TopicPartition object which is used virtually everywhere. I've attempted to address some of the testing gaps as welll. Reviewers: Colin P. McCabe --- .../main/scala/kafka/cluster/Partition.scala | 2 +- .../server/metadata/MetadataPartitions.scala | 12 +- .../metadata/MetadataPartitionsTest.scala | 54 +++- .../RaftReplicaChangeDelegateTest.scala | 146 +++++++++++ .../kafka/controller/BrokersToIsrs.java | 18 +- .../controller/ReplicationControlManager.java | 57 +++-- .../kafka/controller/BrokersToIsrsTest.java | 36 +-- .../controller/QuorumControllerTest.java | 8 +- .../ReplicationControlManagerTest.java | 232 ++++++++++++++++-- 9 files changed, 479 insertions(+), 86 deletions(-) create mode 100644 core/src/test/scala/unit/kafka/server/RaftReplicaChangeDelegateTest.scala diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala index 99ae4ab85f8ef..9d33381815136 100755 --- a/core/src/main/scala/kafka/cluster/Partition.scala +++ b/core/src/main/scala/kafka/cluster/Partition.scala @@ -1403,7 +1403,7 @@ class Partition(val topicPartition: TopicPartition, case Errors.FENCED_LEADER_EPOCH => debug(s"Failed to update ISR to $proposedIsrState since we sent an old leader epoch. Giving up.") case Errors.INVALID_UPDATE_VERSION => - debug(s"Failed to update ISR to $proposedIsrState due to invalid zk version. Giving up.") + debug(s"Failed to update ISR to $proposedIsrState due to invalid version. Giving up.") case _ => warn(s"Failed to update ISR to $proposedIsrState due to unexpected $error. Retrying.") sendAlterIsrRequest(proposedIsrState) diff --git a/core/src/main/scala/kafka/server/metadata/MetadataPartitions.scala b/core/src/main/scala/kafka/server/metadata/MetadataPartitions.scala index bd84e7a4d348b..96ed8a592fe05 100644 --- a/core/src/main/scala/kafka/server/metadata/MetadataPartitions.scala +++ b/core/src/main/scala/kafka/server/metadata/MetadataPartitions.scala @@ -39,6 +39,7 @@ object MetadataPartition { record.leaderEpoch(), record.replicas(), record.isr(), + record.partitionEpoch(), Collections.emptyList(), // TODO KAFKA-12285 handle offline replicas Collections.emptyList(), Collections.emptyList()) @@ -52,6 +53,7 @@ object MetadataPartition { partition.leaderEpoch(), partition.replicas(), partition.isr(), + partition.zkVersion(), partition.offlineReplicas(), prevPartition.flatMap(p => Some(p.addingReplicas)).getOrElse(Collections.emptyList()), prevPartition.flatMap(p => Some(p.removingReplicas)).getOrElse(Collections.emptyList()) @@ -65,6 +67,7 @@ case class MetadataPartition(topicName: String, leaderEpoch: Int, replicas: util.List[Integer], isr: util.List[Integer], + partitionEpoch: Int, offlineReplicas: util.List[Integer], addingReplicas: util.List[Integer], removingReplicas: util.List[Integer]) { @@ -79,13 +82,13 @@ case class MetadataPartition(topicName: String, setIsr(isr). setAddingReplicas(addingReplicas). setRemovingReplicas(removingReplicas). - setIsNew(isNew) - // Note: we don't set ZKVersion here. + setIsNew(isNew). + setZkVersion(partitionEpoch) } def isReplicaFor(brokerId: Int): Boolean = replicas.contains(Integer.valueOf(brokerId)) - def copyWithChanges(record: PartitionChangeRecord): MetadataPartition = { + def merge(record: PartitionChangeRecord): MetadataPartition = { val (newLeader, newLeaderEpoch) = if (record.leader() == MetadataPartition.NO_LEADER_CHANGE) { (leaderId, leaderEpoch) } else { @@ -102,6 +105,7 @@ case class MetadataPartition(topicName: String, newLeaderEpoch, replicas, newIsr, + partitionEpoch + 1, offlineReplicas, addingReplicas, removingReplicas) @@ -132,7 +136,7 @@ class MetadataPartitionsBuilder(val brokerId: Int, case None => throw new RuntimeException(s"Unable to locate topic with name $name") case Some(partitionMap) => Option(partitionMap.get(record.partitionId())) match { case None => throw new RuntimeException(s"Unable to locate $name-${record.partitionId}") - case Some(partition) => set(partition.copyWithChanges(record)) + case Some(partition) => set(partition.merge(record)) } } } diff --git a/core/src/test/scala/kafka/server/metadata/MetadataPartitionsTest.scala b/core/src/test/scala/kafka/server/metadata/MetadataPartitionsTest.scala index 1b4cff7156128..a708d7381d36d 100644 --- a/core/src/test/scala/kafka/server/metadata/MetadataPartitionsTest.scala +++ b/core/src/test/scala/kafka/server/metadata/MetadataPartitionsTest.scala @@ -18,11 +18,14 @@ package kafka.server.metadata import java.util.Collections + import org.apache.kafka.common.Uuid import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{Test, Timeout} - import java.util.concurrent.TimeUnit + +import org.apache.kafka.common.metadata.PartitionChangeRecord + import scala.collection.mutable import scala.jdk.CollectionConverters._ @@ -30,16 +33,14 @@ import scala.jdk.CollectionConverters._ @Timeout(value = 120000, unit = TimeUnit.MILLISECONDS) class MetadataPartitionsTest { - val emptyPartitions = MetadataPartitions(Collections.emptyMap(), Collections.emptyMap()) + private val emptyPartitions = MetadataPartitions(Collections.emptyMap(), Collections.emptyMap()) private def newPartition(topicName: String, partitionIndex: Int, replicas: Option[Seq[Int]] = None, isr: Option[Seq[Int]] = None): MetadataPartition = { - val effectiveReplicas = replicas - .getOrElse(List(partitionIndex, partitionIndex + 1, partitionIndex + 2)) - .map(Int.box) - .toList.asJava + val effectiveReplicas = asJavaList(replicas + .getOrElse(List(partitionIndex, partitionIndex + 1, partitionIndex + 2))) val effectiveIsr = isr match { case None => effectiveReplicas @@ -47,9 +48,11 @@ class MetadataPartitionsTest { } new MetadataPartition(topicName, partitionIndex, - partitionIndex % 3, 100, + effectiveReplicas.asScala.head, + leaderEpoch = 100, effectiveReplicas, effectiveIsr, + partitionEpoch = 200, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()) @@ -149,4 +152,41 @@ class MetadataPartitionsTest { assertEquals(Some("bar"), image.topicIdToName(Uuid.fromString("a1I0JF3yRzWFyOuY3F_vHw"))) assertEquals(None, image.topicIdToName(Uuid.fromString("gdMy05W7QWG4ZjWir1DjBw"))) } + + @Test + def testMergePartitionChangeRecord(): Unit = { + val initialMetadata = newPartition( + topicName = "foo", + partitionIndex = 0, + replicas = Some(Seq(1, 2, 3)), + isr = Some(Seq(1, 2, 3)) + ) + assertEquals(1, initialMetadata.leaderId) + + // If only the ISR changes, then the leader epoch + // remains the same and the partition epoch is bumped. + val updatedIsr = initialMetadata.merge(new PartitionChangeRecord() + .setPartitionId(0) + .setIsr(asJavaList(Seq(1, 2)))) + assertEquals(asJavaList(Seq(1, 2)), updatedIsr.isr) + assertEquals(initialMetadata.leaderEpoch, updatedIsr.leaderEpoch) + assertEquals(initialMetadata.partitionEpoch + 1, updatedIsr.partitionEpoch) + assertEquals(initialMetadata.leaderId, updatedIsr.leaderId) + + // If the leader changes, then both the leader epoch + // and the partition epoch should get bumped. + val updatedLeader = initialMetadata.merge(new PartitionChangeRecord() + .setPartitionId(0) + .setLeader(2) + .setIsr(asJavaList(Seq(2, 3)))) + assertEquals(asJavaList(Seq(2, 3)), updatedLeader.isr) + assertEquals(initialMetadata.leaderEpoch + 1, updatedLeader.leaderEpoch) + assertEquals(initialMetadata.partitionEpoch + 1, updatedLeader.partitionEpoch) + assertEquals(2, updatedLeader.leaderId) + } + + private def asJavaList(replicas: Iterable[Int]): java.util.List[Integer] = { + replicas.map(Int.box).toList.asJava + } + } diff --git a/core/src/test/scala/unit/kafka/server/RaftReplicaChangeDelegateTest.scala b/core/src/test/scala/unit/kafka/server/RaftReplicaChangeDelegateTest.scala new file mode 100644 index 0000000000000..609f1cf3b3625 --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/RaftReplicaChangeDelegateTest.scala @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import java.util.Collections + +import kafka.cluster.Partition +import kafka.controller.StateChangeLogger +import kafka.server.checkpoints.OffsetCheckpoints +import kafka.server.metadata.{MetadataBroker, MetadataBrokers, MetadataPartition} +import org.apache.kafka.common.message.LeaderAndIsrRequestData +import org.apache.kafka.common.network.ListenerName +import org.apache.kafka.common.{Node, TopicPartition} +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.mockito.ArgumentMatchers +import org.mockito.ArgumentMatchers._ +import org.mockito.Mockito._ + +import scala.jdk.CollectionConverters._ + +class RaftReplicaChangeDelegateTest { + private val listenerName = new ListenerName("PLAINTEXT") + + @ParameterizedTest + @ValueSource(booleans = Array(true, false)) + def testLeaderAndIsrPropagation(isLeader: Boolean): Unit = { + val leaderId = 0 + val topicPartition = new TopicPartition("foo", 5) + val replicas = Seq(0, 1, 2).map(Int.box).asJava + + val helper = mockedHelper() + val partition = mock(classOf[Partition]) + when(partition.topicPartition).thenReturn(topicPartition) + + val highWatermarkCheckpoints = mock(classOf[OffsetCheckpoints]) + when(highWatermarkCheckpoints.fetch( + anyString(), + ArgumentMatchers.eq(topicPartition) + )).thenReturn(None) + + val metadataPartition = new MetadataPartition( + topicName = topicPartition.topic, + partitionIndex = topicPartition.partition, + leaderId = leaderId, + leaderEpoch = 27, + replicas = replicas, + isr = replicas, + partitionEpoch = 50, + offlineReplicas = Collections.emptyList(), + addingReplicas = Collections.emptyList(), + removingReplicas = Collections.emptyList() + ) + + val expectedLeaderAndIsr = new LeaderAndIsrRequestData.LeaderAndIsrPartitionState() + .setTopicName(topicPartition.topic) + .setPartitionIndex(topicPartition.partition) + .setIsNew(true) + .setLeader(leaderId) + .setLeaderEpoch(27) + .setReplicas(replicas) + .setIsr(replicas) + .setAddingReplicas(Collections.emptyList()) + .setRemovingReplicas(Collections.emptyList()) + .setZkVersion(50) + + val delegate = new RaftReplicaChangeDelegate(helper) + val updatedPartitions = if (isLeader) { + when(partition.makeLeader(expectedLeaderAndIsr, highWatermarkCheckpoints)) + .thenReturn(true) + delegate.makeLeaders( + prevPartitionsAlreadyExisting = Set.empty, + partitionStates = Map(partition -> metadataPartition), + highWatermarkCheckpoints, + metadataOffset = Some(500) + ) + } else { + when(partition.makeFollower(expectedLeaderAndIsr, highWatermarkCheckpoints)) + .thenReturn(true) + when(partition.leaderReplicaIdOpt).thenReturn(Some(leaderId)) + delegate.makeFollowers( + prevPartitionsAlreadyExisting = Set.empty, + currentBrokers = aliveBrokers(replicas), + partitionStates = Map(partition -> metadataPartition), + highWatermarkCheckpoints, + metadataOffset = Some(500) + ) + } + + assertEquals(Set(partition), updatedPartitions) + } + + private def aliveBrokers(replicas: java.util.List[Integer]): MetadataBrokers = { + def mkNode(replicaId: Int): Node = { + new Node(replicaId, "localhost", 9092 + replicaId, "") + } + + val brokers = replicas.asScala.map { replicaId => + replicaId -> MetadataBroker( + id = replicaId, + rack = "", + endpoints = Map(listenerName.value -> mkNode(replicaId)), + fenced = false + ) + }.toMap + + MetadataBrokers(brokers.values.toList.asJava, brokers.asJava) + } + + private def mockedHelper(): RaftReplicaChangeDelegateHelper = { + val helper = mock(classOf[RaftReplicaChangeDelegateHelper]) + + val stateChangeLogger = mock(classOf[StateChangeLogger]) + when(helper.stateChangeLogger).thenReturn(stateChangeLogger) + when(stateChangeLogger.isDebugEnabled).thenReturn(false) + when(stateChangeLogger.isTraceEnabled).thenReturn(false) + + val replicaFetcherManager = mock(classOf[ReplicaFetcherManager]) + when(helper.replicaFetcherManager).thenReturn(replicaFetcherManager) + + val replicaAlterLogDirsManager = mock(classOf[ReplicaAlterLogDirsManager]) + when(helper.replicaAlterLogDirsManager).thenReturn(replicaAlterLogDirsManager) + + val config = mock(classOf[KafkaConfig]) + when(config.interBrokerListenerName).thenReturn(listenerName) + when(helper.config).thenReturn(config) + + helper + } + +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/BrokersToIsrs.java b/metadata/src/main/java/org/apache/kafka/controller/BrokersToIsrs.java index 6b219eb943ad3..46148e7e09990 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/BrokersToIsrs.java +++ b/metadata/src/main/java/org/apache/kafka/controller/BrokersToIsrs.java @@ -54,11 +54,11 @@ public class BrokersToIsrs { private final static int REPLICA_MASK = 0x7fff_ffff; - static class TopicPartition { + static class TopicIdPartition { private final Uuid topicId; private final int partitionId; - TopicPartition(Uuid topicId, int partitionId) { + TopicIdPartition(Uuid topicId, int partitionId) { this.topicId = topicId; this.partitionId = partitionId; } @@ -73,8 +73,8 @@ public int partitionId() { @Override public boolean equals(Object o) { - if (!(o instanceof TopicPartition)) return false; - TopicPartition other = (TopicPartition) o; + if (!(o instanceof TopicIdPartition)) return false; + TopicIdPartition other = (TopicIdPartition) o; return other.topicId.equals(topicId) && other.partitionId == partitionId; } @@ -89,13 +89,13 @@ public String toString() { } } - static class PartitionsOnReplicaIterator implements Iterator { + static class PartitionsOnReplicaIterator implements Iterator { private final Iterator> iterator; private final boolean leaderOnly; private int offset = 0; Uuid uuid = Uuid.ZERO_UUID; int[] replicas = EMPTY; - private TopicPartition next = null; + private TopicIdPartition next = null; PartitionsOnReplicaIterator(Map topicMap, boolean leaderOnly) { this.iterator = topicMap.entrySet().iterator(); @@ -115,18 +115,18 @@ public boolean hasNext() { } int replica = replicas[offset++]; if ((!leaderOnly) || (replica & LEADER_FLAG) != 0) { - next = new TopicPartition(uuid, replica & REPLICA_MASK); + next = new TopicIdPartition(uuid, replica & REPLICA_MASK); return true; } } } @Override - public TopicPartition next() { + public TopicIdPartition next() { if (!hasNext()) { throw new NoSuchElementException(); } - TopicPartition result = next; + TopicIdPartition result = next; next = null; return result; } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java index aa60f5127de4d..9fd172f486ae6 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @@ -49,7 +49,7 @@ import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiError; import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.controller.BrokersToIsrs.TopicPartition; +import org.apache.kafka.controller.BrokersToIsrs.TopicIdPartition; import org.apache.kafka.metadata.ApiMessageAndVersion; import org.apache.kafka.metadata.BrokerHeartbeatReply; import org.apache.kafka.metadata.BrokerRegistration; @@ -102,13 +102,13 @@ static class TopicControlInfo { } static class PartitionControlInfo { - private final int[] replicas; - private final int[] isr; - private final int[] removingReplicas; - private final int[] addingReplicas; - private final int leader; - private final int leaderEpoch; - private final int partitionEpoch; + public final int[] replicas; + public final int[] isr; + public final int[] removingReplicas; + public final int[] addingReplicas; + public final int leader; + public final int leaderEpoch; + public final int partitionEpoch; PartitionControlInfo(PartitionRecord record) { this(Replicas.toArray(record.replicas()), @@ -581,6 +581,12 @@ ControllerResult alterIsr(AlterIsrRequestData request) { setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code())); continue; } + if (request.brokerId() != partition.leader) { + responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). + setPartitionIndex(partitionData.partitionIndex()). + setErrorCode(Errors.INVALID_REQUEST.code())); + continue; + } if (partitionData.leaderEpoch() != partition.leaderEpoch) { responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). setPartitionIndex(partitionData.partitionIndex()). @@ -611,6 +617,13 @@ ControllerResult alterIsr(AlterIsrRequestData request) { setPartitionId(partitionData.partitionIndex()). setTopicId(topic.id). setIsr(partitionData.newIsr()), (short) 0)); + responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). + setPartitionIndex(partitionData.partitionIndex()). + setErrorCode(Errors.NONE.code()). + setLeaderId(partition.leader). + setLeaderEpoch(partition.leaderEpoch). + setCurrentIsrVersion(partition.partitionEpoch + 1). + setIsr(partitionData.newIsr())); } } return new ControllerResult<>(records, response); @@ -663,21 +676,21 @@ void handleBrokerUnregistered(int brokerId, long brokerEpoch, * @param records The record list to append to. */ void handleNodeDeactivated(int brokerId, List records) { - Iterator iterator = brokersToIsrs.iterator(brokerId, false); + Iterator iterator = brokersToIsrs.iterator(brokerId, false); while (iterator.hasNext()) { - TopicPartition topicPartition = iterator.next(); - TopicControlInfo topic = topics.get(topicPartition.topicId()); + TopicIdPartition topicIdPartition = iterator.next(); + TopicControlInfo topic = topics.get(topicIdPartition.topicId()); if (topic == null) { - throw new RuntimeException("Topic ID " + topicPartition.topicId() + " existed in " + + throw new RuntimeException("Topic ID " + topicIdPartition.topicId() + " existed in " + "isrMembers, but not in the topics map."); } - PartitionControlInfo partition = topic.parts.get(topicPartition.partitionId()); + PartitionControlInfo partition = topic.parts.get(topicIdPartition.partitionId()); if (partition == null) { - throw new RuntimeException("Partition " + topicPartition + + throw new RuntimeException("Partition " + topicIdPartition + " existed in isrMembers, but not in the partitions map."); } PartitionChangeRecord record = new PartitionChangeRecord(). - setPartitionId(topicPartition.partitionId()). + setPartitionId(topicIdPartition.partitionId()). setTopicId(topic.id); int[] newIsr = Replicas.copyWithout(partition.isr, brokerId); if (newIsr.length == 0) { @@ -727,24 +740,24 @@ void handleBrokerUnfenced(int brokerId, long brokerEpoch, List records) { - Iterator iterator = brokersToIsrs.noLeaderIterator(); + Iterator iterator = brokersToIsrs.noLeaderIterator(); while (iterator.hasNext()) { - TopicPartition topicPartition = iterator.next(); - TopicControlInfo topic = topics.get(topicPartition.topicId()); + TopicIdPartition topicIdPartition = iterator.next(); + TopicControlInfo topic = topics.get(topicIdPartition.topicId()); if (topic == null) { - throw new RuntimeException("Topic ID " + topicPartition.topicId() + " existed in " + + throw new RuntimeException("Topic ID " + topicIdPartition.topicId() + " existed in " + "isrMembers, but not in the topics map."); } - PartitionControlInfo partition = topic.parts.get(topicPartition.partitionId()); + PartitionControlInfo partition = topic.parts.get(topicIdPartition.partitionId()); if (partition == null) { - throw new RuntimeException("Partition " + topicPartition + + throw new RuntimeException("Partition " + topicIdPartition + " existed in isrMembers, but not in the partitions map."); } // TODO: if this partition is configured for unclean leader election, // check the replica set rather than the ISR. if (Replicas.contains(partition.isr, brokerId)) { records.add(new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(topicPartition.partitionId()). + setPartitionId(topicIdPartition.partitionId()). setTopicId(topic.id). setLeader(brokerId), (short) 0)); } diff --git a/metadata/src/test/java/org/apache/kafka/controller/BrokersToIsrsTest.java b/metadata/src/test/java/org/apache/kafka/controller/BrokersToIsrsTest.java index 6f124ad8ac4d8..525bf1ee3ae80 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/BrokersToIsrsTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/BrokersToIsrsTest.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.controller.BrokersToIsrs.PartitionsOnReplicaIterator; -import org.apache.kafka.controller.BrokersToIsrs.TopicPartition; +import org.apache.kafka.controller.BrokersToIsrs.TopicIdPartition; import org.apache.kafka.timeline.SnapshotRegistry; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -38,16 +38,16 @@ public class BrokersToIsrsTest { Uuid.fromString("U52uRe20RsGI0RvpcTx33Q") }; - private static Set toSet(TopicPartition... partitions) { - HashSet set = new HashSet<>(); - for (TopicPartition partition : partitions) { + private static Set toSet(TopicIdPartition... partitions) { + HashSet set = new HashSet<>(); + for (TopicIdPartition partition : partitions) { set.add(partition); } return set; } - private static Set toSet(PartitionsOnReplicaIterator iterator) { - HashSet set = new HashSet<>(); + private static Set toSet(PartitionsOnReplicaIterator iterator) { + HashSet set = new HashSet<>(); while (iterator.hasNext()) { set.add(iterator.next()); } @@ -61,18 +61,18 @@ public void testIterator() { assertEquals(toSet(), toSet(brokersToIsrs.iterator(1, false))); brokersToIsrs.update(UUIDS[0], 0, null, new int[] {1, 2, 3}, -1, 1); brokersToIsrs.update(UUIDS[1], 1, null, new int[] {2, 3, 4}, -1, 4); - assertEquals(toSet(new TopicPartition(UUIDS[0], 0)), + assertEquals(toSet(new TopicIdPartition(UUIDS[0], 0)), toSet(brokersToIsrs.iterator(1, false))); - assertEquals(toSet(new TopicPartition(UUIDS[0], 0), - new TopicPartition(UUIDS[1], 1)), + assertEquals(toSet(new TopicIdPartition(UUIDS[0], 0), + new TopicIdPartition(UUIDS[1], 1)), toSet(brokersToIsrs.iterator(2, false))); - assertEquals(toSet(new TopicPartition(UUIDS[1], 1)), + assertEquals(toSet(new TopicIdPartition(UUIDS[1], 1)), toSet(brokersToIsrs.iterator(4, false))); assertEquals(toSet(), toSet(brokersToIsrs.iterator(5, false))); brokersToIsrs.update(UUIDS[1], 2, null, new int[] {3, 2, 1}, -1, 3); - assertEquals(toSet(new TopicPartition(UUIDS[0], 0), - new TopicPartition(UUIDS[1], 1), - new TopicPartition(UUIDS[1], 2)), + assertEquals(toSet(new TopicIdPartition(UUIDS[0], 0), + new TopicIdPartition(UUIDS[1], 1), + new TopicIdPartition(UUIDS[1], 2)), toSet(brokersToIsrs.iterator(2, false))); } @@ -82,14 +82,14 @@ public void testLeadersOnlyIterator() { BrokersToIsrs brokersToIsrs = new BrokersToIsrs(snapshotRegistry); brokersToIsrs.update(UUIDS[0], 0, null, new int[]{1, 2, 3}, -1, 1); brokersToIsrs.update(UUIDS[1], 1, null, new int[]{2, 3, 4}, -1, 4); - assertEquals(toSet(new TopicPartition(UUIDS[0], 0)), + assertEquals(toSet(new TopicIdPartition(UUIDS[0], 0)), toSet(brokersToIsrs.iterator(1, true))); assertEquals(toSet(), toSet(brokersToIsrs.iterator(2, true))); - assertEquals(toSet(new TopicPartition(UUIDS[1], 1)), + assertEquals(toSet(new TopicIdPartition(UUIDS[1], 1)), toSet(brokersToIsrs.iterator(4, true))); brokersToIsrs.update(UUIDS[0], 0, new int[]{1, 2, 3}, new int[]{1, 2, 3}, 1, 2); assertEquals(toSet(), toSet(brokersToIsrs.iterator(1, true))); - assertEquals(toSet(new TopicPartition(UUIDS[0], 0)), + assertEquals(toSet(new TopicIdPartition(UUIDS[0], 0)), toSet(brokersToIsrs.iterator(2, true))); } @@ -98,12 +98,12 @@ public void testNoLeader() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); BrokersToIsrs brokersToIsrs = new BrokersToIsrs(snapshotRegistry); brokersToIsrs.update(UUIDS[0], 2, null, new int[]{1, 2, 3}, -1, 3); - assertEquals(toSet(new TopicPartition(UUIDS[0], 2)), + assertEquals(toSet(new TopicIdPartition(UUIDS[0], 2)), toSet(brokersToIsrs.iterator(3, true))); assertEquals(toSet(), toSet(brokersToIsrs.iterator(2, true))); assertEquals(toSet(), toSet(brokersToIsrs.noLeaderIterator())); brokersToIsrs.update(UUIDS[0], 2, new int[]{1, 2, 3}, new int[]{1, 2, 3}, 3, -1); - assertEquals(toSet(new TopicPartition(UUIDS[0], 2)), + assertEquals(toSet(new TopicIdPartition(UUIDS[0], 2)), toSet(brokersToIsrs.noLeaderIterator())); } } diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index 16b52d1578391..fcc47f1ebb7f5 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -32,7 +32,7 @@ import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiError; -import org.apache.kafka.controller.BrokersToIsrs.TopicPartition; +import org.apache.kafka.controller.BrokersToIsrs.TopicIdPartition; import org.apache.kafka.metadata.BrokerHeartbeatReply; import org.apache.kafka.metadata.BrokerRegistrationReply; import org.apache.kafka.metalog.LocalLogManagerTestEnv; @@ -157,9 +157,9 @@ public void testUnregisterBroker() throws Throwable { setCurrentMetadataOffset(100000L)).get()); assertEquals(Errors.NONE.code(), active.createTopics( createTopicsRequestData).get().topics().find("foo").errorCode()); - CompletableFuture topicPartitionFuture = active.appendReadEvent( + CompletableFuture topicPartitionFuture = active.appendReadEvent( "debugGetPartition", () -> { - Iterator iterator = active. + Iterator iterator = active. replicationControl().brokersToIsrs().iterator(0, true); assertTrue(iterator.hasNext()); return iterator.next(); @@ -168,7 +168,7 @@ public void testUnregisterBroker() throws Throwable { active.unregisterBroker(0).get(); topicPartitionFuture = active.appendReadEvent( "debugGetPartition", () -> { - Iterator iterator = active. + Iterator iterator = active. replicationControl().brokersToIsrs().noLeaderIterator(); assertTrue(iterator.hasNext()); return iterator.next(); diff --git a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java index 9cc4173bf7cf1..a639614cdd52a 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java @@ -17,20 +17,17 @@ package org.apache.kafka.controller; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.StaleBrokerEpochException; +import org.apache.kafka.common.message.AlterIsrRequestData; +import org.apache.kafka.common.message.AlterIsrResponseData; import org.apache.kafka.common.message.BrokerHeartbeatRequestData; +import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment; import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic; import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection; -import org.apache.kafka.common.message.CreateTopicsRequestData; -import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; import org.apache.kafka.common.message.CreateTopicsResponseData; +import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; import org.apache.kafka.common.metadata.RegisterBrokerRecord; import org.apache.kafka.common.metadata.TopicRecord; import org.apache.kafka.common.protocol.Errors; @@ -40,16 +37,30 @@ import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.metadata.ApiMessageAndVersion; import org.apache.kafka.metadata.BrokerHeartbeatReply; +import org.apache.kafka.metadata.BrokerRegistration; import org.apache.kafka.timeline.SnapshotRegistry; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import static org.apache.kafka.common.protocol.Errors.INVALID_TOPIC_EXCEPTION; -import static org.apache.kafka.controller.BrokersToIsrs.TopicPartition; +import static org.apache.kafka.controller.BrokersToIsrs.TopicIdPartition; import static org.apache.kafka.controller.ReplicationControlManager.PartitionControlInfo; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; @Timeout(40) @@ -74,22 +85,22 @@ private static ReplicationControlManager newReplicationControlManager() { clusterControl); } - private static void registerBroker(int brokerId, ClusterControlManager clusterControl) { + private static void registerBroker(int brokerId, ReplicationControlManager replicationControl) { RegisterBrokerRecord brokerRecord = new RegisterBrokerRecord(). - setBrokerEpoch(100).setBrokerId(brokerId); + setBrokerEpoch(brokerId + 100).setBrokerId(brokerId); brokerRecord.endPoints().add(new RegisterBrokerRecord.BrokerEndpoint(). setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). setPort((short) 9092 + brokerId). setName("PLAINTEXT"). setHost("localhost")); - clusterControl.replay(brokerRecord); + replicationControl.clusterControl.replay(brokerRecord); } private static void unfenceBroker(int brokerId, ReplicationControlManager replicationControl) throws Exception { ControllerResult result = replicationControl. processBrokerHeartbeat(new BrokerHeartbeatRequestData(). - setBrokerId(brokerId).setBrokerEpoch(100).setCurrentMetadataOffset(1). + setBrokerId(brokerId).setBrokerEpoch(brokerId + 100).setCurrentMetadataOffset(1). setWantFence(false).setWantShutDown(false), 0); assertEquals(new BrokerHeartbeatReply(true, false, false, false), result.response()); ControllerTestUtils.replayAll(replicationControl.clusterControl, result.records()); @@ -109,11 +120,11 @@ public void testCreateTopics() throws Exception { setErrorMessage("Unable to replicate the partition 3 times: there are only 0 usable brokers")); assertEquals(expectedResponse, result.response()); - registerBroker(0, replicationControl.clusterControl); + registerBroker(0, replicationControl); unfenceBroker(0, replicationControl); - registerBroker(1, replicationControl.clusterControl); + registerBroker(1, replicationControl); unfenceBroker(1, replicationControl); - registerBroker(2, replicationControl.clusterControl); + registerBroker(2, replicationControl); unfenceBroker(2, replicationControl); ControllerResult result2 = replicationControl.createTopics(request); @@ -180,7 +191,7 @@ private static CreatableTopicResult createTestTopic( public void testRemoveLeaderships() throws Exception { ReplicationControlManager replicationControl = newReplicationControlManager(); for (int i = 0; i < 6; i++) { - registerBroker(i, replicationControl.clusterControl); + registerBroker(i, replicationControl); unfenceBroker(i, replicationControl); } CreatableTopicResult result = createTestTopic(replicationControl, "foo", @@ -190,9 +201,9 @@ public void testRemoveLeaderships() throws Exception { new int[] {2, 3, 0}, new int[] {0, 2, 1} }); - Set expectedPartitions = new HashSet<>(); - expectedPartitions.add(new TopicPartition(result.topicId(), 0)); - expectedPartitions.add(new TopicPartition(result.topicId(), 3)); + Set expectedPartitions = new HashSet<>(); + expectedPartitions.add(new TopicIdPartition(result.topicId(), 0)); + expectedPartitions.add(new TopicIdPartition(result.topicId(), 3)); assertEquals(expectedPartitions, ControllerTestUtils. iteratorToSet(replicationControl.brokersToIsrs().iterator(0, true))); List records = new ArrayList<>(); @@ -201,4 +212,183 @@ public void testRemoveLeaderships() throws Exception { assertEquals(Collections.emptySet(), ControllerTestUtils. iteratorToSet(replicationControl.brokersToIsrs().iterator(0, true))); } + + @Test + public void testShrinkAndExpandIsr() throws Exception { + ReplicationControlManager replicationControl = newReplicationControlManager(); + for (int i = 0; i < 3; i++) { + registerBroker(i, replicationControl); + unfenceBroker(i, replicationControl); + } + CreatableTopicResult createTopicResult = createTestTopic(replicationControl, "foo", + new int[][] {new int[] {0, 1, 2}}); + + TopicIdPartition topicIdPartition = new TopicIdPartition(createTopicResult.topicId(), 0); + TopicPartition topicPartition = new TopicPartition("foo", 0); + assertEquals(OptionalInt.of(0), currentLeader(replicationControl, topicIdPartition)); + long brokerEpoch = currentBrokerEpoch(replicationControl, 0); + AlterIsrRequestData.PartitionData shrinkIsrRequest = newAlterIsrPartition( + replicationControl, topicIdPartition, Arrays.asList(0, 1)); + ControllerResult shrinkIsrResult = sendAlterIsr( + replicationControl, 0, brokerEpoch, "foo", shrinkIsrRequest); + AlterIsrResponseData.PartitionData shrinkIsrResponse = assertAlterIsrResponse( + shrinkIsrResult, topicPartition, Errors.NONE); + assertConsistentAlterIsrResponse(replicationControl, topicIdPartition, shrinkIsrResponse); + + AlterIsrRequestData.PartitionData expandIsrRequest = newAlterIsrPartition( + replicationControl, topicIdPartition, Arrays.asList(0, 1, 2)); + ControllerResult expandIsrResult = sendAlterIsr( + replicationControl, 0, brokerEpoch, "foo", expandIsrRequest); + AlterIsrResponseData.PartitionData expandIsrResponse = assertAlterIsrResponse( + expandIsrResult, topicPartition, Errors.NONE); + assertConsistentAlterIsrResponse(replicationControl, topicIdPartition, expandIsrResponse); + } + + @Test + public void testInvalidAlterIsrRequests() throws Exception { + ReplicationControlManager replicationControl = newReplicationControlManager(); + for (int i = 0; i < 3; i++) { + registerBroker(i, replicationControl); + unfenceBroker(i, replicationControl); + } + CreatableTopicResult createTopicResult = createTestTopic(replicationControl, "foo", + new int[][] {new int[] {0, 1, 2}}); + + TopicIdPartition topicIdPartition = new TopicIdPartition(createTopicResult.topicId(), 0); + TopicPartition topicPartition = new TopicPartition("foo", 0); + assertEquals(OptionalInt.of(0), currentLeader(replicationControl, topicIdPartition)); + long brokerEpoch = currentBrokerEpoch(replicationControl, 0); + + // Invalid leader + AlterIsrRequestData.PartitionData invalidLeaderRequest = newAlterIsrPartition( + replicationControl, topicIdPartition, Arrays.asList(0, 1)); + ControllerResult invalidLeaderResult = sendAlterIsr( + replicationControl, 1, currentBrokerEpoch(replicationControl, 1), + "foo", invalidLeaderRequest); + assertAlterIsrResponse(invalidLeaderResult, topicPartition, Errors.INVALID_REQUEST); + + // Stale broker epoch + AlterIsrRequestData.PartitionData invalidBrokerEpochRequest = newAlterIsrPartition( + replicationControl, topicIdPartition, Arrays.asList(0, 1)); + assertThrows(StaleBrokerEpochException.class, () -> sendAlterIsr( + replicationControl, 0, brokerEpoch - 1, "foo", invalidBrokerEpochRequest)); + + // Invalid leader epoch + AlterIsrRequestData.PartitionData invalidLeaderEpochRequest = newAlterIsrPartition( + replicationControl, topicIdPartition, Arrays.asList(0, 1)); + invalidLeaderEpochRequest.setLeaderEpoch(500); + ControllerResult invalidLeaderEpochResult = sendAlterIsr( + replicationControl, 1, currentBrokerEpoch(replicationControl, 1), + "foo", invalidLeaderEpochRequest); + assertAlterIsrResponse(invalidLeaderEpochResult, topicPartition, Errors.INVALID_REQUEST); + + // Invalid ISR (3 is not a valid replica) + AlterIsrRequestData.PartitionData invalidIsrRequest1 = newAlterIsrPartition( + replicationControl, topicIdPartition, Arrays.asList(0, 1)); + invalidIsrRequest1.setNewIsr(Arrays.asList(0, 1, 3)); + ControllerResult invalidIsrResult1 = sendAlterIsr( + replicationControl, 1, currentBrokerEpoch(replicationControl, 1), + "foo", invalidIsrRequest1); + assertAlterIsrResponse(invalidIsrResult1, topicPartition, Errors.INVALID_REQUEST); + + // Invalid ISR (does not include leader 0) + AlterIsrRequestData.PartitionData invalidIsrRequest2 = newAlterIsrPartition( + replicationControl, topicIdPartition, Arrays.asList(0, 1)); + invalidIsrRequest2.setNewIsr(Arrays.asList(1, 2)); + ControllerResult invalidIsrResult2 = sendAlterIsr( + replicationControl, 1, currentBrokerEpoch(replicationControl, 1), + "foo", invalidIsrRequest2); + assertAlterIsrResponse(invalidIsrResult2, topicPartition, Errors.INVALID_REQUEST); + } + + private long currentBrokerEpoch( + ReplicationControlManager replicationControl, + int brokerId + ) { + Map registrations = replicationControl.clusterControl.brokerRegistrations(); + BrokerRegistration registration = registrations.get(brokerId); + assertNotNull(registration, "No current registration for broker " + brokerId); + return registration.epoch(); + } + + private OptionalInt currentLeader( + ReplicationControlManager replicationControl, + TopicIdPartition topicIdPartition + ) { + PartitionControlInfo partitionControl = + replicationControl.getPartition(topicIdPartition.topicId(), topicIdPartition.partitionId()); + if (partitionControl.leader < 0) { + return OptionalInt.empty(); + } else { + return OptionalInt.of(partitionControl.leader); + } + } + + private AlterIsrRequestData.PartitionData newAlterIsrPartition( + ReplicationControlManager replicationControl, + TopicIdPartition topicIdPartition, + List newIsr + ) { + PartitionControlInfo partitionControl = + replicationControl.getPartition(topicIdPartition.topicId(), topicIdPartition.partitionId()); + return new AlterIsrRequestData.PartitionData() + .setPartitionIndex(0) + .setLeaderEpoch(partitionControl.leaderEpoch) + .setCurrentIsrVersion(partitionControl.partitionEpoch) + .setNewIsr(newIsr); + } + + private ControllerResult sendAlterIsr( + ReplicationControlManager replicationControl, + int brokerId, + long brokerEpoch, + String topic, + AlterIsrRequestData.PartitionData partitionData + ) throws Exception { + AlterIsrRequestData request = new AlterIsrRequestData() + .setBrokerId(brokerId) + .setBrokerEpoch(brokerEpoch); + + AlterIsrRequestData.TopicData topicData = new AlterIsrRequestData.TopicData() + .setName(topic); + request.topics().add(topicData); + topicData.partitions().add(partitionData); + + ControllerResult result = replicationControl.alterIsr(request); + ControllerTestUtils.replayAll(replicationControl, result.records()); + return result; + } + + private AlterIsrResponseData.PartitionData assertAlterIsrResponse( + ControllerResult alterIsrResult, + TopicPartition topicPartition, + Errors expectedError + ) { + AlterIsrResponseData response = alterIsrResult.response(); + assertEquals(1, response.topics().size()); + + AlterIsrResponseData.TopicData topicData = response.topics().get(0); + assertEquals(topicPartition.topic(), topicData.name()); + assertEquals(1, topicData.partitions().size()); + + AlterIsrResponseData.PartitionData partitionData = topicData.partitions().get(0); + assertEquals(topicPartition.partition(), partitionData.partitionIndex()); + assertEquals(expectedError, Errors.forCode(partitionData.errorCode())); + return partitionData; + } + + private void assertConsistentAlterIsrResponse( + ReplicationControlManager replicationControl, + TopicIdPartition topicIdPartition, + AlterIsrResponseData.PartitionData partitionData + ) { + PartitionControlInfo partitionControl = + replicationControl.getPartition(topicIdPartition.topicId(), topicIdPartition.partitionId()); + assertEquals(partitionControl.leader, partitionData.leaderId()); + assertEquals(partitionControl.leaderEpoch, partitionData.leaderEpoch()); + assertEquals(partitionControl.partitionEpoch, partitionData.currentIsrVersion()); + List expectedIsr = IntStream.of(partitionControl.isr).boxed().collect(Collectors.toList()); + assertEquals(expectedIsr, partitionData.isr()); + } + } From 7f90eda04720274a0e4f7ec32a7340245f493e9e Mon Sep 17 00:00:00 2001 From: Davor Poldrugo Date: Fri, 26 Feb 2021 02:02:40 +0100 Subject: [PATCH 062/243] KAFKA-8562; SaslChannelBuilder - Avoid (reverse) DNS lookup while building SslTransportLayer (#10059) This patch moves the `peerHost` helper defined in `SslChannelBuilder` into `SslFactor`. `SaslChannelBuilder` is then updated to use a new `createSslEngine` overload which relies on `peerHost` when building its `SslEngine`. The purpose is to avoid the reverse DNS in `getHostName`. Reviewers: Ismael Juma , Manikumar Reddy , Jason Gustafson --- .../common/network/SaslChannelBuilder.java | 3 +- .../common/network/SslChannelBuilder.java | 48 ++----------------- .../kafka/common/security/ssl/SslFactory.java | 48 +++++++++++++++++++ .../kafka/common/network/SslSelectorTest.java | 4 +- .../common/network/SslTransportLayerTest.java | 4 +- .../SaslAuthenticatorFailureDelayTest.java | 2 +- .../authenticator/SaslAuthenticatorTest.java | 10 ++-- 7 files changed, 62 insertions(+), 57 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java index 17988db87a650..8b390d11bc6dd 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java @@ -256,8 +256,7 @@ protected TransportLayer buildTransportLayer(String id, SelectionKey key, Socket ChannelMetadataRegistry metadataRegistry) throws IOException { if (this.securityProtocol == SecurityProtocol.SASL_SSL) { return SslTransportLayer.create(id, key, - sslFactory.createSslEngine(socketChannel.socket().getInetAddress().getHostName(), - socketChannel.socket().getPort()), + sslFactory.createSslEngine(socketChannel.socket()), metadataRegistry); } else { return new PlaintextTransportLayer(key); diff --git a/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java index 909009b329f06..1140ea769a27a 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/network/SslChannelBuilder.java @@ -33,7 +33,6 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.nio.channels.SelectionKey; import java.nio.channels.SocketChannel; import java.util.Map; @@ -103,8 +102,7 @@ public ListenerName listenerName() { public KafkaChannel buildChannel(String id, SelectionKey key, int maxReceiveSize, MemoryPool memoryPool, ChannelMetadataRegistry metadataRegistry) throws KafkaException { try { - SslTransportLayer transportLayer = buildTransportLayer(sslFactory, id, key, - peerHost(key), metadataRegistry); + SslTransportLayer transportLayer = buildTransportLayer(sslFactory, id, key, metadataRegistry); Supplier authenticatorCreator = () -> new SslAuthenticator(configs, transportLayer, listenerName, sslPrincipalMapper); return new KafkaChannel(id, transportLayer, authenticatorCreator, maxReceiveSize, @@ -120,52 +118,12 @@ public void close() { if (sslFactory != null) sslFactory.close(); } - protected SslTransportLayer buildTransportLayer(SslFactory sslFactory, String id, SelectionKey key, - String host, ChannelMetadataRegistry metadataRegistry) throws IOException { + protected SslTransportLayer buildTransportLayer(SslFactory sslFactory, String id, SelectionKey key, ChannelMetadataRegistry metadataRegistry) throws IOException { SocketChannel socketChannel = (SocketChannel) key.channel(); - return SslTransportLayer.create(id, key, sslFactory.createSslEngine(host, socketChannel.socket().getPort()), + return SslTransportLayer.create(id, key, sslFactory.createSslEngine(socketChannel.socket()), metadataRegistry); } - /** - * Returns host/IP address of remote host without reverse DNS lookup to be used as the host - * for creating SSL engine. This is used as a hint for session reuse strategy and also for - * hostname verification of server hostnames. - *

    - * Scenarios: - *

      - *
    • Server-side - *
        - *
      • Server accepts connection from a client. Server knows only client IP - * address. We want to avoid reverse DNS lookup of the client IP address since the server - * does not verify or use client hostname. The IP address can be used directly.
        • - *
      - *
      • - *
    • Client-side - *
        - *
      • Client connects to server using hostname. No lookup is necessary - * and the hostname should be used to create the SSL engine. This hostname is validated - * against the hostname in SubjectAltName (dns) or CommonName in the certificate if - * hostname verification is enabled. Authentication fails if hostname does not match.
        • - *
      • Client connects to server using IP address, but certificate contains only - * SubjectAltName (dns). Use of reverse DNS lookup to determine hostname introduces - * a security vulnerability since authentication would be reliant on a secure DNS. - * Hence hostname verification should fail in this case.
        • - *
      • Client connects to server using IP address and certificate contains - * SubjectAltName (ipaddress). This could be used when Kafka is on a private network. - * If reverse DNS lookup is used, authentication would succeed using IP address if lookup - * fails and IP address is used, but authentication would fail if lookup succeeds and - * dns name is used. For consistency and to avoid dependency on a potentially insecure - * DNS, reverse DNS lookup should be avoided and the IP address specified by the client for - * connection should be used to create the SSL engine.
        • - *
      • - *
    - */ - private String peerHost(SelectionKey key) { - SocketChannel socketChannel = (SocketChannel) key.channel(); - return new InetSocketAddress(socketChannel.socket().getInetAddress(), 0).getHostString(); - } - /** * Note that client SSL authentication is handled in {@link SslTransportLayer}. This class is only used * to transform the derived principal using a {@link KafkaPrincipalBuilder} configured by the user. diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java index 91b23b61a9397..d0cc4cc1e6951 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java +++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java @@ -31,6 +31,8 @@ import javax.net.ssl.SSLEngineResult; import javax.net.ssl.SSLException; import java.io.Closeable; +import java.net.InetSocketAddress; +import java.net.Socket; import java.nio.ByteBuffer; import java.security.cert.Certificate; import java.security.cert.X509Certificate; @@ -183,6 +185,14 @@ private SslEngineFactory createNewSslEngineFactory(Map newConfigs) { } } + public SSLEngine createSslEngine(Socket socket) { + return createSslEngine(peerHost(socket), socket.getPort()); + } + + /** + * Prefer `createSslEngine(Socket)` if a `Socket` instance is available. If using this overload, + * avoid reverse DNS resolution in the computation of `peerHost`. + */ public SSLEngine createSslEngine(String peerHost, int peerPort) { if (sslEngineFactory == null) { throw new IllegalStateException("SslFactory has not been configured."); @@ -194,6 +204,44 @@ public SSLEngine createSslEngine(String peerHost, int peerPort) { } } + /** + * Returns host/IP address of remote host without reverse DNS lookup to be used as the host + * for creating SSL engine. This is used as a hint for session reuse strategy and also for + * hostname verification of server hostnames. + *

    + * Scenarios: + *

      + *
    • Server-side + *
        + *
      • Server accepts connection from a client. Server knows only client IP + * address. We want to avoid reverse DNS lookup of the client IP address since the server + * does not verify or use client hostname. The IP address can be used directly.
        • + *
      + *
      • + *
    • Client-side + *
        + *
      • Client connects to server using hostname. No lookup is necessary + * and the hostname should be used to create the SSL engine. This hostname is validated + * against the hostname in SubjectAltName (dns) or CommonName in the certificate if + * hostname verification is enabled. Authentication fails if hostname does not match.
        • + *
      • Client connects to server using IP address, but certificate contains only + * SubjectAltName (dns). Use of reverse DNS lookup to determine hostname introduces + * a security vulnerability since authentication would be reliant on a secure DNS. + * Hence hostname verification should fail in this case.
        • + *
      • Client connects to server using IP address and certificate contains + * SubjectAltName (ipaddress). This could be used when Kafka is on a private network. + * If reverse DNS lookup is used, authentication would succeed using IP address if lookup + * fails and IP address is used, but authentication would fail if lookup succeeds and + * dns name is used. For consistency and to avoid dependency on a potentially insecure + * DNS, reverse DNS lookup should be avoided and the IP address specified by the client for + * connection should be used to create the SSL engine.
        • + *
      • + *
    + */ + private String peerHost(Socket socket) { + return new InetSocketAddress(socket.getInetAddress(), 0).getHostString(); + } + public SslEngineFactory sslEngineFactory() { return sslEngineFactory; } diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslSelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslSelectorTest.java index 0e94744c84209..7f95566c9f981 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslSelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslSelectorTest.java @@ -383,9 +383,9 @@ public TestSslChannelBuilder(Mode mode) { @Override protected SslTransportLayer buildTransportLayer(SslFactory sslFactory, String id, SelectionKey key, - String host, ChannelMetadataRegistry metadataRegistry) throws IOException { + ChannelMetadataRegistry metadataRegistry) throws IOException { SocketChannel socketChannel = (SocketChannel) key.channel(); - SSLEngine sslEngine = sslFactory.createSslEngine(host, socketChannel.socket().getPort()); + SSLEngine sslEngine = sslFactory.createSslEngine(socketChannel.socket()); TestSslTransportLayer transportLayer = new TestSslTransportLayer(id, key, sslEngine, metadataRegistry); return transportLayer; } diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java index 13f763279f783..44187134225fc 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java @@ -1368,9 +1368,9 @@ public void configureBufferSizes(Integer netReadBufSize, Integer netWriteBufSize @Override protected SslTransportLayer buildTransportLayer(SslFactory sslFactory, String id, SelectionKey key, - String host, ChannelMetadataRegistry metadataRegistry) throws IOException { + ChannelMetadataRegistry metadataRegistry) throws IOException { SocketChannel socketChannel = (SocketChannel) key.channel(); - SSLEngine sslEngine = sslFactory.createSslEngine(host, socketChannel.socket().getPort()); + SSLEngine sslEngine = sslFactory.createSslEngine(socketChannel.socket()); return newTransportLayer(id, key, sslEngine); } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java index 7dfde12bafc9f..db6ba89463057 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java @@ -219,7 +219,7 @@ private NioEchoServer createEchoServer(ListenerName listenerName, SecurityProtoc private void createClientConnection(SecurityProtocol securityProtocol, String node) throws Exception { createSelector(securityProtocol, saslClientConfigs); - InetSocketAddress addr = new InetSocketAddress("127.0.0.1", server.port()); + InetSocketAddress addr = new InetSocketAddress("localhost", server.port()); selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE); } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index 6c836f33c57bc..988a0f2823213 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -258,7 +258,7 @@ public void testMissingUsernameSaslPlain() throws Exception { SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL; server = createEchoServer(securityProtocol); createSelector(securityProtocol, saslClientConfigs); - InetSocketAddress addr = new InetSocketAddress("127.0.0.1", server.port()); + InetSocketAddress addr = new InetSocketAddress("localhost", server.port()); try { selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE); fail("SASL/PLAIN channel created without username"); @@ -282,7 +282,7 @@ public void testMissingPasswordSaslPlain() throws Exception { SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL; server = createEchoServer(securityProtocol); createSelector(securityProtocol, saslClientConfigs); - InetSocketAddress addr = new InetSocketAddress("127.0.0.1", server.port()); + InetSocketAddress addr = new InetSocketAddress("localhost", server.port()); try { selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE); fail("SASL/PLAIN channel created without password"); @@ -399,7 +399,7 @@ public void testMultipleServerMechanisms() throws Exception { saslClientConfigs.put(SaslConfigs.SASL_MECHANISM, "DIGEST-MD5"); createSelector(securityProtocol, saslClientConfigs); selector2 = selector; - InetSocketAddress addr = new InetSocketAddress("127.0.0.1", server.port()); + InetSocketAddress addr = new InetSocketAddress("localhost", server.port()); selector.connect(node2, addr, BUFFER_SIZE, BUFFER_SIZE); NetworkTestUtils.checkClientConnection(selector, node2, 100, 10); selector = null; // keeps it from being closed when next one is created @@ -409,7 +409,7 @@ public void testMultipleServerMechanisms() throws Exception { saslClientConfigs.put(SaslConfigs.SASL_MECHANISM, "SCRAM-SHA-256"); createSelector(securityProtocol, saslClientConfigs); selector3 = selector; - selector.connect(node3, new InetSocketAddress("127.0.0.1", server.port()), BUFFER_SIZE, BUFFER_SIZE); + selector.connect(node3, new InetSocketAddress("localhost", server.port()), BUFFER_SIZE, BUFFER_SIZE); NetworkTestUtils.checkClientConnection(selector, node3, 100, 10); server.verifyAuthenticationMetrics(3, 0); @@ -2016,7 +2016,7 @@ protected void setSaslAuthenticateAndHandshakeVersions(ApiVersionsResponse apiVe }; clientChannelBuilder.configure(saslClientConfigs); this.selector = NetworkTestUtils.createSelector(clientChannelBuilder, time); - InetSocketAddress addr = new InetSocketAddress("127.0.0.1", server.port()); + InetSocketAddress addr = new InetSocketAddress("localhost", server.port()); selector.connect(node, addr, BUFFER_SIZE, BUFFER_SIZE); } From 1657deec37a45dc4c6a932024de4aa5042c9dda2 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Thu, 25 Feb 2021 17:16:37 -0800 Subject: [PATCH 063/243] MINOR: tune KIP-631 configurations (#10179) Since we expect KIP-631 controller fail-overs to be fairly cheap, tune the default raft configuration parameters so that we detect node failures more quickly. Reduce the broker session timeout as well so that broker failures are detected more quickly. Reviewers: Jason Gustafson , Alok Nikhil --- .../src/main/scala/kafka/server/KafkaConfig.scala | 2 +- .../org/apache/kafka/raft/KafkaRaftClient.java | 2 +- .../java/org/apache/kafka/raft/RaftConfig.java | 15 ++++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index e01bf60babb0f..d2e34142519ea 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -72,7 +72,7 @@ object Defaults { val QueuedMaxRequestBytes = -1 val InitialBrokerRegistrationTimeoutMs = 60000 val BrokerHeartbeatIntervalMs = 2000 - val BrokerSessionTimeoutMs = 18000 + val BrokerSessionTimeoutMs = 9000 /** KIP-500 Configuration */ val EmptyNodeId: Int = -1 diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index 560d47098bead..f99ffb6d02298 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -140,7 +140,7 @@ */ public class KafkaRaftClient implements RaftClient { private static final int RETRY_BACKOFF_BASE_MS = 100; - private static final int FETCH_MAX_WAIT_MS = 1000; + private static final int FETCH_MAX_WAIT_MS = 500; static final int MAX_BATCH_SIZE = 8 * 1024 * 1024; private final AtomicReference shutdown = new AtomicReference<>(); diff --git a/raft/src/main/java/org/apache/kafka/raft/RaftConfig.java b/raft/src/main/java/org/apache/kafka/raft/RaftConfig.java index 13dd8794b78d0..358d5f3d5ed3c 100644 --- a/raft/src/main/java/org/apache/kafka/raft/RaftConfig.java +++ b/raft/src/main/java/org/apache/kafka/raft/RaftConfig.java @@ -40,6 +40,11 @@ * For example: `1@0.0.0.0:0,2@0.0.0.0:0,3@0.0.0.0:0` * This will assign an {@link UnknownAddressSpec} to the voter entries * + * The default raft timeouts are relatively low compared to some other timeouts such as + * request.timeout.ms. This is part of a general design philosophy where we see changing + * the leader of a Raft cluster as a relatively quick operation. For example, the KIP-631 + * controller should be able to transition from standby to active without reloading all of + * the metadata. The standby is a "hot" standby, not a "cold" one. */ public class RaftConfig { @@ -58,18 +63,18 @@ public class RaftConfig { public static final String QUORUM_ELECTION_TIMEOUT_MS_CONFIG = QUORUM_PREFIX + "election.timeout.ms"; public static final String QUORUM_ELECTION_TIMEOUT_MS_DOC = "Maximum time in milliseconds to wait " + "without being able to fetch from the leader before triggering a new election"; - public static final int DEFAULT_QUORUM_ELECTION_TIMEOUT_MS = 5_000; + public static final int DEFAULT_QUORUM_ELECTION_TIMEOUT_MS = 1_000; public static final String QUORUM_FETCH_TIMEOUT_MS_CONFIG = QUORUM_PREFIX + "fetch.timeout.ms"; public static final String QUORUM_FETCH_TIMEOUT_MS_DOC = "Maximum time without a successful fetch from " + "the current leader before becoming a candidate and triggering a election for voters; Maximum time without " + "receiving fetch from a majority of the quorum before asking around to see if there's a new epoch for leader"; - public static final int DEFAULT_QUORUM_FETCH_TIMEOUT_MS = 15_000; + public static final int DEFAULT_QUORUM_FETCH_TIMEOUT_MS = 2_000; public static final String QUORUM_ELECTION_BACKOFF_MAX_MS_CONFIG = QUORUM_PREFIX + "election.backoff.max.ms"; public static final String QUORUM_ELECTION_BACKOFF_MAX_MS_DOC = "Maximum time in milliseconds before starting new elections. " + "This is used in the binary exponential backoff mechanism that helps prevent gridlocked elections"; - public static final int DEFAULT_QUORUM_ELECTION_BACKOFF_MAX_MS = 5_000; + public static final int DEFAULT_QUORUM_ELECTION_BACKOFF_MAX_MS = 1_000; public static final String QUORUM_LINGER_MS_CONFIG = QUORUM_PREFIX + "append.linger.ms"; public static final String QUORUM_LINGER_MS_DOC = "The duration in milliseconds that the leader will " + @@ -79,12 +84,12 @@ public class RaftConfig { public static final String QUORUM_REQUEST_TIMEOUT_MS_CONFIG = QUORUM_PREFIX + CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG; public static final String QUORUM_REQUEST_TIMEOUT_MS_DOC = CommonClientConfigs.REQUEST_TIMEOUT_MS_DOC; - public static final int DEFAULT_QUORUM_REQUEST_TIMEOUT_MS = 20_000; + public static final int DEFAULT_QUORUM_REQUEST_TIMEOUT_MS = 2_000; public static final String QUORUM_RETRY_BACKOFF_MS_CONFIG = QUORUM_PREFIX + CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG; public static final String QUORUM_RETRY_BACKOFF_MS_DOC = CommonClientConfigs.RETRY_BACKOFF_MS_DOC; - public static final int DEFAULT_QUORUM_RETRY_BACKOFF_MS = 100; + public static final int DEFAULT_QUORUM_RETRY_BACKOFF_MS = 20; private final int requestTimeoutMs; private final int retryBackoffMs; From 74dfe80bb8b63b3916560143ce2c85f8b41f0f2e Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 25 Feb 2021 19:38:21 -0800 Subject: [PATCH 064/243] KAFKA-12365; Disable APIs not supported by KIP-500 broker/controller (#10194) This patch updates request `listeners` tags to be in line with what the KIP-500 broker/controller support today. We will re-enable these APIs as needed once we have added the support. I have also updated `ControllerApis` to use `ApiVersionManager` and simplified the envelope handling logic. Reviewers: Ron Dagostino , Colin P. McCabe --- .../message/AddOffsetsToTxnRequest.json | 2 +- .../message/AddPartitionsToTxnRequest.json | 2 +- .../common/message/AlterConfigsRequest.json | 2 +- .../AlterPartitionReassignmentsRequest.json | 2 +- .../AlterUserScramCredentialsRequest.json | 2 +- .../common/message/CreateAclsRequest.json | 2 +- .../message/CreateDelegationTokenRequest.json | 2 +- .../message/CreatePartitionsRequest.json | 2 +- .../common/message/DeleteAclsRequest.json | 2 +- .../common/message/DescribeAclsRequest.json | 2 +- .../message/DescribeClientQuotasRequest.json | 2 +- .../message/DescribeConfigsRequest.json | 2 +- .../DescribeDelegationTokenRequest.json | 2 +- .../DescribeUserScramCredentialsRequest.json | 2 +- .../common/message/ElectLeadersRequest.json | 2 +- .../common/message/EndTxnRequest.json | 2 +- .../message/ExpireDelegationTokenRequest.json | 2 +- .../common/message/InitProducerIdRequest.json | 2 +- .../ListPartitionReassignmentsRequest.json | 2 +- .../common/message/MetadataRequest.json | 2 +- .../message/RenewDelegationTokenRequest.json | 2 +- .../message/TxnOffsetCommitRequest.json | 2 +- .../common/message/UpdateFeaturesRequest.json | 2 +- .../scala/kafka/server/ControllerApis.scala | 141 +++++------------- .../scala/kafka/server/ControllerServer.scala | 3 +- .../kafka/server/ControllerApisTest.scala | 7 +- .../core/group_mode_transactions_test.py | 2 +- .../kafkatest/tests/core/transactions_test.py | 3 +- 28 files changed, 68 insertions(+), 134 deletions(-) diff --git a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json index ade3fc72c9a51..7212a02ac2a25 100644 --- a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json +++ b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json @@ -16,7 +16,7 @@ { "apiKey": 25, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker"], "name": "AddOffsetsToTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json index 4920da176c723..99e72a9fd7453 100644 --- a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json +++ b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json @@ -16,7 +16,7 @@ { "apiKey": 24, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker"], "name": "AddPartitionsToTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/AlterConfigsRequest.json b/clients/src/main/resources/common/message/AlterConfigsRequest.json index 31057e3410aaf..fa46656622e8a 100644 --- a/clients/src/main/resources/common/message/AlterConfigsRequest.json +++ b/clients/src/main/resources/common/message/AlterConfigsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 33, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "AlterConfigsRequest", // Version 1 is the same as version 0. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json index 2e124413a3bbc..ee05b42e3a7bb 100644 --- a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json +++ b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 45, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker"], "name": "AlterPartitionReassignmentsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json b/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json index 8937394ef6e51..70e1483097075 100644 --- a/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json +++ b/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 51, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "AlterUserScramCredentialsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/CreateAclsRequest.json b/clients/src/main/resources/common/message/CreateAclsRequest.json index 5b3bfed78162c..109b444c79d66 100644 --- a/clients/src/main/resources/common/message/CreateAclsRequest.json +++ b/clients/src/main/resources/common/message/CreateAclsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 30, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "CreateAclsRequest", // Version 1 adds resource pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json index 0c31d32fe56b7..d65d490a6e066 100644 --- a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json @@ -16,7 +16,7 @@ { "apiKey": 38, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "CreateDelegationTokenRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/CreatePartitionsRequest.json b/clients/src/main/resources/common/message/CreatePartitionsRequest.json index 6e249498659fa..8053628b51457 100644 --- a/clients/src/main/resources/common/message/CreatePartitionsRequest.json +++ b/clients/src/main/resources/common/message/CreatePartitionsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 37, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "CreatePartitionsRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/DeleteAclsRequest.json b/clients/src/main/resources/common/message/DeleteAclsRequest.json index fd7c1522b43bd..1e7aa9a1b1421 100644 --- a/clients/src/main/resources/common/message/DeleteAclsRequest.json +++ b/clients/src/main/resources/common/message/DeleteAclsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 31, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "DeleteAclsRequest", // Version 1 adds the pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/DescribeAclsRequest.json b/clients/src/main/resources/common/message/DescribeAclsRequest.json index 58886da654707..e551ca45be0a8 100644 --- a/clients/src/main/resources/common/message/DescribeAclsRequest.json +++ b/clients/src/main/resources/common/message/DescribeAclsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 29, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "DescribeAclsRequest", // Version 1 adds resource pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json index d14cfc95733d3..5ada552e29cfe 100644 --- a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json @@ -16,7 +16,7 @@ { "apiKey": 48, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker"], "name": "DescribeClientQuotasRequest", // Version 1 enables flexible versions. "validVersions": "0-1", diff --git a/clients/src/main/resources/common/message/DescribeConfigsRequest.json b/clients/src/main/resources/common/message/DescribeConfigsRequest.json index 23be19cb0e625..f48b168e72d62 100644 --- a/clients/src/main/resources/common/message/DescribeConfigsRequest.json +++ b/clients/src/main/resources/common/message/DescribeConfigsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 32, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker"], "name": "DescribeConfigsRequest", // Version 1 adds IncludeSynonyms. // Version 2 is the same as version 1. diff --git a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json index da5bbd046d16c..79c342e14e004 100644 --- a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json @@ -16,7 +16,7 @@ { "apiKey": 41, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "DescribeDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json b/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json index 2f7a1112c4800..cef8929ac24f0 100644 --- a/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json +++ b/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 50, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "DescribeUserScramCredentialsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ElectLeadersRequest.json b/clients/src/main/resources/common/message/ElectLeadersRequest.json index dd9fa21641585..d407f5e3c9d60 100644 --- a/clients/src/main/resources/common/message/ElectLeadersRequest.json +++ b/clients/src/main/resources/common/message/ElectLeadersRequest.json @@ -16,7 +16,7 @@ { "apiKey": 43, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "ElectLeadersRequest", // Version 1 implements multiple leader election types, as described by KIP-460. // diff --git a/clients/src/main/resources/common/message/EndTxnRequest.json b/clients/src/main/resources/common/message/EndTxnRequest.json index f16ef76246d35..7e7d41dbf8b84 100644 --- a/clients/src/main/resources/common/message/EndTxnRequest.json +++ b/clients/src/main/resources/common/message/EndTxnRequest.json @@ -16,7 +16,7 @@ { "apiKey": 26, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker"], "name": "EndTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json index c830a93df398b..736f1dfe2deda 100644 --- a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json @@ -16,7 +16,7 @@ { "apiKey": 40, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "ExpireDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/InitProducerIdRequest.json b/clients/src/main/resources/common/message/InitProducerIdRequest.json index e8795e6582169..9e3450550390b 100644 --- a/clients/src/main/resources/common/message/InitProducerIdRequest.json +++ b/clients/src/main/resources/common/message/InitProducerIdRequest.json @@ -16,7 +16,7 @@ { "apiKey": 22, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker"], "name": "InitProducerIdRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json index f013e3fe9ffab..75e87446f49b2 100644 --- a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json +++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 46, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker"], "name": "ListPartitionReassignmentsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/MetadataRequest.json b/clients/src/main/resources/common/message/MetadataRequest.json index 66908103e99c8..e5083a84ea343 100644 --- a/clients/src/main/resources/common/message/MetadataRequest.json +++ b/clients/src/main/resources/common/message/MetadataRequest.json @@ -16,7 +16,7 @@ { "apiKey": 3, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker", "controller"], "name": "MetadataRequest", "validVersions": "0-11", "flexibleVersions": "9+", diff --git a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json index 182682e4c913a..7240ac3191332 100644 --- a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json @@ -16,7 +16,7 @@ { "apiKey": 39, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "RenewDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json index a832ef7a96832..127ff3d11551f 100644 --- a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json +++ b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json @@ -16,7 +16,7 @@ { "apiKey": 28, "type": "request", - "listeners": ["zkBroker", "broker"], + "listeners": ["zkBroker"], "name": "TxnOffsetCommitRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/UpdateFeaturesRequest.json b/clients/src/main/resources/common/message/UpdateFeaturesRequest.json index 21e1bf663dda5..41be8cf8706ba 100644 --- a/clients/src/main/resources/common/message/UpdateFeaturesRequest.json +++ b/clients/src/main/resources/common/message/UpdateFeaturesRequest.json @@ -16,7 +16,7 @@ { "apiKey": 57, "type": "request", - "listeners": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker"], "name": "UpdateFeaturesRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/core/src/main/scala/kafka/server/ControllerApis.scala b/core/src/main/scala/kafka/server/ControllerApis.scala index 336775c4933e8..53a41dc919a1a 100644 --- a/core/src/main/scala/kafka/server/ControllerApis.scala +++ b/core/src/main/scala/kafka/server/ControllerApis.scala @@ -24,15 +24,15 @@ import kafka.raft.RaftManager import kafka.server.QuotaFactory.QuotaManagers import kafka.utils.Logging import org.apache.kafka.clients.admin.AlterConfigOp +import org.apache.kafka.common.Node import org.apache.kafka.common.acl.AclOperation.{ALTER, ALTER_CONFIGS, CLUSTER_ACTION, CREATE, DESCRIBE} import org.apache.kafka.common.config.ConfigResource -import org.apache.kafka.common.errors.ApiException +import org.apache.kafka.common.errors.{ApiException, ClusterAuthorizationException} import org.apache.kafka.common.internals.FatalExitError -import org.apache.kafka.common.message.ApiVersionsResponseData.{ApiVersion, SupportedFeatureKey} import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseBroker -import org.apache.kafka.common.message.{ApiVersionsResponseData, BeginQuorumEpochResponseData, BrokerHeartbeatResponseData, BrokerRegistrationResponseData, CreateTopicsResponseData, DescribeQuorumResponseData, EndQuorumEpochResponseData, FetchResponseData, MetadataResponseData, UnregisterBrokerResponseData, VoteResponseData} +import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, BrokerHeartbeatResponseData, BrokerRegistrationResponseData, CreateTopicsResponseData, DescribeQuorumResponseData, EndQuorumEpochResponseData, FetchResponseData, MetadataResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, UnregisterBrokerResponseData, VoteResponseData} import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage, Errors} import org.apache.kafka.common.record.BaseRecords import org.apache.kafka.common.requests._ @@ -40,9 +40,8 @@ import org.apache.kafka.common.resource.Resource import org.apache.kafka.common.resource.Resource.CLUSTER_NAME import org.apache.kafka.common.resource.ResourceType.{CLUSTER, TOPIC} import org.apache.kafka.common.utils.Time -import org.apache.kafka.common.Node import org.apache.kafka.controller.Controller -import org.apache.kafka.metadata.{ApiMessageAndVersion, BrokerHeartbeatReply, BrokerRegistrationReply, FeatureMap, FeatureMapAndEpoch, VersionRange} +import org.apache.kafka.metadata.{ApiMessageAndVersion, BrokerHeartbeatReply, BrokerRegistrationReply, VersionRange} import org.apache.kafka.server.authorizer.Authorizer import scala.collection.mutable @@ -60,77 +59,14 @@ class ControllerApis(val requestChannel: RequestChannel, val raftManager: RaftManager[ApiMessageAndVersion], val config: KafkaConfig, val metaProperties: MetaProperties, - val controllerNodes: Seq[Node]) extends ApiRequestHandler with Logging { + val controllerNodes: Seq[Node], + val apiVersionManager: ApiVersionManager) extends ApiRequestHandler with Logging { val authHelper = new AuthHelper(authorizer) val requestHelper = new RequestHandlerHelper(requestChannel, quotas, time) - var supportedApiKeys = Set( - ApiKeys.FETCH, - ApiKeys.METADATA, - //ApiKeys.SASL_HANDSHAKE - ApiKeys.API_VERSIONS, - ApiKeys.CREATE_TOPICS, - //ApiKeys.DELETE_TOPICS, - //ApiKeys.DESCRIBE_ACLS, - //ApiKeys.CREATE_ACLS, - //ApiKeys.DELETE_ACLS, - //ApiKeys.DESCRIBE_CONFIGS, - //ApiKeys.ALTER_CONFIGS, - //ApiKeys.SASL_AUTHENTICATE, - //ApiKeys.CREATE_PARTITIONS, - //ApiKeys.CREATE_DELEGATION_TOKEN - //ApiKeys.RENEW_DELEGATION_TOKEN - //ApiKeys.EXPIRE_DELEGATION_TOKEN - //ApiKeys.DESCRIBE_DELEGATION_TOKEN - //ApiKeys.ELECT_LEADERS - ApiKeys.INCREMENTAL_ALTER_CONFIGS, - //ApiKeys.ALTER_PARTITION_REASSIGNMENTS - //ApiKeys.LIST_PARTITION_REASSIGNMENTS - ApiKeys.ALTER_CLIENT_QUOTAS, - //ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS - //ApiKeys.ALTER_USER_SCRAM_CREDENTIALS - //ApiKeys.UPDATE_FEATURES - ApiKeys.ENVELOPE, - ApiKeys.VOTE, - ApiKeys.BEGIN_QUORUM_EPOCH, - ApiKeys.END_QUORUM_EPOCH, - ApiKeys.DESCRIBE_QUORUM, - ApiKeys.ALTER_ISR, - ApiKeys.BROKER_REGISTRATION, - ApiKeys.BROKER_HEARTBEAT, - ApiKeys.UNREGISTER_BROKER, - ) - - private def maybeHandleInvalidEnvelope( - envelope: RequestChannel.Request, - forwardedApiKey: ApiKeys - ): Boolean = { - def sendEnvelopeError(error: Errors): Unit = { - requestHelper.sendErrorResponseMaybeThrottle(envelope, error.exception) - } - - if (!authHelper.authorize(envelope.context, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) { - // Forwarding request must have CLUSTER_ACTION authorization to reduce the risk of impersonation. - sendEnvelopeError(Errors.CLUSTER_AUTHORIZATION_FAILED) - true - } else if (!forwardedApiKey.forwardable) { - sendEnvelopeError(Errors.INVALID_REQUEST) - true - } else { - false - } - } - override def handle(request: RequestChannel.Request): Unit = { try { - val handled = request.envelope.exists(envelope => { - maybeHandleInvalidEnvelope(envelope, request.header.apiKey) - }) - - if (handled) - return - request.header.apiKey match { case ApiKeys.FETCH => handleFetch(request) case ApiKeys.METADATA => handleMetadataRequest(request) @@ -146,7 +82,9 @@ class ControllerApis(val requestChannel: RequestChannel, case ApiKeys.UNREGISTER_BROKER => handleUnregisterBroker(request) case ApiKeys.ALTER_CLIENT_QUOTAS => handleAlterClientQuotas(request) case ApiKeys.INCREMENTAL_ALTER_CONFIGS => handleIncrementalAlterConfigs(request) - case ApiKeys.ENVELOPE => EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle) + case ApiKeys.ENVELOPE => handleEnvelopeRequest(request) + case ApiKeys.SASL_HANDSHAKE => handleSaslHandshakeRequest(request) + case ApiKeys.SASL_AUTHENTICATE => handleSaslAuthenticateRequest(request) case _ => throw new ApiException(s"Unsupported ApiKey ${request.context.header.apiKey()}") } } catch { @@ -155,6 +93,27 @@ class ControllerApis(val requestChannel: RequestChannel, } } + def handleEnvelopeRequest(request: RequestChannel.Request): Unit = { + if (!authHelper.authorize(request.context, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) { + requestHelper.sendErrorResponseMaybeThrottle(request, new ClusterAuthorizationException( + s"Principal ${request.context.principal} does not have required CLUSTER_ACTION for envelope")) + } else { + EnvelopeUtils.handleEnvelopeRequest(request, requestChannel.metrics, handle) + } + } + + def handleSaslHandshakeRequest(request: RequestChannel.Request): Unit = { + val responseData = new SaslHandshakeResponseData().setErrorCode(Errors.ILLEGAL_SASL_STATE.code) + requestHelper.sendResponseMaybeThrottle(request, _ => new SaslHandshakeResponse(responseData)) + } + + def handleSaslAuthenticateRequest(request: RequestChannel.Request): Unit = { + val responseData = new SaslAuthenticateResponseData() + .setErrorCode(Errors.ILLEGAL_SASL_STATE.code) + .setErrorMessage("SaslAuthenticate request received after successful authentication") + requestHelper.sendResponseMaybeThrottle(request, _ => new SaslAuthenticateResponse(responseData)) + } + private def handleFetch(request: RequestChannel.Request): Unit = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) handleRaftRequest(request, response => new FetchResponse[BaseRecords](response.asInstanceOf[FetchResponseData])) @@ -251,45 +210,17 @@ class ControllerApis(val requestChannel: RequestChannel, // If this is considered to leak information about the broker version a workaround is to use SSL // with client authentication which is performed at an earlier stage of the connection where the // ApiVersionRequest is not available. - def createResponseCallback(features: FeatureMapAndEpoch, - requestThrottleMs: Int): ApiVersionsResponse = { + def createResponseCallback(requestThrottleMs: Int): ApiVersionsResponse = { val apiVersionRequest = request.body[ApiVersionsRequest] - if (apiVersionRequest.hasUnsupportedRequestVersion) + if (apiVersionRequest.hasUnsupportedRequestVersion) { apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.UNSUPPORTED_VERSION.exception) - else if (!apiVersionRequest.isValid) + } else if (!apiVersionRequest.isValid) { apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.INVALID_REQUEST.exception) - else { - val data = new ApiVersionsResponseData(). - setErrorCode(0.toShort). - setThrottleTimeMs(requestThrottleMs). - setFinalizedFeaturesEpoch(features.epoch()) - supportedFeatures.foreach { - case (k, v) => data.supportedFeatures().add(new SupportedFeatureKey(). - setName(k).setMaxVersion(v.max()).setMinVersion(v.min())) - } - // features.finalizedFeatures().asScala.foreach { - // case (k, v) => data.finalizedFeatures().add(new FinalizedFeatureKey(). - // setName(k).setMaxVersionLevel(v.max()).setMinVersionLevel(v.min())) - // } - ApiKeys.values().foreach { - key => - if (supportedApiKeys.contains(key)) { - data.apiKeys().add(new ApiVersion(). - setApiKey(key.id). - setMaxVersion(key.latestVersion()). - setMinVersion(key.oldestVersion())) - } - } - new ApiVersionsResponse(data) + } else { + apiVersionManager.apiVersionResponse(requestThrottleMs) } } - // FutureConverters.toScala(controller.finalizedFeatures()).onComplete { - // case Success(features) => - requestHelper.sendResponseMaybeThrottle(request, - requestThrottleMs => createResponseCallback(new FeatureMapAndEpoch( - new FeatureMap(new util.HashMap()), 0), requestThrottleMs)) - // case Failure(e) => requestHelper.handleError(request, e) - // } + requestHelper.sendResponseMaybeThrottle(request, createResponseCallback) } private def handleVote(request: RequestChannel.Request): Unit = { diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index 50fc6e2c4f774..18ea2c4e8b6c6 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -168,7 +168,8 @@ class ControllerServer( raftManager, config, metaProperties, - controllerNodes.toSeq) + controllerNodes.toSeq, + apiVersionManager) controllerApisHandlerPool = new KafkaRequestHandlerPool(config.nodeId, socketServer.dataPlaneRequestChannel, controllerApis, diff --git a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala index 0ca44f3cc17f2..2f1c91ee91b56 100644 --- a/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/ControllerApisTest.scala @@ -23,10 +23,11 @@ import java.util.Properties import kafka.network.RequestChannel import kafka.raft.RaftManager import kafka.server.QuotaFactory.QuotaManagers -import kafka.server.{ClientQuotaManager, ClientRequestQuotaManager, ControllerApis, ControllerMutationQuotaManager, KafkaConfig, MetaProperties, ReplicationQuotaManager} +import kafka.server.{ClientQuotaManager, ClientRequestQuotaManager, ControllerApis, ControllerMutationQuotaManager, KafkaConfig, MetaProperties, ReplicationQuotaManager, SimpleApiVersionManager} import kafka.utils.MockTime import org.apache.kafka.common.Uuid import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.BrokerRegistrationRequestData import org.apache.kafka.common.network.{ClientInformation, ListenerName} import org.apache.kafka.common.protocol.Errors @@ -40,6 +41,7 @@ import org.junit.jupiter.api.{AfterEach, Test} import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ import org.mockito.{ArgumentCaptor, ArgumentMatchers} + import scala.jdk.CollectionConverters._ class ControllerApisTest { @@ -81,7 +83,8 @@ class ControllerApisTest { raftManager, new KafkaConfig(props), MetaProperties(Uuid.fromString("JgxuGe9URy-E-ceaL04lEw"), nodeId = nodeId), - Seq.empty + Seq.empty, + new SimpleApiVersionManager(ListenerType.CONTROLLER) ) } diff --git a/tests/kafkatest/tests/core/group_mode_transactions_test.py b/tests/kafkatest/tests/core/group_mode_transactions_test.py index e9638d50ef4f1..18e6d97ecb412 100644 --- a/tests/kafkatest/tests/core/group_mode_transactions_test.py +++ b/tests/kafkatest/tests/core/group_mode_transactions_test.py @@ -269,7 +269,7 @@ def setup_topics(self): @cluster(num_nodes=10) @matrix(failure_mode=["hard_bounce", "clean_bounce"], - bounce_target=["brokers", "clients"], metadata_quorum=quorum.all_non_upgrade) + bounce_target=["brokers", "clients"]) def test_transactions(self, failure_mode, bounce_target, metadata_quorum=quorum.zk): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol diff --git a/tests/kafkatest/tests/core/transactions_test.py b/tests/kafkatest/tests/core/transactions_test.py index 2891e70ff19a0..8989d83c601ac 100644 --- a/tests/kafkatest/tests/core/transactions_test.py +++ b/tests/kafkatest/tests/core/transactions_test.py @@ -244,8 +244,7 @@ def setup_topics(self): @matrix(failure_mode=["hard_bounce", "clean_bounce"], bounce_target=["brokers", "clients"], check_order=[True, False], - use_group_metadata=[True, False], - metadata_quorum=quorum.all_non_upgrade) + use_group_metadata=[True, False]) def test_transactions(self, failure_mode, bounce_target, check_order, use_group_metadata, metadata_quorum=quorum.zk): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol From c9832aabae5d55dc22b962867d5c1bcc3804b605 Mon Sep 17 00:00:00 2001 From: dengziming Date: Fri, 26 Feb 2021 15:04:25 +0800 Subject: [PATCH 065/243] DOCS: Update protocol doc for missing data type (#10162) Reviewers: Chia-Ping Tsai --- .../main/resources/common/message/README.md | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/clients/src/main/resources/common/message/README.md b/clients/src/main/resources/common/message/README.md index f7575fd536462..29a8491a10b73 100644 --- a/clients/src/main/resources/common/message/README.md +++ b/clients/src/main/resources/common/message/README.md @@ -75,16 +75,24 @@ There are several primitive field types available. * "int16": a 16-bit integer. +* "uint16": a 16-bit unsigned integer. + * "int32": a 32-bit integer. +* "uint32": a 32-bit unsigned integer. + * "int64": a 64-bit integer. * "float64": is a double-precision floating point number (IEEE 754). * "string": a UTF-8 string. +* "uuid": a type 4 immutable universally unique identifier. + * "bytes": binary data. +* "records": recordset such as memory recordset. + In addition to these primitive field types, there is also an array type. Array types start with a "[]" and end with the name of the element type. For example, []Foo declares an array of "Foo" objects. Array fields have their own @@ -96,12 +104,12 @@ Guide](https://kafka.apache.org/protocol.html). Nullable Fields --------------- Booleans, ints, and floats can never be null. However, fields that are strings, -bytes, or arrays may optionally be "nullable." When a field is "nullable," that -simply means that we are prepared to serialize and deserialize null entries for -that field. +bytes, uuid, records, or arrays may optionally be "nullable". When a field is +"nullable", that simply means that we are prepared to serialize and deserialize +null entries for that field. If you want to declare a field as nullable, you set "nullableVersions" for that -field. Nullability is implemented as a version range in order to accomodate a +field. Nullability is implemented as a version range in order to accommodate a very common pattern in Kafka where a field that was originally not nullable becomes nullable in a later version. @@ -176,6 +184,10 @@ been set: * Bytes fields default to the empty byte array. +* Uuid fields default to zero uuid. + +* Records fields default to null. + * Array fields default to empty. You can specify "null" as a default value for a string field by specifing the From 4c0be2080d554f19cbc7b45e8e980c73df163bc8 Mon Sep 17 00:00:00 2001 From: tinawenqiao <315524513@qq.com> Date: Fri, 26 Feb 2021 19:02:44 +0800 Subject: [PATCH 066/243] KAFKA-10449: Add some important parameter desc in connect-distributed.properties (#9235) Reviewers: Mickael Maison --- config/connect-distributed.properties | 10 ++++++++++ .../org/apache/kafka/connect/runtime/WorkerConfig.java | 10 ++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/config/connect-distributed.properties b/config/connect-distributed.properties index 72db145f3f892..45cc85159cb8a 100644 --- a/config/connect-distributed.properties +++ b/config/connect-distributed.properties @@ -66,14 +66,24 @@ status.storage.replication.factor=1 # Flush much faster than normal, which is useful for testing/debugging offset.flush.interval.ms=10000 +# List of comma-separated URIs the REST API will listen on. The supported protocols are HTTP and HTTPS. +# Specify hostname as 0.0.0.0 to bind to all interfaces. +# Leave hostname empty to bind to default interface. +# After this parameter is set, rest.host.name/port will not take effect. +# Examples of legal listener lists: HTTP://myhost:8083,HTTPS://myhost:8084" +#listeners=HTTP://:8083 + # These are provided to inform the user about the presence of the REST host and port configs # Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests. +# DEPRECATED As of 1.1.0: only used when listeners is not set. Use listeners instead. #rest.host.name= #rest.port=8083 # The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers. +# If not set, it uses the value for "listeners" or the rest.host.name/port if configured. #rest.advertised.host.name= #rest.advertised.port= +#rest.advertised.listener= # Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins # (connectors, converters, transformations). The list should consist of top level directories that include diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java index e140e594f3784..980aafa4ff494 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java @@ -147,20 +147,22 @@ public class WorkerConfig extends AbstractConfig { public static final long OFFSET_COMMIT_TIMEOUT_MS_DEFAULT = 5000L; /** - * @deprecated As of 1.1.0. + * @deprecated As of 1.1.0. Only used when listeners is not set. Use listeners instead. */ @Deprecated public static final String REST_HOST_NAME_CONFIG = "rest.host.name"; private static final String REST_HOST_NAME_DOC - = "Hostname for the REST API. If this is set, it will only bind to this interface."; + = "Hostname for the REST API. If this is set, it will only bind to this interface.\n" + + "Deprecated, only used when listeners is not set. Use listeners instead."; /** - * @deprecated As of 1.1.0. + * @deprecated As of 1.1.0. Only used when listeners is not set. Use listeners instead. */ @Deprecated public static final String REST_PORT_CONFIG = "rest.port"; private static final String REST_PORT_DOC - = "Port for the REST API to listen on."; + = "Port for the REST API to listen on.\n" + + "Deprecated, only used when listeners is not set. Use listeners instead."; public static final int REST_PORT_DEFAULT = 8083; public static final String LISTENERS_CONFIG = "listeners"; From 5dae3648abdb64969680ad2313756277a63530c1 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Fri, 26 Feb 2021 11:26:43 -0800 Subject: [PATCH 067/243] MINOR: Clean up group instance id handling in `GroupCoordinator` (#9958) This is a continuation of a refactor started in #9952. The logic in `GroupCoordinator` is loose and inconsistent in the handling of the `groupInstanceId`. In some cases, such as in the JoinGroup hander, we verify that the groupInstanceId from the request is mapped to the memberId precisely. In other cases, such as Heartbeat, we check the mapping, but only to validate fencing. The patch consolidates the member validation so that all handlers follow the same logic. A second problem is that many of the APIs where a `groupInstanceId` is expected use optional arguments. For example: ```scala def hasStaticMember(groupInstanceId: Option[String]): Boolean def addStaticMember(groupInstanceId: Option[String], newMemberId: String): Unit ``` If `groupInstanceId` is `None`, then `hasStaticMember` is guaranteed to return `false` while `addStaticMember` raises an `IllegalStateException`. So the APIs suggest a generality which is not supported and does not make sense. Finally, the patch attempts to introduce stronger internal invariants inside `GroupMetadata`. Currently it is possible for an inconsistent `groupInstanceId` to `memberId` mapping to exist because we expose separate APIs to modify `members` and `staticMembers`. We rely on the caller to ensure this doesn't happen. Similarly, it is possible for a member to be in the `pendingMembers` set as well as the stable `members` map. The patch fixes this by consolidating the paths to addition and removal from these collections and adding assertions to ensure that invariants are maintained. Reviewers: David Jacot --- .../coordinator/group/GroupCoordinator.scala | 525 ++++++++++++------ .../coordinator/group/GroupMetadata.scala | 135 +++-- .../coordinator/group/MemberMetadata.scala | 9 +- .../group/GroupCoordinatorTest.scala | 249 +++------ .../group/GroupMetadataManagerTest.scala | 20 +- .../coordinator/group/GroupMetadataTest.scala | 106 +++- 6 files changed, 609 insertions(+), 435 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index 2c5bf70fe7963..dfe3c950cf844 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -184,9 +184,31 @@ class GroupCoordinator(val brokerId: Int, group.remove(memberId) responseCallback(JoinGroupResult(JoinGroupRequest.UNKNOWN_MEMBER_ID, Errors.GROUP_MAX_SIZE_REACHED)) } else if (isUnknownMember) { - doUnknownJoinGroup(group, groupInstanceId, requireKnownMemberId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, protocols, responseCallback) + doNewMemberJoinGroup( + group, + groupInstanceId, + requireKnownMemberId, + clientId, + clientHost, + rebalanceTimeoutMs, + sessionTimeoutMs, + protocolType, + protocols, + responseCallback + ) } else { - doJoinGroup(group, memberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, protocols, responseCallback) + doCurrentMemberJoinGroup( + group, + memberId, + groupInstanceId, + clientId, + clientHost, + rebalanceTimeoutMs, + sessionTimeoutMs, + protocolType, + protocols, + responseCallback + ) } // attempt to complete JoinGroup @@ -198,16 +220,18 @@ class GroupCoordinator(val brokerId: Int, } } - private def doUnknownJoinGroup(group: GroupMetadata, - groupInstanceId: Option[String], - requireKnownMemberId: Boolean, - clientId: String, - clientHost: String, - rebalanceTimeoutMs: Int, - sessionTimeoutMs: Int, - protocolType: String, - protocols: List[(String, Array[Byte])], - responseCallback: JoinCallback): Unit = { + private def doNewMemberJoinGroup( + group: GroupMetadata, + groupInstanceId: Option[String], + requireKnownMemberId: Boolean, + clientId: String, + clientHost: String, + rebalanceTimeoutMs: Int, + sessionTimeoutMs: Int, + protocolType: String, + protocols: List[(String, Array[Byte])], + responseCallback: JoinCallback + ): Unit = { group.inLock { if (group.is(Dead)) { // if the group is marked as dead, it means some other thread has just removed the group @@ -219,75 +243,176 @@ class GroupCoordinator(val brokerId: Int, responseCallback(JoinGroupResult(JoinGroupRequest.UNKNOWN_MEMBER_ID, Errors.INCONSISTENT_GROUP_PROTOCOL)) } else { val newMemberId = group.generateMemberId(clientId, groupInstanceId) + groupInstanceId match { + case Some(instanceId) => + doStaticNewMemberJoinGroup( + group, + instanceId, + newMemberId, + clientId, + clientHost, + rebalanceTimeoutMs, + sessionTimeoutMs, + protocolType, + protocols, + responseCallback + ) - if (group.hasStaticMember(groupInstanceId)) { - updateStaticMemberAndRebalance(group, newMemberId, groupInstanceId, protocols, responseCallback) - } else if (requireKnownMemberId) { - // If member id required (dynamic membership), register the member in the pending member list - // and send back a response to call for another join group request with allocated member id. - debug(s"Dynamic member with unknown member id joins group ${group.groupId} in " + - s"${group.currentState} state. Created a new member id $newMemberId and request the member to rejoin with this id.") - group.addPendingMember(newMemberId) - addPendingMemberExpiration(group, newMemberId, sessionTimeoutMs) - responseCallback(JoinGroupResult(newMemberId, Errors.MEMBER_ID_REQUIRED)) - } else { - info(s"${if (groupInstanceId.isDefined) "Static" else "Dynamic"} Member with unknown member id joins group ${group.groupId} in " + - s"${group.currentState} state. Created a new member id $newMemberId for this member and add to the group.") - addMemberAndRebalance(rebalanceTimeoutMs, sessionTimeoutMs, newMemberId, groupInstanceId, - clientId, clientHost, protocolType, protocols, group, responseCallback) + case None => + doDynamicNewMemberJoinGroup( + group, + requireKnownMemberId, + newMemberId, + clientId, + clientHost, + rebalanceTimeoutMs, + sessionTimeoutMs, + protocolType, + protocols, + responseCallback + ) } } } } - private def doJoinGroup(group: GroupMetadata, - memberId: String, - groupInstanceId: Option[String], - clientId: String, - clientHost: String, - rebalanceTimeoutMs: Int, - sessionTimeoutMs: Int, - protocolType: String, - protocols: List[(String, Array[Byte])], - responseCallback: JoinCallback): Unit = { + private def doStaticNewMemberJoinGroup( + group: GroupMetadata, + groupInstanceId: String, + newMemberId: String, + clientId: String, + clientHost: String, + rebalanceTimeoutMs: Int, + sessionTimeoutMs: Int, + protocolType: String, + protocols: List[(String, Array[Byte])], + responseCallback: JoinCallback + ): Unit = { + group.currentStaticMemberId(groupInstanceId) match { + case Some(oldMemberId) => + info(s"Static member with groupInstanceId=$groupInstanceId and unknown member id joins " + + s"group ${group.groupId} in ${group.currentState} state. Replacing previously mapped " + + s"member $oldMemberId with this groupInstanceId.") + updateStaticMemberAndRebalance(group, oldMemberId, newMemberId, groupInstanceId, protocols, responseCallback) + + case None => + info(s"Static member with groupInstanceId=$groupInstanceId and unknown member id joins " + + s"group ${group.groupId} in ${group.currentState} state. Created a new member id $newMemberId " + + s"for this member and add to the group.") + addMemberAndRebalance(rebalanceTimeoutMs, sessionTimeoutMs, newMemberId, Some(groupInstanceId), + clientId, clientHost, protocolType, protocols, group, responseCallback) + } + } + + private def doDynamicNewMemberJoinGroup( + group: GroupMetadata, + requireKnownMemberId: Boolean, + newMemberId: String, + clientId: String, + clientHost: String, + rebalanceTimeoutMs: Int, + sessionTimeoutMs: Int, + protocolType: String, + protocols: List[(String, Array[Byte])], + responseCallback: JoinCallback + ): Unit = { + if (requireKnownMemberId) { + // If member id required, register the member in the pending member list and send + // back a response to call for another join group request with allocated member id. + info(s"Dynamic member with unknown member id joins group ${group.groupId} in " + + s"${group.currentState} state. Created a new member id $newMemberId and request the " + + s"member to rejoin with this id.") + group.addPendingMember(newMemberId) + addPendingMemberExpiration(group, newMemberId, sessionTimeoutMs) + responseCallback(JoinGroupResult(newMemberId, Errors.MEMBER_ID_REQUIRED)) + } else { + info(s"Dynamic Member with unknown member id joins group ${group.groupId} in " + + s"${group.currentState} state. Created a new member id $newMemberId for this member " + + s"and add to the group.") + addMemberAndRebalance(rebalanceTimeoutMs, sessionTimeoutMs, newMemberId, None, + clientId, clientHost, protocolType, protocols, group, responseCallback) + } + } + + private def validateCurrentMember( + group: GroupMetadata, + memberId: String, + groupInstanceId: Option[String], + operation: String + ): Option[Errors] = { + // We are validating two things: + // 1. If `groupInstanceId` is present, then it exists and is mapped to `memberId` + // 2. The `memberId` exists in the group + groupInstanceId.flatMap { instanceId => + group.currentStaticMemberId(instanceId) match { + case Some(currentMemberId) if currentMemberId != memberId => + info(s"Request memberId=$memberId for static member with groupInstanceId=$instanceId " + + s"is fenced by current memberId=$currentMemberId during operation $operation") + Some(Errors.FENCED_INSTANCE_ID) + case Some(_) => + None + case None => + Some(Errors.UNKNOWN_MEMBER_ID) + } + }.orElse { + if (!group.has(memberId)) { + Some(Errors.UNKNOWN_MEMBER_ID) + } else { + None + } + } + } + + private def doCurrentMemberJoinGroup( + group: GroupMetadata, + memberId: String, + groupInstanceId: Option[String], + clientId: String, + clientHost: String, + rebalanceTimeoutMs: Int, + sessionTimeoutMs: Int, + protocolType: String, + protocols: List[(String, Array[Byte])], + responseCallback: JoinCallback + ): Unit = { group.inLock { if (group.is(Dead)) { // if the group is marked as dead, it means some other thread has just removed the group - // from the coordinator metadata; this is likely that the group has migrated to some other + // from the coordinator metadata; it is likely that the group has migrated to some other // coordinator OR the group is in a transient unstable phase. Let the member retry // finding the correct coordinator and rejoin. responseCallback(JoinGroupResult(memberId, Errors.COORDINATOR_NOT_AVAILABLE)) } else if (!group.supportsProtocols(protocolType, MemberMetadata.plainProtocolSet(protocols))) { responseCallback(JoinGroupResult(memberId, Errors.INCONSISTENT_GROUP_PROTOCOL)) } else if (group.isPendingMember(memberId)) { - // A rejoining pending member will be accepted. Note that pending member will never be a static member. - if (groupInstanceId.isDefined) { - throw new IllegalStateException(s"the static member $groupInstanceId was not expected to be assigned " + - s"into pending member bucket with member id $memberId") - } else { - debug(s"Dynamic Member with specific member id $memberId joins group ${group.groupId} in " + - s"${group.currentState} state. Adding to the group now.") - addMemberAndRebalance(rebalanceTimeoutMs, sessionTimeoutMs, memberId, groupInstanceId, - clientId, clientHost, protocolType, protocols, group, responseCallback) + // A rejoining pending member will be accepted. Note that pending member cannot be a static member. + groupInstanceId.foreach { instanceId => + throw new IllegalStateException(s"Received unexpected JoinGroup with groupInstanceId=$instanceId " + + s"for pending member with memberId=$memberId") } + + debug(s"Pending dynamic member with id $memberId joins group ${group.groupId} in " + + s"${group.currentState} state. Adding to the group now.") + addMemberAndRebalance(rebalanceTimeoutMs, sessionTimeoutMs, memberId, None, + clientId, clientHost, protocolType, protocols, group, responseCallback) } else { - val groupInstanceIdNotFound = groupInstanceId.isDefined && !group.hasStaticMember(groupInstanceId) - if (group.isStaticMemberFenced(memberId, groupInstanceId, "join-group")) { - // given member id doesn't match with the groupInstanceId. Inform duplicate instance to shut down immediately. - responseCallback(JoinGroupResult(memberId, Errors.FENCED_INSTANCE_ID)) - } else if (!group.has(memberId) || groupInstanceIdNotFound) { - // If the dynamic member trying to register with an unrecognized id, or - // the static member joins with unknown group instance id, send the response to let - // it reset its member id and retry. - responseCallback(JoinGroupResult(memberId, Errors.UNKNOWN_MEMBER_ID)) - } else { - val member = group.get(memberId) + val memberErrorOpt = validateCurrentMember( + group, + memberId, + groupInstanceId, + operation = "join-group" + ) - group.currentState match { + memberErrorOpt match { + case Some(error) => responseCallback(JoinGroupResult(memberId, error)) + + case None => group.currentState match { case PreparingRebalance => + val member = group.get(memberId) updateMemberAndRebalance(group, member, protocols, s"Member ${member.memberId} joining group during ${group.currentState}", responseCallback) case CompletingRebalance => + val member = group.get(memberId) if (member.matches(protocols)) { // member is joining with the same metadata (which could be because it failed to // receive the initial JoinGroup response), so just return current group information @@ -369,6 +494,40 @@ class GroupCoordinator(val brokerId: Int, } } + private def validateSyncGroup( + group: GroupMetadata, + generationId: Int, + memberId: String, + protocolType: Option[String], + protocolName: Option[String], + groupInstanceId: Option[String], + ): Option[Errors] = { + if (group.is(Dead)) { + // if the group is marked as dead, it means some other thread has just removed the group + // from the coordinator metadata; this is likely that the group has migrated to some other + // coordinator OR the group is in a transient unstable phase. Let the member retry + // finding the correct coordinator and rejoin. + Some(Errors.COORDINATOR_NOT_AVAILABLE) + } else { + validateCurrentMember( + group, + memberId, + groupInstanceId, + operation = "sync-group" + ).orElse { + if (generationId != group.generationId) { + Some(Errors.ILLEGAL_GENERATION) + } else if (protocolType.isDefined && !group.protocolType.contains(protocolType.get)) { + Some(Errors.INCONSISTENT_GROUP_PROTOCOL) + } else if (protocolName.isDefined && !group.protocolName.contains(protocolName.get)) { + Some(Errors.INCONSISTENT_GROUP_PROTOCOL) + } else { + None + } + } + } + } + private def doSyncGroup(group: GroupMetadata, generationId: Int, memberId: String, @@ -378,24 +537,19 @@ class GroupCoordinator(val brokerId: Int, groupAssignment: Map[String, Array[Byte]], responseCallback: SyncCallback): Unit = { group.inLock { - if (group.is(Dead)) { - // if the group is marked as dead, it means some other thread has just removed the group - // from the coordinator metadata; this is likely that the group has migrated to some other - // coordinator OR the group is in a transient unstable phase. Let the member retry - // finding the correct coordinator and rejoin. - responseCallback(SyncGroupResult(Errors.COORDINATOR_NOT_AVAILABLE)) - } else if (group.isStaticMemberFenced(memberId, groupInstanceId, "sync-group")) { - responseCallback(SyncGroupResult(Errors.FENCED_INSTANCE_ID)) - } else if (!group.has(memberId)) { - responseCallback(SyncGroupResult(Errors.UNKNOWN_MEMBER_ID)) - } else if (generationId != group.generationId) { - responseCallback(SyncGroupResult(Errors.ILLEGAL_GENERATION)) - } else if (protocolType.isDefined && !group.protocolType.contains(protocolType.get)) { - responseCallback(SyncGroupResult(Errors.INCONSISTENT_GROUP_PROTOCOL)) - } else if (protocolName.isDefined && !group.protocolName.contains(protocolName.get)) { - responseCallback(SyncGroupResult(Errors.INCONSISTENT_GROUP_PROTOCOL)) - } else { - group.currentState match { + val validationErrorOpt = validateSyncGroup( + group, + generationId, + memberId, + protocolType, + protocolName, + groupInstanceId + ) + + validationErrorOpt match { + case Some(error) => responseCallback(SyncGroupResult(error)) + + case None => group.currentState match { case Empty => responseCallback(SyncGroupResult(Errors.UNKNOWN_MEMBER_ID)) @@ -452,6 +606,14 @@ class GroupCoordinator(val brokerId: Int, def handleLeaveGroup(groupId: String, leavingMembers: List[MemberIdentity], responseCallback: LeaveGroupResult => Unit): Unit = { + + def removeCurrentMemberFromGroup(group: GroupMetadata, memberId: String): Unit = { + val member = group.get(memberId) + removeMemberAndUpdateGroup(group, member, s"removing member $memberId on LeaveGroup") + removeHeartbeatForLeavingMember(group, member) + info(s"Member $member has left group $groupId through explicit `LeaveGroup` request") + } + validateGroupStatus(groupId, ApiKeys.LEAVE_GROUP) match { case Some(error) => responseCallback(leaveError(error, List.empty)) @@ -469,33 +631,34 @@ class GroupCoordinator(val brokerId: Int, val memberErrors = leavingMembers.map { leavingMember => val memberId = leavingMember.memberId val groupInstanceId = Option(leavingMember.groupInstanceId) - if (memberId != JoinGroupRequest.UNKNOWN_MEMBER_ID - && group.isStaticMemberFenced(memberId, groupInstanceId, "leave-group")) { - memberLeaveError(leavingMember, Errors.FENCED_INSTANCE_ID) - } else if (group.isPendingMember(memberId)) { - if (groupInstanceId.isDefined) { - throw new IllegalStateException(s"the static member $groupInstanceId was not expected to be leaving " + - s"from pending member bucket with member id $memberId") - } else { - // if a pending member is leaving, it needs to be removed from the pending list, heartbeat cancelled - // and if necessary, prompt a JoinGroup completion. - info(s"Pending member $memberId is leaving group ${group.groupId}.") - removePendingMemberAndUpdateGroup(group, memberId) - heartbeatPurgatory.checkAndComplete(MemberKey(group.groupId, memberId)) - memberLeaveError(leavingMember, Errors.NONE) + + // The LeaveGroup API allows administrative removal of members by GroupInstanceId + // in which case we expect the MemberId to be undefined. + if (memberId == JoinGroupRequest.UNKNOWN_MEMBER_ID) { + groupInstanceId.flatMap(group.currentStaticMemberId) match { + case Some(currentMemberId) => + removeCurrentMemberFromGroup(group, currentMemberId) + memberLeaveError(leavingMember, Errors.NONE) + case None => + memberLeaveError(leavingMember, Errors.UNKNOWN_MEMBER_ID) } - } else if (!group.has(memberId) && !group.hasStaticMember(groupInstanceId)) { - memberLeaveError(leavingMember, Errors.UNKNOWN_MEMBER_ID) - } else { - val member = if (group.hasStaticMember(groupInstanceId)) - group.get(group.getStaticMemberId(groupInstanceId)) - else - group.get(memberId) - removeHeartbeatForLeavingMember(group, member) - info(s"Member[group.instance.id ${member.groupInstanceId}, member.id ${member.memberId}] " + - s"in group ${group.groupId} has left, removing it from the group") - removeMemberAndUpdateGroup(group, member, s"removing member $memberId on LeaveGroup") + } else if (group.isPendingMember(memberId)) { + removePendingMemberAndUpdateGroup(group, memberId) + heartbeatPurgatory.checkAndComplete(MemberKey(group.groupId, memberId)) + info(s"Pending member with memberId=$memberId has left group ${group.groupId} " + + s"through explicit `LeaveGroup` request") memberLeaveError(leavingMember, Errors.NONE) + } else { + val memberError = validateCurrentMember( + group, + memberId, + groupInstanceId, + operation = "leave-group" + ).getOrElse { + removeCurrentMemberFromGroup(group, memberId) + Errors.NONE + } + memberLeaveError(leavingMember, memberError) } } responseCallback(leaveError(Errors.NONE, memberErrors)) @@ -602,6 +765,30 @@ class GroupCoordinator(val brokerId: Int, groupError -> partitionErrors } + private def validateHeartbeat( + group: GroupMetadata, + generationId: Int, + memberId: String, + groupInstanceId: Option[String] + ): Option[Errors] = { + if (group.is(Dead)) { + Some(Errors.COORDINATOR_NOT_AVAILABLE) + } else { + validateCurrentMember( + group, + memberId, + groupInstanceId, + operation = "heartbeat" + ).orElse { + if (generationId != group.generationId) { + Some(Errors.ILLEGAL_GENERATION) + } else { + None + } + } + } + } + def handleHeartbeat(groupId: String, memberId: String, groupInstanceId: Option[String], @@ -621,18 +808,15 @@ class GroupCoordinator(val brokerId: Int, responseCallback(Errors.UNKNOWN_MEMBER_ID) case Some(group) => group.inLock { - if (group.is(Dead)) { - // if the group is marked as dead, it means some other thread has just removed the group - // from the coordinator metadata; this is likely that the group has migrated to some other - // coordinator OR the group is in a transient unstable phase. Let the member retry - // finding the correct coordinator and rejoin. - responseCallback(Errors.COORDINATOR_NOT_AVAILABLE) - } else if (group.isStaticMemberFenced(memberId, groupInstanceId, "heartbeat")) { - responseCallback(Errors.FENCED_INSTANCE_ID) - } else if (!group.has(memberId)) { - responseCallback(Errors.UNKNOWN_MEMBER_ID) - } else if (generationId != group.generationId) { - responseCallback(Errors.ILLEGAL_GENERATION) + val validationErrorOpt = validateHeartbeat( + group, + generationId, + memberId, + groupInstanceId + ) + + if (validationErrorOpt.isDefined) { + responseCallback(validationErrorOpt.get) } else { group.currentState match { case Empty => @@ -724,26 +908,55 @@ class GroupCoordinator(val brokerId: Int, offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata], responseCallback: immutable.Map[TopicPartition, Errors] => Unit): Unit = { group.inLock { - if (group.is(Dead)) { - // if the group is marked as dead, it means some other thread has just removed the group - // from the coordinator metadata; it is likely that the group has migrated to some other - // coordinator OR the group is in a transient unstable phase. Let the member retry - // finding the correct coordinator and rejoin. - responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.COORDINATOR_NOT_AVAILABLE }) - } else if (group.isStaticMemberFenced(memberId, groupInstanceId, "txn-commit-offsets")) { - responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.FENCED_INSTANCE_ID }) - } else if (memberId != JoinGroupRequest.UNKNOWN_MEMBER_ID && !group.has(memberId)) { - // Enforce member id when it is set. - responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.UNKNOWN_MEMBER_ID }) - } else if (generationId >= 0 && generationId != group.generationId) { - // Enforce generation check when it is set. - responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.ILLEGAL_GENERATION }) + val validationErrorOpt = validateOffsetCommit( + group, + generationId, + memberId, + groupInstanceId, + isTransactional = true + ) + + if (validationErrorOpt.isDefined) { + responseCallback(offsetMetadata.map { case (k, _) => k -> validationErrorOpt.get }) } else { groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback, producerId, producerEpoch) } } } + private def validateOffsetCommit( + group: GroupMetadata, + generationId: Int, + memberId: String, + groupInstanceId: Option[String], + isTransactional: Boolean + ): Option[Errors] = { + if (group.is(Dead)) { + Some(Errors.COORDINATOR_NOT_AVAILABLE) + } else if (generationId >= 0 || memberId != JoinGroupRequest.UNKNOWN_MEMBER_ID || groupInstanceId.isDefined) { + validateCurrentMember( + group, + memberId, + groupInstanceId, + operation = if (isTransactional) "txn-offset-commit" else "offset-commit" + ).orElse { + if (generationId != group.generationId) { + Some(Errors.ILLEGAL_GENERATION) + } else { + None + } + } + } else if (!isTransactional && !group.is(Empty)) { + // When the group is non-empty, only members can commit offsets. + // This does not apply to transactional offset commits, since the + // older versions of this protocol do not require memberId and + // generationId. + Some(Errors.UNKNOWN_MEMBER_ID) + } else { + None + } + } + private def doCommitOffsets(group: GroupMetadata, memberId: String, groupInstanceId: Option[String], @@ -751,23 +964,21 @@ class GroupCoordinator(val brokerId: Int, offsetMetadata: immutable.Map[TopicPartition, OffsetAndMetadata], responseCallback: immutable.Map[TopicPartition, Errors] => Unit): Unit = { group.inLock { - if (group.is(Dead)) { - // if the group is marked as dead, it means some other thread has just removed the group - // from the coordinator metadata; it is likely that the group has migrated to some other - // coordinator OR the group is in a transient unstable phase. Let the member retry - // finding the correct coordinator and rejoin. - responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.COORDINATOR_NOT_AVAILABLE }) - } else if (group.isStaticMemberFenced(memberId, groupInstanceId, "commit-offsets")) { - responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.FENCED_INSTANCE_ID }) - } else if (generationId < 0 && group.is(Empty)) { - // The group is only using Kafka to store offsets. - groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback) - } else if (!group.has(memberId)) { - responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.UNKNOWN_MEMBER_ID }) - } else if (generationId != group.generationId) { - responseCallback(offsetMetadata.map { case (k, _) => k -> Errors.ILLEGAL_GENERATION }) + val validationErrorOpt = validateOffsetCommit( + group, + generationId, + memberId, + groupInstanceId, + isTransactional = false + ) + + if (validationErrorOpt.isDefined) { + responseCallback(offsetMetadata.map { case (k, _) => k -> validationErrorOpt.get }) } else { group.currentState match { + case Empty => + groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback) + case Stable | PreparingRebalance => // During PreparingRebalance phase, we still allow a commit request since we rely // on heartbeat response to eventually notify the rebalance in progress signal to the consumer @@ -982,7 +1193,6 @@ class GroupCoordinator(val brokerId: Int, } private def removeHeartbeatForLeavingMember(group: GroupMetadata, member: MemberMetadata): Unit = { - member.isLeaving = true val memberKey = MemberKey(group.groupId, member.memberId) heartbeatPurgatory.checkAndComplete(memberKey) } @@ -1016,26 +1226,17 @@ class GroupCoordinator(val brokerId: Int, // for new members. If the new member is still there, we expect it to retry. completeAndScheduleNextExpiration(group, member, NewMemberJoinTimeoutMs) - if (member.isStaticMember) { - info(s"Adding new static member $groupInstanceId to group ${group.groupId} with member id $memberId.") - group.addStaticMember(groupInstanceId, memberId) - } else { - group.removePendingMember(memberId) - } maybePrepareRebalance(group, s"Adding new member $memberId with group instance id $groupInstanceId") } private def updateStaticMemberAndRebalance(group: GroupMetadata, + oldMemberId: String, newMemberId: String, - groupInstanceId: Option[String], + groupInstanceId: String, protocols: List[(String, Array[Byte])], responseCallback: JoinCallback): Unit = { - val oldMemberId = group.getStaticMemberId(groupInstanceId) - info(s"Static member $groupInstanceId of group ${group.groupId} with unknown member id rejoins, assigning new member id $newMemberId, while " + - s"old member id $oldMemberId will be removed.") - val currentLeader = group.leaderOrNull - val member = group.replaceGroupInstance(oldMemberId, newMemberId, groupInstanceId) + val member = group.replaceStaticMember(groupInstanceId, oldMemberId, newMemberId) // Heartbeat of old member id will expire without effect since the group no longer contains that member id. // New heartbeat shall be scheduled with new member id. completeAndScheduleNextHeartbeatExpiration(group, member) @@ -1058,7 +1259,7 @@ class GroupCoordinator(val brokerId: Int, // Failed to persist member.id of the given static member, revert the update of the static member in the group. group.updateMember(knownStaticMember, oldProtocols, null) - val oldMember = group.replaceGroupInstance(newMemberId, oldMemberId, groupInstanceId) + val oldMember = group.replaceStaticMember(groupInstanceId, newMemberId, oldMemberId) completeAndScheduleNextHeartbeatExpiration(group, oldMember) responseCallback(JoinGroupResult( List.empty, @@ -1158,14 +1359,14 @@ class GroupCoordinator(val brokerId: Int, } private def removePendingMemberAndUpdateGroup(group: GroupMetadata, memberId: String): Unit = { - group.removePendingMember(memberId) + group.remove(memberId) if (group.is(PreparingRebalance)) { joinPurgatory.checkAndComplete(GroupKey(group.groupId)) } } - def tryCompleteJoin(group: GroupMetadata, forceComplete: () => Boolean) = { + def tryCompleteJoin(group: GroupMetadata, forceComplete: () => Boolean): Boolean = { group.inLock { if (group.hasAllMembersJoined) forceComplete() @@ -1181,8 +1382,8 @@ class GroupCoordinator(val brokerId: Int, s"who haven't joined: ${notYetRejoinedDynamicMembers.keySet}") notYetRejoinedDynamicMembers.values foreach { failedMember => - removeHeartbeatForLeavingMember(group, failedMember) group.remove(failedMember.memberId) + removeHeartbeatForLeavingMember(group, failedMember) } } @@ -1260,9 +1461,9 @@ class GroupCoordinator(val brokerId: Int, def shouldCompleteNonPendingHeartbeat(group: GroupMetadata, memberId: String): Boolean = { if (group.has(memberId)) { val member = group.get(memberId) - member.hasSatisfiedHeartbeat || member.isLeaving + member.hasSatisfiedHeartbeat } else { - info(s"Member id $memberId was not found in ${group.groupId} during heartbeat completion check") + debug(s"Member id $memberId was not found in ${group.groupId} during heartbeat completion check") true } } diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala index 5bb92bba1dacc..50328f0516300 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala @@ -143,9 +143,6 @@ private object GroupMetadata extends Logging { group.currentStateTimestamp = currentStateTimestamp members.foreach { member => group.add(member, null) - if (member.isStaticMember) { - group.addStaticMember(member.groupInstanceId, member.memberId) - } info(s"Loaded member $member in group $groupId with generation ${group.generationId}.") } group.subscribedTopics = group.computeSubscribedTopics() @@ -226,11 +223,10 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState def inLock[T](fun: => T): T = CoreUtils.inLock(lock)(fun) - def is(groupState: GroupState) = state == groupState - def not(groupState: GroupState) = state != groupState - def has(memberId: String) = members.contains(memberId) - def get(memberId: String) = members(memberId) - def size = members.size + def is(groupState: GroupState): Boolean = state == groupState + def has(memberId: String): Boolean = members.contains(memberId) + def get(memberId: String): MemberMetadata = members(memberId) + def size: Int = members.size def isLeader(memberId: String): Boolean = leaderId.contains(memberId) def leaderOrNull: String = leaderId.orNull @@ -239,6 +235,13 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState def isConsumerGroup: Boolean = protocolType.contains(ConsumerProtocol.PROTOCOL_TYPE) def add(member: MemberMetadata, callback: JoinCallback = null): Unit = { + member.groupInstanceId.foreach { instanceId => + if (staticMembers.contains(instanceId)) + throw new IllegalStateException(s"Static member with groupInstanceId=$instanceId " + + s"cannot be added to group $groupId since it is already a member") + staticMembers.put(instanceId, member.memberId) + } + if (members.isEmpty) this.protocolType = Some(member.protocolType) @@ -247,16 +250,20 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState if (leaderId.isEmpty) leaderId = Some(member.memberId) + members.put(member.memberId, member) - member.supportedProtocols.foreach{ case (protocol, _) => supportedProtocols(protocol) += 1 } + member.supportedProtocols.foreach { case (protocol, _) => supportedProtocols(protocol) += 1 } member.awaitingJoinCallback = callback + if (member.isAwaitingJoin) numMembersAwaitingJoin += 1 + + pendingMembers.remove(member.memberId) } def remove(memberId: String): Unit = { members.remove(memberId).foreach { member => - member.supportedProtocols.foreach{ case (protocol, _) => supportedProtocols(protocol) -= 1 } + member.supportedProtocols.foreach { case (protocol, _) => supportedProtocols(protocol) -= 1 } if (member.isAwaitingJoin) numMembersAwaitingJoin -= 1 @@ -265,6 +272,8 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState if (isLeader(memberId)) leaderId = members.keys.headOption + + pendingMembers.remove(memberId) } /** @@ -302,76 +311,72 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState * [For static members only]: Replace the old member id with the new one, * keep everything else unchanged and return the updated member. */ - def replaceGroupInstance(oldMemberId: String, - newMemberId: String, - groupInstanceId: Option[String]): MemberMetadata = { - if(groupInstanceId.isEmpty) { - throw new IllegalArgumentException(s"unexpected null group.instance.id in replaceGroupInstance") - } - val oldMember = members.remove(oldMemberId) + def replaceStaticMember( + groupInstanceId: String, + oldMemberId: String, + newMemberId: String + ): MemberMetadata = { + val memberMetadata = members.remove(oldMemberId) .getOrElse(throw new IllegalArgumentException(s"Cannot replace non-existing member id $oldMemberId")) // Fence potential duplicate member immediately if someone awaits join/sync callback. - maybeInvokeJoinCallback(oldMember, JoinGroupResult(oldMemberId, Errors.FENCED_INSTANCE_ID)) + maybeInvokeJoinCallback(memberMetadata, JoinGroupResult(oldMemberId, Errors.FENCED_INSTANCE_ID)) + maybeInvokeSyncCallback(memberMetadata, SyncGroupResult(Errors.FENCED_INSTANCE_ID)) - maybeInvokeSyncCallback(oldMember, SyncGroupResult(Errors.FENCED_INSTANCE_ID)) + memberMetadata.memberId = newMemberId + members.put(newMemberId, memberMetadata) - oldMember.memberId = newMemberId - members.put(newMemberId, oldMember) - - if (isLeader(oldMemberId)) + if (isLeader(oldMemberId)) { leaderId = Some(newMemberId) - addStaticMember(groupInstanceId, newMemberId) - oldMember - } - - def isPendingMember(memberId: String): Boolean = pendingMembers.contains(memberId) && !has(memberId) - - def addPendingMember(memberId: String) = pendingMembers.add(memberId) + } - def removePendingMember(memberId: String) = pendingMembers.remove(memberId) + staticMembers.put(groupInstanceId, newMemberId) + memberMetadata + } - def hasStaticMember(groupInstanceId: Option[String]) = groupInstanceId.isDefined && staticMembers.contains(groupInstanceId.get) + def isPendingMember(memberId: String): Boolean = pendingMembers.contains(memberId) - def getStaticMemberId(groupInstanceId: Option[String]) = { - if(groupInstanceId.isEmpty) { - throw new IllegalArgumentException(s"unexpected null group.instance.id in getStaticMemberId") + def addPendingMember(memberId: String): Boolean = { + if (has(memberId)) { + throw new IllegalStateException(s"Attempt to add pending member $memberId which is already " + + s"a stable member of the group") } - staticMembers(groupInstanceId.get) + pendingMembers.add(memberId) } - def addStaticMember(groupInstanceId: Option[String], newMemberId: String) = { - if(groupInstanceId.isEmpty) { - throw new IllegalArgumentException(s"unexpected null group.instance.id in addStaticMember") - } - staticMembers.put(groupInstanceId.get, newMemberId) + def hasStaticMember(groupInstanceId: String): Boolean = { + staticMembers.contains(groupInstanceId) } - def currentState = state + def currentStaticMemberId(groupInstanceId: String): Option[String] = { + staticMembers.get(groupInstanceId) + } + + def currentState: GroupState = state - def notYetRejoinedMembers = members.filter(!_._2.isAwaitingJoin).toMap + def notYetRejoinedMembers: Map[String, MemberMetadata] = members.filter(!_._2.isAwaitingJoin).toMap - def hasAllMembersJoined = members.size == numMembersAwaitingJoin && pendingMembers.isEmpty + def hasAllMembersJoined: Boolean = members.size == numMembersAwaitingJoin && pendingMembers.isEmpty - def allMembers = members.keySet + def allMembers: collection.Set[String] = members.keySet - def allStaticMembers = staticMembers.keySet + def allStaticMembers: collection.Set[String] = staticMembers.keySet // For testing only. - def allDynamicMembers = { + private[group] def allDynamicMembers: Set[String] = { val dynamicMemberSet = new mutable.HashSet[String] allMembers.foreach(memberId => dynamicMemberSet.add(memberId)) staticMembers.values.foreach(memberId => dynamicMemberSet.remove(memberId)) dynamicMemberSet.toSet } - def numPending = pendingMembers.size + def numPending: Int = pendingMembers.size def numAwaiting: Int = numMembersAwaitingJoin - def allMemberMetadata = members.values.toList + def allMemberMetadata: List[MemberMetadata] = members.values.toList - def rebalanceTimeoutMs = members.values.foldLeft(0) { (timeout, member) => + def rebalanceTimeoutMs: Int = members.values.foldLeft(0) { (timeout, member) => timeout.max(member.rebalanceTimeoutMs) } @@ -390,20 +395,14 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState * 1. given member is a known static member to group * 2. group stored member.id doesn't match with given member.id */ - def isStaticMemberFenced(memberId: String, - groupInstanceId: Option[String], - operation: String): Boolean = { - if (hasStaticMember(groupInstanceId) - && getStaticMemberId(groupInstanceId) != memberId) { - error(s"given member.id $memberId is identified as a known static member ${groupInstanceId.get}, " + - s"but not matching the expected member.id ${getStaticMemberId(groupInstanceId)} during $operation, will " + - s"respond with instance fenced error") - true - } else - false + def isStaticMemberFenced( + groupInstanceId: String, + memberId: String + ): Boolean = { + currentStaticMemberId(groupInstanceId).exists(_ != memberId) } - def canRebalance = PreparingRebalance.validPreviousStates.contains(state) + def canRebalance: Boolean = PreparingRebalance.validPreviousStates.contains(state) def transitionTo(groupState: GroupState): Unit = { assertValidTransition(groupState) @@ -430,7 +429,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState private def candidateProtocols: Set[String] = { // get the set of protocols that are commonly supported by all members val numMembers = members.size - supportedProtocols.filter(_._2 == numMembers).map(_._1).toSet + supportedProtocols.filter(_._2 == numMembers).keys.toSet } def supportsProtocols(memberProtocolType: String, memberProtocols: Set[String]): Boolean = { @@ -490,8 +489,8 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState def updateMember(member: MemberMetadata, protocols: List[(String, Array[Byte])], callback: JoinCallback): Unit = { - member.supportedProtocols.foreach{ case (protocol, _) => supportedProtocols(protocol) -= 1 } - protocols.foreach{ case (protocol, _) => supportedProtocols(protocol) += 1 } + member.supportedProtocols.foreach { case (protocol, _) => supportedProtocols(protocol) -= 1 } + protocols.foreach { case (protocol, _) => supportedProtocols(protocol) += 1 } member.supportedProtocols = protocols if (callback != null && !member.isAwaitingJoin) { @@ -762,7 +761,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState expiredOffsets } - def allOffsets = offsets.map { case (topicPartition, commitRecordMetadataAndOffset) => + def allOffsets: Map[TopicPartition, OffsetAndMetadata] = offsets.map { case (topicPartition, commitRecordMetadataAndOffset) => (topicPartition, commitRecordMetadataAndOffset.offsetAndMetadata) }.toMap @@ -771,9 +770,9 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState // visible for testing private[group] def offsetWithRecordMetadata(topicPartition: TopicPartition): Option[CommitRecordMetadataAndOffset] = offsets.get(topicPartition) - def numOffsets = offsets.size + def numOffsets: Int = offsets.size - def hasOffsets = offsets.nonEmpty || pendingOffsetCommits.nonEmpty || pendingTransactionalOffsetCommits.nonEmpty + def hasOffsets: Boolean = offsets.nonEmpty || pendingOffsetCommits.nonEmpty || pendingTransactionalOffsetCommits.nonEmpty private def assertValidTransition(targetState: GroupState): Unit = { if (!targetState.validPreviousStates.contains(state)) diff --git a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala index 73ec0e14ec36e..514dbfbd3887d 100644 --- a/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala +++ b/core/src/main/scala/kafka/coordinator/group/MemberMetadata.scala @@ -63,9 +63,8 @@ private[group] class MemberMetadata(var memberId: String, var supportedProtocols: List[(String, Array[Byte])], var assignment: Array[Byte] = Array.empty[Byte]) { - var awaitingJoinCallback: JoinGroupResult => Unit = null - var awaitingSyncCallback: SyncGroupResult => Unit = null - var isLeaving: Boolean = false + var awaitingJoinCallback: JoinGroupResult => Unit = _ + var awaitingSyncCallback: SyncGroupResult => Unit = _ var isNew: Boolean = false def isStaticMember: Boolean = groupInstanceId.isDefined @@ -77,8 +76,8 @@ private[group] class MemberMetadata(var memberId: String, // delayed heartbeat can be completed. var heartbeatSatisfied: Boolean = false - def isAwaitingJoin = awaitingJoinCallback != null - def isAwaitingSync = awaitingSyncCallback != null + def isAwaitingJoin: Boolean = awaitingJoinCallback != null + def isAwaitingSync: Boolean = awaitingSyncCallback != null /** * Get metadata corresponding to the provided protocol. diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index 3e1f82074edac..39826894c8f1a 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -78,9 +78,9 @@ class GroupCoordinatorTest { private val protocolType = "consumer" private val protocolName = "range" private val memberId = "memberId" - private val groupInstanceId = Some("groupInstanceId") - private val leaderInstanceId = Some("leader") - private val followerInstanceId = Some("follower") + private val groupInstanceId = "groupInstanceId" + private val leaderInstanceId = "leader" + private val followerInstanceId = "follower" private val invalidMemberId = "invalidMember" private val metadata = Array[Byte]() private val protocols = List((protocolName, metadata)) @@ -770,13 +770,13 @@ class GroupCoordinatorTest { timer.advanceClock(1) // Old follower rejoins group will be matching current member.id. val oldFollowerJoinGroupFuture = - sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = followerInstanceId) + sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = Some(followerInstanceId)) EasyMock.reset(replicaManager) timer.advanceClock(1) // Duplicate follower joins group with unknown member id will trigger member.id replacement. val duplicateFollowerJoinFuture = - sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = followerInstanceId) + sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = Some(followerInstanceId)) timer.advanceClock(1) // Old member shall be fenced immediately upon duplicate follower joins. @@ -785,7 +785,6 @@ class GroupCoordinatorTest { Errors.FENCED_INSTANCE_ID, -1, Set.empty, - groupId, PreparingRebalance, None) verifyDelayedTaskNotCompleted(duplicateFollowerJoinFuture) @@ -797,7 +796,7 @@ class GroupCoordinatorTest { // Known leader rejoins will trigger rebalance. val leaderJoinGroupFuture = - sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, protocols, groupInstanceId = leaderInstanceId) + sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, protocols, groupInstanceId = Some(leaderInstanceId)) timer.advanceClock(1) assertTrue(getGroup(groupId).is(PreparingRebalance)) @@ -805,7 +804,7 @@ class GroupCoordinatorTest { timer.advanceClock(1) // Old follower rejoins group will match current member.id. val oldFollowerJoinGroupFuture = - sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = followerInstanceId) + sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = Some(followerInstanceId)) timer.advanceClock(1) val leaderJoinGroupResult = Await.result(leaderJoinGroupFuture, Duration(1, TimeUnit.MILLISECONDS)) @@ -813,7 +812,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 1, Set(leaderInstanceId, followerInstanceId), - groupId, CompletingRebalance, Some(protocolType)) assertEquals(rebalanceResult.leaderId, leaderJoinGroupResult.memberId) @@ -825,7 +823,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 1, Set.empty, - groupId, CompletingRebalance, Some(protocolType), expectedLeaderId = leaderJoinGroupResult.memberId) @@ -838,11 +835,11 @@ class GroupCoordinatorTest { // will return fenced exception while broker replaces the member identity with the duplicate follower joins. EasyMock.reset(replicaManager) val oldFollowerSyncGroupFuture = sendSyncGroupFollower(groupId, oldFollowerJoinGroupResult.generationId, - oldFollowerJoinGroupResult.memberId, Some(protocolType), Some(protocolName), followerInstanceId) + oldFollowerJoinGroupResult.memberId, Some(protocolType), Some(protocolName), Some(followerInstanceId)) EasyMock.reset(replicaManager) val duplicateFollowerJoinFuture = - sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = followerInstanceId) + sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = Some(followerInstanceId)) timer.advanceClock(1) val oldFollowerSyncGroupResult = Await.result(oldFollowerSyncGroupFuture, Duration(1, TimeUnit.MILLISECONDS)) @@ -857,7 +854,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 2, Set(followerInstanceId), // this follower will become the new leader, and hence it would have the member list - groupId, CompletingRebalance, Some(protocolType), expectedLeaderId = duplicateFollowerJoinGroupResult.memberId) @@ -870,27 +866,26 @@ class GroupCoordinatorTest { // Known leader rejoins will trigger rebalance. val leaderJoinGroupFuture = - sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, protocols, groupInstanceId = leaderInstanceId) + sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, protocols, groupInstanceId = Some(leaderInstanceId)) timer.advanceClock(1) assertTrue(getGroup(groupId).is(PreparingRebalance)) EasyMock.reset(replicaManager) // Duplicate follower joins group will trigger member.id replacement. val duplicateFollowerJoinGroupFuture = - sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = followerInstanceId) + sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocols, groupInstanceId = Some(followerInstanceId)) EasyMock.reset(replicaManager) timer.advanceClock(1) // Old follower rejoins group will fail because member.id already updated. val oldFollowerJoinGroupFuture = - sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = followerInstanceId) + sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, groupInstanceId = Some(followerInstanceId)) val leaderRejoinGroupResult = Await.result(leaderJoinGroupFuture, Duration(1, TimeUnit.MILLISECONDS)) checkJoinGroupResult(leaderRejoinGroupResult, Errors.NONE, rebalanceResult.generation + 1, Set(leaderInstanceId, followerInstanceId), - groupId, CompletingRebalance, Some(protocolType)) @@ -899,7 +894,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 1, Set.empty, - groupId, CompletingRebalance, Some(protocolType)) assertNotEquals(rebalanceResult.followerId, duplicateFollowerJoinGroupResult.memberId) @@ -909,7 +903,6 @@ class GroupCoordinatorTest { Errors.FENCED_INSTANCE_ID, -1, Set.empty, - groupId, CompletingRebalance, None) } @@ -922,14 +915,15 @@ class GroupCoordinatorTest { EasyMock.reset(replicaManager) val assignedMemberId = joinGroupResult.memberId // The second join group should return immediately since we are using the same metadata during CompletingRebalance. - val rejoinResponseFuture = sendJoinGroup(groupId, assignedMemberId, protocolType, protocols, groupInstanceId) + val rejoinResponseFuture = sendJoinGroup(groupId, assignedMemberId, protocolType, protocols, Some(groupInstanceId)) timer.advanceClock(1) joinGroupResult = Await.result(rejoinResponseFuture, Duration(1, TimeUnit.MILLISECONDS)) assertEquals(Errors.NONE, joinGroupResult.error) assertTrue(getGroup(groupId).is(CompletingRebalance)) EasyMock.reset(replicaManager) - val syncGroupFuture = sendSyncGroupLeader(groupId, joinGroupResult.generationId, assignedMemberId, Some(protocolType), Some(protocolName), groupInstanceId, Map(assignedMemberId -> Array[Byte]())) + val syncGroupFuture = sendSyncGroupLeader(groupId, joinGroupResult.generationId, assignedMemberId, + Some(protocolType), Some(protocolName), Some(groupInstanceId), Map(assignedMemberId -> Array[Byte]())) timer.advanceClock(1) val syncGroupResult = Await.result(syncGroupFuture, Duration(1, TimeUnit.MILLISECONDS)) assertEquals(Errors.NONE, syncGroupResult.error) @@ -941,13 +935,13 @@ class GroupCoordinatorTest { val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) // A static leader rejoin with unknown id will not trigger rebalance, and no assignment will be returned. - val joinGroupResult = staticJoinGroupWithPersistence(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocolSuperset, clockAdvance = 1) + val joinGroupResult = staticJoinGroupWithPersistence(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, + leaderInstanceId, protocolType, protocolSuperset, clockAdvance = 1) checkJoinGroupResult(joinGroupResult, Errors.NONE, rebalanceResult.generation, // The group should be at the same generation Set.empty, - groupId, Stable, Some(protocolType), rebalanceResult.leaderId) @@ -958,7 +952,8 @@ class GroupCoordinatorTest { EasyMock.reset(replicaManager) // Old leader will get fenced. - val oldLeaderSyncGroupResult = syncGroupLeader(groupId, rebalanceResult.generation, rebalanceResult.leaderId, Map.empty, None, None, leaderInstanceId) + val oldLeaderSyncGroupResult = syncGroupLeader(groupId, rebalanceResult.generation, rebalanceResult.leaderId, + Map.empty, None, None, Some(leaderInstanceId)) assertEquals(Errors.FENCED_INSTANCE_ID, oldLeaderSyncGroupResult.error) // Calling sync on old leader.id will fail because that leader.id is no longer valid and replaced. @@ -981,7 +976,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 1, // The group has promoted to the new generation. Set(leaderInstanceId), - groupId, CompletingRebalance, Some(protocolType), rebalanceResult.leaderId, @@ -1025,7 +1019,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 1, // The group has promoted to the new generation, and leader has changed because old one times out. Set(leaderInstanceId, followerInstanceId), - groupId, CompletingRebalance, Some(protocolType), rebalanceResult.followerId, @@ -1048,7 +1041,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 1, Set(leaderInstanceId, followerInstanceId), - groupId, CompletingRebalance, Some(protocolType)) @@ -1064,13 +1056,13 @@ class GroupCoordinatorTest { val selectedProtocol = getGroup(groupId).selectProtocol val newProtocols = List((selectedProtocol, metadata)) // Timeout old leader in the meantime. - val joinGroupResult = staticJoinGroupWithPersistence(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, followerInstanceId, protocolType, newProtocols, clockAdvance = 1, appendRecordError = Errors.MESSAGE_TOO_LARGE) + val joinGroupResult = staticJoinGroupWithPersistence(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, + followerInstanceId, protocolType, newProtocols, clockAdvance = 1, appendRecordError = Errors.MESSAGE_TOO_LARGE) checkJoinGroupResult(joinGroupResult, Errors.UNKNOWN_SERVER_ERROR, rebalanceResult.generation, Set.empty, - groupId, Stable, Some(protocolType)) @@ -1082,7 +1074,8 @@ class GroupCoordinatorTest { EasyMock.reset(replicaManager) // Sync with old member id will also not fail because the member id is not updated because of persistence failure - val syncGroupWithOldMemberIdResult = syncGroupFollower(groupId, rebalanceResult.generation, rebalanceResult.followerId, None, None, followerInstanceId) + val syncGroupWithOldMemberIdResult = syncGroupFollower(groupId, rebalanceResult.generation, + rebalanceResult.followerId, None, None, Some(followerInstanceId)) assertEquals(Errors.NONE, syncGroupWithOldMemberIdResult.error) } @@ -1095,13 +1088,13 @@ class GroupCoordinatorTest { val selectedProtocol = getGroup(groupId).selectProtocol val newProtocols = List((selectedProtocol, metadata)) // Timeout old leader in the meantime. - val joinGroupResult = staticJoinGroupWithPersistence(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, followerInstanceId, protocolType, newProtocols, clockAdvance = 1) + val joinGroupResult = staticJoinGroupWithPersistence(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, + followerInstanceId, protocolType, newProtocols, clockAdvance = 1) checkJoinGroupResult(joinGroupResult, Errors.NONE, rebalanceResult.generation, Set.empty, - groupId, Stable, Some(protocolType)) @@ -1113,11 +1106,13 @@ class GroupCoordinatorTest { EasyMock.reset(replicaManager) // Sync with old member id will fail because the member id is updated - val syncGroupWithOldMemberIdResult = syncGroupFollower(groupId, rebalanceResult.generation, rebalanceResult.followerId, None, None, followerInstanceId) + val syncGroupWithOldMemberIdResult = syncGroupFollower(groupId, rebalanceResult.generation, + rebalanceResult.followerId, None, None, Some(followerInstanceId)) assertEquals(Errors.FENCED_INSTANCE_ID, syncGroupWithOldMemberIdResult.error) EasyMock.reset(replicaManager) - val syncGroupWithNewMemberIdResult = syncGroupFollower(groupId, rebalanceResult.generation, joinGroupResult.memberId, None, None, followerInstanceId) + val syncGroupWithNewMemberIdResult = syncGroupFollower(groupId, rebalanceResult.generation, + joinGroupResult.memberId, None, None, Some(followerInstanceId)) assertEquals(Errors.NONE, syncGroupWithNewMemberIdResult.error) assertEquals(rebalanceResult.followerAssignment, syncGroupWithNewMemberIdResult.memberAssignment) } @@ -1127,10 +1122,12 @@ class GroupCoordinatorTest { val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) // A static leader rejoin with known member id will trigger rebalance. - val leaderRejoinGroupFuture = sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, protocolSuperset, leaderInstanceId) + val leaderRejoinGroupFuture = sendJoinGroup(groupId, rebalanceResult.leaderId, protocolType, + protocolSuperset, Some(leaderInstanceId)) // Rebalance complete immediately after follower rejoin. EasyMock.reset(replicaManager) - val followerRejoinWithFuture = sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocolSuperset, followerInstanceId) + val followerRejoinWithFuture = sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, + protocolSuperset, Some(followerInstanceId)) timer.advanceClock(1) @@ -1139,7 +1136,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 1, // The group has promoted to the new generation. Set(leaderInstanceId, followerInstanceId), - groupId, CompletingRebalance, Some(protocolType), rebalanceResult.leaderId, @@ -1149,7 +1145,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 1, // The group has promoted to the new generation. Set.empty, - groupId, CompletingRebalance, Some(protocolType), rebalanceResult.leaderId, @@ -1157,7 +1152,8 @@ class GroupCoordinatorTest { EasyMock.reset(replicaManager) // The follower protocol changed from protocolSuperset to general protocols. - val followerRejoinWithProtocolChangeFuture = sendJoinGroup(groupId, rebalanceResult.followerId, protocolType, protocols, followerInstanceId) + val followerRejoinWithProtocolChangeFuture = sendJoinGroup(groupId, rebalanceResult.followerId, + protocolType, protocols, Some(followerInstanceId)) // The group will transit to PreparingRebalance due to protocol change from follower. assertTrue(getGroup(groupId).is(PreparingRebalance)) @@ -1166,7 +1162,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation + 2, // The group has promoted to the new generation. Set(followerInstanceId), - groupId, CompletingRebalance, Some(protocolType), rebalanceResult.followerId, @@ -1186,7 +1181,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation, // The group has no change. Set.empty, - groupId, Stable, Some(protocolType)) @@ -1211,7 +1205,6 @@ class GroupCoordinatorTest { Errors.NONE, rebalanceResult.generation, // The group has no change. Set.empty, - groupId, Stable, Some(protocolType), rebalanceResult.leaderId, @@ -1238,7 +1231,8 @@ class GroupCoordinatorTest { def staticMemberSyncAsLeaderWithInvalidMemberId(): Unit = { val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) - val syncGroupResult = syncGroupLeader(groupId, rebalanceResult.generation, "invalid", Map.empty, None, None, leaderInstanceId) + val syncGroupResult = syncGroupLeader(groupId, rebalanceResult.generation, "invalid", + Map.empty, None, None, Some(leaderInstanceId)) assertEquals(Errors.FENCED_INSTANCE_ID, syncGroupResult.error) } @@ -1254,7 +1248,7 @@ class GroupCoordinatorTest { assertEquals(Errors.NONE, validHeartbeatResult) EasyMock.reset(replicaManager) - val invalidHeartbeatResult = heartbeat(groupId, invalidMemberId, rebalanceResult.generation, leaderInstanceId) + val invalidHeartbeatResult = heartbeat(groupId, invalidMemberId, rebalanceResult.generation, Some(leaderInstanceId)) assertEquals(Errors.FENCED_INSTANCE_ID, invalidHeartbeatResult) } @@ -1268,7 +1262,7 @@ class GroupCoordinatorTest { EasyMock.reset(replicaManager) val joinGroupResult = staticJoinGroupWithPersistence(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId, protocolType, protocols, clockAdvance = timeAdvance) - assertTrue(joinGroupResult.memberId.startsWith(leaderInstanceId.get)) + assertTrue(joinGroupResult.memberId.startsWith(leaderInstanceId)) assertNotEquals(lastMemberId, joinGroupResult.memberId) lastMemberId = joinGroupResult.memberId } @@ -1301,7 +1295,8 @@ class GroupCoordinatorTest { assertEquals(Errors.NONE, validOffsetCommitResult(tp)) EasyMock.reset(replicaManager) - val invalidOffsetCommitResult = commitOffsets(groupId, invalidMemberId, rebalanceResult.generation, Map(tp -> offset), leaderInstanceId) + val invalidOffsetCommitResult = commitOffsets(groupId, invalidMemberId, rebalanceResult.generation, + Map(tp -> offset), Some(leaderInstanceId)) assertEquals(Errors.FENCED_INSTANCE_ID, invalidOffsetCommitResult(tp)) } @@ -1309,45 +1304,12 @@ class GroupCoordinatorTest { def staticMemberJoinWithUnknownInstanceIdAndKnownMemberId(): Unit = { val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) - val joinGroupResult = staticJoinGroup(groupId, rebalanceResult.leaderId, Some("unknown_instance"), protocolType, protocolSuperset, clockAdvance = 1) + val joinGroupResult = staticJoinGroup(groupId, rebalanceResult.leaderId, "unknown_instance", + protocolType, protocolSuperset, clockAdvance = 1) assertEquals(Errors.UNKNOWN_MEMBER_ID, joinGroupResult.error) } - @Test - def staticMemberJoinWithIllegalStateAsPendingMember(): Unit = { - val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) - val group = groupCoordinator.groupManager.getGroup(groupId).get - group.addPendingMember(rebalanceResult.followerId) - group.remove(rebalanceResult.followerId) - EasyMock.reset(replicaManager) - - // Illegal state exception shall trigger since follower id resides in pending member bucket. - val expectedException = assertThrows(classOf[IllegalStateException], - () => staticJoinGroup(groupId, rebalanceResult.followerId, followerInstanceId, protocolType, protocolSuperset, clockAdvance = 1)) - - val message = expectedException.getMessage - assertTrue(message.contains(rebalanceResult.followerId)) - assertTrue(message.contains(followerInstanceId.get)) - } - - @Test - def staticMemberLeaveWithIllegalStateAsPendingMember(): Unit = { - val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) - val group = groupCoordinator.groupManager.getGroup(groupId).get - group.addPendingMember(rebalanceResult.followerId) - group.remove(rebalanceResult.followerId) - EasyMock.reset(replicaManager) - - // Illegal state exception shall trigger since follower id resides in pending member bucket. - val expectedException = assertThrows(classOf[IllegalStateException], - () => singleLeaveGroup(groupId, rebalanceResult.followerId, followerInstanceId)) - - val message = expectedException.getMessage - assertTrue(message.contains(rebalanceResult.followerId)) - assertTrue(message.contains(followerInstanceId.get)) - } - @Test def staticMemberReJoinWithIllegalStateAsUnknownMember(): Unit = { staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) @@ -1363,23 +1325,7 @@ class GroupCoordinatorTest { val message = expectedException.getMessage assertTrue(message.contains(group.groupId)) - assertTrue(message.contains(followerInstanceId.get)) - } - - @Test - def staticMemberReJoinWithIllegalArgumentAsMissingOldMember(): Unit = { - staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) - val group = groupCoordinator.groupManager.getGroup(groupId).get - val invalidMemberId = "invalid_member_id" - group.addStaticMember(followerInstanceId, invalidMemberId) - EasyMock.reset(replicaManager) - - // Illegal state exception shall trigger since follower corresponding id is not defined in member list. - val expectedException = assertThrows(classOf[IllegalArgumentException], - () => staticJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, followerInstanceId, protocolType, protocolSuperset, clockAdvance = 1)) - - val message = expectedException.getMessage - assertTrue(message.contains(invalidMemberId)) + assertTrue(message.contains(followerInstanceId)) } @Test @@ -1405,7 +1351,6 @@ class GroupCoordinatorTest { Errors.NONE, 3, Set(leaderInstanceId), - groupId, CompletingRebalance, Some(protocolType) ) @@ -1431,7 +1376,7 @@ class GroupCoordinatorTest { assertEquals(Set(rebalanceResult.leaderId, rebalanceResult.followerId, dynamicJoinResult.memberId), getGroup(groupId).allMembers) - assertEquals(Set(leaderInstanceId.get, followerInstanceId.get), + assertEquals(Set(leaderInstanceId, followerInstanceId), getGroup(groupId).allStaticMembers) assertEquals(Set(dynamicJoinResult.memberId), getGroup(groupId).allDynamicMembers) @@ -1455,11 +1400,12 @@ class GroupCoordinatorTest { // Increase session timeout so that the follower won't be evicted when rebalance timeout is reached. val initialRebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId, sessionTimeout = DefaultRebalanceTimeout * 2) - val newMemberInstanceId = Some("newMember") + val newMemberInstanceId = "newMember" val leaderId = initialRebalanceResult.leaderId - val newMemberJoinGroupFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocolSuperset, newMemberInstanceId) + val newMemberJoinGroupFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, + protocolSuperset, Some(newMemberInstanceId)) assertGroupState(groupState = PreparingRebalance) EasyMock.reset(replicaManager) @@ -1468,7 +1414,6 @@ class GroupCoordinatorTest { Errors.NONE, initialRebalanceResult.generation + 1, Set(leaderInstanceId, followerInstanceId, newMemberInstanceId), - groupId, CompletingRebalance, Some(protocolType), expectedLeaderId = leaderId, @@ -1480,7 +1425,6 @@ class GroupCoordinatorTest { Errors.NONE, initialRebalanceResult.generation + 1, Set.empty, - groupId, CompletingRebalance, Some(protocolType), expectedLeaderId = leaderId) @@ -1491,9 +1435,10 @@ class GroupCoordinatorTest { // Increase session timeout so that the leader won't be evicted when rebalance timeout is reached. val initialRebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId, sessionTimeout = DefaultRebalanceTimeout * 2) - val newMemberInstanceId = Some("newMember") + val newMemberInstanceId = "newMember" - val newMemberJoinGroupFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocolSuperset, newMemberInstanceId) + val newMemberJoinGroupFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, + protocolSuperset, Some(newMemberInstanceId)) timer.advanceClock(1) assertGroupState(groupState = PreparingRebalance) @@ -1510,7 +1455,6 @@ class GroupCoordinatorTest { Errors.NONE, initialRebalanceResult.generation + 1, Set(leaderInstanceId, followerInstanceId, newMemberInstanceId), - groupId, CompletingRebalance, Some(protocolType)) @@ -1518,7 +1462,6 @@ class GroupCoordinatorTest { Errors.NONE, initialRebalanceResult.generation + 1, Set.empty, - groupId, CompletingRebalance, Some(protocolType), expectedLeaderId = newLeaderResult.memberId) @@ -1529,7 +1472,7 @@ class GroupCoordinatorTest { // JoinGroup(leader) EasyMock.reset(replicaManager) val leaderResponseFuture = sendJoinGroup(groupId, "fake-id", protocolType, - protocolSuperset, leaderInstanceId, DefaultSessionTimeout) + protocolSuperset, Some(leaderInstanceId), DefaultSessionTimeout) // The Protocol Type is None when there is an error val leaderJoinGroupResult = await(leaderResponseFuture, 1) @@ -1542,12 +1485,12 @@ class GroupCoordinatorTest { // JoinGroup(leader) EasyMock.reset(replicaManager) val leaderResponseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, - protocolSuperset, leaderInstanceId, DefaultSessionTimeout) + protocolSuperset, Some(leaderInstanceId), DefaultSessionTimeout) // JoinGroup(follower) EasyMock.reset(replicaManager) val followerResponseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, - protocolSuperset, followerInstanceId, DefaultSessionTimeout) + protocolSuperset, Some(followerInstanceId), DefaultSessionTimeout) timer.advanceClock(GroupInitialRebalanceDelay + 1) timer.advanceClock(DefaultRebalanceTimeout + 1) @@ -1595,12 +1538,12 @@ class GroupCoordinatorTest { // JoinGroup(leader) with the Protocol Type of the group EasyMock.reset(replicaManager) val leaderResponseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, this.protocolType, - protocolSuperset, leaderInstanceId, DefaultSessionTimeout) + protocolSuperset, Some(leaderInstanceId), DefaultSessionTimeout) // JoinGroup(follower) with the Protocol Type of the group EasyMock.reset(replicaManager) val followerResponseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, this.protocolType, - protocolSuperset, followerInstanceId, DefaultSessionTimeout) + protocolSuperset, Some(followerInstanceId), DefaultSessionTimeout) timer.advanceClock(GroupInitialRebalanceDelay + 1) timer.advanceClock(DefaultRebalanceTimeout + 1) @@ -1641,14 +1584,16 @@ class GroupCoordinatorTest { * - follower id * - follower assignment */ - private def staticMembersJoinAndRebalance(leaderInstanceId: Option[String], - followerInstanceId: Option[String], + private def staticMembersJoinAndRebalance(leaderInstanceId: String, + followerInstanceId: String, sessionTimeout: Int = DefaultSessionTimeout): RebalanceResult = { EasyMock.reset(replicaManager) - val leaderResponseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocolSuperset, leaderInstanceId, sessionTimeout) + val leaderResponseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, + protocolSuperset, Some(leaderInstanceId), sessionTimeout) EasyMock.reset(replicaManager) - val followerResponseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, protocolSuperset, followerInstanceId, sessionTimeout) + val followerResponseFuture = sendJoinGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, protocolType, + protocolSuperset, Some(followerInstanceId), sessionTimeout) // The goal for two timer advance is to let first group initial join complete and set newMemberAdded flag to false. Next advance is // to trigger the rebalance as needed for follower delayed join. One large time advance won't help because we could only populate one // delayed join from purgatory and the new delayed op is created at that time and never be triggered. @@ -1687,8 +1632,7 @@ class GroupCoordinatorTest { private def checkJoinGroupResult(joinGroupResult: JoinGroupResult, expectedError: Errors, expectedGeneration: Int, - expectedGroupInstanceIds: Set[Option[String]], - groupId: String, + expectedGroupInstanceIds: Set[String], expectedGroupState: GroupState, expectedProtocolType: Option[String], expectedLeaderId: String = JoinGroupRequest.UNKNOWN_MEMBER_ID, @@ -1696,7 +1640,7 @@ class GroupCoordinatorTest { assertEquals(expectedError, joinGroupResult.error) assertEquals(expectedGeneration, joinGroupResult.generationId) assertEquals(expectedGroupInstanceIds.size, joinGroupResult.members.size) - val resultedGroupInstanceIds = joinGroupResult.members.map(member => Some(member.groupInstanceId())).toSet + val resultedGroupInstanceIds = joinGroupResult.members.map(member => member.groupInstanceId).toSet assertEquals(expectedGroupInstanceIds, resultedGroupInstanceIds) assertGroupState(groupState = expectedGroupState) assertEquals(expectedProtocolType, joinGroupResult.protocolType) @@ -1737,7 +1681,7 @@ class GroupCoordinatorTest { val memberId = "memberId" val group = new GroupMetadata(groupId, Empty, new MockTime()) - val member = new MemberMetadata(memberId, groupInstanceId, + val member = new MemberMetadata(memberId, Some(groupInstanceId), ClientId, ClientHost, DefaultRebalanceTimeout, DefaultSessionTimeout, protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) @@ -2963,15 +2907,15 @@ class GroupCoordinatorTest { val rebalanceResult = staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) val leaderNoMemberIdCommitOffsetResult = commitTransactionalOffsets(groupId, producerId, producerEpoch, - Map(tp -> offset), memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID, groupInstanceId = leaderInstanceId) - assertEquals(Errors.FENCED_INSTANCE_ID, leaderNoMemberIdCommitOffsetResult (tp)) + Map(tp -> offset), memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID, groupInstanceId = Some(leaderInstanceId)) + assertEquals(Errors.FENCED_INSTANCE_ID, leaderNoMemberIdCommitOffsetResult(tp)) val leaderInvalidMemberIdCommitOffsetResult = commitTransactionalOffsets(groupId, producerId, producerEpoch, - Map(tp -> offset), memberId = "invalid-member", groupInstanceId = leaderInstanceId) + Map(tp -> offset), memberId = "invalid-member", groupInstanceId = Some(leaderInstanceId)) assertEquals(Errors.FENCED_INSTANCE_ID, leaderInvalidMemberIdCommitOffsetResult (tp)) val leaderCommitOffsetResult = commitTransactionalOffsets(groupId, producerId, producerEpoch, - Map(tp -> offset), rebalanceResult.leaderId, leaderInstanceId) + Map(tp -> offset), rebalanceResult.leaderId, Some(leaderInstanceId), rebalanceResult.generation) assertEquals(Errors.NONE, leaderCommitOffsetResult (tp)) } @@ -3003,10 +2947,11 @@ class GroupCoordinatorTest { val joinGroupError = joinGroupResult.error assertEquals(Errors.NONE, joinGroupError) + EasyMock.reset(replicaManager) val assignedConsumerId = joinGroupResult.memberId val leaderCommitOffsetResult = commitTransactionalOffsets(groupId, producerId, producerEpoch, - Map(tp -> offset), assignedConsumerId) + Map(tp -> offset), assignedConsumerId, generationId = joinGroupResult.generationId) assertEquals(Errors.NONE, leaderCommitOffsetResult (tp)) } @@ -3027,7 +2972,7 @@ class GroupCoordinatorTest { val initialGenerationId = joinGroupResult.generationId val illegalGenerationCommitOffsetResult = commitTransactionalOffsets(groupId, producerId, producerEpoch, Map(tp -> offset), memberId = assignedConsumerId, generationId = initialGenerationId + 5) - assertEquals(Errors.ILLEGAL_GENERATION, illegalGenerationCommitOffsetResult (tp)) + assertEquals(Errors.ILLEGAL_GENERATION, illegalGenerationCommitOffsetResult(tp)) } @Test @@ -3159,7 +3104,7 @@ class GroupCoordinatorTest { assertEquals(Errors.NONE, joinGroupResult.error) EasyMock.reset(replicaManager) - val leaveGroupResults = singleLeaveGroup(groupId, "some_member", leaderInstanceId) + val leaveGroupResults = singleLeaveGroup(groupId, "some_member", Some(leaderInstanceId)) verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.FENCED_INSTANCE_ID)) } @@ -3170,7 +3115,7 @@ class GroupCoordinatorTest { EasyMock.reset(replicaManager) // Having unknown member id will not affect the request processing. - val leaveGroupResults = singleLeaveGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, leaderInstanceId) + val leaveGroupResults = singleLeaveGroup(groupId, JoinGroupRequest.UNKNOWN_MEMBER_ID, Some(leaderInstanceId)) verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.NONE)) } @@ -3179,7 +3124,7 @@ class GroupCoordinatorTest { staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity() - .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity().setGroupInstanceId(followerInstanceId.get))) + .setGroupInstanceId(leaderInstanceId), new MemberIdentity().setGroupInstanceId(followerInstanceId))) verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.NONE, Errors.NONE)) } @@ -3189,7 +3134,7 @@ class GroupCoordinatorTest { staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) val leaveGroupResults = batchLeaveGroup("invalid-group", List(new MemberIdentity() - .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity().setGroupInstanceId(followerInstanceId.get))) + .setGroupInstanceId(leaderInstanceId), new MemberIdentity().setGroupInstanceId(followerInstanceId))) verifyLeaveGroupResult(leaveGroupResults, Errors.NOT_COORDINATOR) } @@ -3197,7 +3142,7 @@ class GroupCoordinatorTest { @Test def testStaticMembersUnknownGroupBatchLeaveGroup(): Unit = { val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity() - .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity().setGroupInstanceId(followerInstanceId.get))) + .setGroupInstanceId(leaderInstanceId), new MemberIdentity().setGroupInstanceId(followerInstanceId))) verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID, Errors.UNKNOWN_MEMBER_ID)) } @@ -3207,8 +3152,8 @@ class GroupCoordinatorTest { staticMembersJoinAndRebalance(leaderInstanceId, followerInstanceId) val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity() - .setGroupInstanceId(leaderInstanceId.get), new MemberIdentity() - .setGroupInstanceId(followerInstanceId.get) + .setGroupInstanceId(leaderInstanceId), new MemberIdentity() + .setGroupInstanceId(followerInstanceId) .setMemberId("invalid-member"))) verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.NONE, Errors.FENCED_INSTANCE_ID)) @@ -3220,7 +3165,7 @@ class GroupCoordinatorTest { val leaveGroupResults = batchLeaveGroup(groupId, List(new MemberIdentity() .setGroupInstanceId("unknown-instance"), new MemberIdentity() - .setGroupInstanceId(followerInstanceId.get))) + .setGroupInstanceId(followerInstanceId))) verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID, Errors.NONE)) } @@ -3237,25 +3182,6 @@ class GroupCoordinatorTest { verifyLeaveGroupResult(leaveGroupResults, Errors.NONE, List(Errors.UNKNOWN_MEMBER_ID, Errors.NONE)) } - @Test - def testPendingMemberWithUnexpectedInstanceIdBatchLeaveGroup(): Unit = { - val pendingMember = setupGroupWithPendingMember() - - EasyMock.reset(replicaManager) - - // Bypass the FENCED_INSTANCE_ID check by defining pending member as a static member. - val instanceId = "instanceId" - val pendingMemberId = pendingMember.memberId - getGroup(groupId).addStaticMember(Option(instanceId), pendingMemberId) - val expectedException = assertThrows(classOf[IllegalStateException], - () => batchLeaveGroup(groupId, List(new MemberIdentity().setGroupInstanceId("unknown-instance"), - new MemberIdentity().setGroupInstanceId(instanceId).setMemberId(pendingMemberId)))) - - val message = expectedException.getMessage - assertTrue(message.contains(instanceId)) - assertTrue(message.contains(pendingMemberId)) - } - @Test def testListGroupsIncludesStableGroups(): Unit = { val memberId = JoinGroupRequest.UNKNOWN_MEMBER_ID @@ -3385,7 +3311,7 @@ class GroupCoordinatorTest { assertEquals(protocolType, summary.protocolType) assertEquals("range", summary.protocol) assertEquals(List(assignedMemberId), summary.members.map(_.memberId)) - assertEquals(List(leaderInstanceId), summary.members.map(_.groupInstanceId)) + assertEquals(List(leaderInstanceId), summary.members.flatMap(_.groupInstanceId)) } @Test @@ -3847,7 +3773,7 @@ class GroupCoordinatorTest { memberId: String, protocolType: String, protocols: List[(String, Array[Byte])], - groupInstanceId: Option[String], + groupInstanceId: String, sessionTimeout: Int, rebalanceTimeout: Int, appendRecordError: Errors, @@ -3872,7 +3798,7 @@ class GroupCoordinatorTest { EasyMock.expect(replicaManager.getMagic(EasyMock.anyObject())).andReturn(Some(RecordBatch.MAGIC_VALUE_V1)).anyTimes() EasyMock.replay(replicaManager) - groupCoordinator.handleJoinGroup(groupId, memberId, groupInstanceId, + groupCoordinator.handleJoinGroup(groupId, memberId, Some(groupInstanceId), requireKnownMemberId, "clientId", "clientHost", rebalanceTimeout, sessionTimeout, protocolType, protocols, responseCallback) responseFuture } @@ -3950,13 +3876,13 @@ class GroupCoordinatorTest { private def staticJoinGroup(groupId: String, memberId: String, - groupInstanceId: Option[String], + groupInstanceId: String, protocolType: String, protocols: List[(String, Array[Byte])], clockAdvance: Int = GroupInitialRebalanceDelay + 1, sessionTimeout: Int = DefaultSessionTimeout, rebalanceTimeout: Int = DefaultRebalanceTimeout): JoinGroupResult = { - val responseFuture = sendJoinGroup(groupId, memberId, protocolType, protocols, groupInstanceId, sessionTimeout, rebalanceTimeout) + val responseFuture = sendJoinGroup(groupId, memberId, protocolType, protocols, Some(groupInstanceId), sessionTimeout, rebalanceTimeout) timer.advanceClock(clockAdvance) // should only have to wait as long as session timeout, but allow some extra time in case of an unexpected delay @@ -3965,14 +3891,15 @@ class GroupCoordinatorTest { private def staticJoinGroupWithPersistence(groupId: String, memberId: String, - groupInstanceId: Option[String], + groupInstanceId: String, protocolType: String, protocols: List[(String, Array[Byte])], clockAdvance: Int, sessionTimeout: Int = DefaultSessionTimeout, rebalanceTimeout: Int = DefaultRebalanceTimeout, appendRecordError: Errors = Errors.NONE): JoinGroupResult = { - val responseFuture = sendStaticJoinGroupWithPersistence(groupId, memberId, protocolType, protocols, groupInstanceId, sessionTimeout, rebalanceTimeout, appendRecordError) + val responseFuture = sendStaticJoinGroupWithPersistence(groupId, memberId, protocolType, protocols, + groupInstanceId, sessionTimeout, rebalanceTimeout, appendRecordError) timer.advanceClock(clockAdvance) // should only have to wait as long as session timeout, but allow some extra time in case of an unexpected delay diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index 45eccbe952940..231382e515a6b 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -60,7 +60,7 @@ class GroupMetadataManagerTest { var metrics: kMetrics = null val groupId = "foo" - val groupInstanceId = Some("bar") + val groupInstanceId = "bar" val groupPartitionId = 0 val groupTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) val protocolType = "protocolType" @@ -845,7 +845,7 @@ class GroupMetadataManagerTest { val staticMemberId = "staticMemberId" val dynamicMemberId = "dynamicMemberId" - val staticMember = new MemberMetadata(staticMemberId, groupInstanceId, "", "", rebalanceTimeout, sessionTimeout, + val staticMember = new MemberMetadata(staticMemberId, Some(groupInstanceId), "", "", rebalanceTimeout, sessionTimeout, protocolType, List(("protocol", Array[Byte]()))) val dynamicMember = new MemberMetadata(dynamicMemberId, None, "", "", rebalanceTimeout, sessionTimeout, @@ -861,7 +861,7 @@ class GroupMetadataManagerTest { assertTrue(group.has(staticMemberId)) assertTrue(group.has(dynamicMemberId)) assertTrue(group.hasStaticMember(groupInstanceId)) - assertEquals(staticMemberId, group.getStaticMemberId(groupInstanceId)) + assertEquals(Some(staticMemberId), group.currentStaticMemberId(groupInstanceId)) } @Test @@ -876,7 +876,7 @@ class GroupMetadataManagerTest { ("protocol", ConsumerProtocol.serializeSubscription(new Subscription(List(topic).asJava)).array()) ) - val member = new MemberMetadata(memberId, groupInstanceId, "", "", rebalanceTimeout, + val member = new MemberMetadata(memberId, Some(groupInstanceId), "", "", rebalanceTimeout, sessionTimeout, protocolType, subscriptions) val members = Seq(member) @@ -916,7 +916,7 @@ class GroupMetadataManagerTest { val subscriptions = List(("protocol", Array[Byte]())) - val member = new MemberMetadata(memberId, groupInstanceId, "", "", rebalanceTimeout, + val member = new MemberMetadata(memberId, Some(groupInstanceId), "", "", rebalanceTimeout, sessionTimeout, protocolType, subscriptions) val members = Seq(member) @@ -1093,7 +1093,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) - val member = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeout, sessionTimeout, + val member = new MemberMetadata(memberId, Some(groupInstanceId), clientId, clientHost, rebalanceTimeout, sessionTimeout, protocolType, List(("protocol", Array[Byte]()))) group.add(member, _ => ()) group.transitionTo(PreparingRebalance) @@ -1122,7 +1122,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) - val member = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeout, sessionTimeout, + val member = new MemberMetadata(memberId, Some(groupInstanceId), clientId, clientHost, rebalanceTimeout, sessionTimeout, protocolType, List(("protocol", Array[Byte]()))) group.add(member, _ => ()) group.transitionTo(PreparingRebalance) @@ -1621,7 +1621,7 @@ class GroupMetadataManagerTest { groupMetadataManager.addGroup(group) val subscription = new Subscription(List(topic).asJava) - val member = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeout, sessionTimeout, + val member = new MemberMetadata(memberId, Some(groupInstanceId), clientId, clientHost, rebalanceTimeout, sessionTimeout, protocolType, List(("protocol", ConsumerProtocol.serializeSubscription(subscription).array()))) group.add(member, _ => ()) group.transitionTo(PreparingRebalance) @@ -1866,7 +1866,7 @@ class GroupMetadataManagerTest { val member = new MemberMetadata( memberId, - groupInstanceId, + Some(groupInstanceId), clientId, clientHost, rebalanceTimeout, @@ -2304,7 +2304,7 @@ class GroupMetadataManagerTest { assignmentBytes: Array[Byte] = Array.emptyByteArray, apiVersion: ApiVersion = ApiVersion.latestVersion): SimpleRecord = { val memberProtocols = List((protocol, Array.emptyByteArray)) - val member = new MemberMetadata(memberId, groupInstanceId, "clientId", "clientHost", 30000, 10000, protocolType, memberProtocols) + val member = new MemberMetadata(memberId, Some(groupInstanceId), "clientId", "clientHost", 30000, 10000, protocolType, memberProtocols) val group = GroupMetadata.loadGroup(groupId, Stable, generation, protocolType, protocol, memberId, if (apiVersion >= KAFKA_2_1_IV0) Some(time.milliseconds()) else None, Seq(member), time) val groupMetadataKey = GroupMetadataManager.groupMetadataKey(groupId) diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala index eecfbdd8f55e2..ce6128abd5759 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala @@ -33,7 +33,7 @@ import scala.jdk.CollectionConverters._ */ class GroupMetadataTest { private val protocolType = "consumer" - private val groupInstanceId = Some("groupInstanceId") + private val groupInstanceId = "groupInstanceId" private val memberId = "memberId" private val clientId = "clientId" private val clientHost = "clientHost" @@ -41,13 +41,10 @@ class GroupMetadataTest { private val sessionTimeoutMs = 10000 private var group: GroupMetadata = null - private var member: MemberMetadata = null @BeforeEach def setUp(): Unit = { group = new GroupMetadata("groupId", Empty, Time.SYSTEM) - member = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, - protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) } @Test @@ -194,14 +191,14 @@ class GroupMetadataTest { @Test def testSelectProtocol(): Unit = { val memberId = "memberId" - val member = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) group.add(member) assertEquals("range", group.selectProtocol) val otherMemberId = "otherMemberId" - val otherMember = new MemberMetadata(otherMemberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, + val otherMember = new MemberMetadata(otherMemberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("roundrobin", Array.empty[Byte]), ("range", Array.empty[Byte]))) group.add(otherMember) @@ -209,7 +206,7 @@ class GroupMetadataTest { assertTrue(Set("range", "roundrobin")(group.selectProtocol)) val lastMemberId = "lastMemberId" - val lastMember = new MemberMetadata(lastMemberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, + val lastMember = new MemberMetadata(lastMemberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("roundrobin", Array.empty[Byte]), ("range", Array.empty[Byte]))) group.add(lastMember) @@ -225,11 +222,11 @@ class GroupMetadataTest { @Test def testSelectProtocolChoosesCompatibleProtocol(): Unit = { val memberId = "memberId" - val member = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) val otherMemberId = "otherMemberId" - val otherMember = new MemberMetadata(otherMemberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, + val otherMember = new MemberMetadata(otherMemberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("roundrobin", Array.empty[Byte]), ("blah", Array.empty[Byte]))) group.add(member) @@ -239,6 +236,9 @@ class GroupMetadataTest { @Test def testSupportsProtocols(): Unit = { + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) + // by default, the group supports everything assertTrue(group.supportsProtocols(protocolType, Set("roundrobin", "range"))) @@ -249,7 +249,7 @@ class GroupMetadataTest { assertFalse(group.supportsProtocols(protocolType, Set("foo", "bar"))) val otherMemberId = "otherMemberId" - val otherMember = new MemberMetadata(otherMemberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, + val otherMember = new MemberMetadata(otherMemberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("roundrobin", Array.empty[Byte]), ("blah", Array.empty[Byte]))) group.add(otherMember) @@ -265,7 +265,7 @@ class GroupMetadataTest { assertEquals(None, group.getSubscribedTopics) val memberId = "memberId" - val member = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("range", ConsumerProtocol.serializeSubscription(new Subscription(List("foo").asJava)).array()))) group.transitionTo(PreparingRebalance) @@ -282,7 +282,7 @@ class GroupMetadataTest { assertEquals(Some(Set.empty), group.getSubscribedTopics) - val memberWithFaultyProtocol = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, + val memberWithFaultyProtocol = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("range", Array.empty[Byte]))) group.transitionTo(PreparingRebalance) @@ -299,7 +299,7 @@ class GroupMetadataTest { assertEquals(None, group.getSubscribedTopics) val memberId = "memberId" - val member = new MemberMetadata(memberId, groupInstanceId, clientId, clientHost, rebalanceTimeoutMs, + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, "My Protocol", List(("range", Array.empty[Byte]))) group.transitionTo(PreparingRebalance) @@ -312,6 +312,9 @@ class GroupMetadataTest { @Test def testInitNextGeneration(): Unit = { + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) + member.supportedProtocols = List(("roundrobin", Array.empty[Byte])) group.transitionTo(PreparingRebalance) @@ -512,25 +515,17 @@ class GroupMetadataTest { assertFalse(group.hasPendingOffsetCommitsFromProducer(producerId)) } - @Test - def testReplaceGroupInstanceWithEmptyGroupInstanceId(): Unit = { - group.add(member) - group.addStaticMember(groupInstanceId, memberId) - assertTrue(group.isLeader(memberId)) - assertEquals(memberId, group.getStaticMemberId(groupInstanceId)) - - val newMemberId = "newMemberId" - assertThrows(classOf[IllegalArgumentException], () => group.replaceGroupInstance(memberId, newMemberId, Option.empty)) - } - @Test def testReplaceGroupInstanceWithNonExistingMember(): Unit = { val newMemberId = "newMemberId" - assertThrows(classOf[IllegalArgumentException], () => group.replaceGroupInstance(memberId, newMemberId, groupInstanceId)) + assertThrows(classOf[IllegalArgumentException], () => group.replaceStaticMember(groupInstanceId, memberId, newMemberId)) } @Test def testReplaceGroupInstance(): Unit = { + val member = new MemberMetadata(memberId, Some(groupInstanceId), clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) + var joinAwaitingMemberFenced = false group.add(member, joinGroupResult => { joinAwaitingMemberFenced = joinGroupResult.error == Errors.FENCED_INSTANCE_ID @@ -539,14 +534,13 @@ class GroupMetadataTest { member.awaitingSyncCallback = syncGroupResult => { syncAwaitingMemberFenced = syncGroupResult.error == Errors.FENCED_INSTANCE_ID } - group.addStaticMember(groupInstanceId, memberId) assertTrue(group.isLeader(memberId)) - assertEquals(memberId, group.getStaticMemberId(groupInstanceId)) + assertEquals(Some(memberId), group.currentStaticMemberId(groupInstanceId)) val newMemberId = "newMemberId" - group.replaceGroupInstance(memberId, newMemberId, groupInstanceId) + group.replaceStaticMember(groupInstanceId, memberId, newMemberId) assertTrue(group.isLeader(newMemberId)) - assertEquals(newMemberId, group.getStaticMemberId(groupInstanceId)) + assertEquals(Some(newMemberId), group.currentStaticMemberId(groupInstanceId)) assertTrue(joinAwaitingMemberFenced) assertTrue(syncAwaitingMemberFenced) assertFalse(member.isAwaitingJoin) @@ -555,6 +549,9 @@ class GroupMetadataTest { @Test def testInvokeJoinCallback(): Unit = { + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) + var invoked = false group.add(member, _ => { invoked = true @@ -568,6 +565,8 @@ class GroupMetadataTest { @Test def testNotInvokeJoinCallback(): Unit = { + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) group.add(member) assertFalse(member.isAwaitingJoin) @@ -577,6 +576,9 @@ class GroupMetadataTest { @Test def testInvokeSyncCallback(): Unit = { + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) + group.add(member) member.awaitingSyncCallback = _ => {} @@ -587,6 +589,8 @@ class GroupMetadataTest { @Test def testNotInvokeSyncCallback(): Unit = { + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) group.add(member) val invoked = group.maybeInvokeSyncCallback(member, SyncGroupResult(Errors.NONE)) @@ -615,6 +619,50 @@ class GroupMetadataTest { assertFalse(group.hasPendingOffsetCommitsForTopicPartition(new TopicPartition("non-exist", 0))) } + @Test + def testCannotAddPendingMemberIfStable(): Unit = { + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) + group.add(member) + assertThrows(classOf[IllegalStateException], () => group.addPendingMember(memberId)) + } + + @Test + def testRemovalFromPendingAfterMemberIsStable(): Unit = { + group.addPendingMember(memberId) + assertFalse(group.has(memberId)) + assertTrue(group.isPendingMember(memberId)) + + val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs, + protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte]))) + group.add(member) + assertTrue(group.has(memberId)) + assertFalse(group.isPendingMember(memberId)) + } + + @Test + def testRemovalFromPendingWhenMemberIsRemoved(): Unit = { + group.addPendingMember(memberId) + assertFalse(group.has(memberId)) + assertTrue(group.isPendingMember(memberId)) + + group.remove(memberId) + assertFalse(group.has(memberId)) + assertFalse(group.isPendingMember(memberId)) + } + + @Test + def testCannotAddStaticMemberIfAlreadyPresent(): Unit = { + val member = new MemberMetadata(memberId, Some(groupInstanceId), clientId, clientHost, + rebalanceTimeoutMs, sessionTimeoutMs, protocolType, List(("range", Array.empty[Byte]))) + group.add(member) + assertTrue(group.has(memberId)) + assertTrue(group.hasStaticMember(groupInstanceId)) + + // We aren ot permitted to add the member again if it is already present + assertThrows(classOf[IllegalStateException], () => group.add(member)) + } + private def assertState(group: GroupMetadata, targetState: GroupState): Unit = { val states: Set[GroupState] = Set(Stable, PreparingRebalance, CompletingRebalance, Dead) val otherStates = states - targetState From 32e6a5b320194c40824ac3cdd5ecc015b0254cce Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Fri, 26 Feb 2021 11:28:11 -0800 Subject: [PATCH 068/243] MINOR: Add cluster-metadata-decoder to DumpLogSegments (#10212) Add the --cluster-metadata-decoder and --skip-record-metadata options to the DumpLogSegments command-line tool, as described in KIP-631. Co-authored-by: David Arthur Reviewers: Jason Gustafson --- .../scala/kafka/tools/DumpLogSegments.scala | 90 ++++++++++++++----- .../kafka/tools/DumpLogSegmentsTest.scala | 68 +++++++++++++- 2 files changed, 132 insertions(+), 26 deletions(-) diff --git a/core/src/main/scala/kafka/tools/DumpLogSegments.scala b/core/src/main/scala/kafka/tools/DumpLogSegments.scala index 8263176438f9c..8bc07da73e51c 100755 --- a/core/src/main/scala/kafka/tools/DumpLogSegments.scala +++ b/core/src/main/scala/kafka/tools/DumpLogSegments.scala @@ -19,14 +19,18 @@ package kafka.tools import java.io._ +import com.fasterxml.jackson.databind.node.{IntNode, JsonNodeFactory, ObjectNode, TextNode} import kafka.coordinator.group.GroupMetadataManager import kafka.coordinator.transaction.TransactionLog import kafka.log._ import kafka.serializer.Decoder import kafka.utils._ import kafka.utils.Implicits._ +import org.apache.kafka.common.metadata.{MetadataJsonConverters, MetadataRecordType} +import org.apache.kafka.common.protocol.ByteBufferAccessor import org.apache.kafka.common.record._ import org.apache.kafka.common.utils.Utils +import org.apache.kafka.raft.metadata.MetadataRecordSerde import scala.jdk.CollectionConverters._ import scala.collection.mutable @@ -55,7 +59,7 @@ object DumpLogSegments { suffix match { case Log.LogFileSuffix => dumpLog(file, opts.shouldPrintDataLog, nonConsecutivePairsForLogFilesMap, opts.isDeepIteration, - opts.maxMessageSize, opts.messageParser) + opts.maxMessageSize, opts.messageParser, opts.skipRecordMetadata) case Log.IndexFileSuffix => dumpIndex(file, opts.indexSanityOnly, opts.verifyOnly, misMatchesForIndexFilesMap, opts.maxMessageSize) case Log.TimeIndexFileSuffix => @@ -242,7 +246,8 @@ object DumpLogSegments { nonConsecutivePairsForLogFilesMap: mutable.Map[String, List[(Long, Long)]], isDeepIteration: Boolean, maxMessageSize: Int, - parser: MessageParser[_, _]): Unit = { + parser: MessageParser[_, _], + skipRecordMetadata: Boolean): Unit = { val startOffset = file.getName.split("\\.")(0).toLong println("Starting offset: " + startOffset) val fileRecords = FileRecords.open(file, false) @@ -263,31 +268,38 @@ object DumpLogSegments { } lastOffset = record.offset - print(s"$RecordIndent offset: ${record.offset} isValid: ${record.isValid} crc: ${record.checksumOrNull}" + - s" keySize: ${record.keySize} valueSize: ${record.valueSize} ${batch.timestampType}: ${record.timestamp}" + - s" baseOffset: ${batch.baseOffset} lastOffset: ${batch.lastOffset} baseSequence: ${batch.baseSequence}" + - s" lastSequence: ${batch.lastSequence} producerEpoch: ${batch.producerEpoch} partitionLeaderEpoch: ${batch.partitionLeaderEpoch}" + - s" batchSize: ${batch.sizeInBytes} magic: ${batch.magic} compressType: ${batch.compressionType} position: ${validBytes}") - - - if (batch.magic >= RecordBatch.MAGIC_VALUE_V2) { - print(" sequence: " + record.sequence + " headerKeys: " + record.headers.map(_.key).mkString("[", ",", "]")) - } else { - print(s" crc: ${record.checksumOrNull} isvalid: ${record.isValid}") - } + var prefix = s"${RecordIndent} " + if (!skipRecordMetadata) { + print(s"${prefix}offset: ${record.offset} isValid: ${record.isValid} crc: ${record.checksumOrNull}" + + s" keySize: ${record.keySize} valueSize: ${record.valueSize} ${batch.timestampType}: ${record.timestamp}" + + s" baseOffset: ${batch.baseOffset} lastOffset: ${batch.lastOffset} baseSequence: ${batch.baseSequence}" + + s" lastSequence: ${batch.lastSequence} producerEpoch: ${batch.producerEpoch} partitionLeaderEpoch: ${batch.partitionLeaderEpoch}" + + s" batchSize: ${batch.sizeInBytes} magic: ${batch.magic} compressType: ${batch.compressionType} position: ${validBytes}") + prefix = " " + + if (batch.magic >= RecordBatch.MAGIC_VALUE_V2) { + print(" sequence: " + record.sequence + " headerKeys: " + record.headers.map(_.key).mkString("[", ",", "]")) + } else { + print(s" crc: ${record.checksumOrNull} isvalid: ${record.isValid}") + } - if (batch.isControlBatch) { - val controlTypeId = ControlRecordType.parseTypeId(record.key) - ControlRecordType.fromTypeId(controlTypeId) match { - case ControlRecordType.ABORT | ControlRecordType.COMMIT => - val endTxnMarker = EndTransactionMarker.deserialize(record) - print(s" endTxnMarker: ${endTxnMarker.controlType} coordinatorEpoch: ${endTxnMarker.coordinatorEpoch}") - case controlType => - print(s" controlType: $controlType($controlTypeId)") + if (batch.isControlBatch) { + val controlTypeId = ControlRecordType.parseTypeId(record.key) + ControlRecordType.fromTypeId(controlTypeId) match { + case ControlRecordType.ABORT | ControlRecordType.COMMIT => + val endTxnMarker = EndTransactionMarker.deserialize(record) + print(s" endTxnMarker: ${endTxnMarker.controlType} coordinatorEpoch: ${endTxnMarker.coordinatorEpoch}") + case controlType => + print(s" controlType: $controlType($controlTypeId)") + } } - } else if (printContents) { + } + if (printContents && !batch.isControlBatch) { val (key, payload) = parser.parse(record) - key.foreach(key => print(s" key: $key")) + key.foreach { key => + print(s"${prefix}key: $key") + prefix = " " + } payload.foreach(payload => print(s" payload: $payload")) } println() @@ -382,6 +394,30 @@ object DumpLogSegments { } } + private class ClusterMetadataLogMessageParser extends MessageParser[String, String] { + val metadataRecordSerde = new MetadataRecordSerde() + + override def parse(record: Record): (Option[String], Option[String]) = { + val output = try { + val messageAndVersion = metadataRecordSerde. + read(new ByteBufferAccessor(record.value), record.valueSize()) + val json = new ObjectNode(JsonNodeFactory.instance) + json.set("type", new TextNode(MetadataRecordType.fromId( + messageAndVersion.message().apiKey()).toString)) + json.set("version", new IntNode(messageAndVersion.version())) + json.set("data", MetadataJsonConverters.writeJson( + messageAndVersion.message(), messageAndVersion.version())) + json.toString() + } catch { + case e: Throwable => { + s"Error at ${record.offset}, skipping. ${e.getMessage}" + } + } + // No keys for metadata records + (None, Some(output)) + } + } + private class DumpLogSegmentsOptions(args: Array[String]) extends CommandDefaultOptions(args) { val printOpt = parser.accepts("print-data-log", "if set, printing the messages content when dumping data logs. Automatically set if any decoder option is specified.") val verifyOpt = parser.accepts("verify-index-only", "if set, just verify the index log without printing its content.") @@ -409,6 +445,8 @@ object DumpLogSegments { "__consumer_offsets topic.") val transactionLogOpt = parser.accepts("transaction-log-decoder", "if set, log data will be parsed as " + "transaction metadata from the __transaction_state topic.") + val clusterMetadataOpt = parser.accepts("cluster-metadata-decoder", "if set, log data will be parsed as cluster metadata records.") + val skipRecordMetadataOpt = parser.accepts("skip-record-metadata", "whether to skip printing metadata for each record.") options = parser.parse(args : _*) def messageParser: MessageParser[_, _] = @@ -416,6 +454,8 @@ object DumpLogSegments { new OffsetsMessageParser } else if (options.has(transactionLogOpt)) { new TransactionLogMessageParser + } else if (options.has(clusterMetadataOpt)) { + new ClusterMetadataLogMessageParser } else { val valueDecoder: Decoder[_] = CoreUtils.createObject[Decoder[_]](options.valueOf(valueDecoderOpt), new VerifiableProperties) val keyDecoder: Decoder[_] = CoreUtils.createObject[Decoder[_]](options.valueOf(keyDecoderOpt), new VerifiableProperties) @@ -425,9 +465,11 @@ object DumpLogSegments { lazy val shouldPrintDataLog: Boolean = options.has(printOpt) || options.has(offsetsOpt) || options.has(transactionLogOpt) || + options.has(clusterMetadataOpt) || options.has(valueDecoderOpt) || options.has(keyDecoderOpt) + lazy val skipRecordMetadata = options.has(skipRecordMetadataOpt) lazy val isDeepIteration: Boolean = options.has(deepIterationOpt) || shouldPrintDataLog lazy val verifyOnly: Boolean = options.has(verifyOpt) lazy val indexSanityOnly: Boolean = options.has(indexSanityOpt) diff --git a/core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala b/core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala index 7714c7f085d30..1a5e51e5fe3da 100644 --- a/core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala +++ b/core/src/test/scala/unit/kafka/tools/DumpLogSegmentsTest.scala @@ -18,14 +18,21 @@ package kafka.tools import java.io.{ByteArrayOutputStream, File, PrintWriter} +import java.nio.ByteBuffer +import java.util import java.util.Properties -import kafka.log.{Log, LogConfig, LogManager} +import kafka.log.{Log, LogConfig, LogManager, LogTest} import kafka.server.{BrokerTopicStats, LogDirFailureChannel} import kafka.tools.DumpLogSegments.TimeIndexDumpErrors import kafka.utils.{MockTime, TestUtils} +import org.apache.kafka.common.Uuid +import org.apache.kafka.common.metadata.{PartitionChangeRecord, RegisterBrokerRecord, TopicRecord} +import org.apache.kafka.common.protocol.{ByteBufferAccessor, ObjectSerializationCache} import org.apache.kafka.common.record.{CompressionType, MemoryRecords, SimpleRecord} import org.apache.kafka.common.utils.Utils +import org.apache.kafka.metadata.ApiMessageAndVersion +import org.apache.kafka.raft.metadata.MetadataRecordSerde import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} @@ -55,7 +62,9 @@ class DumpLogSegmentsTest { time = time, brokerTopicStats = new BrokerTopicStats, maxProducerIdExpirationMs = 60 * 60 * 1000, producerIdExpirationCheckIntervalMs = LogManager.ProducerIdExpirationCheckIntervalMs, logDirFailureChannel = new LogDirFailureChannel(10)) + } + def addSimpleRecords(): Unit = { val now = System.currentTimeMillis() val firstBatchRecords = (0 until 10).map { i => new SimpleRecord(now + i * 2, s"message key $i".getBytes, s"message value $i".getBytes)} batches += BatchInfo(firstBatchRecords, true, true) @@ -82,7 +91,7 @@ class DumpLogSegmentsTest { @Test def testPrintDataLog(): Unit = { - + addSimpleRecords() def verifyRecordsInOutput(checkKeysAndValues: Boolean, args: Array[String]): Unit = { def isBatch(index: Int): Boolean = { var i = 0 @@ -152,6 +161,7 @@ class DumpLogSegmentsTest { @Test def testDumpIndexMismatches(): Unit = { + addSimpleRecords() val offsetMismatches = mutable.Map[String, List[(Long, Long)]]() DumpLogSegments.dumpIndex(new File(indexFilePath), indexSanityOnly = false, verifyOnly = true, offsetMismatches, Int.MaxValue) @@ -160,6 +170,7 @@ class DumpLogSegmentsTest { @Test def testDumpTimeIndexErrors(): Unit = { + addSimpleRecords() val errors = new TimeIndexDumpErrors DumpLogSegments.dumpTimeIndex(new File(timeIndexFilePath), indexSanityOnly = false, verifyOnly = true, errors, Int.MaxValue) @@ -168,6 +179,59 @@ class DumpLogSegmentsTest { assertEquals(Map.empty, errors.shallowOffsetNotFound) } + @Test + def testDumpMetadataRecords(): Unit = { + val mockTime = new MockTime + val logConfig = LogTest.createLogConfig(segmentBytes = 1024 * 1024) + val log = LogTest.createLog(logDir, logConfig, new BrokerTopicStats, mockTime.scheduler, mockTime) + + val metadataRecords = Seq( + new ApiMessageAndVersion( + new RegisterBrokerRecord().setBrokerId(0).setBrokerEpoch(10), 0.toShort), + new ApiMessageAndVersion( + new RegisterBrokerRecord().setBrokerId(1).setBrokerEpoch(20), 0.toShort), + new ApiMessageAndVersion( + new TopicRecord().setName("test-topic").setTopicId(Uuid.randomUuid()), 0.toShort), + new ApiMessageAndVersion( + new PartitionChangeRecord().setTopicId(Uuid.randomUuid()).setLeader(1). + setPartitionId(0).setIsr(util.Arrays.asList(0, 1, 2)), 0.toShort) + ) + + val records: Array[SimpleRecord] = metadataRecords.map(message => { + val serde = new MetadataRecordSerde() + val cache = new ObjectSerializationCache + val size = serde.recordSize(message, cache) + val buf = ByteBuffer.allocate(size) + val writer = new ByteBufferAccessor(buf) + serde.write(message, cache, writer) + buf.flip() + new SimpleRecord(null, buf.array) + }).toArray + log.appendAsLeader(MemoryRecords.withRecords(CompressionType.NONE, records:_*), leaderEpoch = 1) + log.flush() + + var output = runDumpLogSegments(Array("--cluster-metadata-decoder", "false", "--files", logFilePath)) + assert(output.contains("TOPIC_RECORD")) + assert(output.contains("BROKER_RECORD")) + + output = runDumpLogSegments(Array("--cluster-metadata-decoder", "--skip-record-metadata", "false", "--files", logFilePath)) + assert(output.contains("TOPIC_RECORD")) + assert(output.contains("BROKER_RECORD")) + + // Bogus metadata record + val buf = ByteBuffer.allocate(4) + val writer = new ByteBufferAccessor(buf) + writer.writeUnsignedVarint(10000) + writer.writeUnsignedVarint(10000) + log.appendAsLeader(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord(null, buf.array)), leaderEpoch = 2) + log.appendAsLeader(MemoryRecords.withRecords(CompressionType.NONE, records:_*), leaderEpoch = 2) + + output = runDumpLogSegments(Array("--cluster-metadata-decoder", "--skip-record-metadata", "false", "--files", logFilePath)) + assert(output.contains("TOPIC_RECORD")) + assert(output.contains("BROKER_RECORD")) + assert(output.contains("skipping")) + } + @Test def testDumpEmptyIndex(): Unit = { val indexFile = new File(indexFilePath) From a66f154370980783add1973954cff120cf3825e1 Mon Sep 17 00:00:00 2001 From: Ivan Yurchenko Date: Fri, 26 Feb 2021 21:33:21 +0200 Subject: [PATCH 069/243] KAFKA-12235: Fix a bug where describeConfigs does not work on 2+ config keys (#9990) Fix a bug where if more than one configuration key is supplied, describeConfigs fails to return any results at all. Reviewers: Colin P. McCabe --- .../scala/kafka/server/ConfigHelper.scala | 4 ++-- .../kafka/server/ZkAdminManagerTest.scala | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/kafka/server/ConfigHelper.scala b/core/src/main/scala/kafka/server/ConfigHelper.scala index 2bc8dd9232bce..779bea8426878 100644 --- a/core/src/main/scala/kafka/server/ConfigHelper.scala +++ b/core/src/main/scala/kafka/server/ConfigHelper.scala @@ -47,11 +47,11 @@ class ConfigHelper(metadataCache: MetadataCache, config: KafkaConfig, configRepo def createResponseConfig(configs: Map[String, Any], createConfigEntry: (String, Any) => DescribeConfigsResponseData.DescribeConfigsResourceResult): DescribeConfigsResponseData.DescribeConfigsResult = { - val filteredConfigPairs = if (resource.configurationKeys == null) + val filteredConfigPairs = if (resource.configurationKeys == null || resource.configurationKeys.isEmpty) configs.toBuffer else configs.filter { case (configName, _) => - resource.configurationKeys.asScala.forall(_.contains(configName)) + resource.configurationKeys.asScala.contains(configName) }.toBuffer val configEntries = filteredConfigPairs.map { case (name, value) => createConfigEntry(name, value) } diff --git a/core/src/test/scala/unit/kafka/server/ZkAdminManagerTest.scala b/core/src/test/scala/unit/kafka/server/ZkAdminManagerTest.scala index ae0ac79c41c8a..353313364c46a 100644 --- a/core/src/test/scala/unit/kafka/server/ZkAdminManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ZkAdminManagerTest.scala @@ -34,6 +34,8 @@ import java.util.Properties import kafka.server.metadata.ZkConfigRepository +import scala.jdk.CollectionConverters._ + class ZkAdminManagerTest { private val zkClient: KafkaZkClient = EasyMock.createNiceMock(classOf[KafkaZkClient]) @@ -85,6 +87,25 @@ class ZkAdminManagerTest { assertFalse(results.head.configs().isEmpty, "Should return configs") } + @Test + def testDescribeConfigsWithConfigurationKeys(): Unit = { + EasyMock.expect(zkClient.getEntityConfigs(ConfigType.Topic, topic)).andReturn(TestUtils.createBrokerConfig(brokerId, "zk")) + EasyMock.expect(metadataCache.contains(topic)).andReturn(true) + + EasyMock.replay(zkClient, metadataCache) + + val resources = List(new DescribeConfigsRequestData.DescribeConfigsResource() + .setResourceName(topic) + .setResourceType(ConfigResource.Type.TOPIC.id) + .setConfigurationKeys(List("retention.ms", "retention.bytes", "segment.bytes").asJava) + ) + val configHelper = createConfigHelper(metadataCache, zkClient) + val results: List[DescribeConfigsResponseData.DescribeConfigsResult] = configHelper.describeConfigs(resources, true, true) + assertEquals(Errors.NONE.code, results.head.errorCode()) + val resultConfigKeys = results.head.configs().asScala.map(r => r.name()).toSet + assertEquals(Set("retention.ms", "retention.bytes", "segment.bytes"), resultConfigKeys) + } + @Test def testDescribeConfigsWithDocumentation(): Unit = { EasyMock.expect(zkClient.getEntityConfigs(ConfigType.Topic, topic)).andReturn(new Properties) From 958f90e710f0de164dce1dda5f45d75a8b8fb8d4 Mon Sep 17 00:00:00 2001 From: Walker Carlson <18128741+wcarlson5@users.noreply.github.com> Date: Fri, 26 Feb 2021 12:17:28 -0800 Subject: [PATCH 070/243] KAFKA-12375: fix concurrency issue in application shutdown (#10213) Need to ensure that `enforceRebalance` is used in a thread safe way Reviewers: Bruno Cadonna , Anna Sophie Blee-Goldman --- .../kafka/streams/processor/internals/StreamThread.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java index b967f5af6f7c3..9fadd53569be5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java @@ -571,6 +571,11 @@ boolean runLoop() { // until the rebalance is completed before we close and commit the tasks while (isRunning() || taskManager.isRebalanceInProgress()) { try { + if (assignmentErrorCode.get() == AssignorError.SHUTDOWN_REQUESTED.code()) { + log.warn("Detected that shutdown was requested. " + + "All clients in this app will now begin to shutdown"); + mainConsumer.enforceRebalance(); + } runOnce(); if (nextProbingRebalanceMs.get() < time.milliseconds()) { log.info("Triggering the followup rebalance scheduled for {} ms.", nextProbingRebalanceMs.get()); @@ -660,10 +665,7 @@ public void shutdownToError() { } public void sendShutdownRequest(final AssignorError assignorError) { - log.warn("Detected that shutdown was requested. " + - "All clients in this app will now begin to shutdown"); assignmentErrorCode.set(assignorError.code()); - mainConsumer.enforceRebalance(); } private void handleTaskMigrated(final TaskMigratedException e) { From 02226fa090513882b9229ac834fd493d71ae6d96 Mon Sep 17 00:00:00 2001 From: Ron Dagostino Date: Fri, 26 Feb 2021 16:54:21 -0500 Subject: [PATCH 071/243] MINOR: disable test_produce_bench_transactions for Raft metadata quorum (#10222) Reviewers: Colin P. McCabe --- tests/kafkatest/tests/core/produce_bench_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kafkatest/tests/core/produce_bench_test.py b/tests/kafkatest/tests/core/produce_bench_test.py index 734dfb580b8df..5ac08d7b3f307 100644 --- a/tests/kafkatest/tests/core/produce_bench_test.py +++ b/tests/kafkatest/tests/core/produce_bench_test.py @@ -66,7 +66,6 @@ def test_produce_bench(self, metadata_quorum=quorum.zk): self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2)) @cluster(num_nodes=8) - @matrix(metadata_quorum=quorum.all_non_upgrade) def test_produce_bench_transactions(self, metadata_quorum=quorum.zk): spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, self.workload_service.producer_node, From 068d8fedcb34dcb2d07a6375bd267d3323c7e06c Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Fri, 26 Feb 2021 14:40:46 -0800 Subject: [PATCH 072/243] KAFKA-10101: Fix edge cases in Log.recoverLog and LogManager.loadLogs (#8812) 1. Don't advance recovery point in `recoverLog` unless there was a clean shutdown. 2. Ensure the recovery point is not ahead of the log end offset. 3. Clean and flush leader epoch cache and truncate produce state manager if deleting segments due to log end offset being smaller than log start offset. 4. If we are unable to delete clean shutdown file that exists, mark the directory as offline (this was the intent, but the code was wrong). Updated one test that was failing after this change to verify the new behavior. Reviewers: Jun Rao , Jason Gustafson --- core/src/main/scala/kafka/log/Log.scala | 46 +++++++++++++------ .../src/main/scala/kafka/log/LogManager.scala | 2 +- .../test/scala/unit/kafka/log/LogTest.scala | 6 ++- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index 2249b5eccbbe3..2729154bab4ac 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -849,9 +849,25 @@ class Log(@volatile private var _dir: File, * @throws LogSegmentOffsetOverflowException if we encountered a legacy segment with offset overflow */ private[log] def recoverLog(): Long = { + /** return the log end offset if valid */ + def deleteSegmentsIfLogStartGreaterThanLogEnd(): Option[Long] = { + if (logSegments.nonEmpty) { + val logEndOffset = activeSegment.readNextOffset + if (logEndOffset >= logStartOffset) + Some(logEndOffset) + else { + warn(s"Deleting all segments because logEndOffset ($logEndOffset) is smaller than logStartOffset ($logStartOffset). " + + "This could happen if segment files were deleted from the file system.") + removeAndDeleteSegments(logSegments, asyncDelete = true, LogRecovery) + leaderEpochCache.foreach(_.clearAndFlush()) + producerStateManager.truncateFullyAndStartAt(logStartOffset) + None + } + } else None + } + // if we have the clean shutdown marker, skip recovery if (!hadCleanShutdown) { - // okay we need to actually recover this log val unflushed = logSegments(this.recoveryPoint, Long.MaxValue).iterator var truncated = false @@ -879,16 +895,7 @@ class Log(@volatile private var _dir: File, } } - if (logSegments.nonEmpty) { - val logEndOffset = activeSegment.readNextOffset - if (logEndOffset < logStartOffset) { - warn(s"Deleting all segments because logEndOffset ($logEndOffset) is smaller than logStartOffset ($logStartOffset). " + - "This could happen if segment files were deleted from the file system.") - removeAndDeleteSegments(logSegments, - asyncDelete = true, - reason = LogRecovery) - } - } + val logEndOffsetOption = deleteSegmentsIfLogStartGreaterThanLogEnd() if (logSegments.isEmpty) { // no existing segments, create a new mutable segment beginning at logStartOffset @@ -900,8 +907,21 @@ class Log(@volatile private var _dir: File, preallocate = config.preallocate)) } - recoveryPoint = activeSegment.readNextOffset - recoveryPoint + // Update the recovery point if there was a clean shutdown and did not perform any changes to + // the segment. Otherwise, we just ensure that the recovery point is not ahead of the log end + // offset. To ensure correctness and to make it easier to reason about, it's best to only advance + // the recovery point in flush(Long). If we advanced the recovery point here, we could skip recovery for + // unflushed segments if the broker crashed after we checkpoint the recovery point and before we flush the + // segment. + (hadCleanShutdown, logEndOffsetOption) match { + case (true, Some(logEndOffset)) => + recoveryPoint = logEndOffset + logEndOffset + case _ => + val logEndOffset = logEndOffsetOption.getOrElse(activeSegment.readNextOffset) + recoveryPoint = Math.min(recoveryPoint, logEndOffset) + logEndOffset + } } // Rebuild producer state until lastOffset. This method may be called from the recovery code path, and thus must be diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala index acb9d34c60be9..1ca4d7ec33f2c 100755 --- a/core/src/main/scala/kafka/log/LogManager.scala +++ b/core/src/main/scala/kafka/log/LogManager.scala @@ -318,7 +318,7 @@ class LogManager(logDirs: Seq[File], info(s"Skipping recovery for all logs in $logDirAbsolutePath since clean shutdown file was found") // Cache the clean shutdown status and use that for rest of log loading workflow. Delete the CleanShutdownFile // so that if broker crashes while loading the log, it is considered hard shutdown during the next boot up. KAFKA-10471 - cleanShutdownFile.delete() + Files.deleteIfExists(cleanShutdownFile.toPath) hadCleanShutdown = true } else { // log recovery itself is being performed by `Log` class during initialization diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index 1a953c53d5d9b..66ee5d2538c20 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -2518,7 +2518,11 @@ class LogTest { log.close() // test recovery case - log = createLog(logDir, logConfig, lastShutdownClean = false) + val recoveryPoint = 10 + log = createLog(logDir, logConfig, recoveryPoint = recoveryPoint, lastShutdownClean = false) + // the recovery point should not be updated after unclean shutdown until the log is flushed + verifyRecoveredLog(log, recoveryPoint) + log.flush() verifyRecoveredLog(log, lastOffset) log.close() } From 5d37901500d554cc5602ae141a98fd87bda3dcbf Mon Sep 17 00:00:00 2001 From: Ron Dagostino Date: Fri, 26 Feb 2021 19:56:11 -0500 Subject: [PATCH 073/243] KAFKA-12374: Add missing config sasl.mechanism.controller.protocol (#10199) Fix some cases where we were erroneously using the configuration of the inter broker listener instead of the controller listener. Add the sasl.mechanism.controller.protocol configuration key specified by KIP-631. Add some ducktape tests. Reviewers: Colin P. McCabe , David Arthur , Boyang Chen --- .../main/scala/kafka/raft/RaftManager.scala | 11 ++- .../BrokerToControllerChannelManager.scala | 25 +++-- .../main/scala/kafka/server/KafkaConfig.scala | 5 + .../server/BrokerLifecycleManagerTest.scala | 3 + .../unit/kafka/server/KafkaConfigTest.scala | 1 + .../sanity_checks/test_verifiable_producer.py | 73 ++++++++++++++ tests/kafkatest/services/kafka/kafka.py | 67 ++++++++++--- .../services/kafka/templates/kafka.properties | 7 ++ .../services/security/security_config.py | 52 ++++++++-- tests/kafkatest/tests/core/security_test.py | 96 +++++++++++++++---- 10 files changed, 293 insertions(+), 47 deletions(-) diff --git a/core/src/main/scala/kafka/raft/RaftManager.scala b/core/src/main/scala/kafka/raft/RaftManager.scala index ecf89348ccaf9..1881a1db9a70d 100644 --- a/core/src/main/scala/kafka/raft/RaftManager.scala +++ b/core/src/main/scala/kafka/raft/RaftManager.scala @@ -30,10 +30,11 @@ import kafka.utils.{KafkaScheduler, Logging, ShutdownableThread} import org.apache.kafka.clients.{ApiVersions, ClientDnsLookup, ManualMetadataUpdater, NetworkClient} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.network.{ChannelBuilders, NetworkReceive, Selectable, Selector} +import org.apache.kafka.common.network.{ChannelBuilders, ListenerName, NetworkReceive, Selectable, Selector} import org.apache.kafka.common.protocol.ApiMessage import org.apache.kafka.common.requests.RequestHeader import org.apache.kafka.common.security.JaasContext +import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.{LogContext, Time} import org.apache.kafka.raft.RaftConfig.{AddressSpec, InetAddressSpec, NON_ROUTABLE_ADDRESS, UnknownAddressSpec} import org.apache.kafka.raft.{FileBasedStateStore, KafkaRaftClient, RaftClient, RaftConfig, RaftRequest, RecordSerde} @@ -241,12 +242,14 @@ class KafkaRaftManager[T]( } private def buildNetworkClient(): NetworkClient = { + val controllerListenerName = new ListenerName(config.controllerListenerNames.head) + val controllerSecurityProtocol = config.listenerSecurityProtocolMap.getOrElse(controllerListenerName, SecurityProtocol.forName(controllerListenerName.value())) val channelBuilder = ChannelBuilders.clientChannelBuilder( - config.interBrokerSecurityProtocol, + controllerSecurityProtocol, JaasContext.Type.SERVER, config, - config.interBrokerListenerName, - config.saslMechanismInterBrokerProtocol, + controllerListenerName, + config.saslMechanismControllerProtocol, time, config.saslInterBrokerHandshakeRequestEnable, logContext diff --git a/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala b/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala index 621c8671b9ec4..16c4a5b95d5f8 100644 --- a/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala +++ b/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala @@ -40,6 +40,7 @@ trait ControllerNodeProvider { def get(): Option[Node] def listenerName: ListenerName def securityProtocol: SecurityProtocol + def saslMechanism: String } object MetadataCacheControllerNodeProvider { @@ -56,7 +57,8 @@ object MetadataCacheControllerNodeProvider { new MetadataCacheControllerNodeProvider( metadataCache, listenerName, - securityProtocol + securityProtocol, + config.saslMechanismInterBrokerProtocol ) } } @@ -64,7 +66,8 @@ object MetadataCacheControllerNodeProvider { class MetadataCacheControllerNodeProvider( val metadataCache: kafka.server.MetadataCache, val listenerName: ListenerName, - val securityProtocol: SecurityProtocol + val securityProtocol: SecurityProtocol, + val saslMechanism: String ) extends ControllerNodeProvider { override def get(): Option[Node] = { metadataCache.getControllerId @@ -78,9 +81,16 @@ object RaftControllerNodeProvider { config: KafkaConfig, controllerQuorumVoterNodes: Seq[Node]): RaftControllerNodeProvider = { - val listenerName = new ListenerName(config.controllerListenerNames.head) - val securityProtocol = config.listenerSecurityProtocolMap.getOrElse(listenerName, SecurityProtocol.forName(listenerName.value())) - new RaftControllerNodeProvider(metaLogManager, controllerQuorumVoterNodes, listenerName, securityProtocol) + val controllerListenerName = new ListenerName(config.controllerListenerNames.head) + val controllerSecurityProtocol = config.listenerSecurityProtocolMap.getOrElse(controllerListenerName, SecurityProtocol.forName(controllerListenerName.value())) + val controllerSaslMechanism = config.saslMechanismControllerProtocol + new RaftControllerNodeProvider( + metaLogManager, + controllerQuorumVoterNodes, + controllerListenerName, + controllerSecurityProtocol, + controllerSaslMechanism + ) } } @@ -91,7 +101,8 @@ object RaftControllerNodeProvider { class RaftControllerNodeProvider(val metaLogManager: MetaLogManager, controllerQuorumVoterNodes: Seq[Node], val listenerName: ListenerName, - val securityProtocol: SecurityProtocol + val securityProtocol: SecurityProtocol, + val saslMechanism: String ) extends ControllerNodeProvider with Logging { val idToNode = controllerQuorumVoterNodes.map(node => node.id() -> node).toMap @@ -179,7 +190,7 @@ class BrokerToControllerChannelManagerImpl( JaasContext.Type.SERVER, config, controllerNodeProvider.listenerName, - config.saslMechanismInterBrokerProtocol, + controllerNodeProvider.saslMechanism, time, config.saslInterBrokerHandshakeRequestEnable, logContext diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index d2e34142519ea..39a93aa47acf5 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -378,6 +378,7 @@ object KafkaConfig { val NodeIdProp = "node.id" val MetadataLogDirProp = "metadata.log.dir" val ControllerListenerNamesProp = "controller.listener.names" + val SaslMechanismControllerProtocolProp = "sasl.mechanism.controller.protocol" /************* Authorizer Configuration ***********/ val AuthorizerClassNameProp = "authorizer.class.name" @@ -675,6 +676,7 @@ object KafkaConfig { "KIP-500. If it is not set, the metadata log is placed in the first log directory from log.dirs." val ControllerListenerNamesDoc = "A comma-separated list of the names of the listeners used by the KIP-500 controller. This is required " + "if this process is a KIP-500 controller. The ZK-based controller will not use this configuration." + val SaslMechanismControllerProtocolDoc = "SASL mechanism used for communication with controllers. Default is GSSAPI." /************* Authorizer Configuration ***********/ val AuthorizerClassNameDoc = s"The fully qualified name of a class that implements s${classOf[Authorizer].getName}" + @@ -1081,6 +1083,7 @@ object KafkaConfig { .defineInternal(BrokerSessionTimeoutMsProp, INT, Defaults.BrokerSessionTimeoutMs, null, MEDIUM, BrokerSessionTimeoutMsDoc) .defineInternal(MetadataLogDirProp, STRING, null, null, HIGH, MetadataLogDirDoc) .defineInternal(ControllerListenerNamesProp, STRING, null, null, HIGH, ControllerListenerNamesDoc) + .defineInternal(SaslMechanismControllerProtocolProp, STRING, SaslConfigs.DEFAULT_SASL_MECHANISM, null, HIGH, SaslMechanismControllerProtocolDoc) /************* Authorizer Configuration ***********/ .define(AuthorizerClassNameProp, STRING, Defaults.AuthorizerClassName, LOW, AuthorizerClassNameDoc) @@ -1814,6 +1817,8 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO def controllerListeners: Seq[EndPoint] = listeners.filter(l => controllerListenerNames.contains(l.listenerName.value())) + def saslMechanismControllerProtocol = getString(KafkaConfig.SaslMechanismControllerProtocolProp) + def controlPlaneListener: Option[EndPoint] = { controlPlaneListenerName.map { listenerName => listeners.filter(endpoint => endpoint.listenerName.value() == listenerName.value()).head diff --git a/core/src/test/scala/unit/kafka/server/BrokerLifecycleManagerTest.scala b/core/src/test/scala/unit/kafka/server/BrokerLifecycleManagerTest.scala index 7544a463bed29..d3bcfef9342ee 100644 --- a/core/src/test/scala/unit/kafka/server/BrokerLifecycleManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/BrokerLifecycleManagerTest.scala @@ -22,6 +22,7 @@ import java.util.concurrent.atomic.{AtomicLong, AtomicReference} import kafka.utils.{MockTime, TestUtils} import org.apache.kafka.clients.{Metadata, MockClient, NodeApiVersions} +import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.{Node, Uuid} import org.apache.kafka.common.internals.ClusterResourceListeners import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion @@ -58,6 +59,8 @@ class BrokerLifecycleManagerTest { override def listenerName: ListenerName = new ListenerName("PLAINTEXT") override def securityProtocol: SecurityProtocol = SecurityProtocol.PLAINTEXT; + + override def saslMechanism: String = SaslConfigs.DEFAULT_SASL_MECHANISM } class BrokerLifecycleManagerTestContext(properties: Properties) { diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala index 6271105c8ca35..987a1fd8ef4dc 100755 --- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala @@ -779,6 +779,7 @@ class KafkaConfigTest { case KafkaConfig.SslPrincipalMappingRulesProp => // ignore string //Sasl Configs + case KafkaConfig.SaslMechanismControllerProtocolProp => // ignore case KafkaConfig.SaslMechanismInterBrokerProtocolProp => // ignore case KafkaConfig.SaslEnabledMechanismsProp => case KafkaConfig.SaslClientCallbackHandlerClassProp => diff --git a/tests/kafkatest/sanity_checks/test_verifiable_producer.py b/tests/kafkatest/sanity_checks/test_verifiable_producer.py index 32961f1995dff..7fcb603d598c4 100644 --- a/tests/kafkatest/sanity_checks/test_verifiable_producer.py +++ b/tests/kafkatest/sanity_checks/test_verifiable_producer.py @@ -96,4 +96,77 @@ def test_simple_run(self, producer_version, security_protocol = 'PLAINTEXT', sas num_produced = self.producer.num_acked assert num_produced == self.num_messages, "num_produced: %d, num_messages: %d" % (num_produced, self.num_messages) + @cluster(num_nodes=4) + @matrix(inter_broker_security_protocol=['PLAINTEXT', 'SSL'], metadata_quorum=[quorum.remote_raft]) + @matrix(inter_broker_security_protocol=['SASL_SSL'], inter_broker_sasl_mechanism=['PLAIN', 'GSSAPI'], + metadata_quorum=[quorum.remote_raft]) + def test_multiple_raft_security_protocols( + self, inter_broker_security_protocol, inter_broker_sasl_mechanism='GSSAPI', metadata_quorum=quorum.remote_raft): + """ + Test for remote Raft cases that we can start VerifiableProducer on the current branch snapshot version, and + verify that we can produce a small number of messages. The inter-controller and broker-to-controller + security protocols are defined to be different (which differs from the above test, where they were the same). + """ + self.kafka.security_protocol = self.kafka.interbroker_security_protocol = inter_broker_security_protocol + self.kafka.client_sasl_mechanism = self.kafka.interbroker_sasl_mechanism = inter_broker_sasl_mechanism + controller_quorum = self.kafka.controller_quorum + sasl_mechanism = 'PLAIN' if inter_broker_sasl_mechanism == 'GSSAPI' else 'GSSAPI' + if inter_broker_security_protocol == 'PLAINTEXT': + controller_security_protocol = 'SSL' + intercontroller_security_protocol = 'SASL_SSL' + elif inter_broker_security_protocol == 'SSL': + controller_security_protocol = 'SASL_SSL' + intercontroller_security_protocol = 'PLAINTEXT' + else: # inter_broker_security_protocol == 'SASL_SSL' + controller_security_protocol = 'PLAINTEXT' + intercontroller_security_protocol = 'SSL' + controller_quorum.controller_security_protocol = controller_security_protocol + controller_quorum.controller_sasl_mechanism = sasl_mechanism + controller_quorum.intercontroller_security_protocol = intercontroller_security_protocol + controller_quorum.intercontroller_sasl_mechanism = sasl_mechanism + self.kafka.start() + + node = self.producer.nodes[0] + node.version = KafkaVersion(str(DEV_BRANCH)) + self.producer.start() + wait_until(lambda: self.producer.num_acked > 5, timeout_sec=15, + err_msg="Producer failed to start in a reasonable amount of time.") + + # See above comment above regarding use of version.vstring (distutils.version.LooseVersion) + assert is_version(node, [node.version.vstring], logger=self.logger) + + self.producer.wait() + num_produced = self.producer.num_acked + assert num_produced == self.num_messages, "num_produced: %d, num_messages: %d" % (num_produced, self.num_messages) + + @cluster(num_nodes=4) + @parametrize(metadata_quorum=quorum.remote_raft) + def test_multiple_raft_sasl_mechanisms(self, metadata_quorum): + """ + Test for remote Raft cases that we can start VerifiableProducer on the current branch snapshot version, and + verify that we can produce a small number of messages. The inter-controller and broker-to-controller + security protocols are both SASL_PLAINTEXT but the SASL mechanisms are different (we set + GSSAPI for the inter-controller mechanism and PLAIN for the broker-to-controller mechanism). + This test differs from the above tests -- he ones above used the same SASL mechanism for both paths. + """ + self.kafka.security_protocol = self.kafka.interbroker_security_protocol = 'PLAINTEXT' + controller_quorum = self.kafka.controller_quorum + controller_quorum.controller_security_protocol = 'SASL_PLAINTEXT' + controller_quorum.controller_sasl_mechanism = 'PLAIN' + controller_quorum.intercontroller_security_protocol = 'SASL_PLAINTEXT' + controller_quorum.intercontroller_sasl_mechanism = 'GSSAPI' + self.kafka.start() + + node = self.producer.nodes[0] + node.version = KafkaVersion(str(DEV_BRANCH)) + self.producer.start() + wait_until(lambda: self.producer.num_acked > 5, timeout_sec=15, + err_msg="Producer failed to start in a reasonable amount of time.") + + # See above comment above regarding use of version.vstring (distutils.version.LooseVersion) + assert is_version(node, [node.version.vstring], logger=self.logger) + + self.producer.wait() + num_produced = self.producer.num_acked + assert num_produced == self.num_messages, "num_produced: %d, num_messages: %d" % (num_produced, self.num_messages) diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py index c83319082ca50..6a5cb6278517f 100644 --- a/tests/kafkatest/services/kafka/kafka.py +++ b/tests/kafkatest/services/kafka/kafka.py @@ -232,7 +232,7 @@ def __init__(self, context, num_nodes, zk, security_protocol=SecurityConfig.PLAI :param str tls_version: version of the TLS protocol. :param str interbroker_security_protocol: security protocol to use for broker-to-broker (and Raft controller-to-controller) communication :param str client_sasl_mechanism: sasl mechanism for clients to use - :param str interbroker_sasl_mechanism: sasl mechanism to use for broker-to-broker communication + :param str interbroker_sasl_mechanism: sasl mechanism to use for broker-to-broker (and to-controller) communication :param str authorizer_class_name: which authorizer class to use :param str version: which kafka version to use. Defaults to "dev" branch :param jmx_object_names: @@ -443,14 +443,55 @@ def setup_interbroker_listener(self, security_protocol, use_separate_listener=Fa @property def security_config(self): if not self._security_config: - client_sasl_mechanism_to_use = self.client_sasl_mechanism if self.quorum_info.using_zk or self.quorum_info.has_brokers else self.controller_sasl_mechanism - interbroker_sasl_mechanism_to_use = self.interbroker_sasl_mechanism if self.quorum_info.using_zk or self.quorum_info.has_brokers else self.intercontroller_sasl_mechanism - self._security_config = SecurityConfig(self.context, self.security_protocol, self.interbroker_security_protocol, + # we will later change the security protocols to PLAINTEXT if this is a remote Raft controller case since + # those security protocols are irrelevant there and we don't want to falsely indicate the use of SASL or TLS + security_protocol_to_use=self.security_protocol + interbroker_security_protocol_to_use=self.interbroker_security_protocol + # determine uses/serves controller sasl mechanisms + serves_controller_sasl_mechanism=None + serves_intercontroller_sasl_mechanism=None + uses_controller_sasl_mechanism=None + if self.quorum_info.has_brokers: + if self.controller_quorum.controller_security_protocol in SecurityConfig.SASL_SECURITY_PROTOCOLS: + uses_controller_sasl_mechanism = self.controller_quorum.controller_sasl_mechanism + if self.quorum_info.has_controllers: + if self.intercontroller_security_protocol in SecurityConfig.SASL_SECURITY_PROTOCOLS: + serves_intercontroller_sasl_mechanism = self.intercontroller_sasl_mechanism + uses_controller_sasl_mechanism = self.intercontroller_sasl_mechanism # won't change from above in co-located case + if self.controller_security_protocol in SecurityConfig.SASL_SECURITY_PROTOCOLS: + serves_controller_sasl_mechanism = self.controller_sasl_mechanism + # determine if raft uses TLS + raft_tls = False + if self.quorum_info.has_brokers and not self.quorum_info.has_controllers: + # Raft-based broker only + raft_tls = self.controller_quorum.controller_security_protocol in SecurityConfig.SSL_SECURITY_PROTOCOLS + if self.quorum_info.has_controllers: + # remote or co-located raft controller + raft_tls = self.controller_security_protocol in SecurityConfig.SSL_SECURITY_PROTOCOLS \ + or self.intercontroller_security_protocol in SecurityConfig.SSL_SECURITY_PROTOCOLS + # clear irrelevant security protocols of SASL/TLS implications for remote controller quorum case + if self.quorum_info.has_controllers and not self.quorum_info.has_brokers: + security_protocol_to_use=SecurityConfig.PLAINTEXT + interbroker_security_protocol_to_use=SecurityConfig.PLAINTEXT + + self._security_config = SecurityConfig(self.context, security_protocol_to_use, interbroker_security_protocol_to_use, zk_sasl=self.zk.zk_sasl if self.quorum_info.using_zk else False, zk_tls=self.zk_client_secure, - client_sasl_mechanism=client_sasl_mechanism_to_use, - interbroker_sasl_mechanism=interbroker_sasl_mechanism_to_use, + client_sasl_mechanism=self.client_sasl_mechanism, + interbroker_sasl_mechanism=self.interbroker_sasl_mechanism, listener_security_config=self.listener_security_config, - tls_version=self.tls_version) + tls_version=self.tls_version, + serves_controller_sasl_mechanism=serves_controller_sasl_mechanism, + serves_intercontroller_sasl_mechanism=serves_intercontroller_sasl_mechanism, + uses_controller_sasl_mechanism=uses_controller_sasl_mechanism, + raft_tls=raft_tls) + # Ensure we have the right inter-broker security protocol because it may have been mutated + # since we cached our security config (ignore if this is a remote raft controller quorum case; the + # inter-broker security protocol is not used there). + if (self.quorum_info.using_zk or self.quorum_info.has_brokers) and \ + self._security_config.interbroker_security_protocol != self.interbroker_security_protocol: + self._security_config.interbroker_security_protocol = self.interbroker_security_protocol + self._security_config.calc_has_sasl() + self._security_config.calc_has_ssl() for port in self.port_mappings.values(): if port.open: self._security_config.enable_security_protocol(port.security_protocol) @@ -470,9 +511,7 @@ def close_port(self, listener_name): self.port_mappings[listener_name].open = False def start_minikdc_if_necessary(self, add_principals=""): - has_sasl = self.security_config.has_sasl if self.quorum_info.using_zk else \ - self.security_config.has_sasl or self.controller_quorum.security_config.has_sasl if self.quorum_info.has_brokers else \ - self.security_config.has_sasl or self.remote_kafka.security_config.has_sasl + has_sasl = self.security_config.has_sasl if has_sasl: if self.minikdc is None: other_service = self.remote_kafka if self.remote_kafka else self.controller_quorum if self.quorum_info.using_raft else None @@ -497,7 +536,10 @@ def start(self, add_principals=""): raise Exception("Unable to start Kafka: TLS to Zookeeper requested but Zookeeper secure port not enabled") if self.quorum_info.has_brokers_and_controllers and ( self.controller_security_protocol != self.intercontroller_security_protocol or - self.controller_sasl_mechanism != self.intercontroller_sasl_mechanism): + self.controller_security_protocol in SecurityConfig.SASL_SECURITY_PROTOCOLS and self.controller_sasl_mechanism != self.intercontroller_sasl_mechanism): + # This is not supported because both the broker and the controller take the first entry from + # controller.listener.names and the value from sasl.mechanism.controller.protocol; + # they share a single config, so they must both see/use identical values. raise Exception("Co-located Raft-based Brokers (%s/%s) and Controllers (%s/%s) cannot talk to Controllers via different security protocols" % (self.controller_security_protocol, self.controller_sasl_mechanism, self.intercontroller_security_protocol, self.intercontroller_sasl_mechanism)) @@ -666,6 +708,9 @@ def start_node(self, node, timeout_sec=60): for node in self.controller_quorum.nodes[:self.controller_quorum.num_nodes_controller_role]]) # define controller.listener.names self.controller_listener_names = ','.join(self.controller_listener_name_list()) + # define sasl.mechanism.controller.protocol to match remote quorum if one exists + if self.remote_controller_quorum: + self.controller_sasl_mechanism = self.remote_controller_quorum.controller_sasl_mechanism prop_file = self.prop_file(node) self.logger.info("kafka.properties:") diff --git a/tests/kafkatest/services/kafka/templates/kafka.properties b/tests/kafkatest/services/kafka/templates/kafka.properties index d7fa2d2167e09..f5c9a74abc86f 100644 --- a/tests/kafkatest/services/kafka/templates/kafka.properties +++ b/tests/kafkatest/services/kafka/templates/kafka.properties @@ -95,6 +95,13 @@ zookeeper.ssl.truststore.password=test-ts-passwd {% if quorum_info.using_zk or quorum_info.has_brokers %} sasl.mechanism.inter.broker.protocol={{ security_config.interbroker_sasl_mechanism }} {% endif %} +{% if quorum_info.using_raft %} +{% if not quorum_info.has_brokers %} +sasl.mechanism.controller.protocol={{ intercontroller_sasl_mechanism }} +{% else %} +sasl.mechanism.controller.protocol={{ controller_quorum.controller_sasl_mechanism }} +{% endif %} +{% endif %} sasl.enabled.mechanisms={{ ",".join(security_config.enabled_sasl_mechanisms) }} sasl.kerberos.service.name=kafka {% if authorizer_class_name is not none %} diff --git a/tests/kafkatest/services/security/security_config.py b/tests/kafkatest/services/security/security_config.py index f68b93d4f79ce..9ab7e6fa9c02f 100644 --- a/tests/kafkatest/services/security/security_config.py +++ b/tests/kafkatest/services/security/security_config.py @@ -120,6 +120,8 @@ class SecurityConfig(TemplateRenderer): SSL = 'SSL' SASL_PLAINTEXT = 'SASL_PLAINTEXT' SASL_SSL = 'SASL_SSL' + SASL_SECURITY_PROTOCOLS = [SASL_PLAINTEXT, SASL_SSL] + SSL_SECURITY_PROTOCOLS = [SSL, SASL_SSL] SASL_MECHANISM_GSSAPI = 'GSSAPI' SASL_MECHANISM_PLAIN = 'PLAIN' SASL_MECHANISM_SCRAM_SHA_256 = 'SCRAM-SHA-256' @@ -145,7 +147,11 @@ class SecurityConfig(TemplateRenderer): def __init__(self, context, security_protocol=None, interbroker_security_protocol=None, client_sasl_mechanism=SASL_MECHANISM_GSSAPI, interbroker_sasl_mechanism=SASL_MECHANISM_GSSAPI, zk_sasl=False, zk_tls=False, template_props="", static_jaas_conf=True, jaas_override_variables=None, - listener_security_config=ListenerSecurityConfig(), tls_version=None): + listener_security_config=ListenerSecurityConfig(), tls_version=None, + serves_controller_sasl_mechanism=None, # Raft Controller does this + serves_intercontroller_sasl_mechanism=None, # Raft Controller does this + uses_controller_sasl_mechanism=None, # communication to Raft Controller (broker and controller both do this) + raft_tls=False): """ Initialize the security properties for the node and copy keystore and truststore to the remote node if the transport protocol @@ -173,8 +179,17 @@ def __init__(self, context, security_protocol=None, interbroker_security_protoco if interbroker_security_protocol is None: interbroker_security_protocol = security_protocol self.interbroker_security_protocol = interbroker_security_protocol - self.has_sasl = self.is_sasl(security_protocol) or self.is_sasl(interbroker_security_protocol) or zk_sasl - self.has_ssl = self.is_ssl(security_protocol) or self.is_ssl(interbroker_security_protocol) or zk_tls + serves_raft_sasl = [] + if serves_controller_sasl_mechanism is not None: + serves_raft_sasl += [serves_controller_sasl_mechanism] + if serves_intercontroller_sasl_mechanism is not None: + serves_raft_sasl += [serves_intercontroller_sasl_mechanism] + self.serves_raft_sasl = set(serves_raft_sasl) + uses_raft_sasl = [] + if uses_controller_sasl_mechanism is not None: + uses_raft_sasl += [uses_controller_sasl_mechanism] + self.uses_raft_sasl = set(uses_raft_sasl) + self.zk_sasl = zk_sasl self.zk_tls = zk_tls self.static_jaas_conf = static_jaas_conf @@ -191,6 +206,7 @@ def __init__(self, context, security_protocol=None, interbroker_security_protoco 'sasl.mechanism.inter.broker.protocol' : interbroker_sasl_mechanism, 'sasl.kerberos.service.name' : 'kafka' } + self.raft_tls = raft_tls if tls_version is not None: self.properties.update({'tls.version' : tls_version}) @@ -198,6 +214,21 @@ def __init__(self, context, security_protocol=None, interbroker_security_protoco self.properties.update(self.listener_security_config.client_listener_overrides) self.jaas_override_variables = jaas_override_variables or {} + self.calc_has_sasl() + self.calc_has_ssl() + + def calc_has_sasl(self): + self.has_sasl = self.is_sasl(self.properties['security.protocol']) \ + or self.is_sasl(self.interbroker_security_protocol) \ + or self.zk_sasl \ + or self.serves_raft_sasl or self.uses_raft_sasl + + def calc_has_ssl(self): + self.has_ssl = self.is_ssl(self.properties['security.protocol']) \ + or self.is_ssl(self.interbroker_security_protocol) \ + or self.zk_tls \ + or self.raft_tls + def client_config(self, template_props="", node=None, jaas_override_variables=None, use_inter_broker_mechanism_for_client = False): # If node is not specified, use static jaas config which will be created later. @@ -315,10 +346,10 @@ def get_property(self, prop_name, template_props=""): return value def is_ssl(self, security_protocol): - return security_protocol == SecurityConfig.SSL or security_protocol == SecurityConfig.SASL_SSL + return security_protocol in SecurityConfig.SSL_SECURITY_PROTOCOLS def is_sasl(self, security_protocol): - return security_protocol == SecurityConfig.SASL_PLAINTEXT or security_protocol == SecurityConfig.SASL_SSL + return security_protocol in SecurityConfig.SASL_SECURITY_PROTOCOLS def is_sasl_scram(self, sasl_mechanism): return sasl_mechanism == SecurityConfig.SASL_MECHANISM_SCRAM_SHA_256 or sasl_mechanism == SecurityConfig.SASL_MECHANISM_SCRAM_SHA_512 @@ -341,7 +372,16 @@ def interbroker_sasl_mechanism(self): @property def enabled_sasl_mechanisms(self): - return set([self.client_sasl_mechanism, self.interbroker_sasl_mechanism]) + sasl_mechanisms = [] + if self.is_sasl(self.security_protocol): + sasl_mechanisms += [self.client_sasl_mechanism] + if self.is_sasl(self.interbroker_security_protocol): + sasl_mechanisms += [self.interbroker_sasl_mechanism] + if self.serves_raft_sasl: + sasl_mechanisms += list(self.serves_raft_sasl) + if self.uses_raft_sasl: + sasl_mechanisms += list(self.uses_raft_sasl) + return set(sasl_mechanisms) @property def has_sasl_kerberos(self): diff --git a/tests/kafkatest/tests/core/security_test.py b/tests/kafkatest/tests/core/security_test.py index 8dcc264d831c0..0ce12c9c71ba7 100644 --- a/tests/kafkatest/tests/core/security_test.py +++ b/tests/kafkatest/tests/core/security_test.py @@ -57,7 +57,7 @@ def producer_consumer_have_expected_error(self, error): return True - @cluster(num_nodes=7) + @cluster(num_nodes=6) @matrix(security_protocol=['PLAINTEXT'], interbroker_security_protocol=['SSL'], metadata_quorum=quorum.all_non_upgrade) @matrix(security_protocol=['SSL'], interbroker_security_protocol=['PLAINTEXT'], metadata_quorum=quorum.all_non_upgrade) def test_client_ssl_endpoint_validation_failure(self, security_protocol, interbroker_security_protocol, metadata_quorum=quorum.zk): @@ -78,34 +78,48 @@ def test_client_ssl_endpoint_validation_failure(self, security_protocol, interbr self.create_kafka(security_protocol=security_protocol, interbroker_security_protocol=interbroker_security_protocol) + if self.kafka.quorum_info.using_raft and interbroker_security_protocol == 'SSL': + # we don't want to interfere with communication to the controller quorum + # (we separately test this below) so make sure it isn't using TLS + # (it uses the inter-broker security information by default) + controller_quorum = self.kafka.controller_quorum + controller_quorum.controller_security_protocol = 'PLAINTEXT' + controller_quorum.intercontroller_security_protocol = 'PLAINTEXT' self.kafka.start() # now set the certs to have invalid hostnames so we can run the actual test SecurityConfig.ssl_stores.valid_hostname = False self.kafka.restart_cluster() - # We need more verbose logging to catch the expected errors - self.create_and_start_clients(log_level="DEBUG") + if self.kafka.quorum_info.using_raft and security_protocol == 'PLAINTEXT': + # the inter-broker security protocol using TLS with a hostname verification failure + # doesn't impact a producer in case of a single broker with a Raft Controller, + # so confirm that this is in fact the observed behavior + self.create_and_start_clients(log_level="INFO") + self.run_validation() + else: + # We need more verbose logging to catch the expected errors + self.create_and_start_clients(log_level="DEBUG") - try: - wait_until(lambda: self.producer.num_acked > 0, timeout_sec=30) + try: + wait_until(lambda: self.producer.num_acked > 0, timeout_sec=30) - # Fail quickly if messages are successfully acked - raise RuntimeError("Messages published successfully but should not have!" - " Endpoint validation did not fail with invalid hostname") - except TimeoutError: - # expected - pass + # Fail quickly if messages are successfully acked + raise RuntimeError("Messages published successfully but should not have!" + " Endpoint validation did not fail with invalid hostname") + except TimeoutError: + # expected + pass - error = 'SSLHandshakeException' if security_protocol == 'SSL' else 'LEADER_NOT_AVAILABLE' - wait_until(lambda: self.producer_consumer_have_expected_error(error), timeout_sec=30) - self.producer.stop() - self.consumer.stop() + error = 'SSLHandshakeException' if security_protocol == 'SSL' else 'LEADER_NOT_AVAILABLE' + wait_until(lambda: self.producer_consumer_have_expected_error(error), timeout_sec=30) + self.producer.stop() + self.consumer.stop() - SecurityConfig.ssl_stores.valid_hostname = True - self.kafka.restart_cluster() - self.create_and_start_clients(log_level="INFO") - self.run_validation() + SecurityConfig.ssl_stores.valid_hostname = True + self.kafka.restart_cluster() + self.create_and_start_clients(log_level="INFO") + self.run_validation() def create_and_start_clients(self, log_level): self.create_producer(log_level=log_level) @@ -113,3 +127,47 @@ def create_and_start_clients(self, log_level): self.create_consumer(log_level=log_level) self.consumer.start() + + @cluster(num_nodes=2) + @matrix(metadata_quorum=[quorum.zk, quorum.remote_raft]) + def test_quorum_ssl_endpoint_validation_failure(self, metadata_quorum=quorum.zk): + """ + Test that invalid hostname in ZooKeeper or Raft Controller results in broker inability to start. + """ + # Start ZooKeeper/Raft-based Controller with valid hostnames in the certs' SANs + # so that we can start Kafka + SecurityConfig.ssl_stores = TestSslStores(self.test_context.local_scratch_dir, + valid_hostname=True) + + self.create_zookeeper_if_necessary(num_nodes=1, + zk_client_port = False, + zk_client_secure_port = True, + zk_tls_encrypt_only = True, + ) + if self.zk: + self.zk.start() + + self.create_kafka(num_nodes=1, + interbroker_security_protocol='SSL', # also sets the broker-to-raft-controller security protocol for the Raft case + zk_client_secure=True, # ignored if we aren't using ZooKeeper + ) + self.kafka.start() + + # now stop the Kafka broker + # and set the cert for ZooKeeper/Raft-based Controller to have an invalid hostname + # so we can restart Kafka and ensure it is unable to start + self.kafka.stop_node(self.kafka.nodes[0]) + + SecurityConfig.ssl_stores.valid_hostname = False + if quorum.for_test(self.test_context) == quorum.zk: + self.kafka.zk.restart_cluster() + else: + self.kafka.remote_controller_quorum.restart_cluster() + + try: + self.kafka.start_node(self.kafka.nodes[0], timeout_sec=30) + raise RuntimeError("Kafka restarted successfully but should not have!" + " Endpoint validation did not fail with invalid hostname") + except TimeoutError: + # expected + pass From 581344673047c126f2e5f2c00c00221d12a9b7d5 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Fri, 26 Feb 2021 17:13:20 -0800 Subject: [PATCH 074/243] MINOR: fix kafka-metadata-shell.sh (#10226) * Fix CLASSPATH issues in the startup script * Fix overly verbose log messages during loading * Update to use the new MetadataRecordSerde (this is needed now that we have a frame version) * Fix initialization Reviewers: Jason Gustafson --- bin/kafka-run-class.sh | 12 ++++++++++++ .../apache/kafka/shell/MetadataNodeManager.java | 4 ++-- .../org/apache/kafka/shell/MetadataShell.java | 8 +++++++- .../apache/kafka/shell/SnapshotFileReader.java | 17 +++++------------ 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index 0d322854a504e..3889be7e3e5f8 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -135,6 +135,18 @@ do CLASSPATH="$CLASSPATH":"$file" done +for file in "$base_dir"/shell/build/libs/kafka-shell*.jar; +do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi +done + +for dir in "$base_dir"/shell/build/dependant-libs-${SCALA_VERSION}*; +do + CLASSPATH="$CLASSPATH:$dir/*" +done + for file in "$base_dir"/tools/build/libs/kafka-tools*.jar; do if should_include_file "$file"; then diff --git a/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java b/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java index fafccfae99dee..739e0278d5cda 100644 --- a/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java +++ b/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java @@ -96,7 +96,7 @@ public void handleCommit(BatchReader reader) { @Override public void handleCommits(long lastOffset, List messages) { appendEvent("handleCommits", () -> { - log.error("handleCommits " + messages + " at offset " + lastOffset); + log.debug("handleCommits " + messages + " at offset " + lastOffset); DirectoryNode dir = data.root.mkdirs("metadataQuorum"); dir.create("offset").setContents(String.valueOf(lastOffset)); for (ApiMessage message : messages) { @@ -108,7 +108,7 @@ public void handleCommits(long lastOffset, List messages) { @Override public void handleNewLeader(MetaLogLeader leader) { appendEvent("handleNewLeader", () -> { - log.error("handleNewLeader " + leader); + log.debug("handleNewLeader " + leader); DirectoryNode dir = data.root.mkdirs("metadataQuorum"); dir.create("leader").setContents(leader.toString()); }, null); diff --git a/shell/src/main/java/org/apache/kafka/shell/MetadataShell.java b/shell/src/main/java/org/apache/kafka/shell/MetadataShell.java index b701310efb774..9ba70d83064e4 100644 --- a/shell/src/main/java/org/apache/kafka/shell/MetadataShell.java +++ b/shell/src/main/java/org/apache/kafka/shell/MetadataShell.java @@ -54,6 +54,9 @@ public Builder setSnapshotPath(String snapshotPath) { } public MetadataShell build() throws Exception { + if (snapshotPath == null) { + throw new RuntimeException("You must supply the log path via --snapshot"); + } MetadataNodeManager nodeManager = null; SnapshotFileReader reader = null; try { @@ -99,11 +102,15 @@ public void run(List args) throws Exception { } if (args == null || args.isEmpty()) { // Interactive mode. + System.out.println("Loading..."); + waitUntilCaughtUp(); + System.out.println("Starting..."); try (InteractiveShell shell = new InteractiveShell(nodeManager)) { shell.runMainLoop(); } } else { // Non-interactive mode. + waitUntilCaughtUp(); Commands commands = new Commands(false); try (PrintWriter writer = new PrintWriter(new BufferedWriter( new OutputStreamWriter(System.out, StandardCharsets.UTF_8)))) { @@ -150,7 +157,6 @@ public static void main(String[] args) throws Exception { } }); MetadataShell shell = builder.build(); - shell.waitUntilCaughtUp(); try { shell.run(res.getList("command")); } finally { diff --git a/shell/src/main/java/org/apache/kafka/shell/SnapshotFileReader.java b/shell/src/main/java/org/apache/kafka/shell/SnapshotFileReader.java index e566be67d7d95..907b4db467ae1 100644 --- a/shell/src/main/java/org/apache/kafka/shell/SnapshotFileReader.java +++ b/shell/src/main/java/org/apache/kafka/shell/SnapshotFileReader.java @@ -18,7 +18,6 @@ package org.apache.kafka.shell; import org.apache.kafka.common.message.LeaderChangeMessage; -import org.apache.kafka.common.metadata.MetadataRecordType; import org.apache.kafka.common.protocol.ApiMessage; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.record.ControlRecordType; @@ -27,8 +26,10 @@ import org.apache.kafka.common.record.Record; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.metadata.ApiMessageAndVersion; import org.apache.kafka.metalog.MetaLogLeader; import org.apache.kafka.metalog.MetaLogListener; +import org.apache.kafka.raft.metadata.MetadataRecordSerde; import org.apache.kafka.queue.EventQueue; import org.apache.kafka.queue.KafkaEventQueue; import org.slf4j.Logger; @@ -53,6 +54,7 @@ public final class SnapshotFileReader implements AutoCloseable { private final CompletableFuture caughtUpFuture; private FileRecords fileRecords; private Iterator batchIterator; + private final MetadataRecordSerde serde = new MetadataRecordSerde(); public SnapshotFileReader(String snapshotPath, MetaLogListener listener) { this.snapshotPath = snapshotPath; @@ -140,17 +142,8 @@ private void handleMetadataBatch(FileChannelRecordBatch batch) { Record record = iter.next(); ByteBufferAccessor accessor = new ByteBufferAccessor(record.value()); try { - int apiKey = accessor.readUnsignedVarint(); - if (apiKey > Short.MAX_VALUE || apiKey < 0) { - throw new RuntimeException("Invalid apiKey value " + apiKey); - } - int apiVersion = accessor.readUnsignedVarint(); - if (apiVersion > Short.MAX_VALUE || apiVersion < 0) { - throw new RuntimeException("Invalid apiVersion value " + apiVersion); - } - ApiMessage message = MetadataRecordType.fromId((short) apiKey).newMetadataRecord(); - message.read(accessor, (short) apiVersion); - messages.add(message); + ApiMessageAndVersion messageAndVersion = serde.read(accessor, record.valueSize()); + messages.add(messageAndVersion.message()); } catch (Throwable e) { log.error("unable to read metadata record at offset {}", record.offset(), e); } From cc088c5abe367989e833e444171dd546370c356f Mon Sep 17 00:00:00 2001 From: Dhruvil Shah Date: Mon, 1 Mar 2021 01:30:30 -0800 Subject: [PATCH 075/243] KAFKA-12254: Ensure MM2 creates topics with source topic configs (#10217) MM2 creates new topics on the destination cluster with default configurations. It has an async periodic task to refresh topic configurations from the source to destination. However, this opens up a window where the destination cluster has data produced to it with default configurations. In the worst case, this could cause data loss if the destination topic is created without the right cleanup.policy. This commit fixes the above issue by ensuring that the right configurations are supplied to AdminClient#createTopics when MM2 creates topics on the destination cluster. Reviewers: Rajini Sivaram --- .../connect/mirror/MirrorSourceConnector.java | 90 ++++++++++++++----- .../mirror/MirrorSourceConnectorTest.java | 31 +++++-- 2 files changed, 92 insertions(+), 29 deletions(-) diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConnector.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConnector.java index 3e7f0c7bea77a..7b844c84d2f56 100644 --- a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConnector.java +++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConnector.java @@ -49,6 +49,7 @@ import java.util.HashSet; import java.util.Collection; import java.util.Collections; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.concurrent.ExecutionException; @@ -306,40 +307,82 @@ private void createOffsetSyncsTopic() { MirrorUtils.createSinglePartitionCompactedTopic(config.offsetSyncsTopic(), config.offsetSyncsTopicReplicationFactor(), config.sourceAdminConfig()); } - // visible for testing - void computeAndCreateTopicPartitions() - throws InterruptedException, ExecutionException { - Map partitionCounts = knownSourceTopicPartitions.stream() - .collect(Collectors.groupingBy(TopicPartition::topic, Collectors.counting())).entrySet().stream() - .collect(Collectors.toMap(x -> formatRemoteTopic(x.getKey()), Entry::getValue)); - Set knownTargetTopics = toTopics(knownTargetTopicPartitions); - List newTopics = partitionCounts.entrySet().stream() - .filter(x -> !knownTargetTopics.contains(x.getKey())) - .map(x -> new NewTopic(x.getKey(), x.getValue().intValue(), (short) replicationFactor)) - .collect(Collectors.toList()); - Map newPartitions = partitionCounts.entrySet().stream() - .filter(x -> knownTargetTopics.contains(x.getKey())) - .collect(Collectors.toMap(Entry::getKey, x -> NewPartitions.increaseTo(x.getValue().intValue()))); - createTopicPartitions(partitionCounts, newTopics, newPartitions); + void computeAndCreateTopicPartitions() throws ExecutionException, InterruptedException { + // get source and target topics with respective partition counts + Map sourceTopicToPartitionCounts = knownSourceTopicPartitions.stream() + .collect(Collectors.groupingBy(TopicPartition::topic, Collectors.counting())).entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + Map targetTopicToPartitionCounts = knownTargetTopicPartitions.stream() + .collect(Collectors.groupingBy(TopicPartition::topic, Collectors.counting())).entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + + Set knownSourceTopics = sourceTopicToPartitionCounts.keySet(); + Set knownTargetTopics = targetTopicToPartitionCounts.keySet(); + Map sourceToRemoteTopics = knownSourceTopics.stream() + .collect(Collectors.toMap(Function.identity(), sourceTopic -> formatRemoteTopic(sourceTopic))); + + // compute existing and new source topics + Map> partitionedSourceTopics = knownSourceTopics.stream() + .collect(Collectors.partitioningBy(sourceTopic -> knownTargetTopics.contains(sourceToRemoteTopics.get(sourceTopic)), + Collectors.toSet())); + Set existingSourceTopics = partitionedSourceTopics.get(true); + Set newSourceTopics = partitionedSourceTopics.get(false); + + // create new topics + if (!newSourceTopics.isEmpty()) + createNewTopics(newSourceTopics, sourceTopicToPartitionCounts); + + // compute topics with new partitions + Map sourceTopicsWithNewPartitions = existingSourceTopics.stream() + .filter(sourceTopic -> { + String targetTopic = sourceToRemoteTopics.get(sourceTopic); + return sourceTopicToPartitionCounts.get(sourceTopic) > targetTopicToPartitionCounts.get(targetTopic); + }) + .collect(Collectors.toMap(Function.identity(), sourceTopicToPartitionCounts::get)); + + // create new partitions + if (!sourceTopicsWithNewPartitions.isEmpty()) { + Map newTargetPartitions = sourceTopicsWithNewPartitions.entrySet().stream() + .collect(Collectors.toMap(sourceTopicAndPartitionCount -> sourceToRemoteTopics.get(sourceTopicAndPartitionCount.getKey()), + sourceTopicAndPartitionCount -> NewPartitions.increaseTo(sourceTopicAndPartitionCount.getValue().intValue()))); + createNewPartitions(newTargetPartitions); + } + } + + private void createNewTopics(Set newSourceTopics, Map sourceTopicToPartitionCounts) + throws ExecutionException, InterruptedException { + Map sourceTopicToConfig = describeTopicConfigs(newSourceTopics); + Map newTopics = newSourceTopics.stream() + .map(sourceTopic -> { + String remoteTopic = formatRemoteTopic(sourceTopic); + int partitionCount = sourceTopicToPartitionCounts.get(sourceTopic).intValue(); + Map configs = configToMap(sourceTopicToConfig.get(sourceTopic)); + return new NewTopic(remoteTopic, partitionCount, (short) replicationFactor) + .configs(configs); + }) + .collect(Collectors.toMap(NewTopic::name, Function.identity())); + createNewTopics(newTopics); } // visible for testing - void createTopicPartitions(Map partitionCounts, List newTopics, - Map newPartitions) { - targetAdminClient.createTopics(newTopics, new CreateTopicsOptions()).values().forEach((k, v) -> v.whenComplete((x, e) -> { + void createNewTopics(Map newTopics) { + targetAdminClient.createTopics(newTopics.values(), new CreateTopicsOptions()).values().forEach((k, v) -> v.whenComplete((x, e) -> { if (e != null) { log.warn("Could not create topic {}.", k, e); } else { - log.info("Created remote topic {} with {} partitions.", k, partitionCounts.get(k)); + log.info("Created remote topic {} with {} partitions.", k, newTopics.get(k).numPartitions()); } })); + } + + void createNewPartitions(Map newPartitions) { targetAdminClient.createPartitions(newPartitions).values().forEach((k, v) -> v.whenComplete((x, e) -> { if (e instanceof InvalidPartitionsException) { // swallow, this is normal } else if (e != null) { log.warn("Could not create topic-partitions for {}.", k, e); } else { - log.info("Increased size of {} to {} partitions.", k, partitionCounts.get(k)); + log.info("Increased size of {} to {} partitions.", k, newPartitions.get(k).totalCount()); } })); } @@ -359,6 +402,11 @@ private static Collection describeTopics(AdminClient adminClie return adminClient.describeTopics(topics).all().get().values(); } + static Map configToMap(Config config) { + return config.entries().stream() + .collect(Collectors.toMap(ConfigEntry::name, ConfigEntry::value)); + } + @SuppressWarnings("deprecation") // use deprecated alterConfigs API for broker compatibility back to 0.11.0 private void updateTopicConfigs(Map topicConfigs) @@ -390,7 +438,7 @@ private static Stream expandTopicDescription(TopicDescription de .map(x -> new TopicPartition(topic, x.partition())); } - private Map describeTopicConfigs(Set topics) + Map describeTopicConfigs(Set topics) throws InterruptedException, ExecutionException { Set resources = topics.stream() .map(x -> new ConfigResource(ConfigResource.Type.TOPIC, x)) diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java index a9633915fb979..42d7951cd60fc 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConnectorTest.java @@ -183,10 +183,16 @@ public void testRefreshTopicPartitions() throws Exception { connector.initialize(mock(ConnectorContext.class)); connector = spy(connector); + Config topicConfig = new Config(Arrays.asList( + new ConfigEntry("cleanup.policy", "compact"), + new ConfigEntry("segment.bytes", "100"))); + Map configs = Collections.singletonMap("topic", topicConfig); + List sourceTopicPartitions = Collections.singletonList(new TopicPartition("topic", 0)); doReturn(sourceTopicPartitions).when(connector).findSourceTopicPartitions(); doReturn(Collections.emptyList()).when(connector).findTargetTopicPartitions(); - doNothing().when(connector).createTopicPartitions(any(), any(), any()); + doReturn(configs).when(connector).describeTopicConfigs(Collections.singleton("topic")); + doNothing().when(connector).createNewTopics(any()); connector.refreshTopicPartitions(); // if target topic is not created, refreshTopicPartitions() will call createTopicPartitions() again @@ -194,13 +200,15 @@ public void testRefreshTopicPartitions() throws Exception { Map expectedPartitionCounts = new HashMap<>(); expectedPartitionCounts.put("source.topic", 1L); - List expectedNewTopics = Arrays.asList(new NewTopic("source.topic", 1, (short) 0)); + Map configMap = MirrorSourceConnector.configToMap(topicConfig); + assertEquals(2, configMap.size()); + + Map expectedNewTopics = new HashMap<>(); + expectedNewTopics.put("source.topic", new NewTopic("source.topic", 1, (short) 0).configs(configMap)); verify(connector, times(2)).computeAndCreateTopicPartitions(); - verify(connector, times(2)).createTopicPartitions( - eq(expectedPartitionCounts), - eq(expectedNewTopics), - eq(Collections.emptyMap())); + verify(connector, times(2)).createNewTopics(eq(expectedNewTopics)); + verify(connector, times(0)).createNewPartitions(any()); List targetTopicPartitions = Collections.singletonList(new TopicPartition("source.topic", 0)); doReturn(targetTopicPartitions).when(connector).findTargetTopicPartitions(); @@ -217,11 +225,19 @@ public void testRefreshTopicPartitionsTopicOnTargetFirst() throws Exception { connector.initialize(mock(ConnectorContext.class)); connector = spy(connector); + Config topicConfig = new Config(Arrays.asList( + new ConfigEntry("cleanup.policy", "compact"), + new ConfigEntry("segment.bytes", "100"))); + Map configs = Collections.singletonMap("source.topic", topicConfig); + List sourceTopicPartitions = Collections.emptyList(); List targetTopicPartitions = Collections.singletonList(new TopicPartition("source.topic", 0)); doReturn(sourceTopicPartitions).when(connector).findSourceTopicPartitions(); doReturn(targetTopicPartitions).when(connector).findTargetTopicPartitions(); - doNothing().when(connector).createTopicPartitions(any(), any(), any()); + doReturn(configs).when(connector).describeTopicConfigs(Collections.singleton("source.topic")); + doReturn(Collections.emptyMap()).when(connector).describeTopicConfigs(Collections.emptySet()); + doNothing().when(connector).createNewTopics(any()); + doNothing().when(connector).createNewPartitions(any()); // partitions appearing on the target cluster should not cause reconfiguration connector.refreshTopicPartitions(); @@ -234,6 +250,5 @@ public void testRefreshTopicPartitionsTopicOnTargetFirst() throws Exception { // when partitions are added to the source cluster, reconfiguration is triggered connector.refreshTopicPartitions(); verify(connector, times(1)).computeAndCreateTopicPartitions(); - } } From d78a923a16d5a211b2bc984aa8026cd4441c4da9 Mon Sep 17 00:00:00 2001 From: David Jacot Date: Mon, 1 Mar 2021 14:59:54 +0100 Subject: [PATCH 076/243] KAFKA-12329; kafka-reassign-partitions command should give a better error message when a topic does not exist (#10141) This patch updates `kafka-reassign-partitions` to provide a meaningful error message to the user when a topic does not exist. Reviewers: Chia-Ping Tsai --- .../admin/ReassignPartitionsCommand.scala | 47 +++++++++++++------ .../admin/ReassignPartitionsUnitTest.scala | 14 ++++++ 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index 65d6428d90415..16dd34cc3736b 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -812,6 +812,26 @@ object ReassignPartitionsCommand extends Logging { proposedAssignments } + private def describeTopics(adminClient: Admin, + topics: Set[String]) + : Map[String, TopicDescription] = { + adminClient.describeTopics(topics.asJava).values.asScala.map { + case (topicName, topicDescriptionFuture) => + try { + topicName -> topicDescriptionFuture.get + } + catch { + case t: ExecutionException => + if (classOf[UnknownTopicOrPartitionException].isInstance(t.getCause)) { + throw new ExecutionException( + new UnknownTopicOrPartitionException(s"Topic $topicName not found.")) + } else { + throw t + } + } + } + } + /** * Get the current replica assignments for some topics. * @@ -823,12 +843,10 @@ object ReassignPartitionsCommand extends Logging { def getReplicaAssignmentForTopics(adminClient: Admin, topics: Seq[String]) : Map[TopicPartition, Seq[Int]] = { - adminClient.describeTopics(topics.asJava).all().get().asScala.flatMap { - case (topicName, topicDescription) => - topicDescription.partitions.asScala.map { - info => (new TopicPartition(topicName, info.partition), - info.replicas.asScala.map(_.id)) - } + describeTopics(adminClient, topics.toSet).flatMap { + case (topicName, topicDescription) => topicDescription.partitions.asScala.map { info => + (new TopicPartition(topicName, info.partition), info.replicas.asScala.map(_.id)) + } } } @@ -843,15 +861,14 @@ object ReassignPartitionsCommand extends Logging { def getReplicaAssignmentForPartitions(adminClient: Admin, partitions: Set[TopicPartition]) : Map[TopicPartition, Seq[Int]] = { - adminClient.describeTopics(partitions.map(_.topic).asJava).all().get().asScala.flatMap { - case (topicName, topicDescription) => - topicDescription.partitions.asScala.flatMap { - info => if (partitions.contains(new TopicPartition(topicName, info.partition))) { - Some(new TopicPartition(topicName, info.partition()), - info.replicas.asScala.map(_.id)) - } else { - None - } + describeTopics(adminClient, partitions.map(_.topic)).flatMap { + case (topicName, topicDescription) => topicDescription.partitions.asScala.flatMap { info => + val tp = new TopicPartition(topicName, info.partition) + if (partitions.contains(tp)) { + Some(tp, info.replicas.asScala.map(_.id)) + } else { + None + } } } } diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala index 2187a0fcdc811..cbbebe7c825ce 100644 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala @@ -286,6 +286,20 @@ class ReassignPartitionsUnitTest { } } + @Test + def testGenerateAssignmentWithInvalidPartitionsFails(): Unit = { + val adminClient = new MockAdminClient.Builder().numBrokers(5).build() + try { + addTopics(adminClient) + assertStartsWith("Topic quux not found", + assertThrows(classOf[ExecutionException], + () => generateAssignment(adminClient, """{"topics":[{"topic":"foo"},{"topic":"quux"}]}""", "0,1", false), + () => "Expected generateAssignment to fail").getCause.getMessage) + } finally { + adminClient.close() + } + } + @Test def testGenerateAssignmentWithInconsistentRacks(): Unit = { val adminClient = new MockAdminClient.Builder(). From a63e5be4195e97e5b825b5912291144d2d0283a3 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Mon, 1 Mar 2021 11:03:34 -0500 Subject: [PATCH 077/243] KAFKA-10340: Proactively close producer when cancelling source tasks (#10016) Close the producer in `WorkerSourceTask` when the latter is cancelled. If the broker do not autocreate the topic, and the connector is not configured to create topics written by the source connector, then the `WorkerSourceTask` main thread will block forever until the topic is created, and will not stop if cancelled or scheduled for shutdown by the worker. Expanded an existing unit test for the WorkerSourceTask class to ensure that the producer is closed when the task is abandoned, and added a new integration test that guarantees that tasks are still shut down even when their producers are trying to write to topics that do not exist. Author: Chris Egerton Reviewed: Greg Harris , Randall Hauch --- .../apache/kafka/connect/runtime/Worker.java | 2 +- .../connect/runtime/WorkerSourceTask.java | 38 ++++++++++++---- .../kafka/connect/runtime/WorkerTask.java | 4 ++ .../ConnectWorkerIntegrationTest.java | 45 +++++++++++++++++-- .../connect/integration/ConnectorHandle.java | 8 ++-- .../MonitorableSourceConnector.java | 1 + .../runtime/ErrorHandlingTaskTest.java | 3 +- ...rrorHandlingTaskWithTopicCreationTest.java | 3 +- .../connect/runtime/WorkerSourceTaskTest.java | 5 ++- ...WorkerSourceTaskWithTopicCreationTest.java | 5 ++- .../kafka/connect/runtime/WorkerTest.java | 4 +- .../runtime/WorkerWithTopicCreationTest.java | 4 +- 12 files changed, 98 insertions(+), 24 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java index 1f6a8d151511c..3dc8c198f5ca1 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java @@ -626,7 +626,7 @@ private WorkerTask buildWorkerTask(ClusterConfigState configState, // Note we pass the configState as it performs dynamic transformations under the covers return new WorkerSourceTask(id, (SourceTask) task, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain, producer, admin, topicCreationGroups, - offsetReader, offsetWriter, config, configState, metrics, loader, time, retryWithToleranceOperator, herder.statusBackingStore()); + offsetReader, offsetWriter, config, configState, metrics, loader, time, retryWithToleranceOperator, herder.statusBackingStore(), executor); } else if (task instanceof SinkTask) { TransformationChain transformationChain = new TransformationChain<>(connConfig.transformations(), retryWithToleranceOperator); log.info("Initializing: {}", transformationChain); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java index 342fa73d8724a..48660f3b33233 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java @@ -60,6 +60,7 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -86,6 +87,7 @@ class WorkerSourceTask extends WorkerTask { private final TopicAdmin admin; private final CloseableOffsetStorageReader offsetReader; private final OffsetStorageWriter offsetWriter; + private final Executor closeExecutor; private final SourceTaskMetricsGroup sourceTaskMetricsGroup; private final AtomicReference producerSendException; private final boolean isTopicTrackingEnabled; @@ -123,7 +125,8 @@ public WorkerSourceTask(ConnectorTaskId id, ClassLoader loader, Time time, RetryWithToleranceOperator retryWithToleranceOperator, - StatusBackingStore statusBackingStore) { + StatusBackingStore statusBackingStore, + Executor closeExecutor) { super(id, statusListener, initialState, loader, connectMetrics, retryWithToleranceOperator, time, statusBackingStore); @@ -139,6 +142,7 @@ public WorkerSourceTask(ConnectorTaskId id, this.admin = admin; this.offsetReader = offsetReader; this.offsetWriter = offsetWriter; + this.closeExecutor = closeExecutor; this.toSend = null; this.lastSendFailed = false; @@ -171,13 +175,9 @@ protected void close() { log.warn("Could not stop task", t); } } - if (producer != null) { - try { - producer.close(Duration.ofSeconds(30)); - } catch (Throwable t) { - log.warn("Could not close producer", t); - } - } + + closeProducer(Duration.ofSeconds(30)); + if (admin != null) { try { admin.close(Duration.ofSeconds(30)); @@ -202,6 +202,14 @@ public void removeMetrics() { public void cancel() { super.cancel(); offsetReader.close(); + // We proactively close the producer here as the main work thread for the task may + // be blocked indefinitely in a call to Producer::send if automatic topic creation is + // not enabled on either the connector or the Kafka cluster. Closing the producer should + // unblock it in that case and allow shutdown to proceed normally. + // With a duration of 0, the producer's own shutdown logic should be fairly quick, + // but closing user-pluggable classes like interceptors may lag indefinitely. So, we + // call close on a separate thread in order to avoid blocking the herder's tick thread. + closeExecutor.execute(() -> closeProducer(Duration.ZERO)); } @Override @@ -259,6 +267,16 @@ public void execute() { } } + private void closeProducer(Duration duration) { + if (producer != null) { + try { + producer.close(duration); + } catch (Throwable t) { + log.warn("Could not close producer for {}", id, t); + } + } + } + private void maybeThrowProducerSendException() { if (producerSendException.get() != null) { throw new ConnectException( @@ -488,7 +506,9 @@ public boolean commitOffsets() { while (!outstandingMessages.isEmpty()) { try { long timeoutMs = timeout - time.milliseconds(); - if (timeoutMs <= 0) { + // If the task has been cancelled, no more records will be sent from the producer; in that case, if any outstanding messages remain, + // we can stop flushing immediately + if (isCancelled() || timeoutMs <= 0) { log.error("{} Failed to flush, timed out while waiting for producer to flush outstanding {} messages", this, outstandingMessages.size()); finishFailedFlush(); recordCommitFailure(time.milliseconds() - started, null); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java index 5cd992052ba01..a717af2343220 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java @@ -156,6 +156,10 @@ protected boolean isStopping() { return stopping; } + protected boolean isCancelled() { + return cancelled; + } + private void doClose() { try { close(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java index bd981832e1174..5cd794e7c83b1 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java @@ -65,13 +65,14 @@ public class ConnectWorkerIntegrationTest { private static final long OFFSET_COMMIT_INTERVAL_MS = TimeUnit.SECONDS.toMillis(30); private static final int NUM_WORKERS = 3; private static final int NUM_TASKS = 4; + private static final int MESSAGES_PER_POLL = 10; private static final String CONNECTOR_NAME = "simple-source"; private static final String TOPIC_NAME = "test-topic"; private EmbeddedConnectCluster.Builder connectBuilder; private EmbeddedConnectCluster connect; - Map workerProps = new HashMap<>(); - Properties brokerProps = new Properties(); + private Map workerProps; + private Properties brokerProps; @Rule public TestRule watcher = ConnectIntegrationTestUtils.newTestWatcher(log); @@ -79,10 +80,12 @@ public class ConnectWorkerIntegrationTest { @Before public void setup() { // setup Connect worker properties + workerProps = new HashMap<>(); workerProps.put(OFFSET_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(OFFSET_COMMIT_INTERVAL_MS)); workerProps.put(CONNECTOR_CLIENT_POLICY_CLASS_CONFIG, "All"); // setup Kafka broker properties + brokerProps = new Properties(); brokerProps.put("auto.create.topics.enable", String.valueOf(false)); // build a Connect cluster backed by Kafka and Zk @@ -288,14 +291,48 @@ public void testTaskStatuses() throws Exception { decreasedNumTasks, "Connector task statuses did not update in time."); } + @Test + public void testSourceTaskNotBlockedOnShutdownWithNonExistentTopic() throws Exception { + // When automatic topic creation is disabled on the broker + brokerProps.put("auto.create.topics.enable", "false"); + connect = connectBuilder + .brokerProps(brokerProps) + .numWorkers(1) + .numBrokers(1) + .build(); + connect.start(); + + connect.assertions().assertAtLeastNumWorkersAreUp(1, "Initial group of workers did not start in time."); + + // and when the connector is not configured to create topics + Map props = defaultSourceConnectorProps("nonexistenttopic"); + props.remove(DEFAULT_TOPIC_CREATION_PREFIX + REPLICATION_FACTOR_CONFIG); + props.remove(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG); + props.put("throughput", "-1"); + + ConnectorHandle connector = RuntimeHandles.get().connectorHandle(CONNECTOR_NAME); + connector.expectedRecords(NUM_TASKS * MESSAGES_PER_POLL); + connect.configureConnector(CONNECTOR_NAME, props); + connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, + NUM_TASKS, "Connector tasks did not start in time"); + connector.awaitRecords(TimeUnit.MINUTES.toMillis(1)); + + // Then if we delete the connector, it and each of its tasks should be stopped by the framework + // even though the producer is blocked because there is no topic + StartAndStopLatch stopCounter = connector.expectedStops(1); + connect.deleteConnector(CONNECTOR_NAME); + + assertTrue("Connector and all tasks were not stopped in time", stopCounter.await(1, TimeUnit.MINUTES)); + } + private Map defaultSourceConnectorProps(String topic) { // setup up props for the source connector Map props = new HashMap<>(); props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName()); props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS)); props.put(TOPIC_CONFIG, topic); - props.put("throughput", String.valueOf(10)); - props.put("messages.per.poll", String.valueOf(10)); + props.put("throughput", "10"); + props.put("messages.per.poll", String.valueOf(MESSAGES_PER_POLL)); props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); props.put(DEFAULT_TOPIC_CREATION_PREFIX + REPLICATION_FACTOR_CONFIG, String.valueOf(1)); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java index 06bc37352e455..ffc9e7a7a4fb0 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java @@ -292,8 +292,8 @@ public StartAndStopLatch expectedStarts(int expectedStarts, boolean includeTasks * {@link StartAndStopLatch#await(long, TimeUnit)} to wait up to a specified duration for the * connector and all tasks to be started at least the specified number of times. * - *

    This method does not track the number of times the connector and tasks are stopped, and - * only tracks the number of times the connector and tasks are started. + *

    This method does not track the number of times the connector and tasks are started, and + * only tracks the number of times the connector and tasks are stopped. * * @param expectedStops the minimum number of starts that are expected once this method is * called @@ -315,8 +315,8 @@ public StartAndStopLatch expectedStops(int expectedStops) { * {@link StartAndStopLatch#await(long, TimeUnit)} to wait up to a specified duration for the * connector and all tasks to be started at least the specified number of times. * - *

    This method does not track the number of times the connector and tasks are stopped, and - * only tracks the number of times the connector and tasks are started. + *

    This method does not track the number of times the connector and tasks are started, and + * only tracks the number of times the connector and tasks are stopped. * * @param expectedStops the minimum number of starts that are expected once this method is * called diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java index c11b9bd79d0fb..aaada374cb2b6 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java @@ -130,6 +130,7 @@ public List poll() { throttler.throttle(); } taskHandle.record(batchSize); + log.info("Returning batch of {} records", batchSize); return LongStream.range(0, batchSize) .mapToObj(i -> new SourceRecord( Collections.singletonMap("task.id", taskId), diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java index 70bbfc6590cd1..daa9eddac9ca9 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java @@ -77,6 +77,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.Executor; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -563,7 +564,7 @@ private void createSourceTask(TargetState initialState, RetryWithToleranceOperat producer, admin, null, offsetReader, offsetWriter, workerConfig, ClusterConfigState.EMPTY, metrics, pluginLoader, time, retryWithToleranceOperator, - statusBackingStore); + statusBackingStore, (Executor) Runnable::run); } private ConsumerRecords records(ConsumerRecord record) { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java index 692f4d24b18b1..5cdfab96d22ee 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java @@ -80,6 +80,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.Executor; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -581,7 +582,7 @@ private void createSourceTask(TargetState initialState, RetryWithToleranceOperat producer, admin, TopicCreationGroup.configuredGroups(sourceConfig), offsetReader, offsetWriter, workerConfig, ClusterConfigState.EMPTY, metrics, pluginLoader, time, retryWithToleranceOperator, - statusBackingStore); + statusBackingStore, (Executor) Runnable::run); } private ConsumerRecords records(ConsumerRecord record) { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java index 8c0988735369a..0f9e6562fb99f 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java @@ -197,7 +197,7 @@ private void createWorkerTask(TargetState initialState, Converter keyConverter, workerTask = new WorkerSourceTask(taskId, sourceTask, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain, producer, admin, null, offsetReader, offsetWriter, config, clusterConfigState, metrics, plugins.delegatingLoader(), Time.SYSTEM, - RetryWithToleranceOperatorTest.NOOP_OPERATOR, statusBackingStore); + RetryWithToleranceOperatorTest.NOOP_OPERATOR, statusBackingStore, Runnable::run); } @Test @@ -710,6 +710,9 @@ public void testCancel() { offsetReader.close(); PowerMock.expectLastCall(); + producer.close(Duration.ZERO); + PowerMock.expectLastCall(); + PowerMock.replayAll(); workerTask.cancel(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskWithTopicCreationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskWithTopicCreationTest.java index c8fdfa72bfe98..d26faa40707dd 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskWithTopicCreationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskWithTopicCreationTest.java @@ -222,7 +222,7 @@ private void createWorkerTask(TargetState initialState, Converter keyConverter, workerTask = new WorkerSourceTask(taskId, sourceTask, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain, producer, admin, TopicCreationGroup.configuredGroups(sourceConfig), offsetReader, offsetWriter, config, clusterConfigState, metrics, plugins.delegatingLoader(), Time.SYSTEM, - RetryWithToleranceOperatorTest.NOOP_OPERATOR, statusBackingStore); + RetryWithToleranceOperatorTest.NOOP_OPERATOR, statusBackingStore, Runnable::run); } @Test @@ -754,6 +754,9 @@ public void testCancel() { offsetReader.close(); PowerMock.expectLastCall(); + producer.close(Duration.ZERO); + PowerMock.expectLastCall(); + PowerMock.replayAll(); workerTask.cancel(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java index 1ac262589bdd3..b5c9122756a3b 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java @@ -94,6 +94,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -1481,7 +1482,8 @@ private void expectNewWorkerTask() throws Exception { EasyMock.eq(pluginLoader), anyObject(Time.class), anyObject(RetryWithToleranceOperator.class), - anyObject(StatusBackingStore.class)) + anyObject(StatusBackingStore.class), + anyObject(Executor.class)) .andReturn(workerTask); } /* Name here needs to be unique as we are testing the aliasing mechanism */ diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerWithTopicCreationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerWithTopicCreationTest.java index 912dfacdd9912..a6745ec99749b 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerWithTopicCreationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerWithTopicCreationTest.java @@ -85,6 +85,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -1389,7 +1390,8 @@ private void expectNewWorkerTask() throws Exception { EasyMock.eq(pluginLoader), anyObject(Time.class), anyObject(RetryWithToleranceOperator.class), - anyObject(StatusBackingStore.class)) + anyObject(StatusBackingStore.class), + anyObject(Executor.class)) .andReturn(workerTask); } From 2633db9b61ab68862556e1c17d90f9baba6666f8 Mon Sep 17 00:00:00 2001 From: David Jacot Date: Mon, 1 Mar 2021 18:25:23 +0100 Subject: [PATCH 078/243] MINOR; Small refactor in `GroupMetadata` (#10236) Reviewers: Jason Gustafson --- .../coordinator/group/GroupMetadata.scala | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala index 50328f0516300..53bce0b8cb0ee 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala @@ -252,7 +252,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState leaderId = Some(member.memberId) members.put(member.memberId, member) - member.supportedProtocols.foreach { case (protocol, _) => supportedProtocols(protocol) += 1 } + incSupportedProtocols(member) member.awaitingJoinCallback = callback if (member.isAwaitingJoin) @@ -263,7 +263,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState def remove(memberId: String): Unit = { members.remove(memberId).foreach { member => - member.supportedProtocols.foreach { case (protocol, _) => supportedProtocols(protocol) -= 1 } + decSupportedProtocols(member) if (member.isAwaitingJoin) numMembersAwaitingJoin -= 1 @@ -426,6 +426,14 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState protocol } + private def incSupportedProtocols(member: MemberMetadata): Unit = { + member.supportedProtocols.foreach { case (protocol, _) => supportedProtocols(protocol) += 1 } + } + + private def decSupportedProtocols(member: MemberMetadata): Unit = { + member.supportedProtocols.foreach { case (protocol, _) => supportedProtocols(protocol) -= 1 } + } + private def candidateProtocols: Set[String] = { // get the set of protocols that are commonly supported by all members val numMembers = members.size @@ -434,7 +442,7 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState def supportsProtocols(memberProtocolType: String, memberProtocols: Set[String]): Boolean = { if (is(Empty)) - !memberProtocolType.isEmpty && memberProtocols.nonEmpty + memberProtocolType.nonEmpty && memberProtocols.nonEmpty else protocolType.contains(memberProtocolType) && memberProtocols.exists(supportedProtocols(_) == members.size) } @@ -489,9 +497,9 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState def updateMember(member: MemberMetadata, protocols: List[(String, Array[Byte])], callback: JoinCallback): Unit = { - member.supportedProtocols.foreach { case (protocol, _) => supportedProtocols(protocol) -= 1 } - protocols.foreach { case (protocol, _) => supportedProtocols(protocol) += 1 } + decSupportedProtocols(member) member.supportedProtocols = protocols + incSupportedProtocols(member) if (callback != null && !member.isAwaitingJoin) { numMembersAwaitingJoin += 1 From e2bffb9086823230c60c4838d38f32917dd8d03d Mon Sep 17 00:00:00 2001 From: dengziming Date: Tue, 2 Mar 2021 03:02:28 +0800 Subject: [PATCH 079/243] MINOR: Word count should account for extra whitespaces between words (#10229) Reviewers: Matthias J. Sax --- .../streams/examples/wordcount/WordCountProcessorDemo.java | 2 +- .../streams/examples/wordcount/WordCountTransformerDemo.java | 2 +- .../streams/examples/wordcount/WordCountProcessorTest.java | 2 +- .../streams/examples/wordcount/WordCountTransformerTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java index 646a01653224d..8835c109e94f3 100644 --- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java +++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorDemo.java @@ -85,7 +85,7 @@ public void init(final ProcessorContext context) { @Override public void process(final Record record) { - final String[] words = record.value().toLowerCase(Locale.getDefault()).split(" "); + final String[] words = record.value().toLowerCase(Locale.getDefault()).split("\\W+"); for (final String word : words) { final Integer oldValue = kvStore.get(word); diff --git a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountTransformerDemo.java b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountTransformerDemo.java index bef0a626f98fc..33f8499d91da2 100644 --- a/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountTransformerDemo.java +++ b/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountTransformerDemo.java @@ -93,7 +93,7 @@ public void init(final ProcessorContext context) { @Override public KeyValue transform(final String dummy, final String line) { - final String[] words = line.toLowerCase(Locale.getDefault()).split(" "); + final String[] words = line.toLowerCase(Locale.getDefault()).split("\\W+"); for (final String word : words) { final Integer oldValue = this.kvStore.get(word); diff --git a/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorTest.java b/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorTest.java index 0df64b7b88605..eb62630a80329 100644 --- a/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorTest.java +++ b/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountProcessorTest.java @@ -52,7 +52,7 @@ public void test() { processor.init(context); // send a record to the processor - processor.process(new Record<>("key", "alpha beta gamma alpha", 0L)); + processor.process(new Record<>("key", "alpha beta\tgamma\n\talpha", 0L)); // note that the processor does not forward during process() assertTrue(context.forwarded().isEmpty()); diff --git a/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountTransformerTest.java b/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountTransformerTest.java index 82e916645a25d..662f0a2e494c7 100644 --- a/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountTransformerTest.java +++ b/streams/examples/src/test/java/org/apache/kafka/streams/examples/wordcount/WordCountTransformerTest.java @@ -52,7 +52,7 @@ public void test() { transformer.init(context); // send a record to the transformer - transformer.transform("key", "alpha beta gamma alpha"); + transformer.transform("key", "alpha beta\tgamma\n\talpha"); // note that the transformer does not forward during transform() assertTrue(context.forwarded().isEmpty()); From 7a0e371e0ed0988f5f66f207d5ec665ecfaaac88 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 1 Mar 2021 11:32:05 -0800 Subject: [PATCH 080/243] MINOR: Remove stack trace of the lock exception in a debug log4j (#10231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the lock exception log is at the DEBUG level only, many people were confused with stack traces that something serious happened; plus, in the source code there is only one call path that can lead to the capture of LockException at task manager/stream thread, so even for debugging purposes there’s no extra information we can get from anyways. Reviewers: Kamal Chandraprakash , Ismael Juma --- .../kafka/streams/processor/internals/TaskManager.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java index d74e10c98556f..5b9ebadb5da3b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java @@ -426,14 +426,11 @@ boolean tryToCompleteRestoration(final long now) { try { task.initializeIfNeeded(); task.clearTaskTimeout(); - } catch (final LockException retriableException) { + } catch (final LockException lockException) { // it is possible that if there are multiple threads within the instance that one thread // trying to grab the task from the other, while the other has not released the lock since // it did not participate in the rebalance. In this case we can just retry in the next iteration - log.debug( - String.format("Could not initialize %s due to the following exception; will retry", task.id()), - retriableException - ); + log.debug("Could not initialize task {} since: {}; will retry", task.id(), lockException.getMessage()); allRunning = false; } catch (final TimeoutException timeoutException) { task.maybeInitTaskTimeoutOrThrow(now, timeoutException); From 22dbf89886974c27c705bba653f9cfe36f70904c Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 1 Mar 2021 12:11:11 -0800 Subject: [PATCH 081/243] KAFKA-12323 Follow-up: Refactor the unit test a bit (#10205) Reviewers: Matthias J. Sax --- .../processor/internals/StreamThreadTest.java | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java index e2b1549b905c0..3043423edd12c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java @@ -1844,19 +1844,17 @@ public void process(final Object key, final Object value) {} assertEquals(0, punctuatedWallClockTime.size()); mockTime.sleep(100L); - for (long i = 0L; i < 10L; i++) { - clientSupplier.consumer.addRecord(new ConsumerRecord<>( - topic1, - 1, - i, - i * 100L, - TimestampType.CREATE_TIME, - ConsumerRecord.NULL_CHECKSUM, - ("K" + i).getBytes().length, - ("V" + i).getBytes().length, - ("K" + i).getBytes(), - ("V" + i).getBytes())); - } + clientSupplier.consumer.addRecord(new ConsumerRecord<>( + topic1, + 1, + 100L, + 100L, + TimestampType.CREATE_TIME, + ConsumerRecord.NULL_CHECKSUM, + "K".getBytes().length, + "V".getBytes().length, + "K".getBytes(), + "V".getBytes())); thread.runOnce(); @@ -1936,19 +1934,19 @@ public void process(final Object key, final Object value) { clientSupplier.consumer.addRecord(new ConsumerRecord<>( topic1, 1, - 0L, - 100L, - TimestampType.CREATE_TIME, - ConsumerRecord.NULL_CHECKSUM, - "K".getBytes().length, - "V".getBytes().length, - "K".getBytes(), - "V".getBytes())); + 110L, + 110L, + TimestampType.CREATE_TIME, + ConsumerRecord.NULL_CHECKSUM, + "K".getBytes().length, + "V".getBytes().length, + "K".getBytes(), + "V".getBytes())); thread.runOnce(); assertEquals(2, peekedContextTime.size()); - assertEquals(0L, peekedContextTime.get(1).longValue()); + assertEquals(110L, peekedContextTime.get(1).longValue()); } @Test From 020ead4b97ed96b9f7e4c7c3ac2523bd18ac930f Mon Sep 17 00:00:00 2001 From: Luke Chen <43372967+showuon@users.noreply.github.com> Date: Tue, 2 Mar 2021 17:32:21 +0800 Subject: [PATCH 082/243] MINOR: Format the revoking active log output in `StreamsPartitionAssignor` (#10242) Reviewers: Chia-Ping Tsai , David Jacot --- .../streams/processor/internals/StreamsPartitionAssignor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java index ac1d97bc2c5d3..841dcc61da0db 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java @@ -885,7 +885,7 @@ private Map computeNewAssignment(final Set statefulT "\tprev owned active {}\n" + "\tprev owned standby {}\n" + "\tassigned active {}\n" + - "\trevoking active {}" + + "\trevoking active {}\n" + "\tassigned standby {}\n", clientId, clientMetadata.state.prevOwnedActiveTasksByConsumer(), From 652cca377a1559d1e0143283779ae90bde5d1e7d Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Tue, 2 Mar 2021 17:51:47 +0800 Subject: [PATCH 083/243] MINOR: correct the error message of validating uint32 (#10193) Reviewers: Tom Bentley , David Jacot --- .../java/org/apache/kafka/common/protocol/types/Type.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java index d801d8a7a8f11..46a59bd08210e 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java @@ -248,7 +248,7 @@ public Integer validate(Object item) { if (item instanceof Integer) return (Integer) item; else - throw new SchemaException(item + " is not an unsigned short."); + throw new SchemaException(item + " is not an a Integer (encoding an unsigned short)"); } @Override @@ -320,7 +320,7 @@ public Long validate(Object item) { if (item instanceof Long) return (Long) item; else - throw new SchemaException(item + " is not a Long."); + throw new SchemaException(item + " is not an a Long (encoding an unsigned integer)."); } @Override From 38fee5fce515e2186c9ea708a50b515a8fffcaf2 Mon Sep 17 00:00:00 2001 From: Lucas Bradstreet Date: Tue, 2 Mar 2021 18:14:43 +0800 Subject: [PATCH 084/243] MINOR: Time and log producer state recovery phases (#10241) During a slow log recovery it's easy to think that loading .snapshot files is a multi-second process. Often it isn't the snapshot loading that takes most of the time, rather it's the time taken to further rebuild the producer state from segment files. This patch times both snapshot load and segment recovery phases to better indicate what is taking time. Reviewers: David Jacot --- core/src/main/scala/kafka/log/Log.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index 2729154bab4ac..f9486bb3dbd97 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -962,8 +962,11 @@ class Log(@volatile private var _dir: File, producerStateManager.takeSnapshot() } } else { + info(s"Reloading from producer snapshot and rebuilding producer state from offset $lastOffset") val isEmptyBeforeTruncation = producerStateManager.isEmpty && producerStateManager.mapEndOffset >= lastOffset + val producerStateLoadStart = time.milliseconds() producerStateManager.truncateAndReload(logStartOffset, lastOffset, time.milliseconds()) + val segmentRecoveryStart = time.milliseconds() // Only do the potentially expensive reloading if the last snapshot offset is lower than the log end // offset (which would be the case on first startup) and there were active producers prior to truncation @@ -999,6 +1002,8 @@ class Log(@volatile private var _dir: File, } producerStateManager.updateMapEndOffset(lastOffset) producerStateManager.takeSnapshot() + info(s"Producer state recovery took ${producerStateLoadStart - segmentRecoveryStart}ms for snapshot load " + + s"and ${time.milliseconds() - segmentRecoveryStart}ms for segment recovery from offset $lastOffset") } } From a92b986c855592d630fbabf49d1e9a160ad5b230 Mon Sep 17 00:00:00 2001 From: John Roesler Date: Tue, 2 Mar 2021 08:20:47 -0600 Subject: [PATCH 085/243] KAFKA-12268: Implement task idling semantics via currentLag API (#10137) Implements KIP-695 Reverts a previous behavior change to Consumer.poll and replaces it with a new Consumer.currentLag API, which returns the client's currently cached lag. Uses this new API to implement the desired task idling semantics improvement from KIP-695. Reverts fdcf8fbf72bee9e672d0790cdbe5539846f7dc8e / KAFKA-10866: Add metadata to ConsumerRecords (#9836) Reviewers: Chia-Ping Tsai , Guozhang Wang --- build.gradle | 1 - checkstyle/suppressions.xml | 5 +- .../kafka/clients/consumer/Consumer.java | 6 + .../clients/consumer/ConsumerRecords.java | 103 +---------- .../kafka/clients/consumer/KafkaConsumer.java | 36 +++- .../kafka/clients/consumer/MockConsumer.java | 32 ++-- .../consumer/internals/FetchedRecords.java | 102 ----------- .../clients/consumer/internals/Fetcher.java | 50 +++--- .../consumer/internals/SubscriptionState.java | 14 +- .../clients/consumer/KafkaConsumerTest.java | 163 +++++------------- .../consumer/internals/FetcherTest.java | 28 +-- .../kafka/api/PlaintextConsumerTest.scala | 36 ++-- .../processor/internals/PartitionGroup.java | 99 +++++------ .../processor/internals/StandbyTask.java | 6 - .../processor/internals/StreamTask.java | 12 +- .../processor/internals/StreamThread.java | 9 +- .../streams/processor/internals/Task.java | 6 - .../processor/internals/TaskManager.java | 3 +- .../internals/ActiveTaskCreatorTest.java | 2 +- .../internals/PartitionGroupTest.java | 29 ++-- .../processor/internals/StreamTaskTest.java | 10 -- .../processor/internals/TaskManagerTest.java | 6 - .../kafka/streams/TopologyTestDriver.java | 16 -- 23 files changed, 203 insertions(+), 571 deletions(-) delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchedRecords.java diff --git a/build.gradle b/build.gradle index 05f77e054f04a..4838a35bf7c03 100644 --- a/build.gradle +++ b/build.gradle @@ -1125,7 +1125,6 @@ project(':clients') { testCompile libs.bcpkix testCompile libs.junitJupiter testCompile libs.mockitoCore - testCompile libs.hamcrest testRuntime libs.slf4jlog4j testRuntime libs.jacksonDatabind diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 46fb97b3e197e..33273fef15c85 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -101,10 +101,7 @@ files="RequestResponseTest.java|FetcherTest.java|KafkaAdminClientTest.java"/> - - + files="MemoryRecordsTest|MetricsTest|RequestResponseTest|TestSslUtils|AclAuthorizerBenchmark"/> diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java index ee4a70770af97..b324773093aad 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -243,6 +244,11 @@ public interface Consumer extends Closeable { */ Map endOffsets(Collection partitions, Duration timeout); + /** + * @see KafkaConsumer#currentLag(TopicPartition) + */ + OptionalLong currentLag(TopicPartition topicPartition); + /** * @see KafkaConsumer#groupMetadata() */ diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecords.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecords.java index cc5317030bfa4..92390e91907e3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecords.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecords.java @@ -16,13 +16,11 @@ */ package org.apache.kafka.clients.consumer; -import org.apache.kafka.clients.consumer.internals.FetchedRecords; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.AbstractIterator; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -34,99 +32,12 @@ * partition returned by a {@link Consumer#poll(java.time.Duration)} operation. */ public class ConsumerRecords implements Iterable> { - public static final ConsumerRecords EMPTY = new ConsumerRecords<>( - Collections.emptyMap(), - Collections.emptyMap() - ); + public static final ConsumerRecords EMPTY = new ConsumerRecords<>(Collections.emptyMap()); private final Map>> records; - private final Map metadata; - public static final class Metadata { - - private final long receivedTimestamp; - private final Long position; - private final Long endOffset; - - public Metadata(final long receivedTimestamp, - final Long position, - final Long endOffset) { - this.receivedTimestamp = receivedTimestamp; - this.position = position; - this.endOffset = endOffset; - } - - /** - * @return The timestamp of the broker response that contained this metadata - */ - public long receivedTimestamp() { - return receivedTimestamp; - } - - /** - * @return The next position the consumer will fetch, or null if the consumer has no position. - */ - public Long position() { - return position; - } - - /** - * @return The lag between the next position to fetch and the current end of the partition, or - * null if the end offset is not known or there is no position. - */ - public Long lag() { - return endOffset == null || position == null ? null : endOffset - position; - } - - /** - * @return The current last offset in the partition. The determination of the "last" offset - * depends on the Consumer's isolation level. Under "read_uncommitted," this is the last successfully - * replicated offset plus one. Under "read_committed," this is the minimum of the last successfully - * replicated offset plus one or the smallest offset of any open transaction. Null if the end offset - * is not known. - */ - public Long endOffset() { - return endOffset; - } - - @Override - public String toString() { - return "Metadata{" + - "receivedTimestamp=" + receivedTimestamp + - ", position=" + position + - ", endOffset=" + endOffset + - '}'; - } - } - - private static Map extractMetadata(final FetchedRecords fetchedRecords) { - final Map metadata = new HashMap<>(); - for (final Map.Entry entry : fetchedRecords.metadata().entrySet()) { - metadata.put( - entry.getKey(), - new Metadata( - entry.getValue().receivedTimestamp(), - entry.getValue().position() == null ? null : entry.getValue().position().offset, - entry.getValue().endOffset() - ) - ); - } - return metadata; - } - - public ConsumerRecords(final Map>> records) { - this.records = records; - this.metadata = new HashMap<>(); - } - - public ConsumerRecords(final Map>> records, - final Map metadata) { + public ConsumerRecords(Map>> records) { this.records = records; - this.metadata = metadata; - } - - ConsumerRecords(final FetchedRecords fetchedRecords) { - this(fetchedRecords.records(), extractMetadata(fetchedRecords)); } /** @@ -142,16 +53,6 @@ public List> records(TopicPartition partition) { return Collections.unmodifiableList(recs); } - /** - * Get the updated metadata returned by the brokers along with this record set. - * May be empty or partial depending on the responses from the broker during this particular poll. - * May also include metadata for additional partitions than the ones for which there are records - * in this {@code ConsumerRecords} object. - */ - public Map metadata() { - return Collections.unmodifiableMap(metadata); - } - /** * Get just the records for the given topic */ diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index f7760bbe890a4..082d0eecd120d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -27,7 +27,6 @@ import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; -import org.apache.kafka.clients.consumer.internals.FetchedRecords; import org.apache.kafka.clients.consumer.internals.Fetcher; import org.apache.kafka.clients.consumer.internals.FetcherMetricsRegistry; import org.apache.kafka.clients.consumer.internals.KafkaConsumerMetrics; @@ -74,6 +73,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -578,6 +578,7 @@ public class KafkaConsumer implements Consumer { private final Deserializer valueDeserializer; private final Fetcher fetcher; private final ConsumerInterceptors interceptors; + private final IsolationLevel isolationLevel; private final Time time; private final ConsumerNetworkClient client; @@ -736,7 +737,7 @@ public KafkaConsumer(Map configs, FetcherMetricsRegistry metricsRegistry = new FetcherMetricsRegistry(Collections.singleton(CLIENT_ID_METRIC_TAG), metricGrpPrefix); ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config, time, logContext); - IsolationLevel isolationLevel = IsolationLevel.valueOf( + this.isolationLevel = IsolationLevel.valueOf( config.getString(ConsumerConfig.ISOLATION_LEVEL_CONFIG).toUpperCase(Locale.ROOT)); Sensor throttleTimeSensor = Fetcher.throttleTimeSensor(metrics, metricsRegistry); int heartbeatIntervalMs = config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG); @@ -849,6 +850,7 @@ public KafkaConsumer(Map configs, this.keyDeserializer = keyDeserializer; this.valueDeserializer = valueDeserializer; this.fetcher = fetcher; + this.isolationLevel = IsolationLevel.READ_UNCOMMITTED; this.interceptors = Objects.requireNonNull(interceptors); this.time = time; this.client = client; @@ -1235,7 +1237,7 @@ private ConsumerRecords poll(final Timer timer, final boolean includeMetad } } - final FetchedRecords records = pollForFetches(timer); + final Map>> records = pollForFetches(timer); if (!records.isEmpty()) { // before returning the fetched records, we can send off the next round of fetches // and avoid block waiting for their responses to enable pipelining while the user @@ -1269,12 +1271,12 @@ boolean updateAssignmentMetadataIfNeeded(final Timer timer, final boolean waitFo /** * @throws KafkaException if the rebalance callback throws exception */ - private FetchedRecords pollForFetches(Timer timer) { + private Map>> pollForFetches(Timer timer) { long pollTimeout = coordinator == null ? timer.remainingMs() : Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs()); // if data is available already, return it immediately - final FetchedRecords records = fetcher.fetchedRecords(); + final Map>> records = fetcher.fetchedRecords(); if (!records.isEmpty()) { return records; } @@ -2219,6 +2221,30 @@ public Map endOffsets(Collection partition } } + /** + * Get the consumer's current lag on the partition. Returns an "empty" {@link OptionalLong} if the lag is not known, + * for example if there is no position yet, or if the end offset is not known yet. + * + *

    + * This method uses locally cached metadata and never makes a remote call. + * + * @param topicPartition The partition to get the lag for. + * + * @return This {@code Consumer} instance's current lag for the given partition. + * + * @throws IllegalStateException if the {@code topicPartition} is not assigned + **/ + @Override + public OptionalLong currentLag(TopicPartition topicPartition) { + acquireAndEnsureOpen(); + try { + final Long lag = subscriptions.partitionLag(topicPartition, isolationLevel); + return lag == null ? OptionalLong.empty() : OptionalLong.of(lag); + } finally { + release(); + } + } + /** * Return the current group metadata associated with this consumer. * diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java index 7ddda76e416b0..b5e0c1d93fc70 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.Queue; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -44,6 +45,8 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static java.util.Collections.singleton; + /** * A mock of the {@link Consumer} interface you can use for testing code that uses Kafka. This class is not @@ -218,21 +221,7 @@ public synchronized ConsumerRecords poll(final Duration timeout) { } toClear.forEach(p -> this.records.remove(p)); - - final Map metadata = new HashMap<>(); - for (final TopicPartition partition : subscriptions.assignedPartitions()) { - if (subscriptions.hasValidPosition(partition) && endOffsets.containsKey(partition)) { - final SubscriptionState.FetchPosition position = subscriptions.position(partition); - final long offset = position.offset; - final long endOffset = endOffsets.get(partition); - metadata.put( - partition, - new ConsumerRecords.Metadata(System.currentTimeMillis(), offset, endOffset) - ); - } - } - - return new ConsumerRecords<>(results, metadata); + return new ConsumerRecords<>(results); } public synchronized void addRecord(ConsumerRecord record) { @@ -243,7 +232,6 @@ public synchronized void addRecord(ConsumerRecord record) { throw new IllegalStateException("Cannot add records for a partition that is not assigned to the consumer"); List> recs = this.records.computeIfAbsent(tp, k -> new ArrayList<>()); recs.add(record); - endOffsets.compute(tp, (ignore, offset) -> offset == null ? record.offset() : Math.max(offset, record.offset())); } /** @@ -318,7 +306,7 @@ public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) @Deprecated @Override public synchronized OffsetAndMetadata committed(final TopicPartition partition) { - return committed(Collections.singleton(partition)).get(partition); + return committed(singleton(partition)).get(partition); } @Deprecated @@ -556,6 +544,16 @@ public Map endOffsets(Collection partition return endOffsets(partitions); } + @Override + public OptionalLong currentLag(TopicPartition topicPartition) { + if (endOffsets.containsKey(topicPartition)) { + return OptionalLong.of(endOffsets.get(topicPartition) - position(topicPartition)); + } else { + // if the test doesn't bother to set an end offset, we assume it wants to model being caught up. + return OptionalLong.of(0L); + } + } + @Override public ConsumerGroupMetadata groupMetadata() { return new ConsumerGroupMetadata("dummy.group.id", 1, "1", Optional.empty()); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchedRecords.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchedRecords.java deleted file mode 100644 index d8ef92bd6e48a..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchedRecords.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.TopicPartition; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class FetchedRecords { - private final Map>> records; - private final Map metadata; - - public static final class FetchMetadata { - - private final long receivedTimestamp; - private final SubscriptionState.FetchPosition position; - private final Long endOffset; - - public FetchMetadata(final long receivedTimestamp, - final SubscriptionState.FetchPosition position, - final Long endOffset) { - this.receivedTimestamp = receivedTimestamp; - this.position = position; - this.endOffset = endOffset; - } - - public long receivedTimestamp() { - return receivedTimestamp; - } - - public SubscriptionState.FetchPosition position() { - return position; - } - - public Long endOffset() { - return endOffset; - } - - @Override - public String toString() { - return "FetchMetadata{" + - "receivedTimestamp=" + receivedTimestamp + - ", position=" + position + - ", endOffset=" + endOffset + - '}'; - } - } - - public FetchedRecords() { - records = new HashMap<>(); - metadata = new HashMap<>(); - } - - public void addRecords(final TopicPartition topicPartition, final List> records) { - if (this.records.containsKey(topicPartition)) { - // this case shouldn't usually happen because we only send one fetch at a time per partition, - // but it might conceivably happen in some rare cases (such as partition leader changes). - // we have to copy to a new list because the old one may be immutable - final List> currentRecords = this.records.get(topicPartition); - final List> newRecords = new ArrayList<>(records.size() + currentRecords.size()); - newRecords.addAll(currentRecords); - newRecords.addAll(records); - this.records.put(topicPartition, newRecords); - } else { - this.records.put(topicPartition, records); - } - } - - public Map>> records() { - return records; - } - - public void addMetadata(final TopicPartition partition, final FetchMetadata fetchMetadata) { - metadata.put(partition, fetchMetadata); - } - - public Map metadata() { - return metadata; - } - - public boolean isEmpty() { - return records.isEmpty() && metadata.isEmpty(); - } -} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 2b9198fa1e757..5e8ad3c2454e2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -262,8 +262,9 @@ public synchronized int sendFetches() { .toForget(data.toForget()) .rackId(clientRackId); - log.debug("Sending {} {} to broker {}", isolationLevel, data, fetchTarget); - + if (log.isDebugEnabled()) { + log.debug("Sending {} {} to broker {}", isolationLevel, data.toString(), fetchTarget); + } RequestFuture future = client.send(fetchTarget, request); // We add the node to the set of nodes with pending fetch requests before adding the // listener because the future may have been fulfilled on another thread (e.g. during a @@ -318,7 +319,7 @@ public void onSuccess(ClientResponse resp) { short responseVersion = resp.requestHeader().apiVersion(); completedFetches.add(new CompletedFetch(partition, partitionData, - metricAggregator, batches, fetchOffset, responseVersion, resp.receivedTimeMs())); + metricAggregator, batches, fetchOffset, responseVersion)); } } @@ -597,8 +598,8 @@ private Map beginningOrEndOffset(Collection fetchedRecords() { - FetchedRecords fetched = new FetchedRecords<>(); + public Map>> fetchedRecords() { + Map>> fetched = new HashMap<>(); Queue pausedCompletedFetches = new ArrayDeque<>(); int recordsRemaining = maxPollRecords; @@ -636,28 +637,20 @@ public FetchedRecords fetchedRecords() { } else { List> records = fetchRecords(nextInLineFetch, recordsRemaining); - TopicPartition partition = nextInLineFetch.partition; - - // This can be false when a rebalance happened before fetched records - // are returned to the consumer's poll call - if (subscriptions.isAssigned(partition)) { - - // initializeCompletedFetch, above, has already persisted the metadata from the fetch in the - // SubscriptionState, so we can just read it out, which in particular lets us re-use the logic - // for determining the end offset - final long receivedTimestamp = nextInLineFetch.receivedTimestamp; - final Long endOffset = subscriptions.logEndOffset(partition, isolationLevel); - final FetchPosition fetchPosition = subscriptions.position(partition); - - final FetchedRecords.FetchMetadata metadata = - new FetchedRecords.FetchMetadata(receivedTimestamp, fetchPosition, endOffset); - - fetched.addMetadata(partition, metadata); - - } - if (!records.isEmpty()) { - fetched.addRecords(partition, records); + TopicPartition partition = nextInLineFetch.partition; + List> currentRecords = fetched.get(partition); + if (currentRecords == null) { + fetched.put(partition, records); + } else { + // this case shouldn't usually happen because we only send one fetch at a time per partition, + // but it might conceivably happen in some rare cases (such as partition leader changes). + // we have to copy to a new list because the old one may be immutable + List> newRecords = new ArrayList<>(records.size() + currentRecords.size()); + newRecords.addAll(currentRecords); + newRecords.addAll(records); + fetched.put(partition, newRecords); + } recordsRemaining -= records.size(); } } @@ -1466,7 +1459,6 @@ private class CompletedFetch { private final FetchResponse.PartitionData partitionData; private final FetchResponseMetricAggregator metricAggregator; private final short responseVersion; - private final long receivedTimestamp; private int recordsRead; private int bytesRead; @@ -1485,15 +1477,13 @@ private CompletedFetch(TopicPartition partition, FetchResponseMetricAggregator metricAggregator, Iterator batches, Long fetchOffset, - short responseVersion, - final long receivedTimestamp) { + short responseVersion) { this.partition = partition; this.partitionData = partitionData; this.metricAggregator = metricAggregator; this.batches = batches; this.nextFetchOffset = fetchOffset; this.responseVersion = responseVersion; - this.receivedTimestamp = receivedTimestamp; this.lastEpoch = Optional.empty(); this.abortedProducerIds = new HashSet<>(); this.abortedTransactions = abortedTransactions(partitionData); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java index b2a5e51aaae8a..7b971a162c319 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java @@ -537,7 +537,7 @@ public synchronized FetchPosition position(TopicPartition tp) { return assignedState(tp).position; } - synchronized Long partitionLag(TopicPartition tp, IsolationLevel isolationLevel) { + public synchronized Long partitionLag(TopicPartition tp, IsolationLevel isolationLevel) { TopicPartitionState topicPartitionState = assignedState(tp); if (isolationLevel == IsolationLevel.READ_COMMITTED) return topicPartitionState.lastStableOffset == null ? null : topicPartitionState.lastStableOffset - topicPartitionState.position.offset; @@ -562,18 +562,6 @@ synchronized void updateLastStableOffset(TopicPartition tp, long lastStableOffse assignedState(tp).lastStableOffset(lastStableOffset); } - synchronized Long logStartOffset(TopicPartition tp) { - return assignedState(tp).logStartOffset; - } - - synchronized Long logEndOffset(TopicPartition tp, IsolationLevel isolationLevel) { - TopicPartitionState topicPartitionState = assignedState(tp); - if (isolationLevel == IsolationLevel.READ_COMMITTED) - return topicPartitionState.lastStableOffset == null ? null : topicPartitionState.lastStableOffset; - else - return topicPartitionState.highWatermark == null ? null : topicPartitionState.highWatermark; - } - /** * Set the preferred read replica with a lease timeout. After this time, the replica will no longer be valid and * {@link #preferredReadReplica(TopicPartition, long)} will return an empty result. diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index e92e684af0c1c..cfdc2bf29de9f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -45,19 +45,18 @@ import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.message.JoinGroupRequestData; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData; -import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsResponseData; -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; +import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; +import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.message.SyncGroupResponseData; +import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; @@ -92,6 +91,8 @@ import org.apache.kafka.test.MockConsumerInterceptor; import org.apache.kafka.test.MockMetricsReporter; import org.apache.kafka.test.TestUtils; +import org.apache.kafka.common.metrics.stats.Avg; + import org.junit.jupiter.api.Test; import javax.management.MBeanServer; @@ -100,6 +101,7 @@ import java.nio.ByteBuffer; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.ConcurrentModificationException; @@ -110,6 +112,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.Properties; import java.util.Queue; import java.util.Set; @@ -126,18 +129,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static java.util.Arrays.asList; -import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; -import static org.apache.kafka.common.utils.Utils.mkEntry; -import static org.apache.kafka.common.utils.Utils.mkMap; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -607,7 +602,7 @@ public void testFetchProgressWithMissingPartitionPosition() { initMetadata(client, Collections.singletonMap(topic, 2)); KafkaConsumer consumer = newConsumerNoAutoCommit(time, client, subscription, metadata); - consumer.assign(asList(tp0, tp1)); + consumer.assign(Arrays.asList(tp0, tp1)); consumer.seekToEnd(singleton(tp0)); consumer.seekToBeginning(singleton(tp1)); @@ -767,7 +762,7 @@ public void testCommitsFetchedDuringAssign() { client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, offset1), Errors.NONE), coordinator); assertEquals(offset1, consumer.committed(Collections.singleton(tp0)).get(tp0).offset()); - consumer.assign(asList(tp0, tp1)); + consumer.assign(Arrays.asList(tp0, tp1)); // fetch offset for two topics Map offsets = new HashMap<>(); @@ -839,14 +834,14 @@ public void testNoCommittedOffsets() { ConsumerPartitionAssignor assignor = new RoundRobinAssignor(); KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); - consumer.assign(asList(tp0, tp1)); + consumer.assign(Arrays.asList(tp0, tp1)); // lookup coordinator client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, node), node); Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port()); // fetch offset for one topic - client.prepareResponseFrom(offsetResponse(mkMap(mkEntry(tp0, offset1), mkEntry(tp1, -1L)), Errors.NONE), coordinator); + client.prepareResponseFrom(offsetResponse(Utils.mkMap(Utils.mkEntry(tp0, offset1), Utils.mkEntry(tp1, -1L)), Errors.NONE), coordinator); final Map committed = consumer.committed(Utils.mkSet(tp0, tp1)); assertEquals(2, committed.size()); assertEquals(offset1, committed.get(tp0).offset()); @@ -1056,7 +1051,6 @@ public void fetchResponseWithUnexpectedPartitionIsIgnored() { ConsumerRecords records = consumer.poll(Duration.ZERO); assertEquals(0, records.count()); - assertThat(records.metadata(), equalTo(emptyMap())); consumer.close(Duration.ofMillis(0)); } @@ -1086,7 +1080,7 @@ public void testSubscriptionChangesWithAutoCommitEnabled() { KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); // initial subscription - consumer.subscribe(asList(topic, topic2), getConsumerRebalanceListener(consumer)); + consumer.subscribe(Arrays.asList(topic, topic2), getConsumerRebalanceListener(consumer)); // verify that subscription has changed but assignment is still unchanged assertEquals(2, consumer.subscription().size()); @@ -1094,7 +1088,7 @@ public void testSubscriptionChangesWithAutoCommitEnabled() { assertTrue(consumer.assignment().isEmpty()); // mock rebalance responses - Node coordinator = prepareRebalance(client, node, assignor, asList(tp0, t2p0), null); + Node coordinator = prepareRebalance(client, node, assignor, Arrays.asList(tp0, t2p0), null); consumer.updateAssignmentMetadataIfNeeded(time.timer(Long.MAX_VALUE)); consumer.poll(Duration.ZERO); @@ -1123,12 +1117,10 @@ public void testSubscriptionChangesWithAutoCommitEnabled() { // verify that the fetch occurred as expected assertEquals(11, records.count()); assertEquals(1L, consumer.position(tp0)); - assertEquals(1L, (long) records.metadata().get(tp0).position()); assertEquals(10L, consumer.position(t2p0)); - assertEquals(10L, (long) records.metadata().get(t2p0).position()); // subscription change - consumer.subscribe(asList(topic, topic3), getConsumerRebalanceListener(consumer)); + consumer.subscribe(Arrays.asList(topic, topic3), getConsumerRebalanceListener(consumer)); // verify that subscription has changed but assignment is still unchanged assertEquals(2, consumer.subscription().size()); @@ -1143,7 +1135,7 @@ public void testSubscriptionChangesWithAutoCommitEnabled() { AtomicBoolean commitReceived = prepareOffsetCommitResponse(client, coordinator, partitionOffsets1); // mock rebalance responses - prepareRebalance(client, node, assignor, asList(tp0, t3p0), coordinator); + prepareRebalance(client, node, assignor, Arrays.asList(tp0, t3p0), coordinator); // mock a response to the next fetch from the new assignment Map fetches2 = new HashMap<>(); @@ -1156,9 +1148,7 @@ public void testSubscriptionChangesWithAutoCommitEnabled() { // verify that the fetch occurred as expected assertEquals(101, records.count()); assertEquals(2L, consumer.position(tp0)); - assertEquals(2L, (long) records.metadata().get(tp0).position()); assertEquals(100L, consumer.position(t3p0)); - assertEquals(100L, (long) records.metadata().get(t3p0).position()); // verify that the offset commits occurred as expected assertTrue(commitReceived.get()); @@ -1501,7 +1491,7 @@ public void testGracefulClose() throws Exception { response.put(tp0, Errors.NONE); OffsetCommitResponse commitResponse = offsetCommitResponse(response); LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(new LeaveGroupResponseData().setErrorCode(Errors.NONE.code())); - consumerCloseTest(5000, asList(commitResponse, leaveGroupResponse), 0, false); + consumerCloseTest(5000, Arrays.asList(commitResponse, leaveGroupResponse), 0, false); } @Test @@ -1872,12 +1862,12 @@ public void testReturnRecordsDuringRebalance() throws InterruptedException { ConsumerPartitionAssignor assignor = new CooperativeStickyAssignor(); KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); - initMetadata(client, mkMap(mkEntry(topic, 1), mkEntry(topic2, 1), mkEntry(topic3, 1))); + initMetadata(client, Utils.mkMap(Utils.mkEntry(topic, 1), Utils.mkEntry(topic2, 1), Utils.mkEntry(topic3, 1))); - consumer.subscribe(asList(topic, topic2), getConsumerRebalanceListener(consumer)); + consumer.subscribe(Arrays.asList(topic, topic2), getConsumerRebalanceListener(consumer)); Node node = metadata.fetch().nodes().get(0); - Node coordinator = prepareRebalance(client, node, assignor, asList(tp0, t2p0), null); + Node coordinator = prepareRebalance(client, node, assignor, Arrays.asList(tp0, t2p0), null); // a poll with non-zero milliseconds would complete three round-trips (discover, join, sync) TestUtils.waitForCondition(() -> { @@ -1908,7 +1898,7 @@ public void testReturnRecordsDuringRebalance() throws InterruptedException { client.respondFrom(fetchResponse(fetches1), node); // subscription change - consumer.subscribe(asList(topic, topic3), getConsumerRebalanceListener(consumer)); + consumer.subscribe(Arrays.asList(topic, topic3), getConsumerRebalanceListener(consumer)); // verify that subscription has changed but assignment is still unchanged assertEquals(Utils.mkSet(topic, topic3), consumer.subscription()); @@ -1954,7 +1944,7 @@ public void testReturnRecordsDuringRebalance() throws InterruptedException { client.respondFrom(fetchResponse(fetches1), node); // now complete the rebalance - client.respondFrom(syncGroupResponse(asList(tp0, t3p0), Errors.NONE), coordinator); + client.respondFrom(syncGroupResponse(Arrays.asList(tp0, t3p0), Errors.NONE), coordinator); AtomicInteger count = new AtomicInteger(0); TestUtils.waitForCondition(() -> { @@ -2053,7 +2043,7 @@ public void testInvalidGroupMetadata() throws InterruptedException { } @Test - public void testPollMetadata() { + public void testCurrentLag() { final Time time = new MockTime(); final SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST); final ConsumerMetadata metadata = createMetadata(subscription); @@ -2065,101 +2055,28 @@ public void testPollMetadata() { final KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); - consumer.assign(singleton(tp0)); - consumer.seek(tp0, 50L); - - final FetchInfo fetchInfo = new FetchInfo(1L, 99L, 50L, 5); - client.prepareResponse(fetchResponse(singletonMap(tp0, fetchInfo))); - - final ConsumerRecords records = consumer.poll(Duration.ofMillis(1)); - assertEquals(5, records.count()); - assertEquals(55L, consumer.position(tp0)); - - // verify that the consumer computes the correct metadata based on the fetch response - final ConsumerRecords.Metadata actualMetadata = records.metadata().get(tp0); - assertEquals(100L, (long) actualMetadata.endOffset()); - assertEquals(55L, (long) actualMetadata.position()); - assertEquals(45L, (long) actualMetadata.lag()); - consumer.close(Duration.ZERO); - } - - - @Test - public void testPollMetadataWithExtraPartitions() { - final Time time = new MockTime(); - final SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST); - final ConsumerMetadata metadata = createMetadata(subscription); - final MockClient client = new MockClient(time, metadata); + // throws for unassigned partition + assertThrows(IllegalStateException.class, () -> consumer.currentLag(tp0)); - initMetadata(client, singletonMap(topic, 2)); - final ConsumerPartitionAssignor assignor = new RoundRobinAssignor(); + consumer.assign(singleton(tp0)); - final KafkaConsumer consumer = - newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + // no error for no current position + assertEquals(OptionalLong.empty(), consumer.currentLag(tp0)); - consumer.assign(asList(tp0, tp1)); consumer.seek(tp0, 50L); - consumer.seek(tp1, 10L); - client.prepareResponse( - fetchResponse( - mkMap( - mkEntry(tp0, new FetchInfo(1L, 99L, 50L, 5)), - mkEntry(tp1, new FetchInfo(0L, 29L, 10L, 0)) - ) - ) - ); - - final ConsumerRecords records = consumer.poll(Duration.ofMillis(1)); - assertEquals(5, records.count()); - assertEquals(55L, consumer.position(tp0)); - - assertEquals(5, records.records(tp0).size()); - final ConsumerRecords.Metadata tp0Metadata = records.metadata().get(tp0); - assertEquals(100L, (long) tp0Metadata.endOffset()); - assertEquals(55L, (long) tp0Metadata.position()); - assertEquals(45L, (long) tp0Metadata.lag()); - - // we may get back metadata for other assigned partitions even if we don't get records for them - assertEquals(0, records.records(tp1).size()); - final ConsumerRecords.Metadata tp1Metadata = records.metadata().get(tp1); - assertEquals(30L, (long) tp1Metadata.endOffset()); - assertEquals(10L, (long) tp1Metadata.position()); - assertEquals(20L, (long) tp1Metadata.lag()); - - consumer.close(Duration.ZERO); - } - - @Test - public void testPollMetadataWithNoRecords() { - final Time time = new MockTime(); - final SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST); - final ConsumerMetadata metadata = createMetadata(subscription); - final MockClient client = new MockClient(time, metadata); - - initMetadata(client, singletonMap(topic, 1)); - final ConsumerPartitionAssignor assignor = new RoundRobinAssignor(); - - final KafkaConsumer consumer = - newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); - - consumer.assign(singleton(tp0)); - consumer.seek(tp0, 50L); + // no error for no end offset (so unknown lag) + assertEquals(OptionalLong.empty(), consumer.currentLag(tp0)); - final FetchInfo fetchInfo = new FetchInfo(1L, 99L, 50L, 0); + final FetchInfo fetchInfo = new FetchInfo(1L, 99L, 50L, 5); client.prepareResponse(fetchResponse(singletonMap(tp0, fetchInfo))); final ConsumerRecords records = consumer.poll(Duration.ofMillis(1)); + assertEquals(5, records.count()); + assertEquals(55L, consumer.position(tp0)); - // we got no records back ... - assertEquals(0, records.count()); - assertEquals(50L, consumer.position(tp0)); - - // ... but we can still get metadata that was in the fetch response - final ConsumerRecords.Metadata actualMetadata = records.metadata().get(tp0); - assertEquals(100L, (long) actualMetadata.endOffset()); - assertEquals(50L, (long) actualMetadata.position()); - assertEquals(50L, (long) actualMetadata.lag()); + // correct lag result + assertEquals(OptionalLong.of(45L), consumer.currentLag(tp0)); consumer.close(Duration.ZERO); } @@ -2329,7 +2246,7 @@ private OffsetFetchResponse offsetResponse(Map offsets, Er } private ListOffsetsResponse listOffsetsResponse(Map offsets) { - return listOffsetsResponse(offsets, emptyMap()); + return listOffsetsResponse(offsets, Collections.emptyMap()); } private ListOffsetsResponse listOffsetsResponse(Map partitionOffsets, @@ -2514,8 +2431,6 @@ private static class FetchInfo { } FetchInfo(long logFirstOffset, long logLastOffset, long offset, int count) { - assertThat(logFirstOffset, lessThanOrEqualTo(offset)); - assertThat(logLastOffset, greaterThanOrEqualTo(offset + count)); this.logFirstOffset = logFirstOffset; this.logLastOffset = logLastOffset; this.offset = offset; @@ -2656,7 +2571,7 @@ public void testPollIdleRatio() { } private static boolean consumerMetricPresent(KafkaConsumer consumer, String name) { - MetricName metricName = new MetricName(name, "consumer-metrics", "", emptyMap()); + MetricName metricName = new MetricName(name, "consumer-metrics", "", Collections.emptyMap()); return consumer.metrics.metrics().containsKey(metricName); } @@ -2696,11 +2611,11 @@ public void testEnforceRebalanceTriggersRebalanceOnNextPoll() { ConsumerPartitionAssignor assignor = new RoundRobinAssignor(); KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); MockRebalanceListener countingRebalanceListener = new MockRebalanceListener(); - initMetadata(client, mkMap(mkEntry(topic, 1), mkEntry(topic2, 1), mkEntry(topic3, 1))); + initMetadata(client, Utils.mkMap(Utils.mkEntry(topic, 1), Utils.mkEntry(topic2, 1), Utils.mkEntry(topic3, 1))); - consumer.subscribe(asList(topic, topic2), countingRebalanceListener); + consumer.subscribe(Arrays.asList(topic, topic2), countingRebalanceListener); Node node = metadata.fetch().nodes().get(0); - prepareRebalance(client, node, assignor, asList(tp0, t2p0), null); + prepareRebalance(client, node, assignor, Arrays.asList(tp0, t2p0), null); // a first rebalance to get the assignment, we need two poll calls since we need two round trips to finish join / sync-group consumer.poll(Duration.ZERO); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 2c13864fb0589..0a7d5cfde49b4 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -522,7 +522,7 @@ public void testParseCorruptedRecord() throws Exception { consumerClient.poll(time.timer(0)); // the first fetchedRecords() should return the first valid message - assertEquals(1, fetcher.fetchedRecords().records().get(tp0).size()); + assertEquals(1, fetcher.fetchedRecords().get(tp0).size()); assertEquals(1, subscriptions.position(tp0).offset); ensureBlockOnRecord(1L); @@ -926,7 +926,7 @@ public void testInFlightFetchOnPausedPartition() { client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - assertNull(fetcher.fetchedRecords().records().get(tp0)); + assertNull(fetcher.fetchedRecords().get(tp0)); } @Test @@ -1115,7 +1115,7 @@ public void testFetchNotLeaderOrFollower() { assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); consumerClient.poll(time.timer(0)); - assertEquals(0, fetcher.fetchedRecords().records().size()); + assertEquals(0, fetcher.fetchedRecords().size()); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); } @@ -1128,7 +1128,7 @@ public void testFetchUnknownTopicOrPartition() { assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); consumerClient.poll(time.timer(0)); - assertEquals(0, fetcher.fetchedRecords().records().size()); + assertEquals(0, fetcher.fetchedRecords().size()); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); } @@ -1142,7 +1142,7 @@ public void testFetchFencedLeaderEpoch() { client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); consumerClient.poll(time.timer(0)); - assertEquals(0, fetcher.fetchedRecords().records().size(), "Should not return any records"); + assertEquals(0, fetcher.fetchedRecords().size(), "Should not return any records"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should have requested metadata update"); } @@ -1156,7 +1156,7 @@ public void testFetchUnknownLeaderEpoch() { client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); consumerClient.poll(time.timer(0)); - assertEquals(0, fetcher.fetchedRecords().records().size(), "Should not return any records"); + assertEquals(0, fetcher.fetchedRecords().size(), "Should not return any records"); assertNotEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should not have requested metadata update"); } @@ -1198,7 +1198,7 @@ public void testFetchOffsetOutOfRange() { assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); - assertEquals(0, fetcher.fetchedRecords().records().size()); + assertEquals(0, fetcher.fetchedRecords().size()); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertNull(subscriptions.validPosition(tp0)); assertNull(subscriptions.position(tp0)); @@ -1216,7 +1216,7 @@ public void testStaleOutOfRangeError() { client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); subscriptions.seek(tp0, 1); consumerClient.poll(time.timer(0)); - assertEquals(0, fetcher.fetchedRecords().records().size()); + assertEquals(0, fetcher.fetchedRecords().size()); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(1, subscriptions.position(tp0).offset); } @@ -1234,7 +1234,7 @@ public void testFetchedRecordsAfterSeek() { consumerClient.poll(time.timer(0)); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); subscriptions.seek(tp0, 2); - assertEquals(0, fetcher.fetchedRecords().records().size()); + assertEquals(0, fetcher.fetchedRecords().size()); } @Test @@ -1390,7 +1390,7 @@ public void testSeekBeforeException() { client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - assertEquals(2, fetcher.fetchedRecords().records().get(tp0).size()); + assertEquals(2, fetcher.fetchedRecords().get(tp0).size()); subscriptions.assignFromUser(Utils.mkSet(tp0, tp1)); subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); @@ -1401,11 +1401,11 @@ public void testSeekBeforeException() { FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, MemoryRecords.EMPTY)); client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), 0, INVALID_SESSION_ID)); consumerClient.poll(time.timer(0)); - assertEquals(1, fetcher.fetchedRecords().records().get(tp0).size()); + assertEquals(1, fetcher.fetchedRecords().get(tp0).size()); subscriptions.seek(tp1, 10); // Should not throw OffsetOutOfRangeException after the seek - assertEquals(0, fetcher.fetchedRecords().records().size()); + assertEquals(0, fetcher.fetchedRecords().size()); } @Test @@ -1418,7 +1418,7 @@ public void testFetchDisconnected() { assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0), true); consumerClient.poll(time.timer(0)); - assertEquals(0, fetcher.fetchedRecords().records().size()); + assertEquals(0, fetcher.fetchedRecords().size()); // disconnects should have no affect on subscription state assertFalse(subscriptions.isOffsetResetNeeded(tp0)); @@ -4521,7 +4521,7 @@ private MetadataResponse newMetadataResponse(String topic, Errors error) { @SuppressWarnings("unchecked") private Map>> fetchedRecords() { - return (Map) fetcher.fetchedRecords().records(); + return (Map) fetcher.fetchedRecords(); } private void buildFetcher(int maxPollRecords) { diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala index d0b9084227c49..d4c8492e9a45c 100644 --- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala @@ -583,7 +583,7 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumer.seekToEnd(List(tp).asJava) assertEquals(totalRecords, consumer.position(tp)) - assertTrue(pollForRecord(consumer, Duration.ofMillis(50)).isEmpty) + assertTrue(consumer.poll(Duration.ofMillis(50)).isEmpty) consumer.seekToBeginning(List(tp).asJava) assertEquals(0L, consumer.position(tp)) @@ -601,7 +601,7 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumer.seekToEnd(List(tp2).asJava) assertEquals(totalRecords, consumer.position(tp2)) - assertTrue(pollForRecord(consumer, Duration.ofMillis(50)).isEmpty) + assertTrue(consumer.poll(Duration.ofMillis(50)).isEmpty) consumer.seekToBeginning(List(tp2).asJava) assertEquals(0L, consumer.position(tp2)) @@ -670,7 +670,7 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumer.pause(partitions) startingTimestamp = System.currentTimeMillis() sendRecords(producer, numRecords = 5, tp, startingTimestamp = startingTimestamp) - assertTrue(pollForRecord(consumer, Duration.ofMillis(100)).isEmpty) + assertTrue(consumer.poll(Duration.ofMillis(100)).isEmpty) consumer.resume(partitions) consumeAndVerifyRecords(consumer = consumer, numRecords = 5, startingOffset = 5, startingTimestamp = startingTimestamp) } @@ -718,8 +718,7 @@ class PlaintextConsumerTest extends BaseConsumerTest { // consuming a record that is too large should succeed since KIP-74 consumer.assign(List(tp).asJava) - val duration = Duration.ofMillis(20000) - val records = pollForRecord(consumer, duration) + val records = consumer.poll(Duration.ofMillis(20000)) assertEquals(1, records.count) val consumerRecord = records.iterator().next() assertEquals(0L, consumerRecord.offset) @@ -751,7 +750,7 @@ class PlaintextConsumerTest extends BaseConsumerTest { // we should only get the small record in the first `poll` consumer.assign(List(tp).asJava) - val records = pollForRecord(consumer, Duration.ofMillis(20000)) + val records = consumer.poll(Duration.ofMillis(20000)) assertEquals(1, records.count) val consumerRecord = records.iterator().next() assertEquals(0L, consumerRecord.offset) @@ -1805,12 +1804,12 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumer3.assign(asList(tp)) consumer3.seek(tp, 1) - val numRecords1 = pollForRecord(consumer1, Duration.ofMillis(5000)).count() + val numRecords1 = consumer1.poll(Duration.ofMillis(5000)).count() assertThrows(classOf[InvalidGroupIdException], () => consumer1.commitSync()) assertThrows(classOf[InvalidGroupIdException], () => consumer2.committed(Set(tp).asJava)) - val numRecords2 = pollForRecord(consumer2, Duration.ofMillis(5000)).count() - val numRecords3 = pollForRecord(consumer3, Duration.ofMillis(5000)).count() + val numRecords2 = consumer2.poll(Duration.ofMillis(5000)).count() + val numRecords3 = consumer3.poll(Duration.ofMillis(5000)).count() consumer1.unsubscribe() consumer2.unsubscribe() @@ -1859,10 +1858,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumer1.assign(asList(tp)) consumer2.assign(asList(tp)) - val records1 = pollForRecord(consumer1, Duration.ofMillis(5000)) + val records1 = consumer1.poll(Duration.ofMillis(5000)) consumer1.commitSync() - val records2 = pollForRecord(consumer2, Duration.ofMillis(5000)) + val records2 = consumer2.poll(Duration.ofMillis(5000)) consumer2.commitSync() consumer1.close() @@ -1873,19 +1872,4 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertTrue(records2.count() == 1 && records2.records(tp).asScala.head.offset == 1, "Expected consumer2 to consume one message from offset 1, which is the committed offset of consumer1") } - - /** - * Consumer#poll returns early if there is metadata to return even if there are no records, - * so when we intend to wait for records, we can't just rely on long polling in the Consumer. - */ - private def pollForRecord(consumer: KafkaConsumer[Array[Byte], Array[Byte]], duration: Duration) = { - val deadline = System.currentTimeMillis() + duration.toMillis - var durationRemaining = deadline - System.currentTimeMillis() - var result = consumer.poll(Duration.ofMillis(durationRemaining)) - while (result.count() == 0 && durationRemaining > 0) { - result = consumer.poll(Duration.ofMillis(durationRemaining)) - durationRemaining = deadline - System.currentTimeMillis() - } - result - } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java index 46b3429f226e6..199bc0e6456cc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PartitionGroup.java @@ -17,7 +17,6 @@ package org.apache.kafka.streams.processor.internals; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.utils.LogContext; @@ -30,6 +29,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.OptionalLong; import java.util.PriorityQueue; import java.util.Set; import java.util.function.Function; @@ -60,6 +60,7 @@ public class PartitionGroup { private final Logger logger; private final Map partitionQueues; + private final Function lagProvider; private final Sensor enforcedProcessingSensor; private final long maxTaskIdleMs; private final Sensor recordLatenessSensor; @@ -68,7 +69,6 @@ public class PartitionGroup { private long streamTime; private int totalBuffered; private boolean allBuffered; - private final Map fetchedLags = new HashMap<>(); private final Map idlePartitionDeadlines = new HashMap<>(); static class RecordInfo { @@ -89,12 +89,14 @@ RecordQueue queue() { PartitionGroup(final LogContext logContext, final Map partitionQueues, + final Function lagProvider, final Sensor recordLatenessSensor, final Sensor enforcedProcessingSensor, final long maxTaskIdleMs) { this.logger = logContext.logger(PartitionGroup.class); nonEmptyQueuesByTime = new PriorityQueue<>(partitionQueues.size(), Comparator.comparingLong(RecordQueue::headRecordTimestamp)); this.partitionQueues = partitionQueues; + this.lagProvider = lagProvider; this.enforcedProcessingSensor = enforcedProcessingSensor; this.maxTaskIdleMs = maxTaskIdleMs; this.recordLatenessSensor = recordLatenessSensor; @@ -103,31 +105,7 @@ RecordQueue queue() { streamTime = RecordQueue.UNKNOWN; } - public void addFetchedMetadata(final TopicPartition partition, final ConsumerRecords.Metadata metadata) { - final Long lag = metadata.lag(); - if (lag != null) { - logger.trace("added fetched lag {}: {}", partition, lag); - fetchedLags.put(partition, lag); - } - } - public boolean readyToProcess(final long wallClockTime) { - if (logger.isTraceEnabled()) { - for (final Map.Entry entry : partitionQueues.entrySet()) { - logger.trace( - "buffered/lag {}: {}/{}", - entry.getKey(), - entry.getValue().size(), - fetchedLags.get(entry.getKey()) - ); - } - } - // Log-level strategy: - // TRACE for messages that don't wait for fetches - // TRACE when we waited for a fetch and decided to wait some more, as configured - // TRACE when we are ready for processing and didn't have to enforce processing - // INFO when we enforce processing, since this has to wait for fetches AND may result in disorder - if (maxTaskIdleMs == StreamsConfig.MAX_TASK_IDLE_MS_DISABLED) { if (logger.isTraceEnabled() && !allBuffered && totalBuffered > 0) { final Set bufferedPartitions = new HashSet<>(); @@ -156,50 +134,53 @@ public boolean readyToProcess(final long wallClockTime) { final TopicPartition partition = entry.getKey(); final RecordQueue queue = entry.getValue(); - final Long nullableFetchedLag = fetchedLags.get(partition); if (!queue.isEmpty()) { // this partition is ready for processing idlePartitionDeadlines.remove(partition); queued.add(partition); - } else if (nullableFetchedLag == null) { - // must wait to fetch metadata for the partition - idlePartitionDeadlines.remove(partition); - logger.trace("Waiting to fetch data for {}", partition); - return false; - } else if (nullableFetchedLag > 0L) { - // must wait to poll the data we know to be on the broker - idlePartitionDeadlines.remove(partition); - logger.trace( - "Lag for {} is currently {}, but no data is buffered locally. Waiting to buffer some records.", - partition, - nullableFetchedLag - ); - return false; } else { - // p is known to have zero lag. wait for maxTaskIdleMs to see if more data shows up. - // One alternative would be to set the deadline to nullableMetadata.receivedTimestamp + maxTaskIdleMs - // instead. That way, we would start the idling timer as of the freshness of our knowledge about the zero - // lag instead of when we happen to run this method, but realistically it's probably a small difference - // and using wall clock time seems more intuitive for users, - // since the log message will be as of wallClockTime. - idlePartitionDeadlines.putIfAbsent(partition, wallClockTime + maxTaskIdleMs); - final long deadline = idlePartitionDeadlines.get(partition); - if (wallClockTime < deadline) { + final OptionalLong fetchedLag = lagProvider.apply(partition); + + if (!fetchedLag.isPresent()) { + // must wait to fetch metadata for the partition + idlePartitionDeadlines.remove(partition); + logger.trace("Waiting to fetch data for {}", partition); + return false; + } else if (fetchedLag.getAsLong() > 0L) { + // must wait to poll the data we know to be on the broker + idlePartitionDeadlines.remove(partition); logger.trace( - "Lag for {} is currently 0 and current time is {}. Waiting for new data to be produced for configured idle time {} (deadline is {}).", + "Lag for {} is currently {}, but no data is buffered locally. Waiting to buffer some records.", partition, - wallClockTime, - maxTaskIdleMs, - deadline + fetchedLag.getAsLong() ); return false; } else { - // this partition is ready for processing due to the task idling deadline passing - if (enforced == null) { - enforced = new HashMap<>(); + // p is known to have zero lag. wait for maxTaskIdleMs to see if more data shows up. + // One alternative would be to set the deadline to nullableMetadata.receivedTimestamp + maxTaskIdleMs + // instead. That way, we would start the idling timer as of the freshness of our knowledge about the zero + // lag instead of when we happen to run this method, but realistically it's probably a small difference + // and using wall clock time seems more intuitive for users, + // since the log message will be as of wallClockTime. + idlePartitionDeadlines.putIfAbsent(partition, wallClockTime + maxTaskIdleMs); + final long deadline = idlePartitionDeadlines.get(partition); + if (wallClockTime < deadline) { + logger.trace( + "Lag for {} is currently 0 and current time is {}. Waiting for new data to be produced for configured idle time {} (deadline is {}).", + partition, + wallClockTime, + maxTaskIdleMs, + deadline + ); + return false; + } else { + // this partition is ready for processing due to the task idling deadline passing + if (enforced == null) { + enforced = new HashMap<>(); + } + enforced.put(partition, deadline); } - enforced.put(partition, deadline); } } } @@ -211,7 +192,7 @@ public boolean readyToProcess(final long wallClockTime) { return false; } else { enforcedProcessingSensor.record(1.0d, wallClockTime); - logger.info("Continuing to process although some partition timestamps were not buffered locally." + + logger.trace("Continuing to process although some partitions are empty on the broker." + "\n\tThere may be out-of-order processing for this task as a result." + "\n\tPartitions with local data: {}." + "\n\tPartitions we gave up waiting for, with their corresponding deadlines: {}." + diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java index d866954668618..4efb10eb566cf 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTask.java @@ -17,7 +17,6 @@ package org.apache.kafka.streams.processor.internals; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.metrics.Sensor; @@ -287,11 +286,6 @@ public void addRecords(final TopicPartition partition, final Iterable> records); - /** - * Add to this task any metadata returned from the poll. - */ - void addFetchedMetadata(TopicPartition partition, ConsumerRecords.Metadata metadata); - default boolean process(final long wallClockTime) { return false; } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java index 5b9ebadb5da3b..209ebb243168f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java @@ -968,7 +968,7 @@ int commitAll() { * @param records Records, can be null */ void addRecordsToTasks(final ConsumerRecords records) { - for (final TopicPartition partition : union(HashSet::new, records.partitions(), records.metadata().keySet())) { + for (final TopicPartition partition : records.partitions()) { final Task activeTask = tasks.activeTasksForInputPartition(partition); if (activeTask == null) { @@ -978,7 +978,6 @@ void addRecordsToTasks(final ConsumerRecords records) { } activeTask.addRecords(partition, records.records(partition)); - activeTask.addFetchedMetadata(partition, records.metadata().get(partition)); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java index d0e8b0127f58c..e8147a17b5b66 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java @@ -470,7 +470,7 @@ private void createTasks() { assertThat( activeTaskCreator.createTasks( - null, + mockClientSupplier.consumer, mkMap( mkEntry(task00, Collections.singleton(new TopicPartition("topic", 0))), mkEntry(task01, Collections.singleton(new TopicPartition("topic", 1))) diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java index 09558d23854e8..40602b5edf1c7 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PartitionGroupTest.java @@ -17,7 +17,6 @@ package org.apache.kafka.streams.processor.internals; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.metrics.Metrics; @@ -41,7 +40,9 @@ import org.junit.Test; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.OptionalLong; import java.util.UUID; import static org.apache.kafka.common.utils.Utils.mkEntry; @@ -481,6 +482,7 @@ public void shouldUpdatePartitionQueuesExpand() { final PartitionGroup group = new PartitionGroup( logContext, mkMap(mkEntry(partition1, queue1)), + tp -> OptionalLong.of(0L), getValueSensor(metrics, lastLatenessValue), enforcedProcessingSensor, maxTaskIdleMs @@ -513,6 +515,7 @@ public void shouldUpdatePartitionQueuesShrinkAndExpand() { final PartitionGroup group = new PartitionGroup( logContext, mkMap(mkEntry(partition1, queue1)), + tp -> OptionalLong.of(0L), getValueSensor(metrics, lastLatenessValue), enforcedProcessingSensor, maxTaskIdleMs @@ -547,6 +550,7 @@ public void shouldNeverWaitIfIdlingIsDisabled() { mkEntry(partition1, queue1), mkEntry(partition2, queue2) ), + tp -> OptionalLong.of(0L), getValueSensor(metrics, lastLatenessValue), enforcedProcessingSensor, StreamsConfig.MAX_TASK_IDLE_MS_DISABLED @@ -584,6 +588,7 @@ public void shouldBeReadyIfAllPartitionsAreBuffered() { mkEntry(partition1, queue1), mkEntry(partition2, queue2) ), + tp -> OptionalLong.of(0L), getValueSensor(metrics, lastLatenessValue), enforcedProcessingSensor, 0L @@ -615,12 +620,14 @@ public void shouldBeReadyIfAllPartitionsAreBuffered() { @Test public void shouldWaitForFetchesWhenMetadataIsIncomplete() { + final HashMap lags = new HashMap<>(); final PartitionGroup group = new PartitionGroup( logContext, mkMap( mkEntry(partition1, queue1), mkEntry(partition2, queue2) ), + tp -> lags.getOrDefault(tp, OptionalLong.empty()), getValueSensor(metrics, lastLatenessValue), enforcedProcessingSensor, 0L @@ -643,18 +650,20 @@ public void shouldWaitForFetchesWhenMetadataIsIncomplete() { )) ); } - group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L)); + lags.put(partition2, OptionalLong.of(0L)); assertThat(group.readyToProcess(0L), is(true)); } @Test public void shouldWaitForPollWhenLagIsNonzero() { + final HashMap lags = new HashMap<>(); final PartitionGroup group = new PartitionGroup( logContext, mkMap( mkEntry(partition1, queue1), mkEntry(partition2, queue2) ), + tp -> lags.getOrDefault(tp, OptionalLong.empty()), getValueSensor(metrics, lastLatenessValue), enforcedProcessingSensor, 0L @@ -664,7 +673,8 @@ public void shouldWaitForPollWhenLagIsNonzero() { new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue), new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue)); group.addRawRecords(partition1, list1); - group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 1L)); + + lags.put(partition2, OptionalLong.of(1L)); assertThat(group.allPartitionsBufferedLocally(), is(false)); @@ -689,6 +699,7 @@ public void shouldIdleAsSpecifiedWhenLagIsZero() { mkEntry(partition1, queue1), mkEntry(partition2, queue2) ), + tp -> OptionalLong.of(0L), getValueSensor(metrics, lastLatenessValue), enforcedProcessingSensor, 1L @@ -698,7 +709,6 @@ public void shouldIdleAsSpecifiedWhenLagIsZero() { new ConsumerRecord<>("topic", 1, 1L, recordKey, recordValue), new ConsumerRecord<>("topic", 1, 5L, recordKey, recordValue)); group.addRawRecords(partition1, list1); - group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L)); assertThat(group.allPartitionsBufferedLocally(), is(false)); @@ -714,16 +724,15 @@ public void shouldIdleAsSpecifiedWhenLagIsZero() { ); } - group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L)); try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) { LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class); assertThat(group.readyToProcess(1L), is(true)); assertThat( appender.getEvents(), hasItem(Matchers.allOf( - Matchers.hasProperty("level", equalTo("INFO")), + Matchers.hasProperty("level", equalTo("TRACE")), Matchers.hasProperty("message", equalTo( - "[test] Continuing to process although some partition timestamps were not buffered locally.\n" + + "[test] Continuing to process although some partitions are empty on the broker.\n" + "\tThere may be out-of-order processing for this task as a result.\n" + "\tPartitions with local data: [topic-1].\n" + "\tPartitions we gave up waiting for, with their corresponding deadlines: {topic-2=1}.\n" + @@ -734,16 +743,15 @@ public void shouldIdleAsSpecifiedWhenLagIsZero() { ); } - group.addFetchedMetadata(partition2, new ConsumerRecords.Metadata(0L, 0L, 0L)); try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(PartitionGroup.class)) { LogCaptureAppender.setClassLoggerToTrace(PartitionGroup.class); assertThat(group.readyToProcess(2L), is(true)); assertThat( appender.getEvents(), hasItem(Matchers.allOf( - Matchers.hasProperty("level", equalTo("INFO")), + Matchers.hasProperty("level", equalTo("TRACE")), Matchers.hasProperty("message", equalTo( - "[test] Continuing to process although some partition timestamps were not buffered locally.\n" + + "[test] Continuing to process although some partitions are empty on the broker.\n" + "\tThere may be out-of-order processing for this task as a result.\n" + "\tPartitions with local data: [topic-1].\n" + "\tPartitions we gave up waiting for, with their corresponding deadlines: {topic-2=1}.\n" + @@ -762,6 +770,7 @@ private PartitionGroup getBasicGroup() { mkEntry(partition1, queue1), mkEntry(partition2, queue2) ), + tp -> OptionalLong.of(0L), getValueSensor(metrics, lastLatenessValue), enforcedProcessingSensor, maxTaskIdleMs diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java index ea3fbdd91e1cf..6913a45c5f003 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java @@ -18,7 +18,6 @@ import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.MockConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetResetStrategy; @@ -424,9 +423,6 @@ public void shouldProcessInOrder() { assertEquals(asList(101, 102, 103), source1.values); assertEquals(singletonList(201), source2.values); - // tell the task that it doesn't need to wait for more records on partition1 - task.addFetchedMetadata(partition1, new ConsumerRecords.Metadata(0L, 30L, 30L)); - assertTrue(task.process(0L)); assertEquals(1, task.numBuffered()); assertEquals(3, source1.numReceived); @@ -434,8 +430,6 @@ public void shouldProcessInOrder() { assertEquals(asList(101, 102, 103), source1.values); assertEquals(asList(201, 202), source2.values); - // tell the task that it doesn't need to wait for more records on partition1 - task.addFetchedMetadata(partition1, new ConsumerRecords.Metadata(0L, 30L, 30L)); assertTrue(task.process(0L)); assertEquals(0, task.numBuffered()); assertEquals(3, source1.numReceived); @@ -967,9 +961,6 @@ public void shouldPunctuateOnceStreamTimeAfterGap() { assertEquals(3, source2.numReceived); assertTrue(task.maybePunctuateStreamTime()); - // tell the task that it doesn't need to wait for new data on partition1 - task.addFetchedMetadata(partition1, new ConsumerRecords.Metadata(0L, 160L, 160L)); - // st: 161 assertTrue(task.process(0L)); assertEquals(0, task.numBuffered()); @@ -1560,7 +1551,6 @@ public void shouldReturnOffsetsForRepartitionTopicsForPurging() { task.addRecords(repartition, singletonList(getConsumerRecordWithOffsetAsTimestamp(repartition, 10L))); assertTrue(task.process(0L)); - task.addFetchedMetadata(partition1, new ConsumerRecords.Metadata(0L, 5L, 5L)); assertTrue(task.process(0L)); task.prepareCommit(); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java index 2ca734de5a0ed..501dbe051996c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java @@ -24,7 +24,6 @@ import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; @@ -3126,11 +3125,6 @@ public void addRecords(final TopicPartition partition, final Iterable entry : startOffsets.entrySet()) { - final ConsumerRecords.Metadata metadata = new ConsumerRecords.Metadata( - mockWallClockTime.milliseconds(), - 0L, - 0L - ); - task.addFetchedMetadata(entry.getKey(), metadata); - } } else { task = null; } @@ -616,12 +606,6 @@ private void enqueueTaskRecord(final String inputTopic, value, headers)) ); - final ConsumerRecords.Metadata metadata = new ConsumerRecords.Metadata( - mockWallClockTime.milliseconds(), - offset, - offset - ); - task.addFetchedMetadata(topicOrPatternPartition, metadata); } private void completeAllProcessableWork() { From b2075a094688855be4e0cb37504c87bb09d4a576 Mon Sep 17 00:00:00 2001 From: vamossagar12 Date: Tue, 2 Mar 2021 23:23:27 +0530 Subject: [PATCH 086/243] KAFKA-12289: Adding test cases for prefix scan in InMemoryKeyValueStore (#10052) Co-authored-by: Bruno Cadonna Reviewers: Bruno Cadonna , Guozhang Wang --- .../streams/state/ReadOnlyKeyValueStore.java | 5 + .../internals/InMemoryKeyValueStoreTest.java | 166 +++++++++++++++++- 2 files changed, 170 insertions(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlyKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlyKeyValueStore.java index 1adfce1ec8bbd..2cc52c8f4596e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlyKeyValueStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/ReadOnlyKeyValueStore.java @@ -104,6 +104,11 @@ default KeyValueIterator reverseAll() { * prefix into the format in which the keys are stored in the stores needs to be passed to this method. * The returned iterator must be safe from {@link java.util.ConcurrentModificationException}s * and must not return null values. + * Since {@code prefixScan()} relies on byte lexicographical ordering and not on the ordering of the key type, results for some types might be unexpected. + * For example, if the key type is {@code Integer}, and the store contains keys [1, 2, 11, 13], + * then running {@code store.prefixScan(1, new IntegerSerializer())} will return [1] and not [1,11,13]. + * In contrast, if the key type is {@code String} the keys will be sorted [1, 11, 13, 2] in the store and {@code store.prefixScan(1, new StringSerializer())} will return [1,11,13]. + * In both cases {@code prefixScan()} starts the scan at 1 and stops at 2. * * @param prefix The prefix. * @param prefixKeySerializer Serializer for the Prefix key type diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreTest.java index 973093623c1ad..b4e3e1f3c2918 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/InMemoryKeyValueStoreTest.java @@ -17,18 +17,55 @@ package org.apache.kafka.streams.state.internals; import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.processor.StateStoreContext; import org.apache.kafka.streams.state.KeyValueStore; -import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.KeyValueStoreTestDriver; import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.KeyValueIterator; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; public class InMemoryKeyValueStoreTest extends AbstractKeyValueStoreTest { + private KeyValueStore byteStore; + private final Serializer stringSerializer = new StringSerializer(); + private final KeyValueStoreTestDriver byteStoreDriver = KeyValueStoreTestDriver.create(Bytes.class, byte[].class); + + @Before + public void createStringKeyValueStore() { + super.before(); + final StateStoreContext byteStoreContext = byteStoreDriver.context(); + final StoreBuilder> storeBuilder = Stores.keyValueStoreBuilder( + Stores.inMemoryKeyValueStore("in-memory-byte-store"), + new Serdes.BytesSerde(), + new Serdes.ByteArraySerde()); + byteStore = storeBuilder.build(); + byteStore.init(byteStoreContext, byteStore); + } + + @After + public void after() { + super.after(); + byteStore.close(); + byteStoreDriver.clear(); + } + @SuppressWarnings("unchecked") @Override protected KeyValueStore createKeyValueStore(final StateStoreContext context) { @@ -60,4 +97,131 @@ public void shouldRemoveKeysWithNullValues() { assertThat(store.get(0), nullValue()); } + + + @Test + public void shouldReturnKeysWithGivenPrefix() { + + final List> entries = new ArrayList<>(); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "k1")), + stringSerializer.serialize(null, "a"))); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "prefix_3")), + stringSerializer.serialize(null, "b"))); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "k2")), + stringSerializer.serialize(null, "c"))); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "prefix_2")), + stringSerializer.serialize(null, "d"))); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "k3")), + stringSerializer.serialize(null, "e"))); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "prefix_1")), + stringSerializer.serialize(null, "f"))); + + byteStore.putAll(entries); + byteStore.flush(); + + final KeyValueIterator keysWithPrefix = byteStore.prefixScan("prefix", stringSerializer); + final List valuesWithPrefix = new ArrayList<>(); + int numberOfKeysReturned = 0; + + while (keysWithPrefix.hasNext()) { + final KeyValue next = keysWithPrefix.next(); + valuesWithPrefix.add(new String(next.value)); + numberOfKeysReturned++; + } + assertThat(numberOfKeysReturned, is(3)); + assertThat(valuesWithPrefix.get(0), is("f")); + assertThat(valuesWithPrefix.get(1), is("d")); + assertThat(valuesWithPrefix.get(2), is("b")); + } + + @Test + public void shouldReturnKeysWithGivenPrefixExcludingNextKeyLargestKey() { + final List> entries = new ArrayList<>(); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "abc")), + stringSerializer.serialize(null, "f"))); + + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "abcd")), + stringSerializer.serialize(null, "f"))); + + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "abce")), + stringSerializer.serialize(null, "f"))); + + byteStore.putAll(entries); + byteStore.flush(); + + final KeyValueIterator keysWithPrefixAsabcd = byteStore.prefixScan("abcd", stringSerializer); + int numberOfKeysReturned = 0; + + while (keysWithPrefixAsabcd.hasNext()) { + keysWithPrefixAsabcd.next().key.get(); + numberOfKeysReturned++; + } + + assertThat(numberOfKeysReturned, is(1)); + } + + @Test + public void shouldReturnUUIDsWithStringPrefix() { + final List> entries = new ArrayList<>(); + final Serializer uuidSerializer = Serdes.UUID().serializer(); + final UUID uuid1 = UUID.randomUUID(); + final UUID uuid2 = UUID.randomUUID(); + final String prefix = uuid1.toString().substring(0, 4); + entries.add(new KeyValue<>( + new Bytes(uuidSerializer.serialize(null, uuid1)), + stringSerializer.serialize(null, "a"))); + entries.add(new KeyValue<>( + new Bytes(uuidSerializer.serialize(null, uuid2)), + stringSerializer.serialize(null, "b"))); + + byteStore.putAll(entries); + byteStore.flush(); + + final KeyValueIterator keysWithPrefix = byteStore.prefixScan(prefix, stringSerializer); + final List valuesWithPrefix = new ArrayList<>(); + int numberOfKeysReturned = 0; + + while (keysWithPrefix.hasNext()) { + final KeyValue next = keysWithPrefix.next(); + valuesWithPrefix.add(new String(next.value)); + numberOfKeysReturned++; + } + + assertThat(numberOfKeysReturned, is(1)); + assertThat(valuesWithPrefix.get(0), is("a")); + } + + @Test + public void shouldReturnNoKeys() { + final List> entries = new ArrayList<>(); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "a")), + stringSerializer.serialize(null, "a"))); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "b")), + stringSerializer.serialize(null, "c"))); + entries.add(new KeyValue<>( + new Bytes(stringSerializer.serialize(null, "c")), + stringSerializer.serialize(null, "e"))); + byteStore.putAll(entries); + byteStore.flush(); + + final KeyValueIterator keysWithPrefix = byteStore.prefixScan("bb", stringSerializer); + int numberOfKeysReturned = 0; + + while (keysWithPrefix.hasNext()) { + keysWithPrefix.next(); + numberOfKeysReturned++; + } + assertThat(numberOfKeysReturned, is(0)); + } } From 4c5867a39b4f817ae34f74e7e0918ec8eef9fdbb Mon Sep 17 00:00:00 2001 From: vamossagar12 Date: Tue, 2 Mar 2021 23:25:14 +0530 Subject: [PATCH 087/243] KAFKA-10766: Unit test cases for RocksDBRangeIterator (#9717) This PR aims to add unit test cases for RocksDBRangeIterator which were missing. Reviewers: Bruno Cadonna , Guozhang Wang --- .../internals/RocksDBRangeIteratorTest.java | 440 ++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBRangeIteratorTest.java diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBRangeIteratorTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBRangeIteratorTest.java new file mode 100644 index 0000000000000..b4c7d79ac7c4f --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/RocksDBRangeIteratorTest.java @@ -0,0 +1,440 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.state.internals; + +import org.apache.kafka.common.utils.Bytes; +import org.junit.Test; +import org.rocksdb.RocksIterator; + +import java.util.Collections; +import java.util.NoSuchElementException; + +import static org.easymock.EasyMock.mock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThrows; + +public class RocksDBRangeIteratorTest { + + private final String storeName = "store"; + private final String key1 = "a"; + private final String key2 = "b"; + private final String key3 = "c"; + private final String key4 = "d"; + + private final String value = "value"; + private final Bytes key1Bytes = Bytes.wrap(key1.getBytes()); + private final Bytes key2Bytes = Bytes.wrap(key2.getBytes()); + private final Bytes key3Bytes = Bytes.wrap(key3.getBytes()); + private final Bytes key4Bytes = Bytes.wrap(key4.getBytes()); + private final byte[] valueBytes = value.getBytes(); + + @Test + public void shouldReturnAllKeysInTheRangeInForwardDirection() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seek(key1Bytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true) + .andReturn(true) + .andReturn(false); + expect(rocksIterator.key()) + .andReturn(key1Bytes.get()) + .andReturn(key2Bytes.get()) + .andReturn(key3Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(3); + rocksIterator.next(); + expectLastCall().times(3); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + key3Bytes, + true, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key1Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key2Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key3Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + + @Test + public void shouldReturnAllKeysInTheRangeReverseDirection() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seekForPrev(key3Bytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true) + .andReturn(true) + .andReturn(false); + expect(rocksIterator.key()) + .andReturn(key3Bytes.get()) + .andReturn(key2Bytes.get()) + .andReturn(key1Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(3); + rocksIterator.prev(); + expectLastCall().times(3); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + key3Bytes, + false, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key3Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key2Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key1Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + + @Test + public void shouldReturnAllKeysWhenLastKeyIsGreaterThanLargestKeyInStateStoreInForwardDirection() { + final Bytes toBytes = Bytes.increment(key4Bytes); + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seek(key1Bytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true) + .andReturn(true) + .andReturn(true) + .andReturn(false); + expect(rocksIterator.key()) + .andReturn(key1Bytes.get()) + .andReturn(key2Bytes.get()) + .andReturn(key3Bytes.get()) + .andReturn(key4Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(4); + rocksIterator.next(); + expectLastCall().times(4); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + toBytes, + true, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key1Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key2Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key3Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key4Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + + + @Test + public void shouldReturnAllKeysWhenLastKeyIsSmallerThanSmallestKeyInStateStoreInReverseDirection() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seekForPrev(key4Bytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true) + .andReturn(true) + .andReturn(true) + .andReturn(false); + expect(rocksIterator.key()) + .andReturn(key4Bytes.get()) + .andReturn(key3Bytes.get()) + .andReturn(key2Bytes.get()) + .andReturn(key1Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(4); + rocksIterator.prev(); + expectLastCall().times(4); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + key4Bytes, + false, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key4Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key3Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key2Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key1Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + + + @Test + public void shouldReturnNoKeysWhenLastKeyIsSmallerThanSmallestKeyInStateStoreForwardDirection() { + // key range in state store: [c-f] + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seek(key1Bytes.get()); + expect(rocksIterator.isValid()).andReturn(false); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + key2Bytes, + true, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + + @Test + public void shouldReturnNoKeysWhenLastKeyIsLargerThanLargestKeyInStateStoreReverseDirection() { + // key range in state store: [c-f] + final String from = "g"; + final String to = "h"; + final Bytes fromBytes = Bytes.wrap(from.getBytes()); + final Bytes toBytes = Bytes.wrap(to.getBytes()); + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seekForPrev(toBytes.get()); + expect(rocksIterator.isValid()) + .andReturn(false); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + fromBytes, + toBytes, + false, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + + @Test + public void shouldReturnAllKeysInPartiallyOverlappingRangeInForwardDirection() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seek(key1Bytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true) + .andReturn(false); + expect(rocksIterator.key()) + .andReturn(key2Bytes.get()) + .andReturn(key3Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(2); + rocksIterator.next(); + expectLastCall().times(2); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + key3Bytes, + true, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key2Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key3Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + + @Test + public void shouldReturnAllKeysInPartiallyOverlappingRangeInReverseDirection() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + final String to = "e"; + final Bytes toBytes = Bytes.wrap(to.getBytes()); + rocksIterator.seekForPrev(toBytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true) + .andReturn(false); + expect(rocksIterator.key()) + .andReturn(key4Bytes.get()) + .andReturn(key3Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(2); + rocksIterator.prev(); + expectLastCall().times(2); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key3Bytes, + toBytes, + false, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key4Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key3Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + + @Test + public void shouldReturnTheCurrentKeyOnInvokingPeekNextKeyInForwardDirection() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seek(key1Bytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true) + .andReturn(false); + expect(rocksIterator.key()) + .andReturn(key2Bytes.get()) + .andReturn(key3Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(2); + rocksIterator.next(); + expectLastCall().times(2); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + key3Bytes, + true, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.peekNextKey(), is(key2Bytes)); + assertThat(rocksDBRangeIterator.peekNextKey(), is(key2Bytes)); + assertThat(rocksDBRangeIterator.next().key, is(key2Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.peekNextKey(), is(key3Bytes)); + assertThat(rocksDBRangeIterator.peekNextKey(), is(key3Bytes)); + assertThat(rocksDBRangeIterator.next().key, is(key3Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + assertThrows(NoSuchElementException.class, rocksDBRangeIterator::peekNextKey); + verify(rocksIterator); + } + + @Test + public void shouldReturnTheCurrentKeyOnInvokingPeekNextKeyInReverseDirection() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + final Bytes toBytes = Bytes.increment(key4Bytes); + rocksIterator.seekForPrev(toBytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true) + .andReturn(false); + expect(rocksIterator.key()) + .andReturn(key4Bytes.get()) + .andReturn(key3Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(2); + rocksIterator.prev(); + expectLastCall().times(2); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key3Bytes, + toBytes, + false, + true + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.peekNextKey(), is(key4Bytes)); + assertThat(rocksDBRangeIterator.peekNextKey(), is(key4Bytes)); + assertThat(rocksDBRangeIterator.next().key, is(key4Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.peekNextKey(), is(key3Bytes)); + assertThat(rocksDBRangeIterator.peekNextKey(), is(key3Bytes)); + assertThat(rocksDBRangeIterator.next().key, is(key3Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + assertThrows(NoSuchElementException.class, rocksDBRangeIterator::peekNextKey); + verify(rocksIterator); + } + + @Test + public void shouldCloseIterator() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seek(key1Bytes.get()); + rocksIterator.close(); + expectLastCall().times(1); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + key2Bytes, + true, + true + ); + rocksDBRangeIterator.close(); + verify(rocksIterator); + } + + @Test + public void shouldExcludeEndOfRange() { + final RocksIterator rocksIterator = mock(RocksIterator.class); + rocksIterator.seek(key1Bytes.get()); + expect(rocksIterator.isValid()) + .andReturn(true) + .andReturn(true); + expect(rocksIterator.key()) + .andReturn(key1Bytes.get()) + .andReturn(key2Bytes.get()); + expect(rocksIterator.value()).andReturn(valueBytes).times(2); + rocksIterator.next(); + expectLastCall().times(2); + replay(rocksIterator); + final RocksDBRangeIterator rocksDBRangeIterator = new RocksDBRangeIterator( + storeName, + rocksIterator, + Collections.emptySet(), + key1Bytes, + key2Bytes, + true, + false + ); + assertThat(rocksDBRangeIterator.hasNext(), is(true)); + assertThat(rocksDBRangeIterator.next().key, is(key1Bytes)); + assertThat(rocksDBRangeIterator.hasNext(), is(false)); + verify(rocksIterator); + } + +} From 29b4a3d1fe98475666b518be3b68fa93c9b575d7 Mon Sep 17 00:00:00 2001 From: Ron Dagostino Date: Tue, 2 Mar 2021 12:57:12 -0500 Subject: [PATCH 088/243] MINOR: Disable transactional/idempotent system tests for Raft quorums (#10224) --- tests/kafkatest/tests/core/replication_test.py | 3 +-- tests/kafkatest/tests/streams/streams_smoke_test.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/kafkatest/tests/core/replication_test.py b/tests/kafkatest/tests/core/replication_test.py index a0c01567d4e9b..bc40c81a96308 100644 --- a/tests/kafkatest/tests/core/replication_test.py +++ b/tests/kafkatest/tests/core/replication_test.py @@ -122,8 +122,7 @@ def min_cluster_size(self): @matrix(failure_mode=["clean_shutdown", "hard_shutdown", "clean_bounce", "hard_bounce"], broker_type=["leader"], security_protocol=["PLAINTEXT"], - enable_idempotence=[True], - metadata_quorum=quorum.all_non_upgrade) + enable_idempotence=[True]) @matrix(failure_mode=["clean_shutdown", "hard_shutdown", "clean_bounce", "hard_bounce"], broker_type=["leader"], security_protocol=["PLAINTEXT", "SASL_SSL"], diff --git a/tests/kafkatest/tests/streams/streams_smoke_test.py b/tests/kafkatest/tests/streams/streams_smoke_test.py index b1f908ddcf3b2..29955cd0ca38f 100644 --- a/tests/kafkatest/tests/streams/streams_smoke_test.py +++ b/tests/kafkatest/tests/streams/streams_smoke_test.py @@ -47,7 +47,8 @@ def __init__(self, test_context): self.driver = StreamsSmokeTestDriverService(test_context, self.kafka) @cluster(num_nodes=8) - @matrix(processing_guarantee=['at_least_once', 'exactly_once', 'exactly_once_beta'], crash=[True, False], metadata_quorum=quorum.all_non_upgrade) + @matrix(processing_guarantee=['at_least_once'], crash=[True, False], metadata_quorum=quorum.all_non_upgrade) + @matrix(processing_guarantee=['exactly_once', 'exactly_once_beta'], crash=[True, False]) def test_streams(self, processing_guarantee, crash, metadata_quorum=quorum.zk): processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee) processor2 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee) From a4ba73270cbdcb31e92fc54fc8c9858abf4be552 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Tue, 2 Mar 2021 10:20:07 -0800 Subject: [PATCH 089/243] KAFKA-12394; Return `TOPIC_AUTHORIZATION_FAILED` in delete topic response if no describe permission (#10223) We now accept topicIds in the `DeleteTopic` request. If the client principal does not have `Describe` permission, then we return `TOPIC_AUTHORIZATION_FAILED`. This is justified because the topicId is not considered sensitive. However, in this case, we should not return the name of the topic in the response since we do consider it sensitive. Reviewers: David Jacot , dengziming , Justine Olshan , Chia-Ping Tsai --- .../kafka/controller/ControllerContext.scala | 4 + .../main/scala/kafka/server/KafkaApis.scala | 37 ++-- .../kafka/api/AuthorizerIntegrationTest.scala | 132 ++++++--------- .../unit/kafka/server/KafkaApisTest.scala | 159 +++++++++++++++++- 4 files changed, 233 insertions(+), 99 deletions(-) diff --git a/core/src/main/scala/kafka/controller/ControllerContext.scala b/core/src/main/scala/kafka/controller/ControllerContext.scala index 042830118223f..379196aa1d42c 100644 --- a/core/src/main/scala/kafka/controller/ControllerContext.scala +++ b/core/src/main/scala/kafka/controller/ControllerContext.scala @@ -463,6 +463,10 @@ class ControllerContext { }.keySet } + def topicName(topicId: Uuid): Option[String] = { + topicNames.get(topicId) + } + def clearPartitionLeadershipInfo(): Unit = partitionLeadershipInfo.clear() def partitionWithLeadersCount: Int = partitionLeadershipInfo.size diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index d4fd527d3d4e1..bc228403b1088 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -1874,7 +1874,7 @@ class KafkaApis(val requestChannel: RequestChannel, if (topic.name() != null && topic.topicId() != Uuid.ZERO_UUID) throw new InvalidRequestException("Topic name and topic ID can not both be specified.") val name = if (topic.topicId() == Uuid.ZERO_UUID) topic.name() - else zkSupport.controller.controllerContext.topicNames.getOrElse(topic.topicId(), null) + else zkSupport.controller.controllerContext.topicName(topic.topicId).orNull results.add(new DeletableTopicResult() .setName(name) .setTopicId(topic.topicId())) @@ -1884,20 +1884,27 @@ class KafkaApis(val requestChannel: RequestChannel, val authorizedDeleteTopics = authHelper.filterByAuthorized(request.context, DELETE, TOPIC, results.asScala.filter(result => result.name() != null))(_.name) results.forEach { topic => - val unresolvedTopicId = !(topic.topicId() == Uuid.ZERO_UUID) && topic.name() == null - if (!config.usesTopicId && topicIdsFromRequest.contains(topic.topicId)) { - topic.setErrorCode(Errors.UNSUPPORTED_VERSION.code) - topic.setErrorMessage("Topic IDs are not supported on the server.") - } else if (unresolvedTopicId) - topic.setErrorCode(Errors.UNKNOWN_TOPIC_ID.code) - else if (topicIdsFromRequest.contains(topic.topicId) && !authorizedDescribeTopics(topic.name)) - topic.setErrorCode(Errors.UNKNOWN_TOPIC_ID.code) - else if (!authorizedDeleteTopics.contains(topic.name)) - topic.setErrorCode(Errors.TOPIC_AUTHORIZATION_FAILED.code) - else if (!metadataCache.contains(topic.name)) - topic.setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code) - else - toDelete += topic.name + val unresolvedTopicId = topic.topicId() != Uuid.ZERO_UUID && topic.name() == null + if (!config.usesTopicId && topicIdsFromRequest.contains(topic.topicId)) { + topic.setErrorCode(Errors.UNSUPPORTED_VERSION.code) + topic.setErrorMessage("Topic IDs are not supported on the server.") + } else if (unresolvedTopicId) { + topic.setErrorCode(Errors.UNKNOWN_TOPIC_ID.code) + } else if (topicIdsFromRequest.contains(topic.topicId) && !authorizedDescribeTopics.contains(topic.name)) { + + // Because the client does not have Describe permission, the name should + // not be returned in the response. Note, however, that we do not consider + // the topicId itself to be sensitive, so there is no reason to obscure + // this case with `UNKNOWN_TOPIC_ID`. + topic.setName(null) + topic.setErrorCode(Errors.TOPIC_AUTHORIZATION_FAILED.code) + } else if (!authorizedDeleteTopics.contains(topic.name)) { + topic.setErrorCode(Errors.TOPIC_AUTHORIZATION_FAILED.code) + } else if (!metadataCache.contains(topic.name)) { + topic.setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code) + } else { + toDelete += topic.name + } } // If no authorized topics return immediately if (toDelete.isEmpty) diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index b100962e35c9c..ebe4634adf8a2 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -60,6 +60,8 @@ import org.apache.kafka.common.{ElectionType, IsolationLevel, Node, TopicPartiti import org.apache.kafka.test.{TestUtils => JTestUtils} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource import scala.annotation.nowarn import scala.collection.mutable @@ -245,9 +247,14 @@ class AuthorizerIntegrationTest extends BaseRequestTest { }) ) - val requestKeysToErrorWithIds = (id: Uuid) => Map[ApiKeys, Nothing => Errors]( - ApiKeys.DELETE_TOPICS -> ((resp: requests.DeleteTopicsResponse) => Errors.forCode(resp.data.responses.asScala.find(_.topicId == id).get.errorCode())) - ) + def findErrorForTopicId(id: Uuid, response: AbstractResponse): Errors = { + response match { + case res: DeleteTopicsResponse => + Errors.forCode(res.data.responses.asScala.find(_.topicId == id).get.errorCode) + case _ => + fail(s"Unexpected response type $response") + } + } val requestKeysToAcls = Map[ApiKeys, Map[ResourcePattern, Set[AccessControlEntry]]]( ApiKeys.METADATA -> topicDescribeAcl, @@ -534,12 +541,12 @@ class AuthorizerIntegrationTest extends BaseRequestTest { .setTimeoutMs(5000)).build() } - private def deleteTopicsWithIdsRequest(id: Uuid = getTopicIds()(topic)): DeleteTopicsRequest = { + private def deleteTopicsWithIdsRequest(topicId: Uuid): DeleteTopicsRequest = { new DeleteTopicsRequest.Builder( new DeleteTopicsRequestData() .setTopics(Collections.singletonList( new DeleteTopicsRequestData.DeleteTopicState() - .setTopicId(id))) + .setTopicId(topicId))) .setTimeoutMs(5000)).build() } @@ -737,29 +744,53 @@ class AuthorizerIntegrationTest extends BaseRequestTest { sendRequests(requestKeyToRequest) } - @Test - def testAuthorizationDeleteTopicsIdWithTopicExisting(): Unit = { - sendRequests(mutable.Map(ApiKeys.CREATE_TOPICS -> createTopicsRequest)) + @ParameterizedTest + @ValueSource(booleans = Array(true, false)) + def testTopicIdAuthorization(withTopicExisting: Boolean): Unit = { + val topicId = if (withTopicExisting) { + createTopic(topic) + getTopicIds()(topic) + } else { + Uuid.randomUuid() + } + + val requestKeyToRequest = mutable.LinkedHashMap[ApiKeys, AbstractRequest]( + ApiKeys.DELETE_TOPICS -> deleteTopicsWithIdsRequest(topicId) + ) - val id = getTopicIds()(topic) + def sendAndVerify( + request: AbstractRequest, + isAuthorized: Boolean, + isDescribeAuthorized: Boolean + ): Unit = { + val response = connectAndReceive[AbstractResponse](request) + val error = findErrorForTopicId(topicId, response) + if (!withTopicExisting) { + assertEquals(Errors.UNKNOWN_TOPIC_ID, error) + } else if (!isDescribeAuthorized || !isAuthorized) { + assertEquals(Errors.TOPIC_AUTHORIZATION_FAILED, error) + } + } - for ((key, request) <- mutable.Map(ApiKeys.DELETE_TOPICS -> deleteTopicsWithIdsRequest())) { + for ((key, request) <- requestKeyToRequest) { removeAllClientAcls() - val resources = requestKeysToAcls(key).map(_._1.resourceType).toSet - sendRequestWithIdAndVerifyResponseError(request, resources, isAuthorized = false, topicExists = true, describeAuthorized = false, id = id) + sendAndVerify(request, isAuthorized = false, isDescribeAuthorized = false) + + val describeAcls = topicDescribeAcl(topicResource) + addAndVerifyAcls(describeAcls, topicResource) val resourceToAcls = requestKeysToAcls(key) resourceToAcls.get(topicResource).foreach { acls => - val describeAcls = topicDescribeAcl(topicResource) val isAuthorized = describeAcls == acls - addAndVerifyAcls(describeAcls, topicResource) - sendRequestWithIdAndVerifyResponseError(request, resources, isAuthorized = isAuthorized, topicExists = true, describeAuthorized = true, id = id) - removeAllClientAcls() + sendAndVerify(request, isAuthorized = isAuthorized, isDescribeAuthorized = true) } - for ((resource, acls) <- resourceToAcls) + removeAllClientAcls() + for ((resource, acls) <- resourceToAcls) { addAndVerifyAcls(acls, resource) - sendRequestWithIdAndVerifyResponseError(request, resources, isAuthorized = true, topicExists = true, describeAuthorized = true, id = id) + } + + sendAndVerify(request, isAuthorized = true, isDescribeAuthorized = true) } } @@ -789,33 +820,6 @@ class AuthorizerIntegrationTest extends BaseRequestTest { sendRequests(requestKeyToRequest, false) } - @Test - def testAuthorizationDeleteTopicsIdWithTopicNotExisting(): Unit = { - val id = Uuid.randomUuid() - val requestKeyToRequest = mutable.LinkedHashMap[ApiKeys, AbstractRequest]( - ApiKeys.DELETE_TOPICS -> deleteTopicsWithIdsRequest(id), - ) - - for ((key, request) <- requestKeyToRequest) { - removeAllClientAcls() - val resources = requestKeysToAcls(key).map(_._1.resourceType).toSet - sendRequestWithIdAndVerifyResponseError(request, resources, isAuthorized = false, topicExists = false, describeAuthorized = false, id = id) - - val resourceToAcls = requestKeysToAcls(key) - resourceToAcls.get(topicResource).foreach { acls => - val describeAcls = topicDescribeAcl(topicResource) - val isAuthorized = describeAcls == acls - addAndVerifyAcls(describeAcls, topicResource) - sendRequestWithIdAndVerifyResponseError(request, resources, isAuthorized = isAuthorized, topicExists = false, describeAuthorized = true, id = id) - removeAllClientAcls() - } - - for ((resource, acls) <- resourceToAcls) - addAndVerifyAcls(acls, resource) - sendRequestWithIdAndVerifyResponseError(request, resources, isAuthorized = true, topicExists = false, describeAuthorized = true, id = id) - } - } - @Test def testCreateTopicAuthorizationWithClusterCreate(): Unit = { removeAllClientAcls() @@ -2058,44 +2062,6 @@ class AuthorizerIntegrationTest extends BaseRequestTest { } } - private def sendRequestWithIdAndVerifyResponseError(request: AbstractRequest, - resources: Set[ResourceType], - isAuthorized: Boolean, - topicExists: Boolean, - describeAuthorized: Boolean, - id: Uuid): AbstractResponse = { - val apiKey = request.apiKey - val response = connectAndReceive[AbstractResponse](request) - val error = requestKeysToErrorWithIds(id)(apiKey).asInstanceOf[AbstractResponse => Errors](response) - - val authorizationErrors = resources.flatMap { resourceType => - if (resourceType == TOPIC) { - if (isAuthorized) - Set(Errors.UNKNOWN_TOPIC_ID, AclEntry.authorizationError(ResourceType.TOPIC)) - else if (describeAuthorized) - Set(AclEntry.authorizationError(ResourceType.TOPIC)) - else - Set(Errors.UNKNOWN_TOPIC_ID) - } else { - Set(AclEntry.authorizationError(resourceType)) - } - } - - if (topicExists) - if (isAuthorized) - assertFalse(authorizationErrors.contains(error), s"$apiKey should be allowed. Found unexpected authorization error $error") - else - assertTrue(authorizationErrors.contains(error), s"$apiKey should be forbidden. Found error $error but expected one of $authorizationErrors") - else if (resources == Set(TOPIC)) - if (isAuthorized) - assertEquals(Errors.UNKNOWN_TOPIC_ID, error, s"$apiKey had an unexpected error") - else { - assertEquals(Errors.UNKNOWN_TOPIC_ID, error, s"$apiKey had an unexpected error") - } - - response - } - private def sendRequestAndVerifyResponseError(request: AbstractRequest, resources: Set[ResourceType], isAuthorized: Boolean, diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 1fec7892434a8..476fe14cf9a65 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -26,7 +26,7 @@ import java.util.{Collections, Optional, Properties, Random} import kafka.api.{ApiVersion, KAFKA_0_10_2_IV0, KAFKA_2_2_IV1, LeaderAndIsr} import kafka.cluster.{Broker, Partition} -import kafka.controller.KafkaController +import kafka.controller.{ControllerContext, KafkaController} import kafka.coordinator.group.GroupCoordinatorConcurrencyTest.{JoinGroupCallback, SyncGroupCallback} import kafka.coordinator.group._ import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator} @@ -75,6 +75,8 @@ import org.easymock.EasyMock._ import org.easymock.{Capture, EasyMock, IAnswer} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource import org.mockito.{ArgumentMatchers, Mockito} import scala.annotation.nowarn @@ -3479,6 +3481,161 @@ class KafkaApisTest { assertEquals(List(mkTopicData(topic = "foo", Seq(1, 2))), fooState.topics.asScala.toList) } + @Test + def testDeleteTopicsByIdAuthorization(): Unit = { + val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) + val controllerContext: ControllerContext = EasyMock.mock(classOf[ControllerContext]) + + EasyMock.expect(clientControllerQuotaManager.newQuotaFor( + EasyMock.anyObject(classOf[RequestChannel.Request]), + EasyMock.anyShort() + )).andReturn(UnboundedControllerMutationQuota) + EasyMock.expect(controller.isActive).andReturn(true) + EasyMock.expect(controller.controllerContext).andStubReturn(controllerContext) + + // Try to delete three topics: + // 1. One without describe permission + // 2. One without delete permission + // 3. One which is authorized, but doesn't exist + + expectTopicAuthorization(authorizer, AclOperation.DESCRIBE, Map( + "foo" -> AuthorizationResult.DENIED, + "bar" -> AuthorizationResult.ALLOWED + )) + + expectTopicAuthorization(authorizer, AclOperation.DELETE, Map( + "foo" -> AuthorizationResult.DENIED, + "bar" -> AuthorizationResult.DENIED + )) + + val topicIdsMap = Map( + Uuid.randomUuid() -> Some("foo"), + Uuid.randomUuid() -> Some("bar"), + Uuid.randomUuid() -> None + ) + + topicIdsMap.foreach { case (topicId, topicNameOpt) => + EasyMock.expect(controllerContext.topicName(topicId)).andReturn(topicNameOpt) + } + + val topicDatas = topicIdsMap.keys.map { topicId => + new DeleteTopicsRequestData.DeleteTopicState().setTopicId(topicId) + }.toList + val deleteRequest = new DeleteTopicsRequest.Builder(new DeleteTopicsRequestData() + .setTopics(topicDatas.asJava)) + .build(ApiKeys.DELETE_TOPICS.latestVersion) + + val request = buildRequest(deleteRequest) + val capturedResponse = expectNoThrottling(request) + + EasyMock.replay(replicaManager, clientRequestQuotaManager, clientControllerQuotaManager, + requestChannel, txnCoordinator, controller, controllerContext, authorizer) + createKafkaApis(authorizer = Some(authorizer)).handleDeleteTopicsRequest(request) + + val deleteResponse = capturedResponse.getValue.asInstanceOf[DeleteTopicsResponse] + + topicIdsMap.foreach { case (topicId, nameOpt) => + val response = deleteResponse.data.responses.asScala.find(_.topicId == topicId).get + nameOpt match { + case Some("foo") => + assertNull(response.name) + assertEquals(Errors.TOPIC_AUTHORIZATION_FAILED, Errors.forCode(response.errorCode)) + case Some("bar") => + assertEquals("bar", response.name) + assertEquals(Errors.TOPIC_AUTHORIZATION_FAILED, Errors.forCode(response.errorCode)) + case None => + assertNull(response.name) + assertEquals(Errors.UNKNOWN_TOPIC_ID, Errors.forCode(response.errorCode)) + case _ => + fail("Unexpected topic id/name mapping") + } + } + } + + @ParameterizedTest + @ValueSource(booleans = Array(true, false)) + def testDeleteTopicsByNameAuthorization(usePrimitiveTopicNameArray: Boolean): Unit = { + val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) + + EasyMock.expect(clientControllerQuotaManager.newQuotaFor( + EasyMock.anyObject(classOf[RequestChannel.Request]), + EasyMock.anyShort() + )).andReturn(UnboundedControllerMutationQuota) + EasyMock.expect(controller.isActive).andReturn(true) + + // Try to delete three topics: + // 1. One without describe permission + // 2. One without delete permission + // 3. One which is authorized, but doesn't exist + + expectTopicAuthorization(authorizer, AclOperation.DESCRIBE, Map( + "foo" -> AuthorizationResult.DENIED, + "bar" -> AuthorizationResult.ALLOWED, + "baz" -> AuthorizationResult.ALLOWED + )) + + expectTopicAuthorization(authorizer, AclOperation.DELETE, Map( + "foo" -> AuthorizationResult.DENIED, + "bar" -> AuthorizationResult.DENIED, + "baz" -> AuthorizationResult.ALLOWED + )) + + val deleteRequest = if (usePrimitiveTopicNameArray) { + new DeleteTopicsRequest.Builder(new DeleteTopicsRequestData() + .setTopicNames(List("foo", "bar", "baz").asJava)) + .build(5.toShort) + } else { + val topicDatas = List( + new DeleteTopicsRequestData.DeleteTopicState().setName("foo"), + new DeleteTopicsRequestData.DeleteTopicState().setName("bar"), + new DeleteTopicsRequestData.DeleteTopicState().setName("baz") + ) + new DeleteTopicsRequest.Builder(new DeleteTopicsRequestData() + .setTopics(topicDatas.asJava)) + .build(ApiKeys.DELETE_TOPICS.latestVersion) + } + + val request = buildRequest(deleteRequest) + val capturedResponse = expectNoThrottling(request) + + EasyMock.replay(replicaManager, clientRequestQuotaManager, clientControllerQuotaManager, + requestChannel, txnCoordinator, controller, authorizer) + createKafkaApis(authorizer = Some(authorizer)).handleDeleteTopicsRequest(request) + + val deleteResponse = capturedResponse.getValue.asInstanceOf[DeleteTopicsResponse] + + def lookupErrorCode(topic: String): Option[Errors] = { + Option(deleteResponse.data.responses().find(topic)) + .map(result => Errors.forCode(result.errorCode)) + } + + assertEquals(Some(Errors.TOPIC_AUTHORIZATION_FAILED), lookupErrorCode("foo")) + assertEquals(Some(Errors.TOPIC_AUTHORIZATION_FAILED), lookupErrorCode("bar")) + assertEquals(Some(Errors.UNKNOWN_TOPIC_OR_PARTITION), lookupErrorCode("baz")) + } + + def expectTopicAuthorization( + authorizer: Authorizer, + aclOperation: AclOperation, + topicResults: Map[String, AuthorizationResult] + ): Unit = { + val expectedActions = topicResults.keys.map { topic => + val pattern = new ResourcePattern(ResourceType.TOPIC, topic, PatternType.LITERAL) + topic -> new Action(aclOperation, pattern, 1, true, true) + }.toMap + + val actionsCapture: Capture[util.List[Action]] = EasyMock.newCapture() + EasyMock.expect(authorizer.authorize(anyObject[RequestContext], EasyMock.capture(actionsCapture))) + .andAnswer(() => { + actionsCapture.getValue.asScala.map { action => + val topic = action.resourcePattern.name + assertEquals(expectedActions(topic), action) + topicResults(topic) + }.asJava + }) + .once() + } + private def createMockRequest(): RequestChannel.Request = { val request: RequestChannel.Request = EasyMock.createNiceMock(classOf[RequestChannel.Request]) val requestHeader: RequestHeader = EasyMock.createNiceMock(classOf[RequestHeader]) From 420162d190d26130e8ff8699991a32205f320add Mon Sep 17 00:00:00 2001 From: Luke Chen <43372967+showuon@users.noreply.github.com> Date: Wed, 3 Mar 2021 03:46:32 +0800 Subject: [PATCH 090/243] KAFKA-10251: increase timeout for consuming records (#10228) Bump the `pollUntilAtLeastNumRecords` timeout from 15s to 30s Reviewers: Anna Sophie Blee-Goldman --- .../scala/integration/kafka/api/TransactionsBounceTest.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala b/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala index 064424833e792..d7470c54b156f 100644 --- a/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala +++ b/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala @@ -32,6 +32,7 @@ import scala.jdk.CollectionConverters._ import scala.collection.mutable class TransactionsBounceTest extends IntegrationTestHarness { + private val consumeRecordTimeout = 30000 private val producerBufferSize = 65536 private val serverMessageMaxBytes = producerBufferSize/2 private val numPartitions = 3 @@ -106,7 +107,7 @@ class TransactionsBounceTest extends IntegrationTestHarness { while (numMessagesProcessed < numInputRecords) { val toRead = Math.min(200, numInputRecords - numMessagesProcessed) trace(s"$iteration: About to read $toRead messages, processed $numMessagesProcessed so far..") - val records = TestUtils.pollUntilAtLeastNumRecords(consumer, toRead) + val records = TestUtils.pollUntilAtLeastNumRecords(consumer, toRead, waitTimeMs = consumeRecordTimeout) trace(s"Received ${records.size} messages, sending them transactionally to $outputTopic") producer.beginTransaction() @@ -134,7 +135,7 @@ class TransactionsBounceTest extends IntegrationTestHarness { val verifyingConsumer = createConsumerAndSubscribe("randomGroup", List(outputTopic), readCommitted = true) val recordsByPartition = new mutable.HashMap[TopicPartition, mutable.ListBuffer[Int]]() - TestUtils.pollUntilAtLeastNumRecords(verifyingConsumer, numInputRecords).foreach { record => + TestUtils.pollUntilAtLeastNumRecords(verifyingConsumer, numInputRecords, waitTimeMs = consumeRecordTimeout).foreach { record => val value = TestUtils.assertCommittedAndGetValue(record).toInt val topicPartition = new TopicPartition(record.topic(), record.partition()) recordsByPartition.getOrElseUpdate(topicPartition, new mutable.ListBuffer[Int]) From a848e0c4208318e5db305876d14af4be0c3ce5fc Mon Sep 17 00:00:00 2001 From: Bruno Cadonna Date: Tue, 2 Mar 2021 21:00:00 +0100 Subject: [PATCH 091/243] KAFKA-10357: Extract setup of changelog from Streams partition assignor (#10163) To implement the explicit user initialization of Kafka Streams as described in KIP-698, we first need to extract the code for the setup of the changelog topics from the Streams partition assignor so that it can also be called outside of a rebalance. Reviewers: Anna Sophie Blee-Goldman , Guozhang Wang --- .../processor/internals/ChangelogTopics.java | 132 +++++++++++ .../internals/StreamsPartitionAssignor.java | 147 +++---------- .../internals/ChangelogTopicsTest.java | 208 ++++++++++++++++++ 3 files changed, 373 insertions(+), 114 deletions(-) create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/ChangelogTopics.java create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/ChangelogTopicsTest.java diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ChangelogTopics.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ChangelogTopics.java new file mode 100644 index 0000000000000..f47db270f611c --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ChangelogTopics.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.processor.internals; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.streams.processor.TaskId; +import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo; +import org.slf4j.Logger; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.UNKNOWN; + +public class ChangelogTopics { + + private final InternalTopicManager internalTopicManager; + private final Map topicGroups; + private final Map> tasksForTopicGroup; + private final Map> changelogPartitionsForStatefulTask = new HashMap<>(); + private final Map> preExistingChangelogPartitionsForTask = new HashMap<>(); + private final Set preExistingNonSourceTopicBasedChangelogPartitions = new HashSet<>(); + private final Set sourceTopicBasedChangelogTopics = new HashSet<>(); + private final Set preExsitingSourceTopicBasedChangelogPartitions = new HashSet<>(); + private final Logger log; + + public ChangelogTopics(final InternalTopicManager internalTopicManager, + final Map topicGroups, + final Map> tasksForTopicGroup, + final String logPrefix) { + this.internalTopicManager = internalTopicManager; + this.topicGroups = topicGroups; + this.tasksForTopicGroup = tasksForTopicGroup; + final LogContext logContext = new LogContext(logPrefix); + log = logContext.logger(getClass()); + } + + public void setup() { + // add tasks to state change log topic subscribers + final Map changelogTopicMetadata = new HashMap<>(); + for (final Map.Entry entry : topicGroups.entrySet()) { + final int topicGroupId = entry.getKey(); + final TopicsInfo topicsInfo = entry.getValue(); + + final Set topicGroupTasks = tasksForTopicGroup.get(topicGroupId); + if (topicGroupTasks == null) { + log.debug("No tasks found for topic group {}", topicGroupId); + continue; + } else if (topicsInfo.stateChangelogTopics.isEmpty()) { + continue; + } + + for (final TaskId task : topicGroupTasks) { + final Set changelogTopicPartitions = topicsInfo.stateChangelogTopics + .keySet() + .stream() + .map(topic -> new TopicPartition(topic, task.partition)) + .collect(Collectors.toSet()); + changelogPartitionsForStatefulTask.put(task, changelogTopicPartitions); + } + + for (final InternalTopicConfig topicConfig : topicsInfo.nonSourceChangelogTopics()) { + // the expected number of partitions is the max value of TaskId.partition + 1 + int numPartitions = UNKNOWN; + for (final TaskId task : topicGroupTasks) { + if (numPartitions < task.partition + 1) { + numPartitions = task.partition + 1; + } + } + topicConfig.setNumberOfPartitions(numPartitions); + changelogTopicMetadata.put(topicConfig.name(), topicConfig); + } + sourceTopicBasedChangelogTopics.addAll(topicsInfo.sourceTopicChangelogs()); + } + + final Set newlyCreatedChangelogTopics = internalTopicManager.makeReady(changelogTopicMetadata); + log.debug("Created state changelog topics {} from the parsed topology.", changelogTopicMetadata.values()); + + for (final Map.Entry> entry : changelogPartitionsForStatefulTask.entrySet()) { + final TaskId taskId = entry.getKey(); + final Set topicPartitions = entry.getValue(); + for (final TopicPartition topicPartition : topicPartitions) { + if (!newlyCreatedChangelogTopics.contains(topicPartition.topic())) { + preExistingChangelogPartitionsForTask.computeIfAbsent(taskId, task -> new HashSet<>()).add(topicPartition); + if (!sourceTopicBasedChangelogTopics.contains(topicPartition.topic())) { + preExistingNonSourceTopicBasedChangelogPartitions.add(topicPartition); + } else { + preExsitingSourceTopicBasedChangelogPartitions.add(topicPartition); + } + } + } + } + } + + public Set preExistingNonSourceTopicBasedPartitions() { + return Collections.unmodifiableSet(preExistingNonSourceTopicBasedChangelogPartitions); + } + + public Set preExistingPartitionsFor(final TaskId taskId) { + if (preExistingChangelogPartitionsForTask.containsKey(taskId)) { + return Collections.unmodifiableSet(preExistingChangelogPartitionsForTask.get(taskId)); + } + return Collections.emptySet(); + } + + public Set preExistingSourceTopicBasedPartitions() { + return Collections.unmodifiableSet(preExsitingSourceTopicBasedChangelogPartitions); + } + + public Set statefulTaskIds() { + return Collections.unmodifiableSet(changelogPartitionsForStatefulTask.keySet()); + } +} \ No newline at end of file diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java index 841dcc61da0db..c1fcd7159912e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java @@ -548,58 +548,6 @@ private void checkAllPartitions(final Set allSourceTopics, } } - /** - * Resolve changelog topic metadata and create them if necessary. Fills in the changelogsByStatefulTask map and - * the optimizedSourceChangelogs set and returns the set of changelogs which were newly created. - */ - private Set prepareChangelogTopics(final Map topicGroups, - final Map> tasksForTopicGroup, - final Map> changelogsByStatefulTask, - final Set optimizedSourceChangelogs) { - // add tasks to state change log topic subscribers - final Map changelogTopicMetadata = new HashMap<>(); - for (final Map.Entry entry : topicGroups.entrySet()) { - final int topicGroupId = entry.getKey(); - final TopicsInfo topicsInfo = entry.getValue(); - - final Set topicGroupTasks = tasksForTopicGroup.get(topicGroupId); - if (topicGroupTasks == null) { - log.debug("No tasks found for topic group {}", topicGroupId); - continue; - } else if (topicsInfo.stateChangelogTopics.isEmpty()) { - continue; - } - - for (final TaskId task : topicGroupTasks) { - changelogsByStatefulTask.put( - task, - topicsInfo.stateChangelogTopics - .keySet() - .stream() - .map(topic -> new TopicPartition(topic, task.partition)) - .collect(Collectors.toSet())); - } - - for (final InternalTopicConfig topicConfig : topicsInfo.nonSourceChangelogTopics()) { - // the expected number of partitions is the max value of TaskId.partition + 1 - int numPartitions = UNKNOWN; - for (final TaskId task : topicGroupTasks) { - if (numPartitions < task.partition + 1) { - numPartitions = task.partition + 1; - } - } - topicConfig.setNumberOfPartitions(numPartitions); - changelogTopicMetadata.put(topicConfig.name(), topicConfig); - } - - optimizedSourceChangelogs.addAll(topicsInfo.sourceTopicChangelogs()); - } - - final Set newlyCreatedTopics = internalTopicManager.makeReady(changelogTopicMetadata); - log.debug("Created state changelog topics {} from the parsed topology.", changelogTopicMetadata.values()); - return newlyCreatedTopics; - } - /** * Assigns a set of tasks to each client (Streams instance) using the configured task assignor, and also * populate the stateful tasks that have been assigned to the clients @@ -619,23 +567,20 @@ private boolean assignTasksToClients(final Cluster fullMetadata, final Map> tasksForTopicGroup = new HashMap<>(); populateTasksForMaps(taskForPartition, tasksForTopicGroup, allSourceTopics, partitionsForTask, fullMetadata); - final Map> changelogsByStatefulTask = new HashMap<>(); - final Set optimizedSourceChangelogs = new HashSet<>(); - final Set newlyCreatedChangelogs = - prepareChangelogTopics(topicGroups, tasksForTopicGroup, changelogsByStatefulTask, optimizedSourceChangelogs); + final ChangelogTopics changelogTopics = new ChangelogTopics( + internalTopicManager, + topicGroups, + tasksForTopicGroup, + logPrefix + ); + changelogTopics.setup(); final Map clientStates = new HashMap<>(); final boolean lagComputationSuccessful = - populateClientStatesMap(clientStates, - clientMetadataMap, - taskForPartition, - changelogsByStatefulTask, - newlyCreatedChangelogs, - optimizedSourceChangelogs - ); + populateClientStatesMap(clientStates, clientMetadataMap, taskForPartition, changelogTopics); final Set allTasks = partitionsForTask.keySet(); - statefulTasks.addAll(changelogsByStatefulTask.keySet()); + statefulTasks.addAll(changelogTopics.statefulTaskIds()); log.debug("Assigning tasks {} to clients {} with number of replicas {}", allTasks, clientStates, numStandbyReplicas()); @@ -677,55 +622,35 @@ private TaskAssignor createTaskAssignor(final boolean lagComputationSuccessful) * @param clientStates a map from each client to its state, including offset lags. Populated by this method. * @param clientMetadataMap a map from each client to its full metadata * @param taskForPartition map from topic partition to its corresponding task - * @param changelogsByStatefulTask map from each stateful task to its set of changelog topic partitions + * @param changelogTopics object that manages changelog topics * * @return whether we were able to successfully fetch the changelog end offsets and compute each client's lag */ private boolean populateClientStatesMap(final Map clientStates, final Map clientMetadataMap, final Map taskForPartition, - final Map> changelogsByStatefulTask, - final Set newlyCreatedChangelogs, - final Set optimizedSourceChangelogs) { + final ChangelogTopics changelogTopics) { boolean fetchEndOffsetsSuccessful; Map allTaskEndOffsetSums; try { - final Collection allChangelogPartitions = - changelogsByStatefulTask.values().stream() - .flatMap(Collection::stream) - .collect(Collectors.toList()); - - final Set preexistingChangelogPartitions = new HashSet<>(); - final Set preexistingSourceChangelogPartitions = new HashSet<>(); - final Set newlyCreatedChangelogPartitions = new HashSet<>(); - for (final TopicPartition changelog : allChangelogPartitions) { - if (newlyCreatedChangelogs.contains(changelog.topic())) { - newlyCreatedChangelogPartitions.add(changelog); - } else if (optimizedSourceChangelogs.contains(changelog.topic())) { - preexistingSourceChangelogPartitions.add(changelog); - } else { - preexistingChangelogPartitions.add(changelog); - } - } - // Make the listOffsets request first so it can fetch the offsets for non-source changelogs // asynchronously while we use the blocking Consumer#committed call to fetch source-changelog offsets final KafkaFuture> endOffsetsFuture = - fetchEndOffsetsFuture(preexistingChangelogPartitions, adminClient); + fetchEndOffsetsFuture(changelogTopics.preExistingNonSourceTopicBasedPartitions(), adminClient); final Map sourceChangelogEndOffsets = - fetchCommittedOffsets(preexistingSourceChangelogPartitions, mainConsumerSupplier.get()); + fetchCommittedOffsets(changelogTopics.preExistingSourceTopicBasedPartitions(), mainConsumerSupplier.get()); final Map endOffsets = ClientUtils.getEndOffsets(endOffsetsFuture); allTaskEndOffsetSums = computeEndOffsetSumsByTask( - changelogsByStatefulTask, endOffsets, sourceChangelogEndOffsets, - newlyCreatedChangelogPartitions); + changelogTopics + ); fetchEndOffsetsSuccessful = true; } catch (final StreamsException | TimeoutException e) { - allTaskEndOffsetSums = changelogsByStatefulTask.keySet().stream().collect(Collectors.toMap(t -> t, t -> UNKNOWN_OFFSET_SUM)); + allTaskEndOffsetSums = changelogTopics.statefulTaskIds().stream().collect(Collectors.toMap(t -> t, t -> UNKNOWN_OFFSET_SUM)); fetchEndOffsetsSuccessful = false; } @@ -741,41 +666,35 @@ private boolean populateClientStatesMap(final Map clientState } /** - * @param changelogsByStatefulTask map from stateful task to its set of changelog topic partitions * @param endOffsets the listOffsets result from the adminClient * @param sourceChangelogEndOffsets the end (committed) offsets of optimized source changelogs - * @param newlyCreatedChangelogPartitions any changelogs that were just created duringthis assignment + * @param changelogTopics object that manages changelog topics * * @return Map from stateful task to its total end offset summed across all changelog partitions */ - private Map computeEndOffsetSumsByTask(final Map> changelogsByStatefulTask, - final Map endOffsets, + private Map computeEndOffsetSumsByTask(final Map endOffsets, final Map sourceChangelogEndOffsets, - final Collection newlyCreatedChangelogPartitions) { + final ChangelogTopics changelogTopics) { + final Map taskEndOffsetSums = new HashMap<>(); - for (final Map.Entry> taskEntry : changelogsByStatefulTask.entrySet()) { - final TaskId task = taskEntry.getKey(); - final Set changelogs = taskEntry.getValue(); - - taskEndOffsetSums.put(task, 0L); - for (final TopicPartition changelog : changelogs) { - final long changelogEndOffset; - if (newlyCreatedChangelogPartitions.contains(changelog)) { - changelogEndOffset = 0L; - } else if (sourceChangelogEndOffsets.containsKey(changelog)) { - changelogEndOffset = sourceChangelogEndOffsets.get(changelog); - } else if (endOffsets.containsKey(changelog)) { - changelogEndOffset = endOffsets.get(changelog).offset(); + for (final TaskId taskId : changelogTopics.statefulTaskIds()) { + taskEndOffsetSums.put(taskId, 0L); + for (final TopicPartition changelogPartition : changelogTopics.preExistingPartitionsFor(taskId)) { + final long changelogPartitionEndOffset; + if (sourceChangelogEndOffsets.containsKey(changelogPartition)) { + changelogPartitionEndOffset = sourceChangelogEndOffsets.get(changelogPartition); + } else if (endOffsets.containsKey(changelogPartition)) { + changelogPartitionEndOffset = endOffsets.get(changelogPartition).offset(); } else { - log.debug("Fetched offsets did not contain the changelog {} of task {}", changelog, task); - throw new IllegalStateException("Could not get end offset for " + changelog); + log.debug("Fetched offsets did not contain the changelog {} of task {}", changelogPartition, taskId); + throw new IllegalStateException("Could not get end offset for " + changelogPartition); } - final long newEndOffsetSum = taskEndOffsetSums.get(task) + changelogEndOffset; + final long newEndOffsetSum = taskEndOffsetSums.get(taskId) + changelogPartitionEndOffset; if (newEndOffsetSum < 0) { - taskEndOffsetSums.put(task, Long.MAX_VALUE); + taskEndOffsetSums.put(taskId, Long.MAX_VALUE); break; } else { - taskEndOffsetSums.put(task, newEndOffsetSum); + taskEndOffsetSums.put(taskId, newEndOffsetSum); } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ChangelogTopicsTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ChangelogTopicsTest.java new file mode 100644 index 0000000000000..06480b3e1cf25 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ChangelogTopicsTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.processor.internals; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.streams.processor.TaskId; +import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import static org.apache.kafka.common.utils.Utils.mkEntry; +import static org.apache.kafka.common.utils.Utils.mkMap; +import static org.apache.kafka.common.utils.Utils.mkSet; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.mock; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class ChangelogTopicsTest { + + private static final String SOURCE_TOPIC_NAME = "source"; + private static final String SINK_TOPIC_NAME = "sink"; + private static final String REPARTITION_TOPIC_NAME = "repartition"; + private static final String CHANGELOG_TOPIC_NAME1 = "changelog1"; + private static final Map TOPIC_CONFIG = Collections.singletonMap("config1", "val1"); + private static final RepartitionTopicConfig REPARTITION_TOPIC_CONFIG = + new RepartitionTopicConfig(REPARTITION_TOPIC_NAME, TOPIC_CONFIG); + private static final UnwindowedChangelogTopicConfig CHANGELOG_TOPIC_CONFIG = + new UnwindowedChangelogTopicConfig(CHANGELOG_TOPIC_NAME1, TOPIC_CONFIG); + + private static final TopicsInfo TOPICS_INFO1 = new TopicsInfo( + mkSet(SINK_TOPIC_NAME), + mkSet(SOURCE_TOPIC_NAME), + mkMap(mkEntry(REPARTITION_TOPIC_NAME, REPARTITION_TOPIC_CONFIG)), + mkMap(mkEntry(CHANGELOG_TOPIC_NAME1, CHANGELOG_TOPIC_CONFIG)) + ); + private static final TopicsInfo TOPICS_INFO2 = new TopicsInfo( + mkSet(SINK_TOPIC_NAME), + mkSet(SOURCE_TOPIC_NAME), + mkMap(mkEntry(REPARTITION_TOPIC_NAME, REPARTITION_TOPIC_CONFIG)), + mkMap() + ); + private static final TopicsInfo TOPICS_INFO3 = new TopicsInfo( + mkSet(SINK_TOPIC_NAME), + mkSet(SOURCE_TOPIC_NAME), + mkMap(mkEntry(REPARTITION_TOPIC_NAME, REPARTITION_TOPIC_CONFIG)), + mkMap(mkEntry(SOURCE_TOPIC_NAME, CHANGELOG_TOPIC_CONFIG)) + ); + private static final TopicsInfo TOPICS_INFO4 = new TopicsInfo( + mkSet(SINK_TOPIC_NAME), + mkSet(SOURCE_TOPIC_NAME), + mkMap(mkEntry(REPARTITION_TOPIC_NAME, REPARTITION_TOPIC_CONFIG)), + mkMap(mkEntry(SOURCE_TOPIC_NAME, null), mkEntry(CHANGELOG_TOPIC_NAME1, CHANGELOG_TOPIC_CONFIG)) + ); + private static final TaskId TASK_0_0 = new TaskId(0, 0); + private static final TaskId TASK_0_1 = new TaskId(0, 1); + private static final TaskId TASK_0_2 = new TaskId(0, 2); + + final InternalTopicManager internalTopicManager = mock(InternalTopicManager.class); + + @Test + public void shouldNotContainChangelogsForStatelessTasks() { + expect(internalTopicManager.makeReady(Collections.emptyMap())).andStubReturn(Collections.emptySet()); + final Map topicGroups = mkMap(mkEntry(0, TOPICS_INFO2)); + final Map> tasksForTopicGroup = mkMap(mkEntry(0, mkSet(TASK_0_0, TASK_0_1, TASK_0_2))); + replay(internalTopicManager); + + final ChangelogTopics changelogTopics = + new ChangelogTopics(internalTopicManager, topicGroups, tasksForTopicGroup, "[test] "); + changelogTopics.setup(); + + verify(internalTopicManager); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_0), is(Collections.emptySet())); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_1), is(Collections.emptySet())); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_2), is(Collections.emptySet())); + assertThat(changelogTopics.preExistingSourceTopicBasedPartitions(), is(Collections.emptySet())); + assertThat(changelogTopics.preExistingNonSourceTopicBasedPartitions(), is(Collections.emptySet())); + } + + @Test + public void shouldNotContainAnyPreExistingChangelogsIfChangelogIsNewlyCreated() { + expect(internalTopicManager.makeReady(mkMap(mkEntry(CHANGELOG_TOPIC_NAME1, CHANGELOG_TOPIC_CONFIG)))) + .andStubReturn(mkSet(CHANGELOG_TOPIC_NAME1)); + final Map topicGroups = mkMap(mkEntry(0, TOPICS_INFO1)); + final Set tasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2); + final Map> tasksForTopicGroup = mkMap(mkEntry(0, tasks)); + replay(internalTopicManager); + + final ChangelogTopics changelogTopics = + new ChangelogTopics(internalTopicManager, topicGroups, tasksForTopicGroup, "[test] "); + changelogTopics.setup(); + + verify(internalTopicManager); + assertThat(CHANGELOG_TOPIC_CONFIG.numberOfPartitions().orElse(Integer.MIN_VALUE), is(3)); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_0), is(Collections.emptySet())); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_1), is(Collections.emptySet())); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_2), is(Collections.emptySet())); + assertThat(changelogTopics.preExistingSourceTopicBasedPartitions(), is(Collections.emptySet())); + assertThat(changelogTopics.preExistingNonSourceTopicBasedPartitions(), is(Collections.emptySet())); + } + + @Test + public void shouldOnlyContainPreExistingNonSourceBasedChangelogs() { + expect(internalTopicManager.makeReady(mkMap(mkEntry(CHANGELOG_TOPIC_NAME1, CHANGELOG_TOPIC_CONFIG)))) + .andStubReturn(Collections.emptySet()); + final Map topicGroups = mkMap(mkEntry(0, TOPICS_INFO1)); + final Set tasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2); + final Map> tasksForTopicGroup = mkMap(mkEntry(0, tasks)); + replay(internalTopicManager); + + final ChangelogTopics changelogTopics = + new ChangelogTopics(internalTopicManager, topicGroups, tasksForTopicGroup, "[test] "); + changelogTopics.setup(); + + verify(internalTopicManager); + assertThat(CHANGELOG_TOPIC_CONFIG.numberOfPartitions().orElse(Integer.MIN_VALUE), is(3)); + final TopicPartition changelogPartition0 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 0); + final TopicPartition changelogPartition1 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 1); + final TopicPartition changelogPartition2 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 2); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_0), is(mkSet(changelogPartition0))); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_1), is(mkSet(changelogPartition1))); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_2), is(mkSet(changelogPartition2))); + assertThat(changelogTopics.preExistingSourceTopicBasedPartitions(), is(Collections.emptySet())); + assertThat( + changelogTopics.preExistingNonSourceTopicBasedPartitions(), + is(mkSet(changelogPartition0, changelogPartition1, changelogPartition2)) + ); + } + + @Test + public void shouldOnlyContainPreExistingSourceBasedChangelogs() { + expect(internalTopicManager.makeReady(Collections.emptyMap())).andStubReturn(Collections.emptySet()); + final Map topicGroups = mkMap(mkEntry(0, TOPICS_INFO3)); + final Set tasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2); + final Map> tasksForTopicGroup = mkMap(mkEntry(0, tasks)); + replay(internalTopicManager); + + final ChangelogTopics changelogTopics = + new ChangelogTopics(internalTopicManager, topicGroups, tasksForTopicGroup, "[test] "); + changelogTopics.setup(); + + verify(internalTopicManager); + final TopicPartition changelogPartition0 = new TopicPartition(SOURCE_TOPIC_NAME, 0); + final TopicPartition changelogPartition1 = new TopicPartition(SOURCE_TOPIC_NAME, 1); + final TopicPartition changelogPartition2 = new TopicPartition(SOURCE_TOPIC_NAME, 2); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_0), is(mkSet(changelogPartition0))); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_1), is(mkSet(changelogPartition1))); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_2), is(mkSet(changelogPartition2))); + assertThat( + changelogTopics.preExistingSourceTopicBasedPartitions(), + is(mkSet(changelogPartition0, changelogPartition1, changelogPartition2)) + ); + assertThat(changelogTopics.preExistingNonSourceTopicBasedPartitions(), is(Collections.emptySet())); + } + + @Test + public void shouldContainBothTypesOfPreExistingChangelogs() { + expect(internalTopicManager.makeReady(mkMap(mkEntry(CHANGELOG_TOPIC_NAME1, CHANGELOG_TOPIC_CONFIG)))) + .andStubReturn(Collections.emptySet()); + final Map topicGroups = mkMap(mkEntry(0, TOPICS_INFO4)); + final Set tasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2); + final Map> tasksForTopicGroup = mkMap(mkEntry(0, tasks)); + replay(internalTopicManager); + + final ChangelogTopics changelogTopics = + new ChangelogTopics(internalTopicManager, topicGroups, tasksForTopicGroup, "[test] "); + changelogTopics.setup(); + + verify(internalTopicManager); + assertThat(CHANGELOG_TOPIC_CONFIG.numberOfPartitions().orElse(Integer.MIN_VALUE), is(3)); + final TopicPartition changelogPartition0 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 0); + final TopicPartition changelogPartition1 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 1); + final TopicPartition changelogPartition2 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 2); + final TopicPartition sourcePartition0 = new TopicPartition(SOURCE_TOPIC_NAME, 0); + final TopicPartition sourcePartition1 = new TopicPartition(SOURCE_TOPIC_NAME, 1); + final TopicPartition sourcePartition2 = new TopicPartition(SOURCE_TOPIC_NAME, 2); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_0), is(mkSet(sourcePartition0, changelogPartition0))); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_1), is(mkSet(sourcePartition1, changelogPartition1))); + assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_2), is(mkSet(sourcePartition2, changelogPartition2))); + assertThat( + changelogTopics.preExistingSourceTopicBasedPartitions(), + is(mkSet(sourcePartition0, sourcePartition1, sourcePartition2)) + ); + assertThat( + changelogTopics.preExistingNonSourceTopicBasedPartitions(), + is(mkSet(changelogPartition0, changelogPartition1, changelogPartition2)) + ); + } +} From 3708a7c6c1ecf1304f091dda1e79ae53ba2df489 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Tue, 2 Mar 2021 13:01:35 -0800 Subject: [PATCH 092/243] KAFKA-12369; Implement `ListTransactions` API (#10206) This patch implements the `ListTransactions` API as documented in KIP-664: https://cwiki.apache.org/confluence/display/KAFKA/KIP-664%3A+Provide+tooling+to+detect+and+abort+hanging+transactions. Reviewers: Tom Bentley , Chia-Ping Tsai --- .../apache/kafka/common/protocol/ApiKeys.java | 3 +- .../common/requests/AbstractRequest.java | 2 + .../common/requests/AbstractResponse.java | 2 + .../requests/ListTransactionsRequest.java | 77 +++++++++++++++++++ .../requests/ListTransactionsResponse.java | 62 +++++++++++++++ .../message/DescribeTransactionsResponse.json | 2 +- .../message/ListTransactionsRequest.json | 31 ++++++++ .../message/ListTransactionsResponse.json | 35 +++++++++ .../common/requests/RequestResponseTest.java | 34 ++++++++ .../transaction/TransactionCoordinator.scala | 14 +++- .../transaction/TransactionLog.scala | 4 +- .../transaction/TransactionMetadata.scala | 64 +++++++++------ .../transaction/TransactionStateManager.scala | 53 +++++++++++++ .../kafka/network/RequestConvertToJson.scala | 2 + .../main/scala/kafka/server/KafkaApis.scala | 21 +++++ .../kafka/api/AuthorizerIntegrationTest.scala | 35 ++++++++- .../transaction/TransactionMetadataTest.scala | 39 +++++++++- .../TransactionStateManagerTest.scala | 70 ++++++++++++++++- .../unit/kafka/server/KafkaApisTest.scala | 67 ++++++++++++++++ .../unit/kafka/server/RequestQuotaTest.scala | 3 + 20 files changed, 587 insertions(+), 33 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ListTransactionsRequest.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ListTransactionsResponse.java create mode 100644 clients/src/main/resources/common/message/ListTransactionsRequest.json create mode 100644 clients/src/main/resources/common/message/ListTransactionsResponse.json diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 07f1a62aec1f8..3297dc5734aa6 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -106,7 +106,8 @@ public enum ApiKeys { BROKER_REGISTRATION(ApiMessageType.BROKER_REGISTRATION, true, RecordBatch.MAGIC_VALUE_V0, false), BROKER_HEARTBEAT(ApiMessageType.BROKER_HEARTBEAT, true, RecordBatch.MAGIC_VALUE_V0, false), UNREGISTER_BROKER(ApiMessageType.UNREGISTER_BROKER, false, RecordBatch.MAGIC_VALUE_V0, true), - DESCRIBE_TRANSACTIONS(ApiMessageType.DESCRIBE_TRANSACTIONS); + DESCRIBE_TRANSACTIONS(ApiMessageType.DESCRIBE_TRANSACTIONS), + LIST_TRANSACTIONS(ApiMessageType.LIST_TRANSACTIONS); private static final Map> APIS_BY_LISTENER = new EnumMap<>(ApiMessageType.ListenerType.class); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 2b754e056ce0a..7802293c9bbfd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -286,6 +286,8 @@ private static AbstractRequest doParseRequest(ApiKeys apiKey, short apiVersion, return UnregisterBrokerRequest.parse(buffer, apiVersion); case DESCRIBE_TRANSACTIONS: return DescribeTransactionsRequest.parse(buffer, apiVersion); + case LIST_TRANSACTIONS: + return ListTransactionsRequest.parse(buffer, apiVersion); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index e35589f68ec4d..5f7b88f269405 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -243,6 +243,8 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, ByteBuffer response return UnregisterBrokerResponse.parse(responseBuffer, version); case DESCRIBE_TRANSACTIONS: return DescribeTransactionsResponse.parse(responseBuffer, version); + case LIST_TRANSACTIONS: + return ListTransactionsResponse.parse(responseBuffer, version); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListTransactionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListTransactionsRequest.java new file mode 100644 index 0000000000000..0651f1fe6e5cc --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListTransactionsRequest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.message.ListTransactionsRequestData; +import org.apache.kafka.common.message.ListTransactionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; + +import java.nio.ByteBuffer; + +public class ListTransactionsRequest extends AbstractRequest { + public static class Builder extends AbstractRequest.Builder { + public final ListTransactionsRequestData data; + + public Builder(ListTransactionsRequestData data) { + super(ApiKeys.LIST_TRANSACTIONS); + this.data = data; + } + + @Override + public ListTransactionsRequest build(short version) { + return new ListTransactionsRequest(data, version); + } + + @Override + public String toString() { + return data.toString(); + } + } + + private final ListTransactionsRequestData data; + + private ListTransactionsRequest(ListTransactionsRequestData data, short version) { + super(ApiKeys.LIST_TRANSACTIONS, version); + this.data = data; + } + + public ListTransactionsRequestData data() { + return data; + } + + @Override + public ListTransactionsResponse getErrorResponse(int throttleTimeMs, Throwable e) { + Errors error = Errors.forException(e); + ListTransactionsResponseData response = new ListTransactionsResponseData() + .setErrorCode(error.code()) + .setThrottleTimeMs(throttleTimeMs); + return new ListTransactionsResponse(response); + } + + public static ListTransactionsRequest parse(ByteBuffer buffer, short version) { + return new ListTransactionsRequest(new ListTransactionsRequestData( + new ByteBufferAccessor(buffer), version), version); + } + + @Override + public String toString(boolean verbose) { + return data.toString(); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListTransactionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListTransactionsResponse.java new file mode 100644 index 0000000000000..13ed184fc3408 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListTransactionsResponse.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.message.ListTransactionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +public class ListTransactionsResponse extends AbstractResponse { + private final ListTransactionsResponseData data; + + public ListTransactionsResponse(ListTransactionsResponseData data) { + super(ApiKeys.LIST_TRANSACTIONS); + this.data = data; + } + + public ListTransactionsResponseData data() { + return data; + } + + @Override + public Map errorCounts() { + Map errorCounts = new HashMap<>(); + updateErrorCounts(errorCounts, Errors.forCode(data.errorCode())); + return errorCounts; + } + + public static ListTransactionsResponse parse(ByteBuffer buffer, short version) { + return new ListTransactionsResponse(new ListTransactionsResponseData( + new ByteBufferAccessor(buffer), version)); + } + + @Override + public String toString() { + return data.toString(); + } + + @Override + public int throttleTimeMs() { + return data.throttleTimeMs(); + } + +} diff --git a/clients/src/main/resources/common/message/DescribeTransactionsResponse.json b/clients/src/main/resources/common/message/DescribeTransactionsResponse.json index affc5aa4f09a8..15f52a473d25e 100644 --- a/clients/src/main/resources/common/message/DescribeTransactionsResponse.json +++ b/clients/src/main/resources/common/message/DescribeTransactionsResponse.json @@ -20,7 +20,7 @@ "validVersions": "0", "flexibleVersions": "0+", "fields": [ - { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", "ignorable": true, + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, { "name": "TransactionStates", "type": "[]TransactionState", "versions": "0+", "fields": [ { "name": "ErrorCode", "type": "int16", "versions": "0+" }, diff --git a/clients/src/main/resources/common/message/ListTransactionsRequest.json b/clients/src/main/resources/common/message/ListTransactionsRequest.json new file mode 100644 index 0000000000000..716b7530f8ccb --- /dev/null +++ b/clients/src/main/resources/common/message/ListTransactionsRequest.json @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 66, + "type": "request", + "listeners": ["zkBroker", "broker"], + "name": "ListTransactionsRequest", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "StateFilters", "type": "[]string", "versions": "0+", + "about": "The transaction states to filter by: if empty, all transactions are returned; if non-empty, then only transactions matching one of the filtered states will be returned" + }, + { "name": "ProducerIdFilters", "type": "[]int64", "versions": "0+", + "about": "The producerIds to filter by: if empty, all transactions will be returned; if non-empty, only transactions which match one of the filtered producerIds will be returned" + } + ] +} diff --git a/clients/src/main/resources/common/message/ListTransactionsResponse.json b/clients/src/main/resources/common/message/ListTransactionsResponse.json new file mode 100644 index 0000000000000..2f17873239143 --- /dev/null +++ b/clients/src/main/resources/common/message/ListTransactionsResponse.json @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 66, + "type": "response", + "name": "ListTransactionsResponse", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+" }, + { "name": "UnknownStateFilters", "type": "[]string", "versions": "0+", + "about": "Set of state filters provided in the request which were unknown to the transaction coordinator" }, + { "name": "TransactionStates", "type": "[]TransactionState", "versions": "0+", "fields": [ + { "name": "TransactionalId", "type": "string", "versions": "0+", "entityType": "transactionalId" }, + { "name": "ProducerId", "type": "int64", "versions": "0+", "entityType": "producerId" }, + { "name": "TransactionState", "type": "string", "versions": "0+", + "about": "The current transaction state of the producer" } + ]} + ] +} diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 219c5c344a005..d873d15207374 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -137,6 +137,8 @@ import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData; import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData; +import org.apache.kafka.common.message.ListTransactionsRequestData; +import org.apache.kafka.common.message.ListTransactionsResponseData; import org.apache.kafka.common.message.OffsetCommitRequestData; import org.apache.kafka.common.message.OffsetCommitResponseData; import org.apache.kafka.common.message.OffsetDeleteRequestData; @@ -560,6 +562,15 @@ public void testDescribeTransactionsSerialization() { } } + @Test + public void testListTransactionsSerialization() { + for (short v : ApiKeys.LIST_TRANSACTIONS.allVersions()) { + checkRequest(createListTransactionsRequest(v), true); + checkErrorResponse(createListTransactionsRequest(v), unknownServerException, true); + checkResponse(createListTransactionsResponse(), v, true); + } + } + @Test public void testDescribeClusterSerialization() { for (short v : ApiKeys.DESCRIBE_CLUSTER.allVersions()) { @@ -2806,4 +2817,27 @@ private DescribeTransactionsResponse createDescribeTransactionsResponse() { return new DescribeTransactionsResponse(data); } + private ListTransactionsRequest createListTransactionsRequest(short version) { + return new ListTransactionsRequest.Builder(new ListTransactionsRequestData() + .setStateFilters(singletonList("Ongoing")) + .setProducerIdFilters(asList(1L, 2L, 15L)) + ).build(version); + } + + private ListTransactionsResponse createListTransactionsResponse() { + ListTransactionsResponseData response = new ListTransactionsResponseData(); + response.setErrorCode(Errors.NONE.code()); + response.setTransactionStates(Arrays.asList( + new ListTransactionsResponseData.TransactionState() + .setTransactionalId("foo") + .setProducerId(12345L) + .setTransactionState("Ongoing"), + new ListTransactionsResponseData.TransactionState() + .setTransactionalId("bar") + .setProducerId(98765L) + .setTransactionState("PrepareAbort") + )); + return new ListTransactionsResponse(response); + } + } diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala index 19f92435dcc42..8b8fde65a61e6 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala @@ -23,7 +23,7 @@ import kafka.server.{KafkaConfig, MetadataCache, ReplicaManager} import kafka.utils.{Logging, Scheduler} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.internals.Topic -import org.apache.kafka.common.message.DescribeTransactionsResponseData +import org.apache.kafka.common.message.{DescribeTransactionsResponseData, ListTransactionsResponseData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.record.RecordBatch @@ -251,11 +251,21 @@ class TransactionCoordinator(brokerId: Int, s"This is illegal as we should never have transitioned to this state." fatal(errorMsg) throw new IllegalStateException(errorMsg) - } } } + def handleListTransactions( + filteredProducerIds: Set[Long], + filteredStates: Set[String] + ): ListTransactionsResponseData = { + if (!isActive.get()) { + new ListTransactionsResponseData().setErrorCode(Errors.COORDINATOR_NOT_AVAILABLE.code) + } else { + txnManager.listTransactionStates(filteredProducerIds, filteredStates) + } + } + def handleDescribeTransactions( transactionalId: String ): DescribeTransactionsResponseData.TransactionState = { diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala index 68a2bedfc957d..cb501f774fd9d 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionLog.scala @@ -87,7 +87,7 @@ object TransactionLog { .setProducerId(txnMetadata.producerId) .setProducerEpoch(txnMetadata.producerEpoch) .setTransactionTimeoutMs(txnMetadata.txnTimeoutMs) - .setTransactionStatus(txnMetadata.txnState.byte) + .setTransactionStatus(txnMetadata.txnState.id) .setTransactionLastUpdateTimestampMs(txnMetadata.txnLastUpdateTimestamp) .setTransactionStartTimestampMs(txnMetadata.txnStartTimestamp) .setTransactionPartitions(transactionPartitions)) @@ -128,7 +128,7 @@ object TransactionLog { producerEpoch = value.producerEpoch, lastProducerEpoch = RecordBatch.NO_PRODUCER_EPOCH, txnTimeoutMs = value.transactionTimeoutMs, - state = TransactionMetadata.byteToState(value.transactionStatus), + state = TransactionState.fromId(value.transactionStatus), topicPartitions = mutable.Set.empty[TopicPartition], txnStartTimestamp = value.transactionStartTimestampMs, txnLastUpdateTimestamp = value.transactionLastUpdateTimestampMs) diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMetadata.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMetadata.scala index 5269f3e3349a2..b30094384d906 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMetadata.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMetadata.scala @@ -25,8 +25,40 @@ import org.apache.kafka.common.record.RecordBatch import scala.collection.{immutable, mutable} + +object TransactionState { + val AllStates = Set( + Empty, + Ongoing, + PrepareCommit, + PrepareAbort, + CompleteCommit, + CompleteAbort, + Dead, + PrepareEpochFence + ) + + def fromName(name: String): Option[TransactionState] = { + AllStates.find(_.name == name) + } + + def fromId(id: Byte): TransactionState = { + id match { + case 0 => Empty + case 1 => Ongoing + case 2 => PrepareCommit + case 3 => PrepareAbort + case 4 => CompleteCommit + case 5 => CompleteAbort + case 6 => Dead + case 7 => PrepareEpochFence + case _ => throw new IllegalStateException(s"Unknown transaction state id $id from the transaction status message") + } + } +} + private[transaction] sealed trait TransactionState { - def byte: Byte + def id: Byte /** * Get the name of this state. This is exposed through the `DescribeTransactions` API. @@ -41,7 +73,7 @@ private[transaction] sealed trait TransactionState { * received AddOffsetsToTxnRequest => Ongoing */ private[transaction] case object Empty extends TransactionState { - val byte: Byte = 0 + val id: Byte = 0 val name: String = "Empty" } @@ -54,7 +86,7 @@ private[transaction] case object Empty extends TransactionState { * received AddOffsetsToTxnRequest => Ongoing */ private[transaction] case object Ongoing extends TransactionState { - val byte: Byte = 1 + val id: Byte = 1 val name: String = "Ongoing" } @@ -64,7 +96,7 @@ private[transaction] case object Ongoing extends TransactionState { * transition: received acks from all partitions => CompleteCommit */ private[transaction] case object PrepareCommit extends TransactionState { - val byte: Byte = 2 + val id: Byte = 2 val name: String = "PrepareCommit" } @@ -74,7 +106,7 @@ private[transaction] case object PrepareCommit extends TransactionState { * transition: received acks from all partitions => CompleteAbort */ private[transaction] case object PrepareAbort extends TransactionState { - val byte: Byte = 3 + val id: Byte = 3 val name: String = "PrepareAbort" } @@ -84,7 +116,7 @@ private[transaction] case object PrepareAbort extends TransactionState { * Will soon be removed from the ongoing transaction cache */ private[transaction] case object CompleteCommit extends TransactionState { - val byte: Byte = 4 + val id: Byte = 4 val name: String = "CompleteCommit" } @@ -94,7 +126,7 @@ private[transaction] case object CompleteCommit extends TransactionState { * Will soon be removed from the ongoing transaction cache */ private[transaction] case object CompleteAbort extends TransactionState { - val byte: Byte = 5 + val id: Byte = 5 val name: String = "CompleteAbort" } @@ -102,7 +134,7 @@ private[transaction] case object CompleteAbort extends TransactionState { * TransactionalId has expired and is about to be removed from the transaction cache */ private[transaction] case object Dead extends TransactionState { - val byte: Byte = 6 + val id: Byte = 6 val name: String = "Dead" } @@ -111,7 +143,7 @@ private[transaction] case object Dead extends TransactionState { */ private[transaction] case object PrepareEpochFence extends TransactionState { - val byte: Byte = 7 + val id: Byte = 7 val name: String = "PrepareEpochFence" } @@ -130,20 +162,6 @@ private[transaction] object TransactionMetadata { new TransactionMetadata(transactionalId, producerId, lastProducerId, producerEpoch, lastProducerEpoch, txnTimeoutMs, state, collection.mutable.Set.empty[TopicPartition], timestamp, timestamp) - def byteToState(byte: Byte): TransactionState = { - byte match { - case 0 => Empty - case 1 => Ongoing - case 2 => PrepareCommit - case 3 => PrepareAbort - case 4 => CompleteCommit - case 5 => CompleteAbort - case 6 => Dead - case 7 => PrepareEpochFence - case unknown => throw new IllegalStateException("Unknown transaction state byte " + unknown + " from the transaction status message") - } - } - def isValidTransition(oldState: TransactionState, newState: TransactionState): Boolean = TransactionMetadata.validPreviousStates(newState).contains(oldState) diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala index b547896706dce..61fad952dc44f 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala @@ -29,6 +29,7 @@ import kafka.utils.CoreUtils.{inReadLock, inWriteLock} import kafka.utils.{Logging, Pool, Scheduler} import kafka.utils.Implicits._ import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.message.ListTransactionsResponseData import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.metrics.stats.{Avg, Max} import org.apache.kafka.common.protocol.Errors @@ -223,6 +224,58 @@ class TransactionStateManager(brokerId: Int, throw new IllegalStateException(s"Unexpected empty transaction metadata returned while putting $txnMetadata"))) } + def listTransactionStates( + filterProducerIds: Set[Long], + filterStateNames: Set[String] + ): ListTransactionsResponseData = { + inReadLock(stateLock) { + val response = new ListTransactionsResponseData() + if (loadingPartitions.nonEmpty) { + response.setErrorCode(Errors.COORDINATOR_LOAD_IN_PROGRESS.code) + } else { + val filterStates = mutable.Set.empty[TransactionState] + filterStateNames.foreach { stateName => + TransactionState.fromName(stateName) match { + case Some(state) => filterStates += state + case None => response.unknownStateFilters.add(stateName) + } + } + + def shouldInclude(txnMetadata: TransactionMetadata): Boolean = { + if (txnMetadata.state == Dead) { + // We filter the `Dead` state since it is a transient state which + // indicates that the transactionalId and its metadata are in the + // process of expiration and removal. + false + } else if (filterProducerIds.nonEmpty && !filterProducerIds.contains(txnMetadata.producerId)) { + false + } else if (filterStateNames.nonEmpty && !filterStates.contains(txnMetadata.state)) { + false + } else { + true + } + } + + val states = new java.util.ArrayList[ListTransactionsResponseData.TransactionState] + transactionMetadataCache.forKeyValue { (_, cache) => + cache.metadataPerTransactionalId.values.foreach { txnMetadata => + txnMetadata.inLock { + if (shouldInclude(txnMetadata)) { + states.add(new ListTransactionsResponseData.TransactionState() + .setTransactionalId(txnMetadata.transactionalId) + .setProducerId(txnMetadata.producerId) + .setTransactionState(txnMetadata.state.name) + ) + } + } + } + } + response.setErrorCode(Errors.NONE.code) + .setTransactionStates(states) + } + } + } + /** * Get the transaction metadata associated with the given transactional id, or an error if * the coordinator does not own the transaction partition or is still loading it; if not found diff --git a/core/src/main/scala/kafka/network/RequestConvertToJson.scala b/core/src/main/scala/kafka/network/RequestConvertToJson.scala index aacd24ec28524..2f23dbc2b2f79 100644 --- a/core/src/main/scala/kafka/network/RequestConvertToJson.scala +++ b/core/src/main/scala/kafka/network/RequestConvertToJson.scala @@ -93,6 +93,7 @@ object RequestConvertToJson { case req: DescribeClusterRequest => DescribeClusterRequestDataJsonConverter.write(req.data, request.version) case req: DescribeProducersRequest => DescribeProducersRequestDataJsonConverter.write(req.data, request.version) case req: DescribeTransactionsRequest => DescribeTransactionsRequestDataJsonConverter.write(req.data, request.version) + case req: ListTransactionsRequest => ListTransactionsRequestDataJsonConverter.write(req.data, request.version) case _ => throw new IllegalStateException(s"ApiKey ${request.apiKey} is not currently handled in `request`, the " + "code should be updated to do so."); } @@ -166,6 +167,7 @@ object RequestConvertToJson { case res: DescribeClusterResponse => DescribeClusterResponseDataJsonConverter.write(res.data, version) case res: DescribeProducersResponse => DescribeProducersResponseDataJsonConverter.write(res.data, version) case res: DescribeTransactionsResponse => DescribeTransactionsResponseDataJsonConverter.write(res.data, version) + case res: ListTransactionsResponse => ListTransactionsResponseDataJsonConverter.write(res.data, version) case _ => throw new IllegalStateException(s"ApiKey ${response.apiKey} is not currently handled in `response`, the " + "code should be updated to do so."); } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index bc228403b1088..22054a3f57388 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -223,6 +223,7 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.DESCRIBE_PRODUCERS => handleDescribeProducersRequest(request) case ApiKeys.UNREGISTER_BROKER => maybeForwardToController(request, handleUnregisterBrokerRequest) case ApiKeys.DESCRIBE_TRANSACTIONS => handleDescribeTransactionsRequest(request) + case ApiKeys.LIST_TRANSACTIONS => handleListTransactionsRequest(request) case _ => throw new IllegalStateException(s"No handler for request api key ${request.header.apiKey}") } } catch { @@ -3310,6 +3311,26 @@ class KafkaApis(val requestChannel: RequestChannel, new DescribeTransactionsResponse(response.setThrottleTimeMs(requestThrottleMs))) } + def handleListTransactionsRequest(request: RequestChannel.Request): Unit = { + val listTransactionsRequest = request.body[ListTransactionsRequest] + val filteredProducerIds = listTransactionsRequest.data.producerIdFilters.asScala.map(Long.unbox).toSet + val filteredStates = listTransactionsRequest.data.stateFilters.asScala.toSet + val response = txnCoordinator.handleListTransactions(filteredProducerIds, filteredStates) + + // The response should contain only transactionalIds that the principal + // has `Describe` permission to access. + val transactionStateIter = response.transactionStates.iterator() + while (transactionStateIter.hasNext) { + val transactionState = transactionStateIter.next() + if (!authHelper.authorize(request.context, DESCRIBE, TRANSACTIONAL_ID, transactionState.transactionalId)) { + transactionStateIter.remove() + } + } + + requestHelper.sendResponseMaybeThrottle(request, requestThrottleMs => + new ListTransactionsResponse(response.setThrottleTimeMs(requestThrottleMs))) + } + private def updateRecordConversionStats(request: RequestChannel.Request, tp: TopicPartition, conversionStats: RecordConversionStats): Unit = { diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index ebe4634adf8a2..bb4266c98a09b 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -46,7 +46,7 @@ import org.apache.kafka.common.message.ListOffsetsRequestData.{ListOffsetsPartit import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.{OffsetForLeaderPartition, OffsetForLeaderTopic, OffsetForLeaderTopicCollection} import org.apache.kafka.common.message.StopReplicaRequestData.{StopReplicaPartitionState, StopReplicaTopicState} import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState} -import org.apache.kafka.common.message.{AddOffsetsToTxnRequestData, AlterPartitionReassignmentsRequestData, AlterReplicaLogDirsRequestData, ControlledShutdownRequestData, CreateAclsRequestData, CreatePartitionsRequestData, CreateTopicsRequestData, DeleteAclsRequestData, DeleteGroupsRequestData, DeleteRecordsRequestData, DeleteTopicsRequestData, DescribeClusterRequestData, DescribeConfigsRequestData, DescribeGroupsRequestData, DescribeLogDirsRequestData, DescribeProducersRequestData, DescribeTransactionsRequestData, FindCoordinatorRequestData, HeartbeatRequestData, IncrementalAlterConfigsRequestData, JoinGroupRequestData, ListPartitionReassignmentsRequestData, MetadataRequestData, OffsetCommitRequestData, ProduceRequestData, SyncGroupRequestData} +import org.apache.kafka.common.message.{AddOffsetsToTxnRequestData, AlterPartitionReassignmentsRequestData, AlterReplicaLogDirsRequestData, ControlledShutdownRequestData, CreateAclsRequestData, CreatePartitionsRequestData, CreateTopicsRequestData, DeleteAclsRequestData, DeleteGroupsRequestData, DeleteRecordsRequestData, DeleteTopicsRequestData, DescribeClusterRequestData, DescribeConfigsRequestData, DescribeGroupsRequestData, DescribeLogDirsRequestData, DescribeProducersRequestData, DescribeTransactionsRequestData, FindCoordinatorRequestData, HeartbeatRequestData, IncrementalAlterConfigsRequestData, JoinGroupRequestData, ListPartitionReassignmentsRequestData, ListTransactionsRequestData, MetadataRequestData, OffsetCommitRequestData, ProduceRequestData, SyncGroupRequestData} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, Records, SimpleRecord} @@ -1805,6 +1805,39 @@ class AuthorizerIntegrationTest extends BaseRequestTest { assertThrows(classOf[TransactionalIdAuthorizationException], () => producer.commitTransaction()) } + @Test + def testListTransactionsAuthorization(): Unit = { + createTopic(topic) + addAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, WRITE, ALLOW)), transactionalIdResource) + addAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, WRITE, ALLOW)), topicResource) + + // Start a transaction and write to a topic. + val producer = buildTransactionalProducer() + producer.initTransactions() + producer.beginTransaction() + producer.send(new ProducerRecord(tp.topic, tp.partition, "1".getBytes, "1".getBytes)).get + + def assertListTransactionResult( + expectedTransactionalIds: Set[String] + ): Unit = { + val listTransactionsRequest = new ListTransactionsRequest.Builder(new ListTransactionsRequestData()).build() + val listTransactionsResponse = connectAndReceive[ListTransactionsResponse](listTransactionsRequest) + assertEquals(Errors.NONE, Errors.forCode(listTransactionsResponse.data.errorCode)) + assertEquals(expectedTransactionalIds, listTransactionsResponse.data.transactionStates.asScala.map(_.transactionalId).toSet) + } + + // First verify that we can list the transaction + assertListTransactionResult(expectedTransactionalIds = Set(transactionalId)) + + // Now revoke authorization and verify that the transaction is no longer listable + removeAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, WRITE, ALLOW)), transactionalIdResource) + assertListTransactionResult(expectedTransactionalIds = Set()) + + // The minimum permission needed is `Describe` + addAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, DESCRIBE, ALLOW)), transactionalIdResource) + assertListTransactionResult(expectedTransactionalIds = Set(transactionalId)) + } + @Test def shouldNotIncludeUnauthorizedTopicsInDescribeTransactionsResponse(): Unit = { createTopic(topic) diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMetadataTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMetadataTest.scala index 4e8b0fd6a387a..92407dddb5690 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMetadataTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMetadataTest.scala @@ -161,7 +161,7 @@ class TransactionMetadataTest { txnStartTimestamp = time.milliseconds(), txnLastUpdateTimestamp = time.milliseconds()) - // let new time be smaller; when transting from Empty the start time would be updated to the update-time + // let new time be smaller; when transiting from Empty the start time would be updated to the update-time var transitMetadata = txnMetadata.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic1", 0)), time.milliseconds() - 1) txnMetadata.completeTransitionTo(transitMetadata) assertEquals(Set[TopicPartition](new TopicPartition("topic1", 0)), txnMetadata.topicPartitions) @@ -460,6 +460,43 @@ class TransactionMetadataTest { assertEquals(Left(Errors.PRODUCER_FENCED), result) } + @Test + def testTransactionStateIdAndNameMapping(): Unit = { + for (state <- TransactionState.AllStates) { + assertEquals(state, TransactionState.fromId(state.id)) + assertEquals(Some(state), TransactionState.fromName(state.name)) + } + } + + @Test + def testAllTransactionStatesAreMapped(): Unit = { + val unmatchedStates = mutable.Set( + Empty, + Ongoing, + PrepareCommit, + PrepareAbort, + CompleteCommit, + CompleteAbort, + PrepareEpochFence, + Dead + ) + + // The exhaustive match is intentional here to ensure that we are + // forced to update the test case if a new state is added. + TransactionState.AllStates.foreach { + case Empty => assertTrue(unmatchedStates.remove(Empty)) + case Ongoing => assertTrue(unmatchedStates.remove(Ongoing)) + case PrepareCommit => assertTrue(unmatchedStates.remove(PrepareCommit)) + case PrepareAbort => assertTrue(unmatchedStates.remove(PrepareAbort)) + case CompleteCommit => assertTrue(unmatchedStates.remove(CompleteCommit)) + case CompleteAbort => assertTrue(unmatchedStates.remove(CompleteAbort)) + case PrepareEpochFence => assertTrue(unmatchedStates.remove(PrepareEpochFence)) + case Dead => assertTrue(unmatchedStates.remove(Dead)) + } + + assertEquals(Set.empty, unmatchedStates) + } + private def testRotateProducerIdInOngoingState(state: TransactionState): Unit = { val producerEpoch = (Short.MaxValue - 1).toShort diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala index 20ca0c421d024..df576931525d6 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala @@ -34,7 +34,7 @@ import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.common.requests.TransactionResult import org.apache.kafka.common.utils.MockTime import org.easymock.{Capture, EasyMock, IAnswer} -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertThrows, assertTrue, fail} +import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import scala.jdk.CollectionConverters._ @@ -476,13 +476,79 @@ class TransactionStateManagerTest { } @Test - def shouldReturnNotCooridnatorErrorIfTransactionIdPartitionNotOwned(): Unit = { + def shouldReturnNotCoordinatorErrorIfTransactionIdPartitionNotOwned(): Unit = { transactionManager.getTransactionState(transactionalId1).fold( err => assertEquals(Errors.NOT_COORDINATOR, err), _ => fail(transactionalId1 + "'s transaction state is already in the cache") ) } + @Test + def testListTransactionsWithCoordinatorLoadingInProgress(): Unit = { + transactionManager.addLoadingPartition(partitionId = 0, coordinatorEpoch = 15) + val listResponse = transactionManager.listTransactionStates( + filterProducerIds = Set.empty, + filterStateNames = Set.empty + ) + assertEquals(Errors.COORDINATOR_LOAD_IN_PROGRESS, Errors.forCode(listResponse.errorCode)) + } + + @Test + def testListTransactionsFiltering(): Unit = { + for (partitionId <- 0 until numPartitions) { + transactionManager.addLoadedTransactionsToCache(partitionId, 0, new Pool[String, TransactionMetadata]()) + } + + def putTransaction( + transactionalId: String, + producerId: Long, + state: TransactionState + ): Unit = { + val txnMetadata = transactionMetadata(transactionalId, producerId, state) + transactionManager.putTransactionStateIfNotExists(txnMetadata).left.toOption.foreach { error => + fail(s"Failed to insert transaction $txnMetadata due to error $error") + } + } + + putTransaction(transactionalId = "t0", producerId = 0, state = Ongoing) + putTransaction(transactionalId = "t1", producerId = 1, state = Ongoing) + putTransaction(transactionalId = "t2", producerId = 2, state = PrepareCommit) + putTransaction(transactionalId = "t3", producerId = 3, state = PrepareAbort) + putTransaction(transactionalId = "t4", producerId = 4, state = CompleteCommit) + putTransaction(transactionalId = "t5", producerId = 5, state = CompleteAbort) + putTransaction(transactionalId = "t6", producerId = 6, state = CompleteAbort) + putTransaction(transactionalId = "t7", producerId = 7, state = PrepareEpochFence) + // Note that `Dead` transactions are never returned. This is a transient state + // which is used when the transaction state is in the process of being deleted + // (whether though expiration or coordinator unloading). + putTransaction(transactionalId = "t8", producerId = 8, state = Dead) + + def assertListTransactions( + expectedTransactionalIds: Set[String], + filterProducerIds: Set[Long] = Set.empty, + filterStates: Set[String] = Set.empty + ): Unit = { + val listResponse = transactionManager.listTransactionStates(filterProducerIds, filterStates) + assertEquals(Errors.NONE, Errors.forCode(listResponse.errorCode)) + assertEquals(expectedTransactionalIds, listResponse.transactionStates.asScala.map(_.transactionalId).toSet) + val expectedUnknownStates = filterStates.filter(state => TransactionState.fromName(state).isEmpty) + assertEquals(expectedUnknownStates, listResponse.unknownStateFilters.asScala.toSet) + } + + assertListTransactions(Set("t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7")) + assertListTransactions(Set("t0", "t1"), filterStates = Set("Ongoing")) + assertListTransactions(Set("t0", "t1"), filterStates = Set("Ongoing", "UnknownState")) + assertListTransactions(Set("t2", "t4"), filterStates = Set("PrepareCommit", "CompleteCommit")) + assertListTransactions(Set(), filterStates = Set("UnknownState")) + assertListTransactions(Set("t5"), filterProducerIds = Set(5L)) + assertListTransactions(Set("t5", "t6"), filterProducerIds = Set(5L, 6L, 8L, 9L)) + assertListTransactions(Set("t4"), filterProducerIds = Set(4L, 5L), filterStates = Set("CompleteCommit")) + assertListTransactions(Set("t4", "t5"), filterProducerIds = Set(4L, 5L), filterStates = Set("CompleteCommit", "CompleteAbort")) + assertListTransactions(Set(), filterProducerIds = Set(3L, 6L), filterStates = Set("UnknownState")) + assertListTransactions(Set(), filterProducerIds = Set(10L), filterStates = Set("CompleteCommit")) + assertListTransactions(Set(), filterStates = Set("Dead")) + } + @Test def shouldOnlyConsiderTransactionsInTheOngoingStateToAbort(): Unit = { for (partitionId <- 0 until numPartitions) { diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 476fe14cf9a65..e995c69302ce8 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -3481,6 +3481,73 @@ class KafkaApisTest { assertEquals(List(mkTopicData(topic = "foo", Seq(1, 2))), fooState.topics.asScala.toList) } + @Test + def testListTransactionsErrorResponse(): Unit = { + val data = new ListTransactionsRequestData() + val listTransactionsRequest = new ListTransactionsRequest.Builder(data).build() + val request = buildRequest(listTransactionsRequest) + val capturedResponse = expectNoThrottling(request) + + EasyMock.expect(txnCoordinator.handleListTransactions(Set.empty[Long], Set.empty[String])) + .andReturn(new ListTransactionsResponseData() + .setErrorCode(Errors.COORDINATOR_LOAD_IN_PROGRESS.code)) + + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) + createKafkaApis().handleListTransactionsRequest(request) + + val response = capturedResponse.getValue.asInstanceOf[ListTransactionsResponse] + assertEquals(0, response.data.transactionStates.size) + assertEquals(Errors.COORDINATOR_LOAD_IN_PROGRESS, Errors.forCode(response.data.errorCode)) + } + + @Test + def testListTransactionsAuthorization(): Unit = { + val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) + val data = new ListTransactionsRequestData() + val listTransactionsRequest = new ListTransactionsRequest.Builder(data).build() + val request = buildRequest(listTransactionsRequest) + val capturedResponse = expectNoThrottling(request) + + val transactionStates = new util.ArrayList[ListTransactionsResponseData.TransactionState]() + transactionStates.add(new ListTransactionsResponseData.TransactionState() + .setTransactionalId("foo") + .setProducerId(12345L) + .setTransactionState("Ongoing")) + transactionStates.add(new ListTransactionsResponseData.TransactionState() + .setTransactionalId("bar") + .setProducerId(98765) + .setTransactionState("PrepareAbort")) + + EasyMock.expect(txnCoordinator.handleListTransactions(Set.empty[Long], Set.empty[String])) + .andReturn(new ListTransactionsResponseData() + .setErrorCode(Errors.NONE.code) + .setTransactionStates(transactionStates)) + + def buildExpectedActions(transactionalId: String): util.List[Action] = { + val pattern = new ResourcePattern(ResourceType.TRANSACTIONAL_ID, transactionalId, PatternType.LITERAL) + val action = new Action(AclOperation.DESCRIBE, pattern, 1, true, true) + Collections.singletonList(action) + } + + EasyMock.expect(authorizer.authorize(anyObject[RequestContext], EasyMock.eq(buildExpectedActions("foo")))) + .andReturn(Seq(AuthorizationResult.ALLOWED).asJava) + .once() + + EasyMock.expect(authorizer.authorize(anyObject[RequestContext], EasyMock.eq(buildExpectedActions("bar")))) + .andReturn(Seq(AuthorizationResult.DENIED).asJava) + .once() + + EasyMock.replay(replicaManager, clientRequestQuotaManager, requestChannel, txnCoordinator, authorizer) + createKafkaApis(authorizer = Some(authorizer)).handleListTransactionsRequest(request) + + val response = capturedResponse.getValue.asInstanceOf[ListTransactionsResponse] + assertEquals(1, response.data.transactionStates.size()) + val transactionState = response.data.transactionStates.get(0) + assertEquals("foo", transactionState.transactionalId) + assertEquals(12345L, transactionState.producerId) + assertEquals("Ongoing", transactionState.transactionState) + } + @Test def testDeleteTopicsByIdAuthorization(): Unit = { val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index a5d5078a761d8..b992aac7f0631 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -639,6 +639,9 @@ class RequestQuotaTest extends BaseRequestTest { new DescribeTransactionsRequest.Builder(new DescribeTransactionsRequestData() .setTransactionalIds(List("test-transactional-id").asJava)) + case ApiKeys.LIST_TRANSACTIONS => + new ListTransactionsRequest.Builder(new ListTransactionsRequestData()) + case _ => throw new IllegalArgumentException("Unsupported API key " + apiKey) } From 36d61650f4437a40ee599f2d36ad05570c4786f6 Mon Sep 17 00:00:00 2001 From: Lucas Bradstreet Date: Wed, 3 Mar 2021 05:11:16 +0800 Subject: [PATCH 093/243] KAFKA-12177: apply log start offset retention before time and size based retention (#10216) Log start offset retention is the cheapest retention to evaluate and does not require access to maxTimestamp fields for segments, nor segment sizes. In addition, it may unblock other types of retention such as time based retention. Without this change retention is not idempotent. It's possible for one deleteOldSegments call to delete segments due to log start offset retention, and a follow up call to delete due to time based retention, even if the time has not changed. Reviewers: Jun Rao --- core/src/main/scala/kafka/log/Log.scala | 4 +++- .../test/scala/unit/kafka/log/LogTest.scala | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index f9486bb3dbd97..5ab2fac3e75b2 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -1889,7 +1889,9 @@ class Log(@volatile private var _dir: File, */ def deleteOldSegments(): Int = { if (config.delete) { - deleteRetentionMsBreachedSegments() + deleteRetentionSizeBreachedSegments() + deleteLogStartOffsetBreachedSegments() + deleteLogStartOffsetBreachedSegments() + + deleteRetentionSizeBreachedSegments() + + deleteRetentionMsBreachedSegments() } else { deleteLogStartOffsetBreachedSegments() } diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index 66ee5d2538c20..a76345fd730e5 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -1515,6 +1515,27 @@ class LogTest { "expect a single producer state snapshot remaining") } + @Test + def testRetentionIdempotency(): Unit = { + val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5, retentionBytes = -1, retentionMs = 900, fileDeleteDelayMs = 0) + val log = createLog(logDir, logConfig) + + log.appendAsLeader(TestUtils.records(List(new SimpleRecord(mockTime.milliseconds() + 100, "a".getBytes))), leaderEpoch = 0) + log.roll() + log.appendAsLeader(TestUtils.records(List(new SimpleRecord(mockTime.milliseconds(), "b".getBytes))), leaderEpoch = 0) + log.roll() + log.appendAsLeader(TestUtils.records(List(new SimpleRecord(mockTime.milliseconds() + 100, "c".getBytes))), leaderEpoch = 0) + + mockTime.sleep(901) + + log.updateHighWatermark(log.logEndOffset) + log.maybeIncrementLogStartOffset(1L, ClientRecordDeletion) + assertEquals(2, log.deleteOldSegments(), + "Expecting two segment deletions as log start offset retention should unblock time based retention") + assertEquals(0, log.deleteOldSegments()) + } + + @Test def testLogStartOffsetMovementDeletesSnapshots(): Unit = { val logConfig = LogTest.createLogConfig(segmentBytes = 2048 * 5, retentionBytes = -1, fileDeleteDelayMs = 0) From 23b61ba3837e92f1cde9cbbcbb172c545b1ae6da Mon Sep 17 00:00:00 2001 From: "A. Sophie Blee-Goldman" Date: Tue, 2 Mar 2021 16:28:15 -0800 Subject: [PATCH 094/243] KAFKA-12375: don't reuse thread.id until a thread has fully shut down (#10215) Always grab a new thread.id and verify that a thread has fully shut down to DEAD before removing from the `threads` list and making that id available again Reviewers: Walker Carlson , Bruno Cadonna --- checkstyle/checkstyle.xml | 1 + .../apache/kafka/streams/KafkaStreams.java | 69 +++++++++++++++---- .../kafka/streams/KafkaStreamsTest.java | 14 ++-- 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index 91045adc60856..7f912dc428a15 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -120,6 +120,7 @@ + diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java index de048d1994011..f62427d5e125f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java +++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java @@ -92,6 +92,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -463,9 +464,8 @@ private void replaceStreamThread(final Throwable throwable) { closeToError(); } final StreamThread deadThread = (StreamThread) Thread.currentThread(); - threads.remove(deadThread); - addStreamThread(); deadThread.shutdown(); + addStreamThread(); if (throwable instanceof RuntimeException) { throw (RuntimeException) throwable; } else if (throwable instanceof Error) { @@ -970,7 +970,7 @@ public Optional addStreamThread() { final StreamThread streamThread; synchronized (changeThreadCount) { threadIdx = getNextThreadIndex(); - cacheSizePerThread = getCacheSizePerThread(threads.size() + 1); + cacheSizePerThread = getCacheSizePerThread(getNumLiveStreamThreads() + 1); resizeThreadCache(cacheSizePerThread); // Creating thread should hold the lock in order to avoid duplicate thread index. // If the duplicate index happen, the metadata of thread may be duplicate too. @@ -984,7 +984,7 @@ public Optional addStreamThread() { } else { streamThread.shutdown(); threads.remove(streamThread); - resizeThreadCache(getCacheSizePerThread(threads.size())); + resizeThreadCache(getCacheSizePerThread(getNumLiveStreamThreads())); } } } @@ -1038,7 +1038,7 @@ private Optional removeStreamThread(final long timeoutMs) throws Timeout // make a copy of threads to avoid holding lock for (final StreamThread streamThread : new ArrayList<>(threads)) { final boolean callingThreadIsNotCurrentStreamThread = !streamThread.getName().equals(Thread.currentThread().getName()); - if (streamThread.isAlive() && (callingThreadIsNotCurrentStreamThread || threads.size() == 1)) { + if (streamThread.isAlive() && (callingThreadIsNotCurrentStreamThread || getNumLiveStreamThreads() == 1)) { log.info("Removing StreamThread " + streamThread.getName()); final Optional groupInstanceID = streamThread.getGroupInstanceID(); streamThread.requestLeaveGroupDuringShutdown(); @@ -1047,10 +1047,15 @@ private Optional removeStreamThread(final long timeoutMs) throws Timeout if (!streamThread.waitOnThreadState(StreamThread.State.DEAD, timeoutMs - begin)) { log.warn("Thread " + streamThread.getName() + " did not shutdown in the allotted time"); timeout = true; + // Don't remove from threads until shutdown is complete. We will trim it from the + // list once it reaches DEAD, and if for some reason it's hanging indefinitely in the + // shutdown then we should just consider this thread.id to be burned + } else { + threads.remove(streamThread); } } - threads.remove(streamThread); - final long cacheSizePerThread = getCacheSizePerThread(threads.size()); + + final long cacheSizePerThread = getCacheSizePerThread(getNumLiveStreamThreads()); resizeThreadCache(cacheSizePerThread); if (groupInstanceID.isPresent() && callingThreadIsNotCurrentStreamThread) { final MemberToRemove memberToRemove = new MemberToRemove(groupInstanceID.get()); @@ -1093,17 +1098,51 @@ private Optional removeStreamThread(final long timeoutMs) throws Timeout return Optional.empty(); } + // Returns the number of threads that are not in the DEAD state -- use this over threads.size() + private int getNumLiveStreamThreads() { + final AtomicInteger numLiveThreads = new AtomicInteger(0); + synchronized (threads) { + processStreamThread(thread -> { + if (thread.state() == StreamThread.State.DEAD) { + threads.remove(thread); + } else { + numLiveThreads.incrementAndGet(); + } + }); + return numLiveThreads.get(); + } + } + private int getNextThreadIndex() { - final HashSet names = new HashSet<>(); - processStreamThread(thread -> names.add(thread.getName())); - final String baseName = clientId + "-StreamThread-"; - for (int i = 1; i <= threads.size(); i++) { - final String name = baseName + i; - if (!names.contains(name)) { - return i; + final HashSet allLiveThreadNames = new HashSet<>(); + final AtomicInteger maxThreadId = new AtomicInteger(1); + synchronized (threads) { + processStreamThread(thread -> { + // trim any DEAD threads from the list so we can reuse the thread.id + // this is only safe to do once the thread has fully completed shutdown + if (thread.state() == StreamThread.State.DEAD) { + threads.remove(thread); + } else { + allLiveThreadNames.add(thread.getName()); + // Assume threads are always named with the "-StreamThread-" suffix + final int threadId = Integer.parseInt(thread.getName().substring(thread.getName().lastIndexOf("-") + 1)); + if (threadId > maxThreadId.get()) { + maxThreadId.set(threadId); + } + } + }); + + final String baseName = clientId + "-StreamThread-"; + for (int i = 1; i <= maxThreadId.get(); i++) { + final String name = baseName + i; + if (!allLiveThreadNames.contains(name)) { + return i; + } } + // It's safe to use threads.size() rather than getNumLiveStreamThreads() to infer the number of threads + // here since we trimmed any DEAD threads earlier in this method while holding the lock + return threads.size() + 1; } - return threads.size() + 1; } private long getCacheSizePerThread(final int numStreamThreads) { diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java index a4cd8bf22c2ea..b3dd559d88d63 100644 --- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java @@ -232,8 +232,8 @@ private void prepareStreams() throws Exception { EasyMock.expect(StreamThread.processingMode(anyObject(StreamsConfig.class))).andReturn(StreamThread.ProcessingMode.AT_LEAST_ONCE).anyTimes(); EasyMock.expect(streamThreadOne.getId()).andReturn(0L).anyTimes(); EasyMock.expect(streamThreadTwo.getId()).andReturn(1L).anyTimes(); - prepareStreamThread(streamThreadOne, true); - prepareStreamThread(streamThreadTwo, false); + prepareStreamThread(streamThreadOne, 1, true); + prepareStreamThread(streamThreadTwo, 2, false); // setup global threads final AtomicReference globalThreadState = new AtomicReference<>(GlobalStreamThread.State.CREATED); @@ -293,7 +293,7 @@ private void prepareStreams() throws Exception { ); } - private void prepareStreamThread(final StreamThread thread, final boolean terminable) throws Exception { + private void prepareStreamThread(final StreamThread thread, final int threadId, final boolean terminable) throws Exception { final AtomicReference state = new AtomicReference<>(StreamThread.State.CREATED); EasyMock.expect(thread.state()).andAnswer(state::get).anyTimes(); @@ -321,7 +321,7 @@ private void prepareStreamThread(final StreamThread thread, final boolean termin }).anyTimes(); EasyMock.expect(thread.getGroupInstanceID()).andStubReturn(Optional.empty()); EasyMock.expect(thread.threadMetadata()).andReturn(new ThreadMetadata( - "newThead", + "processId-StreamThread-" + threadId, "DEAD", "", "", @@ -337,7 +337,7 @@ private void prepareStreamThread(final StreamThread thread, final boolean termin EasyMock.expectLastCall().anyTimes(); thread.requestLeaveGroupDuringShutdown(); EasyMock.expectLastCall().anyTimes(); - EasyMock.expect(thread.getName()).andStubReturn("newThread"); + EasyMock.expect(thread.getName()).andStubReturn("processId-StreamThread-" + threadId); thread.shutdown(); EasyMock.expectLastCall().andAnswer(() -> { supplier.consumer.close(); @@ -564,7 +564,7 @@ public void shouldAddThreadWhenRunning() throws InterruptedException { streams.start(); final int oldSize = streams.threads.size(); TestUtils.waitForCondition(() -> streams.state() == KafkaStreams.State.RUNNING, 15L, "wait until running"); - assertThat(streams.addStreamThread(), equalTo(Optional.of("newThread"))); + assertThat(streams.addStreamThread(), equalTo(Optional.of("processId-StreamThread-" + 2))); assertThat(streams.threads.size(), equalTo(oldSize + 1)); } @@ -613,7 +613,7 @@ public void shouldRemoveThread() throws InterruptedException { final int oldSize = streams.threads.size(); TestUtils.waitForCondition(() -> streams.state() == KafkaStreams.State.RUNNING, 15L, "Kafka Streams client did not reach state RUNNING"); - assertThat(streams.removeStreamThread(), equalTo(Optional.of("newThread"))); + assertThat(streams.removeStreamThread(), equalTo(Optional.of("processId-StreamThread-" + 1))); assertThat(streams.threads.size(), equalTo(oldSize - 1)); } From 4b3e3a9e86a8293282095d15709c1aa56c526ddf Mon Sep 17 00:00:00 2001 From: Lee Dongjin Date: Wed, 3 Mar 2021 09:45:24 +0530 Subject: [PATCH 095/243] y This security vulnerability was found in netty-codec-http, but [caused by netty itself](https://github.com/netty/netty/commit/c735357bf29d07856ad171c6611a2e1a0e0000ec) and [fixed in 4.1.59.Final](https://github.com/netty/netty/security/advisories/GHSA-5mcr-gq6c-3hq2). So, upgrade the netty version from 4.1.51.Final to 4.1.59.Final. Author: Lee Dongjin Reviewers: Manikumar Reddy Closes #10235 from dongjinleekr/feature/KAFKA-12389 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 709ef013cc1f5..90bbadf32aae7 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -100,7 +100,7 @@ versions += [ mavenArtifact: "3.6.3", metrics: "2.2.0", mockito: "3.6.0", - netty: "4.1.51.Final", + netty: "4.1.59.Final", owaspDepCheckPlugin: "6.0.3", powermock: "2.0.9", reflections: "0.9.12", From cfb60064ec50f52f7deeda3c3581259d07670a59 Mon Sep 17 00:00:00 2001 From: dengziming Date: Wed, 3 Mar 2021 12:38:06 +0800 Subject: [PATCH 096/243] MINOR: Fix null exception in coordinator log (#10250) Reviewers: A. Sophie Blee-Goldman , Chia-Ping Tsai --- .../kafka/clients/consumer/internals/AbstractCoordinator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java index 34f81db306d33..ab3c33beb876e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java @@ -256,8 +256,8 @@ protected synchronized boolean ensureCoordinatorReady(final Timer timer) { log.debug("Coordinator discovery failed, refreshing metadata", future.exception()); client.awaitMetadataUpdate(timer); } else { - log.info("FindCoordinator request hit fatal exception", fatalException); fatalException = future.exception(); + log.info("FindCoordinator request hit fatal exception", fatalException); } } else if (coordinator != null && client.isUnavailable(coordinator)) { // we found the coordinator, but the connection has failed, so mark @@ -267,7 +267,7 @@ protected synchronized boolean ensureCoordinatorReady(final Timer timer) { } clearFindCoordinatorFuture(); - if (fatalException != null) + if (fatalException != null) throw fatalException; } while (coordinatorUnknown() && timer.notExpired()); From b77deece1db3fca5575e336e157677f83bf3b506 Mon Sep 17 00:00:00 2001 From: Lee Dongjin Date: Wed, 3 Mar 2021 10:13:40 +0530 Subject: [PATCH 097/243] KAFKA-12400: Upgrade jetty to fix CVE-2020-27223 Here is the fix. The reason of [CVE-2020-27223](https://nvd.nist.gov/vuln/detail/CVE-2020-27223) was DOS vulnerability for Quoted Quality CSV headers and [patched in 9.4.37.v20210219](https://github.com/eclipse/jetty.project/security/advisories/GHSA-m394-8rww-3jr7). This PR updates Jetty dependency into the following version, 9.4.38.v20210224. Author: Lee Dongjin Reviewers: Manikumar Reddy Closes #10245 from dongjinleekr/feature/KAFKA-12400 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 90bbadf32aae7..5974e76896a93 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -70,7 +70,7 @@ versions += [ jacksonDatabind: "2.10.5.1", jacoco: "0.8.5", javassist: "3.27.0-GA", - jetty: "9.4.36.v20210114", + jetty: "9.4.38.v20210224", jersey: "2.31", jline: "3.12.1", jmh: "1.27", From 0d9a95a7d0ab06aecc4480901707e29dd2a3147e Mon Sep 17 00:00:00 2001 From: Satish Duggana Date: Wed, 3 Mar 2021 22:25:13 +0530 Subject: [PATCH 098/243] KAFKA-9548 Added SPIs and public classes/interfaces introduced in KIP-405 for tiered storage feature in Kafka. (#10173) KIP-405 introduces tiered storage feature in Kafka. With this feature, Kafka cluster is configured with two tiers of storage - local and remote. The local tier is the same as the current Kafka that uses the local disks on the Kafka brokers to store the log segments. The new remote tier uses systems, such as HDFS or S3 or other cloud storages to store the completed log segments. Consumers fetch the records stored in remote storage through the brokers with the existing protocol. We introduced a few SPIs for plugging in log/index store and remote log metadata store. This involves two parts 1. Storing the actual data in remote storage like HDFS, S3, or other cloud storages. 2. Storing the metadata about where the remote segments are stored. The default implementation uses an internal Kafka topic. You can read KIP for more details at https://cwiki.apache.org/confluence/display/KAFKA/KIP-405%3A+Kafka+Tiered+Storage Reviewers: Jun Rao --- build.gradle | 1 + .../apache/kafka/common/TopicIdPartition.java | 74 +++++ .../log/remote/storage/LogSegmentData.java | 139 +++++++++ .../storage/RemoteLogMetadataManager.java | 200 +++++++++++++ .../remote/storage/RemoteLogSegmentId.java | 80 +++++ .../storage/RemoteLogSegmentMetadata.java | 282 ++++++++++++++++++ .../RemoteLogSegmentMetadataUpdate.java | 120 ++++++++ .../remote/storage/RemoteLogSegmentState.java | 90 ++++++ .../RemotePartitionDeleteMetadata.java | 110 +++++++ .../storage/RemotePartitionDeleteState.java | 86 ++++++ .../RemoteResourceNotFoundException.java | 39 +++ .../storage/RemoteStorageException.java | 36 +++ .../remote/storage/RemoteStorageManager.java | 141 +++++++++ 13 files changed, 1398 insertions(+) create mode 100644 clients/src/main/java/org/apache/kafka/common/TopicIdPartition.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java diff --git a/build.gradle b/build.gradle index 4838a35bf7c03..acd8a84e6377b 100644 --- a/build.gradle +++ b/build.gradle @@ -1225,6 +1225,7 @@ project(':clients') { include "**/org/apache/kafka/server/authorizer/*" include "**/org/apache/kafka/server/policy/*" include "**/org/apache/kafka/server/quota/*" + include "**/org/apache/kafka/server/log/remote/storage/*" } } diff --git a/clients/src/main/java/org/apache/kafka/common/TopicIdPartition.java b/clients/src/main/java/org/apache/kafka/common/TopicIdPartition.java new file mode 100644 index 0000000000000..9fb570a67f438 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/TopicIdPartition.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common; + +import java.util.Objects; + +/** + * This represents universally unique identifier with topic id for a topic partition. This makes sure that topics + * recreated with the same name will always have unique topic identifiers. + */ +public class TopicIdPartition { + + private final Uuid topicId; + private final TopicPartition topicPartition; + + public TopicIdPartition(Uuid topicId, TopicPartition topicPartition) { + this.topicId = Objects.requireNonNull(topicId, "topicId can not be null"); + this.topicPartition = Objects.requireNonNull(topicPartition, "topicPartition can not be null"); + } + + /** + * @return Universally unique id representing this topic partition. + */ + public Uuid topicId() { + return topicId; + } + + /** + * @return Topic partition representing this instance. + */ + public TopicPartition topicPartition() { + return topicPartition; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TopicIdPartition that = (TopicIdPartition) o; + return Objects.equals(topicId, that.topicId) && + Objects.equals(topicPartition, that.topicPartition); + } + + @Override + public int hashCode() { + return Objects.hash(topicId, topicPartition); + } + + @Override + public String toString() { + return "TopicIdPartition{" + + "topicId=" + topicId + + ", topicPartition=" + topicPartition + + '}'; + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java new file mode 100644 index 0000000000000..662e396352142 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * This represents all the required data and indexes for a specific log segment that needs to be stored in the remote + * storage. This is passed with {@link RemoteStorageManager#copyLogSegmentData(RemoteLogSegmentMetadata, LogSegmentData)} + * while copying a specific log segment to the remote storage. + */ +@InterfaceStability.Evolving +public class LogSegmentData { + + private final File logSegment; + private final File offsetIndex; + private final File timeIndex; + private final File txnIndex; + private final File producerSnapshotIndex; + private final ByteBuffer leaderEpochIndex; + + /** + * Creates a LogSegmentData instance with data and indexes. + * + * @param logSegment actual log segment file + * @param offsetIndex offset index file + * @param timeIndex time index file + * @param txnIndex transaction index file + * @param producerSnapshotIndex producer snapshot until this segment + * @param leaderEpochIndex leader-epoch-index until this segment + */ + public LogSegmentData(File logSegment, + File offsetIndex, + File timeIndex, + File txnIndex, + File producerSnapshotIndex, + ByteBuffer leaderEpochIndex) { + this.logSegment = Objects.requireNonNull(logSegment, "logSegment can not be null"); + this.offsetIndex = Objects.requireNonNull(offsetIndex, "offsetIndex can not be null"); + this.timeIndex = Objects.requireNonNull(timeIndex, "timeIndex can not be null"); + this.txnIndex = Objects.requireNonNull(txnIndex, "txnIndex can not be null"); + this.producerSnapshotIndex = Objects.requireNonNull(producerSnapshotIndex, "producerSnapshotIndex can not be null"); + this.leaderEpochIndex = Objects.requireNonNull(leaderEpochIndex, "leaderEpochIndex can not be null"); + } + + /** + * @return Log segment file of this segment. + */ + public File logSegment() { + return logSegment; + } + + /** + * @return Offset index file. + */ + public File offsetIndex() { + return offsetIndex; + } + + /** + * @return Time index file of this segment. + */ + public File timeIndex() { + return timeIndex; + } + + /** + * @return Transaction index file of this segment. + */ + public File txnIndex() { + return txnIndex; + } + + /** + * @return Producer snapshot file until this segment. + */ + public File producerSnapshotIndex() { + return producerSnapshotIndex; + } + + /** + * @return Leader epoch index until this segment. + */ + public ByteBuffer leaderEpochIndex() { + return leaderEpochIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LogSegmentData that = (LogSegmentData) o; + return Objects.equals(logSegment, that.logSegment) && Objects + .equals(offsetIndex, that.offsetIndex) && Objects + .equals(timeIndex, that.timeIndex) && Objects + .equals(txnIndex, that.txnIndex) && Objects + .equals(producerSnapshotIndex, that.producerSnapshotIndex) && Objects + .equals(leaderEpochIndex, that.leaderEpochIndex); + } + + @Override + public int hashCode() { + return Objects.hash(logSegment, offsetIndex, timeIndex, txnIndex, producerSnapshotIndex, leaderEpochIndex); + } + + @Override + public String toString() { + return "LogSegmentData{" + + "logSegment=" + logSegment + + ", offsetIndex=" + offsetIndex + + ", timeIndex=" + timeIndex + + ", txnIndex=" + txnIndex + + ", producerSnapshotIndex=" + producerSnapshotIndex + + ", leaderEpochIndex=" + leaderEpochIndex + + '}'; + } +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java new file mode 100644 index 0000000000000..f53858d7a4540 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.Configurable; +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.io.Closeable; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * This interface provides storing and fetching remote log segment metadata with strongly consistent semantics. + *

    + * This class can be plugged in to Kafka cluster by adding the implementation class as + * remote.log.metadata.manager.class.name property value. There is an inbuilt implementation backed by + * topic storage in the local cluster. This is used as the default implementation if + * remote.log.metadata.manager.class.name is not configured. + *

    + *

    + * remote.log.metadata.manager.class.path property is about the class path of the RemoteLogStorageManager + * implementation. If specified, the RemoteLogStorageManager implementation and its dependent libraries will be loaded + * by a dedicated classloader which searches this class path before the Kafka broker class path. The syntax of this + * parameter is same with the standard Java class path string. + *

    + *

    + * remote.log.metadata.manager.listener.name property is about listener name of the local broker to which + * it should get connected if needed by RemoteLogMetadataManager implementation. When this is configured all other + * required properties can be passed as properties with prefix of 'remote.log.metadata.manager.listener. + *

    + * "cluster.id", "broker.id" and all other properties prefixed with "remote.log.metadata." are passed when + * {@link #configure(Map)} is invoked on this instance. + *

    + */ +@InterfaceStability.Evolving +public interface RemoteLogMetadataManager extends Configurable, Closeable { + + /** + * Adds {@link RemoteLogSegmentMetadata} with the containing {@link RemoteLogSegmentId} into {@link RemoteLogMetadataManager}. + *

    + * RemoteLogSegmentMetadata is identified by RemoteLogSegmentId and it should have the initial state which is {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED}. + *

    + * {@link #updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate)} should be used to update an existing RemoteLogSegmentMetadata. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @throws RemoteStorageException if there are any storage related errors occurred. + * @throws IllegalArgumentException if the given metadata instance does not have the state as {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED} + */ + void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata) throws RemoteStorageException; + + /** + * This method is used to update the {@link RemoteLogSegmentMetadata}. Currently, it allows to update with the new + * state based on the life cycle of the segment. It can go through the below state transitions. + *

    + * 

    +     * +---------------------+            +----------------------+
+     * |COPY_SEGMENT_STARTED |----------->|COPY_SEGMENT_FINISHED |
+     * +-------------------+-+            +--+-------------------+
+     *                     |                 |
+     *                     |                 |
+     *                     v                 v
+     *                  +--+-----------------+-+
+     *                  |DELETE_SEGMENT_STARTED|
+     *                  +-----------+----------+
+     *                              |
+     *                              |
+     *                              v
+     *                  +-----------+-----------+
+     *                  |DELETE_SEGMENT_FINISHED|
+     *                  +-----------------------+
+     *
    + *

    + * {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED} - This state indicates that the segment copying to remote storage is started but not yet finished. + * {@link RemoteLogSegmentState#COPY_SEGMENT_FINISHED} - This state indicates that the segment copying to remote storage is finished. + *
    + * The leader broker copies the log segments to the remote storage and puts the remote log segment metadata with the + * state as “COPY_SEGMENT_STARTED” and updates the state as “COPY_SEGMENT_FINISHED” once the copy is successful. + *

    + * {@link RemoteLogSegmentState#DELETE_SEGMENT_STARTED} - This state indicates that the segment deletion is started but not yet finished. + * {@link RemoteLogSegmentState#DELETE_SEGMENT_FINISHED} - This state indicates that the segment is deleted successfully. + *
    + * Leader partitions publish both the above delete segment events when remote log retention is reached for the + * respective segments. Remote Partition Removers also publish these events when a segment is deleted as part of + * the remote partition deletion. + * + * @param remoteLogSegmentMetadataUpdate update of the remote log segment metadata. + * @throws RemoteStorageException if there are any storage related errors occurred. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadataUpdate. + * @throws IllegalArgumentException if the given metadata instance has the state as {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED} + */ + void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate remoteLogSegmentMetadataUpdate) + throws RemoteStorageException; + + /** + * Returns {@link RemoteLogSegmentMetadata} if it exists for the given topic partition containing the offset with + * the given leader-epoch for the offset, else returns {@link Optional#empty()}. + * + * @param topicIdPartition topic partition + * @param offset offset + * @param epochForOffset leader epoch for the given offset + * @return the requested remote log segment metadata if it exists. + * @throws RemoteStorageException if there are any storage related errors occurred. + */ + Optional remoteLogSegmentMetadata(TopicIdPartition topicIdPartition, + long offset, + int epochForOffset) + throws RemoteStorageException; + + /** + * Returns the highest log offset of topic partition for the given leader epoch in remote storage. This is used by + * remote log management subsystem to know up to which offset the segments have been copied to remote storage for + * a given leader epoch. + * + * @param topicIdPartition topic partition + * @param leaderEpoch leader epoch + * @return the requested highest log offset if exists. + * @throws RemoteStorageException if there are any storage related errors occurred. + */ + Optional highestLogOffset(TopicIdPartition topicIdPartition, + int leaderEpoch) throws RemoteStorageException; + + /** + * This method is used to update the metadata about remote partition delete event. Currently, it allows updating the + * state ({@link RemotePartitionDeleteState}) of a topic partition in remote metadata storage. Controller invokes + * this method with {@link RemotePartitionDeleteMetadata} having state as {@link RemotePartitionDeleteState#DELETE_PARTITION_MARKED}. + * So, remote partition removers can act on this event to clean the respective remote log segments of the partition. + *


    + * In the case of default RLMM implementation, remote partition remover processes {@link RemotePartitionDeleteState#DELETE_PARTITION_MARKED} + *

      + *
    • sends an event with state as {@link RemotePartitionDeleteState#DELETE_PARTITION_STARTED} + *
    • gets all the remote log segments and deletes them. + *
    • sends an event with state as {@link RemotePartitionDeleteState#DELETE_PARTITION_FINISHED} once all the remote log segments are + * deleted. + *
    + * + * @param remotePartitionDeleteMetadata update on delete state of a partition. + * @throws RemoteStorageException if there are any storage related errors occurred. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remotePartitionDeleteMetadata. + */ + void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata) + throws RemoteStorageException; + + /** + * List all the remote log segment metadata of the given topicIdPartition. + *

    + * Remote Partition Removers uses this method to fetch all the segments for a given topic partition, so that they + * can delete them. + * + * @return Iterator of remote log segment metadata for the given topic partition. + */ + Iterator listRemoteLogSegments(TopicIdPartition topicIdPartition) + throws RemoteStorageException; + + /** + * Returns iterator of remote log segment metadata, sorted by {@link RemoteLogSegmentMetadata#startOffset()} in + * ascending order which contains the given leader epoch. This is used by remote log retention management subsystem + * to fetch the segment metadata for a given leader epoch. + * + * @param topicIdPartition topic partition + * @param leaderEpoch leader epoch + * @return Iterator of remote segments, sorted by start offset in ascending order. + */ + Iterator listRemoteLogSegments(TopicIdPartition topicIdPartition, + int leaderEpoch) throws RemoteStorageException; + + /** + * This method is invoked only when there are changes in leadership of the topic partitions that this broker is + * responsible for. + * + * @param leaderPartitions partitions that have become leaders on this broker. + * @param followerPartitions partitions that have become followers on this broker. + */ + void onPartitionLeadershipChanges(Set leaderPartitions, + Set followerPartitions); + + /** + * This method is invoked only when the topic partitions are stopped on this broker. This can happen when a + * partition is emigrated to other broker or a partition is deleted. + * + * @param partitions topic partitions that have been stopped. + */ + void onStopPartitions(Set partitions); +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java new file mode 100644 index 0000000000000..cbebd9f606d32 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Objects; + +/** + * This class represents a universally unique identifier associated to a topic partition's log segment. This will be + * regenerated for every attempt of copying a specific log segment in {@link RemoteStorageManager#copyLogSegmentData(RemoteLogSegmentMetadata, LogSegmentData)}. + * Once it is stored in remote storage, it is used to access that segment later from remote log metadata storage. + */ +@InterfaceStability.Evolving +public class RemoteLogSegmentId { + + private final TopicIdPartition topicIdPartition; + private final Uuid id; + + public RemoteLogSegmentId(TopicIdPartition topicIdPartition, Uuid id) { + this.topicIdPartition = Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null"); + this.id = Objects.requireNonNull(id, "id can not be null"); + } + + /** + * @return TopicIdPartition of this remote log segment. + */ + public TopicIdPartition topicIdPartition() { + return topicIdPartition; + } + + /** + * @return Universally Unique Id of this remote log segment. + */ + public Uuid id() { + return id; + } + + @Override + public String toString() { + return "RemoteLogSegmentId{" + + "topicIdPartition=" + topicIdPartition + + ", id=" + id + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemoteLogSegmentId that = (RemoteLogSegmentId) o; + return Objects.equals(topicIdPartition, that.topicIdPartition) && Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(topicIdPartition, id); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java new file mode 100644 index 0000000000000..9f0dd817be849 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Collections; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.TreeMap; + +/** + * It describes the metadata about a topic partition's remote log segment in the remote storage. This is uniquely + * represented with {@link RemoteLogSegmentId}. + *

    + * New instance is always created with the state as {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED}. This can be + * updated by applying {@link RemoteLogSegmentMetadataUpdate} for the respective {@link RemoteLogSegmentId} of the + * {@code RemoteLogSegmentMetadata}. + */ +@InterfaceStability.Evolving +public class RemoteLogSegmentMetadata { + + /** + * Universally unique remote log segment id. + */ + private final RemoteLogSegmentId remoteLogSegmentId; + + /** + * Start offset of this segment. + */ + private final long startOffset; + + /** + * End offset of this segment. + */ + private final long endOffset; + + /** + * Broker id from which this event is generated. + */ + private final int brokerId; + + /** + * Maximum timestamp in the segment + */ + private final long maxTimestamp; + + /** + * Epoch time at which the respective {@link #state} is set. + */ + private final long eventTimestamp; + + /** + * LeaderEpoch vs offset for messages within this segment. + */ + private final NavigableMap segmentLeaderEpochs; + + /** + * Size of the segment in bytes. + */ + private final int segmentSizeInBytes; + + /** + * It indicates the state in which the action is executed on this segment. + */ + private final RemoteLogSegmentState state; + + /** + * Creates an instance with the given metadata of remote log segment. + * + * {@code segmentLeaderEpochs} can not be empty. If all the records in this segment belong to the same leader epoch + * then it should have an entry with epoch mapping to start-offset of this segment. + * + * @param remoteLogSegmentId Universally unique remote log segment id. + * @param startOffset Start offset of this segment (inclusive). + * @param endOffset End offset of this segment (inclusive). + * @param maxTimestamp Maximum timestamp in this segment. + * @param brokerId Broker id from which this event is generated. + * @param eventTimestamp Epoch time in milli seconds at which the remote log segment is copied to the remote tier storage. + * @param segmentSizeInBytes Size of this segment in bytes. + * @param state State of the respective segment of remoteLogSegmentId. + * @param segmentLeaderEpochs leader epochs occurred within this segment. + */ + private RemoteLogSegmentMetadata(RemoteLogSegmentId remoteLogSegmentId, + long startOffset, + long endOffset, + long maxTimestamp, + int brokerId, + long eventTimestamp, + int segmentSizeInBytes, + RemoteLogSegmentState state, + Map segmentLeaderEpochs) { + this.remoteLogSegmentId = Objects.requireNonNull(remoteLogSegmentId, "remoteLogSegmentId can not be null"); + this.state = Objects.requireNonNull(state, "state can not be null"); + + this.startOffset = startOffset; + this.endOffset = endOffset; + this.brokerId = brokerId; + this.maxTimestamp = maxTimestamp; + this.eventTimestamp = eventTimestamp; + this.segmentSizeInBytes = segmentSizeInBytes; + + if (segmentLeaderEpochs == null || segmentLeaderEpochs.isEmpty()) { + throw new IllegalArgumentException("segmentLeaderEpochs can not be null or empty"); + } + + this.segmentLeaderEpochs = Collections.unmodifiableNavigableMap(new TreeMap<>(segmentLeaderEpochs)); + } + + /** + * Creates an instance with the given metadata of remote log segment and its state as {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED}. + * + * {@code segmentLeaderEpochs} can not be empty. If all the records in this segment belong to the same leader epoch + * then it should have an entry with epoch mapping to start-offset of this segment. + * + * @param remoteLogSegmentId Universally unique remote log segment id. + * @param startOffset Start offset of this segment (inclusive). + * @param endOffset End offset of this segment (inclusive). + * @param maxTimestamp Maximum timestamp in this segment + * @param brokerId Broker id from which this event is generated. + * @param eventTimestamp Epoch time in milli seconds at which the remote log segment is copied to the remote tier storage. + * @param segmentSizeInBytes Size of this segment in bytes. + * @param segmentLeaderEpochs leader epochs occurred within this segment + */ + public RemoteLogSegmentMetadata(RemoteLogSegmentId remoteLogSegmentId, + long startOffset, + long endOffset, + long maxTimestamp, + int brokerId, + long eventTimestamp, + int segmentSizeInBytes, + Map segmentLeaderEpochs) { + this(remoteLogSegmentId, + startOffset, + endOffset, + maxTimestamp, + brokerId, + eventTimestamp, segmentSizeInBytes, + RemoteLogSegmentState.COPY_SEGMENT_STARTED, + segmentLeaderEpochs); + } + + + /** + * @return unique id of this segment. + */ + public RemoteLogSegmentId remoteLogSegmentId() { + return remoteLogSegmentId; + } + + /** + * @return Start offset of this segment (inclusive). + */ + public long startOffset() { + return startOffset; + } + + /** + * @return End offset of this segment (inclusive). + */ + public long endOffset() { + return endOffset; + } + + /** + * @return Epoch time at which this event is occurred. + */ + public long eventTimestamp() { + return eventTimestamp; + } + + /** + * @return Total size of this segment in bytes. + */ + public int segmentSizeInBytes() { + return segmentSizeInBytes; + } + + /** + * @return Maximum timestamp of a record within this segment. + */ + public long maxTimestamp() { + return maxTimestamp; + } + + /** + * @return Map of leader epoch vs offset for the records available in this segment. + */ + public NavigableMap segmentLeaderEpochs() { + return segmentLeaderEpochs; + } + + /** + * @return Broker id from which this event is generated. + */ + public int brokerId() { + return brokerId; + } + + /** + * Returns the current state of this remote log segment. It can be any of the below + *

      + * {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED} + * {@link RemoteLogSegmentState#COPY_SEGMENT_FINISHED} + * {@link RemoteLogSegmentState#DELETE_SEGMENT_STARTED} + * {@link RemoteLogSegmentState#DELETE_SEGMENT_FINISHED} + *
    + */ + public RemoteLogSegmentState state() { + return state; + } + + /** + * Creates a new RemoteLogSegmentMetadata applying the given {@code rlsmUpdate} on this instance. This method will + * not update this instance. + * + * @param rlsmUpdate update to be applied. + * @return a new instance created by applying the given update on this instance. + */ + public RemoteLogSegmentMetadata createRemoteLogSegmentWithUpdates(RemoteLogSegmentMetadataUpdate rlsmUpdate) { + if (!remoteLogSegmentId.equals(rlsmUpdate.remoteLogSegmentId())) { + throw new IllegalArgumentException("Given rlsmUpdate does not have this instance's remoteLogSegmentId."); + } + + return new RemoteLogSegmentMetadata(remoteLogSegmentId, startOffset, + endOffset, maxTimestamp, rlsmUpdate.brokerId(), rlsmUpdate.eventTimestamp(), + segmentSizeInBytes, rlsmUpdate.state(), segmentLeaderEpochs); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemoteLogSegmentMetadata that = (RemoteLogSegmentMetadata) o; + return startOffset == that.startOffset && endOffset == that.endOffset && brokerId == that.brokerId + && maxTimestamp == that.maxTimestamp && eventTimestamp == that.eventTimestamp + && segmentSizeInBytes == that.segmentSizeInBytes + && Objects.equals(remoteLogSegmentId, that.remoteLogSegmentId) + && Objects.equals(segmentLeaderEpochs, that.segmentLeaderEpochs) && state == that.state; + } + + @Override + public int hashCode() { + return Objects.hash(remoteLogSegmentId, startOffset, endOffset, brokerId, maxTimestamp, eventTimestamp, + segmentLeaderEpochs, segmentSizeInBytes, state); + } + + @Override + public String toString() { + return "RemoteLogSegmentMetadata{" + + "remoteLogSegmentId=" + remoteLogSegmentId + + ", startOffset=" + startOffset + + ", endOffset=" + endOffset + + ", brokerId=" + brokerId + + ", maxTimestamp=" + maxTimestamp + + ", eventTimestamp=" + eventTimestamp + + ", segmentLeaderEpochs=" + segmentLeaderEpochs + + ", segmentSizeInBytes=" + segmentSizeInBytes + + ", state=" + state + + '}'; + } + +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java new file mode 100644 index 0000000000000..4fc1ea1b4cc95 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Objects; + +/** + * It describes the metadata update about the log segment in the remote storage. This is currently used to update the + * state of the remote log segment by using {@link RemoteLogMetadataManager#updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate)}. + * This also includes the timestamp of this event. + */ +@InterfaceStability.Evolving +public class RemoteLogSegmentMetadataUpdate { + + /** + * Universally unique remote log segment id. + */ + private final RemoteLogSegmentId remoteLogSegmentId; + + /** + * Epoch time at which this event is generated. + */ + private final long eventTimestamp; + + /** + * It indicates the state in which the action is executed on this segment. + */ + private final RemoteLogSegmentState state; + + /** + * Broker id from which this event is generated. + */ + private final int brokerId; + + /** + * @param remoteLogSegmentId Universally unique remote log segment id. + * @param eventTimestamp Epoch time in milli seconds at which the remote log segment is copied to the remote tier storage. + * @param state State of the remote log segment. + * @param brokerId Broker id from which this event is generated. + */ + public RemoteLogSegmentMetadataUpdate(RemoteLogSegmentId remoteLogSegmentId, long eventTimestamp, + RemoteLogSegmentState state, int brokerId) { + this.remoteLogSegmentId = Objects.requireNonNull(remoteLogSegmentId, "remoteLogSegmentId can not be null"); + this.state = Objects.requireNonNull(state, "state can not be null"); + this.brokerId = brokerId; + this.eventTimestamp = eventTimestamp; + } + + /** + * @return Universally unique id of this remote log segment. + */ + public RemoteLogSegmentId remoteLogSegmentId() { + return remoteLogSegmentId; + } + + /** + * @return Epoch time at which this event is generated. + */ + public long eventTimestamp() { + return eventTimestamp; + } + + /** + * It represents the state of the remote log segment. It can be one of the values of {@link RemoteLogSegmentState}. + */ + public RemoteLogSegmentState state() { + return state; + } + + /** + * @return Broker id from which this event is generated. + */ + public int brokerId() { + return brokerId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemoteLogSegmentMetadataUpdate that = (RemoteLogSegmentMetadataUpdate) o; + return eventTimestamp == that.eventTimestamp && Objects + .equals(remoteLogSegmentId, that.remoteLogSegmentId) && state == that.state && brokerId == that.brokerId; + } + + @Override + public int hashCode() { + return Objects.hash(remoteLogSegmentId, eventTimestamp, state, brokerId); + } + + @Override + public String toString() { + return "RemoteLogSegmentMetadataUpdate{" + + "remoteLogSegmentId=" + remoteLogSegmentId + + ", eventTimestamp=" + eventTimestamp + + ", state=" + state + + ", brokerId=" + brokerId + + '}'; + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java new file mode 100644 index 0000000000000..e27fdd19cd43b --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * It indicates the state of the remote log segment. This will be based on the action executed on this + * segment by the remote log service implementation. + *

    + * It goes through the below state transitions. + *

    + * 

    + * +---------------------+            +----------------------+
+ * |COPY_SEGMENT_STARTED |----------->|COPY_SEGMENT_FINISHED |
+ * +-------------------+-+            +--+-------------------+
+ *                     |                 |
+ *                     |                 |
+ *                     v                 v
+ *                  +--+-----------------+-+
+ *                  |DELETE_SEGMENT_STARTED|
+ *                  +-----------+----------+
+ *                              |
+ *                              |
+ *                              v
+ *                  +-----------+-----------+
+ *                  |DELETE_SEGMENT_FINISHED|
+ *                  +-----------------------+
+ *
    + */ +@InterfaceStability.Evolving +public enum RemoteLogSegmentState { + + /** + * This state indicates that the segment copying to remote storage is started but not yet finished. + */ + COPY_SEGMENT_STARTED((byte) 0), + + /** + * This state indicates that the segment copying to remote storage is finished. + */ + COPY_SEGMENT_FINISHED((byte) 1), + + /** + * This state indicates that the segment deletion is started but not yet finished. + */ + DELETE_SEGMENT_STARTED((byte) 2), + + /** + * This state indicates that the segment is deleted successfully. + */ + DELETE_SEGMENT_FINISHED((byte) 3); + + private static final Map STATE_TYPES = Collections.unmodifiableMap( + Arrays.stream(values()).collect(Collectors.toMap(RemoteLogSegmentState::id, Function.identity()))); + + private final byte id; + + RemoteLogSegmentState(byte id) { + this.id = id; + } + + public byte id() { + return id; + } + + public static RemoteLogSegmentState forId(byte id) { + return STATE_TYPES.get(id); + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java new file mode 100644 index 0000000000000..fdf4e61c1dbec --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Objects; + +/** + * This class represents the metadata about the remote partition. It can be updated with {@link RemoteLogMetadataManager#putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata)}. + * Possible state transitions are mentioned at {@link RemotePartitionDeleteState}. + */ +@InterfaceStability.Evolving +public class RemotePartitionDeleteMetadata { + + private final TopicIdPartition topicIdPartition; + private final RemotePartitionDeleteState state; + private final long eventTimestamp; + private final int brokerId; + + /** + * + * @param topicIdPartition + * @param state + * @param eventTimestamp + * @param brokerId + */ + public RemotePartitionDeleteMetadata(TopicIdPartition topicIdPartition, + RemotePartitionDeleteState state, + long eventTimestamp, + int brokerId) { + this.topicIdPartition = Objects.requireNonNull(topicIdPartition); + this.state = Objects.requireNonNull(state); + this.eventTimestamp = eventTimestamp; + this.brokerId = brokerId; + } + + /** + * @return TopicIdPartition for which this event is meant for. + */ + public TopicIdPartition topicIdPartition() { + return topicIdPartition; + } + + /** + * It represents the state of the remote partition. It can be one of the values of {@link RemotePartitionDeleteState}. + */ + public RemotePartitionDeleteState state() { + return state; + } + + /** + * @return Epoch time at which this event is occurred. + */ + public long eventTimestamp() { + return eventTimestamp; + } + + /** + * @return broker id from which this event is generated. + */ + public int brokerId() { + return brokerId; + } + + @Override + public String toString() { + return "RemotePartitionDeleteMetadata{" + + "topicPartition=" + topicIdPartition + + ", state=" + state + + ", eventTimestamp=" + eventTimestamp + + ", brokerId=" + brokerId + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemotePartitionDeleteMetadata that = (RemotePartitionDeleteMetadata) o; + return eventTimestamp == that.eventTimestamp && + brokerId == that.brokerId && + Objects.equals(topicIdPartition, that.topicIdPartition) && + state == that.state; + } + + @Override + public int hashCode() { + return Objects.hash(topicIdPartition, state, eventTimestamp, brokerId); + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java new file mode 100644 index 0000000000000..d5172fc1f942e --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * It indicates the deletion state of the remote topic partition. This will be based on the action executed on this + * partition by the remote log service implementation. + * State transitions are mentioned below. + *

    + * 

    + * +-------------------------+
+ * |DELETE_PARTITION_MARKED  |
+ * +-----------+-------------+
+ * |
+ * |
+ * +-----------v--------------+
+ * |DELETE_PARTITION_STARTED  |
+ * +-----------+--------------+
+ * |
+ * |
+ * +-----------v--------------+
+ * |DELETE_PARTITION_FINISHED |
+ * +--------------------------+
+ *
    + *

    + */ +@InterfaceStability.Evolving +public enum RemotePartitionDeleteState { + + /** + * This is used when a topic/partition is marked for delete by the controller. + * That means, all its remote log segments are eligible for deletion so that remote partition removers can + * start deleting them. + */ + DELETE_PARTITION_MARKED((byte) 0), + + /** + * This state indicates that the partition deletion is started but not yet finished. + */ + DELETE_PARTITION_STARTED((byte) 1), + + /** + * This state indicates that the partition is deleted successfully. + */ + DELETE_PARTITION_FINISHED((byte) 2); + + private static final Map STATE_TYPES = Collections.unmodifiableMap( + Arrays.stream(values()).collect(Collectors.toMap(RemotePartitionDeleteState::id, Function.identity()))); + + private final byte id; + + RemotePartitionDeleteState(byte id) { + this.id = id; + } + + public byte id() { + return id; + } + + public static RemotePartitionDeleteState forId(byte id) { + return STATE_TYPES.get(id); + } + +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java new file mode 100644 index 0000000000000..f6ac4ec5ccc19 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +/** + * Exception thrown when a resource is not found on the remote storage. + *

    + * A resource can be a log segment, any of the indexes or any which was stored in remote storage for a particular log + * segment. + */ +public class RemoteResourceNotFoundException extends RemoteStorageException { + private static final long serialVersionUID = 1L; + + public RemoteResourceNotFoundException(final String message) { + super(message); + } + + public RemoteResourceNotFoundException(final Throwable cause) { + super("Requested remote resource was not found", cause); + } + + public RemoteResourceNotFoundException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java new file mode 100644 index 0000000000000..f2488cb2957fb --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +/** + * Exception thrown when there is a remote storage error. + * This can be used as the base exception by implementors of + * {@link RemoteStorageManager} or {@link RemoteLogMetadataManager} to create extended exceptions. + */ +public class RemoteStorageException extends Exception { + private static final long serialVersionUID = 1L; + + public RemoteStorageException(final String message) { + super(message); + } + + public RemoteStorageException(final String message, + final Throwable cause) { + super(message, cause); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java new file mode 100644 index 0000000000000..64e3fdd026616 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.Configurable; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.io.Closeable; +import java.io.InputStream; + +/** + * This interface provides the lifecycle of remote log segments that includes copy, fetch, and delete from remote + * storage. + *

    + * Each upload or copy of a segment is initiated with {@link RemoteLogSegmentMetadata} containing {@link RemoteLogSegmentId} + * which is universally unique even for the same topic partition and offsets. + *

    + * {@link RemoteLogSegmentMetadata} is stored in {@link RemoteLogMetadataManager} before and after copy/delete operations on + * {@link RemoteStorageManager} with the respective {@link RemoteLogSegmentState}. {@link RemoteLogMetadataManager} is + * responsible for storing and fetching metadata about the remote log segments in a strongly consistent manner. + * This allows {@link RemoteStorageManager} to have eventual consistency on metadata (although the data is stored + * in strongly consistent semantics). + */ +@InterfaceStability.Evolving +public interface RemoteStorageManager extends Configurable, Closeable { + + /** + * Type of the index file. + */ + enum IndexType { + /** + * Represents offset index. + */ + Offset, + + /** + * Represents timestamp index. + */ + Timestamp, + + /** + * Represents producer snapshot index. + */ + ProducerSnapshot, + + /** + * Represents transaction index. + */ + Transaction, + + /** + * Represents leader epoch index. + */ + LeaderEpoch, + } + + /** + * Copies the given {@link LogSegmentData} provided for the given {@code remoteLogSegmentMetadata}. This includes + * log segment and its auxiliary indexes like offset index, time index, transaction index, leader epoch index, and + * producer snapshot index. + *

    + * Invoker of this API should always send a unique id as part of {@link RemoteLogSegmentMetadata#remoteLogSegmentId()} + * even when it retries to invoke this method for the same log segment data. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @param logSegmentData data to be copied to tiered storage. + * @throws RemoteStorageException if there are any errors in storing the data of the segment. + */ + void copyLogSegmentData(RemoteLogSegmentMetadata remoteLogSegmentMetadata, + LogSegmentData logSegmentData) + throws RemoteStorageException; + + /** + * Returns the remote log segment data file/object as InputStream for the given {@link RemoteLogSegmentMetadata} + * starting from the given startPosition. The stream will end at the end of the remote log segment data file/object. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @param startPosition start position of log segment to be read, inclusive. + * @return input stream of the requested log segment data. + * @throws RemoteStorageException if there are any errors while fetching the desired segment. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadata. + */ + InputStream fetchLogSegment(RemoteLogSegmentMetadata remoteLogSegmentMetadata, + int startPosition) throws RemoteStorageException; + + /** + * Returns the remote log segment data file/object as InputStream for the given {@link RemoteLogSegmentMetadata} + * starting from the given startPosition. The stream will end at the smaller of endPosition and the end of the + * remote log segment data file/object. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @param startPosition start position of log segment to be read, inclusive. + * @param endPosition end position of log segment to be read, inclusive. + * @return input stream of the requested log segment data. + * @throws RemoteStorageException if there are any errors while fetching the desired segment. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadata. + */ + InputStream fetchLogSegment(RemoteLogSegmentMetadata remoteLogSegmentMetadata, + int startPosition, + int endPosition) throws RemoteStorageException; + + /** + * Returns the index for the respective log segment of {@link RemoteLogSegmentMetadata}. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @param indexType type of the index to be fetched for the segment. + * @return input stream of the requested index. + * @throws RemoteStorageException if there are any errors while fetching the index. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadata. + */ + InputStream fetchIndex(RemoteLogSegmentMetadata remoteLogSegmentMetadata, + IndexType indexType) throws RemoteStorageException; + + /** + * Deletes the resources associated with the given {@code remoteLogSegmentMetadata}. Deletion is considered as + * successful if this call returns successfully without any errors. It will throw {@link RemoteStorageException} if + * there are any errors in deleting the file. + *

    + * + * @param remoteLogSegmentMetadata metadata about the remote log segment to be deleted. + * @throws RemoteResourceNotFoundException if the requested resource is not found + * @throws RemoteStorageException if there are any storage related errors occurred. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadata. + */ + void deleteLogSegmentData(RemoteLogSegmentMetadata remoteLogSegmentMetadata) throws RemoteStorageException; + +} \ No newline at end of file From f6929637b9222a64e8037348ed85c1088dfc1efb Mon Sep 17 00:00:00 2001 From: Luke Chen <43372967+showuon@users.noreply.github.com> Date: Thu, 4 Mar 2021 03:20:08 +0800 Subject: [PATCH 099/243] KAFKA-12284: Wait for mirrored topics to be created (#10185) Reviewers: Mickael Maison --- .../MirrorConnectorsIntegrationBaseTest.java | 105 ++++++++++++------ 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java index f2cc850a4f07d..7aae7d51eb25b 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java @@ -47,6 +47,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -82,6 +83,7 @@ public abstract class MirrorConnectorsIntegrationBaseTest { private static final int CHECKPOINT_DURATION_MS = 20_000; private static final int RECORD_CONSUME_DURATION_MS = 20_000; private static final int OFFSET_SYNC_DURATION_MS = 30_000; + private static final int TOPIC_SYNC_DURATION_MS = 30_000; private static final int NUM_WORKERS = 3; private static final Duration CONSUMER_POLL_TIMEOUT_MS = Duration.ofMillis(500); protected static final String PRIMARY_CLUSTER_ALIAS = "primary"; @@ -227,7 +229,10 @@ public void testReplication() throws Exception { MirrorClient primaryClient = new MirrorClient(mm2Config.clientConfig(PRIMARY_CLUSTER_ALIAS)); MirrorClient backupClient = new MirrorClient(mm2Config.clientConfig(BACKUP_CLUSTER_ALIAS)); - + + // make sure the topic is auto-created in the other cluster + waitForTopicCreated(primary.kafka(), "backup.test-topic-1"); + waitForTopicCreated(backup.kafka(), "primary.test-topic-1"); assertEquals(TopicConfig.CLEANUP_POLICY_COMPACT, getTopicConfig(backup.kafka(), "primary.test-topic-1", TopicConfig.CLEANUP_POLICY_CONFIG), "topic config was not synced"); @@ -312,6 +317,10 @@ public void testReplication() throws Exception { primary.kafka().createTopic("test-topic-2", NUM_PARTITIONS); backup.kafka().createTopic("test-topic-3", NUM_PARTITIONS); + // make sure the topic is auto-created in the other cluster + waitForTopicCreated(backup.kafka(), "primary.test-topic-2"); + waitForTopicCreated(primary.kafka(), "backup.test-topic-3"); + // only produce messages to the first partition produceMessages(primary, "test-topic-2", 1); produceMessages(backup, "test-topic-3", 1); @@ -360,15 +369,17 @@ public void testReplicationWithEmptyPartition() throws Exception { waitForConsumingAllRecords(backupConsumer, expectedRecords); } - Admin backupClient = backup.kafka().createAdminClient(); - // retrieve the consumer group offset from backup cluster - Map remoteOffsets = + try (Admin backupClient = backup.kafka().createAdminClient()) { + // retrieve the consumer group offset from backup cluster + Map remoteOffsets = backupClient.listConsumerGroupOffsets(consumerGroupName).partitionsToOffsetAndMetadata().get(); - // pinpoint the offset of the last partition which does not receive records - OffsetAndMetadata offset = remoteOffsets.get(new TopicPartition(PRIMARY_CLUSTER_ALIAS + "." + topic, NUM_PARTITIONS - 1)); - // offset of the last partition should exist, but its value should be 0 - assertNotNull(offset, "Offset of last partition was not replicated"); - assertEquals(0, offset.offset(), "Offset of last partition is not zero"); + + // pinpoint the offset of the last partition which does not receive records + OffsetAndMetadata offset = remoteOffsets.get(new TopicPartition(PRIMARY_CLUSTER_ALIAS + "." + topic, NUM_PARTITIONS - 1)); + // offset of the last partition should exist, but its value should be 0 + assertNotNull(offset, "Offset of last partition was not replicated"); + assertEquals(0, offset.offset(), "Offset of last partition is not zero"); + } } @Test @@ -396,6 +407,9 @@ public void testOneWayReplicationWithAutoOffsetSync() throws InterruptedExceptio waitUntilMirrorMakerIsRunning(backup, CONNECTOR_LIST, mm2Config, PRIMARY_CLUSTER_ALIAS, BACKUP_CLUSTER_ALIAS); + // make sure the topic is created in the other cluster + waitForTopicCreated(primary.kafka(), "backup.test-topic-1"); + waitForTopicCreated(backup.kafka(), "primary.test-topic-1"); // create a consumer at backup cluster with same consumer group Id to consume 1 topic Consumer backupConsumer = backup.kafka().createConsumerAndSubscribeTo( consumerProps, "primary.test-topic-1"); @@ -412,7 +426,9 @@ public void testOneWayReplicationWithAutoOffsetSync() throws InterruptedExceptio // now create a new topic in primary cluster primary.kafka().createTopic("test-topic-2", NUM_PARTITIONS); - backup.kafka().createTopic("primary.test-topic-2", 1); + // make sure the topic is created in backup cluster + waitForTopicCreated(backup.kafka(), "primary.test-topic-2"); + // produce some records to the new topic in primary cluster produceMessages(primary, "test-topic-2"); @@ -455,27 +471,41 @@ private static void waitUntilMirrorMakerIsRunning(EmbeddedConnectCluster connect "Connector " + connector.getSimpleName() + " tasks did not start in time on cluster: " + connectCluster); } } - + + /* + * wait for the topic created on the cluster + */ + private static void waitForTopicCreated(EmbeddedKafkaCluster cluster, String topicName) throws InterruptedException { + try (final Admin adminClient = cluster.createAdminClient()) { + waitForCondition(() -> adminClient.listTopics().names().get().contains(topicName), TOPIC_SYNC_DURATION_MS, + "Topic: " + topicName + " didn't get created in the cluster" + ); + } + } + /* * delete all topics of the input kafka cluster */ private static void deleteAllTopics(EmbeddedKafkaCluster cluster) throws Exception { - Admin client = cluster.createAdminClient(); - client.deleteTopics(client.listTopics().names().get()); + try (final Admin adminClient = cluster.createAdminClient()) { + Set topicsToBeDeleted = adminClient.listTopics().names().get(); + log.debug("Deleting topics: {} ", topicsToBeDeleted); + adminClient.deleteTopics(topicsToBeDeleted).all().get(); + } } /* * retrieve the config value based on the input cluster, topic and config name */ - private static String getTopicConfig(EmbeddedKafkaCluster cluster, String topic, String configName) - throws Exception { - Admin client = cluster.createAdminClient(); - Collection cr = Collections.singleton( - new ConfigResource(ConfigResource.Type.TOPIC, topic)); - - DescribeConfigsResult configsResult = client.describeConfigs(cr); - Config allConfigs = (Config) configsResult.all().get().values().toArray()[0]; - return allConfigs.get(configName).value(); + private static String getTopicConfig(EmbeddedKafkaCluster cluster, String topic, String configName) throws Exception { + try (Admin client = cluster.createAdminClient()) { + Collection cr = Collections.singleton( + new ConfigResource(ConfigResource.Type.TOPIC, topic)); + + DescribeConfigsResult configsResult = client.describeConfigs(cr); + Config allConfigs = (Config) configsResult.all().get().values().toArray()[0]; + return allConfigs.get(configName).value(); + } } /* @@ -505,27 +535,28 @@ protected void produceMessages(EmbeddedConnectCluster cluster, String topicName, private static void waitForConsumerGroupOffsetSync(EmbeddedConnectCluster connect, Consumer consumer, List topics, String consumerGroupId, int numRecords) throws InterruptedException { - Admin adminClient = connect.kafka().createAdminClient(); - List tps = new ArrayList<>(NUM_PARTITIONS * topics.size()); - for (int partitionIndex = 0; partitionIndex < NUM_PARTITIONS; partitionIndex++) { - for (String topic : topics) { - tps.add(new TopicPartition(topic, partitionIndex)); + try (Admin adminClient = connect.kafka().createAdminClient()) { + List tps = new ArrayList<>(NUM_PARTITIONS * topics.size()); + for (int partitionIndex = 0; partitionIndex < NUM_PARTITIONS; partitionIndex++) { + for (String topic : topics) { + tps.add(new TopicPartition(topic, partitionIndex)); + } } - } - long expectedTotalOffsets = numRecords * topics.size(); + long expectedTotalOffsets = numRecords * topics.size(); - waitForCondition(() -> { - Map consumerGroupOffsets = + waitForCondition(() -> { + Map consumerGroupOffsets = adminClient.listConsumerGroupOffsets(consumerGroupId).partitionsToOffsetAndMetadata().get(); - long consumerGroupOffsetTotal = consumerGroupOffsets.values().stream() + long consumerGroupOffsetTotal = consumerGroupOffsets.values().stream() .mapToLong(OffsetAndMetadata::offset).sum(); - Map offsets = consumer.endOffsets(tps, CONSUMER_POLL_TIMEOUT_MS); - long totalOffsets = offsets.values().stream().mapToLong(l -> l).sum(); + Map offsets = consumer.endOffsets(tps, CONSUMER_POLL_TIMEOUT_MS); + long totalOffsets = offsets.values().stream().mapToLong(l -> l).sum(); - // make sure the consumer group offsets are synced to expected number - return totalOffsets == expectedTotalOffsets && consumerGroupOffsetTotal > 0; - }, OFFSET_SYNC_DURATION_MS, "Consumer group offset sync is not complete in time"); + // make sure the consumer group offsets are synced to expected number + return totalOffsets == expectedTotalOffsets && consumerGroupOffsetTotal > 0; + }, OFFSET_SYNC_DURATION_MS, "Consumer group offset sync is not complete in time"); + } } /* From f40a82e05440e579c69363dc62c2dd9c8520eb02 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Wed, 3 Mar 2021 15:57:27 -0500 Subject: [PATCH 100/243] KAFKA-10759 Add ARM build stage (#9992) Only validation and unit test stages are enabled Co-authored-by: Peng.Lei <73098678+xiao-penglei@users.noreply.github.com> --- Jenkinsfile | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 87840d54b79a9..557988788cd4d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,12 +33,12 @@ def doValidation() { ''' } -def doTest() { - sh ''' - ./gradlew -PscalaVersion=$SCALA_VERSION unitTest integrationTest \ +def doTest(target = "unitTest integrationTest") { + sh """ + ./gradlew -PscalaVersion=$SCALA_VERSION ${target} \ --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \ -PignoreFailures=true -PmaxParallelForks=2 -PmaxTestRetries=1 -PmaxTestRetryFailures=5 - ''' + """ junit '**/build/test-results/**/TEST-*.xml' } @@ -158,6 +158,25 @@ pipeline { echo 'Skipping Kafka Streams archetype test for Java 15' } } + + stage('ARM') { + agent { label 'arm4' } + options { + timeout(time: 2, unit: 'HOURS') + timestamps() + } + environment { + SCALA_VERSION=2.12 + } + steps { + setupGradle() + doValidation() + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + doTest('unitTest') + } + echo 'Skipping Kafka Streams archetype test for ARM build' + } + } } } } From f06a47a7bba9baf035691feadbfeb3ff21802e82 Mon Sep 17 00:00:00 2001 From: Sven Erik Knop Date: Wed, 3 Mar 2021 22:17:49 +0000 Subject: [PATCH 101/243] KAFKA-12170: Fix for Connect Cast SMT to correctly transform a Byte array into a string (#9950) Cast SMT transformation for bytes -> string. Without this fix, the conversion becomes ByteBuffer.toString(), which always gives this useless result: "java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]" With this change, the byte array is converted into a base64 string of the byte buffer content. Reviewers: Mickael Maison , Randall Hauch , Konstantine Karantasis --- .../apache/kafka/connect/transforms/Cast.java | 14 ++++++++++++-- .../kafka/connect/transforms/CastTest.java | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java index e872b336e8573..0a763cc24f85b 100644 --- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java +++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.cache.SynchronizedCache; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.connector.ConnectRecord; import org.apache.kafka.connect.data.ConnectSchema; import org.apache.kafka.connect.data.Date; @@ -39,6 +40,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; +import java.util.Base64; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -56,7 +59,8 @@ public abstract class Cast> implements Transformation // allow casting nested fields. public static final String OVERVIEW_DOC = "Cast fields or the entire key or value to a specific type, e.g. to force an integer field to a smaller " - + "width. Only simple primitive types are supported -- integers, floats, boolean, and string. " + + "width. Cast from integers, floats, boolean and string to any other type, " + + "and cast binary to string (base64 encoded)." + "

    Use the concrete transformation type designed for the record key (" + Key.class.getName() + ") " + "or value (" + Value.class.getName() + ")."; @@ -82,7 +86,7 @@ public String toString() { ConfigDef.Importance.HIGH, "List of fields and the type to cast them to of the form field1:type,field2:type to cast fields of " + "Maps or Structs. A single type to cast the entire value. Valid types are int8, int16, int32, " - + "int64, float32, float64, boolean, and string."); + + "int64, float32, float64, boolean, and string. Note that binary fields can only be cast to string."); private static final String PURPOSE = "cast types"; @@ -364,6 +368,12 @@ private static String castToString(Object value) { if (value instanceof java.util.Date) { java.util.Date dateValue = (java.util.Date) value; return Values.dateFormatFor(dateValue).format(dateValue); + } else if (value instanceof ByteBuffer) { + ByteBuffer byteBuffer = (ByteBuffer) value; + return Base64.getEncoder().encodeToString(Utils.readBytes(byteBuffer)); + } else if (value instanceof byte[]) { + byte[] rawBytes = (byte[]) value; + return Base64.getEncoder().encodeToString(rawBytes); } else { return value.toString(); } diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java index ae90c1956b9f0..764b904ea3e24 100644 --- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java +++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.connect.transforms; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -425,7 +426,11 @@ public void castLogicalToString() { @Test public void castFieldsWithSchema() { Date day = new Date(MILLIS_PER_DAY); - xformValue.configure(Collections.singletonMap(Cast.SPEC_CONFIG, "int8:int16,int16:int32,int32:int64,int64:boolean,float32:float64,float64:boolean,boolean:int8,string:int32,bigdecimal:string,date:string,optional:int32")); + byte[] byteArray = new byte[] {(byte) 0xFE, (byte) 0xDC, (byte) 0xBA, (byte) 0x98, 0x76, 0x54, 0x32, 0x10}; + ByteBuffer byteBuffer = ByteBuffer.wrap(Arrays.copyOf(byteArray, byteArray.length)); + + xformValue.configure(Collections.singletonMap(Cast.SPEC_CONFIG, + "int8:int16,int16:int32,int32:int64,int64:boolean,float32:float64,float64:boolean,boolean:int8,string:int32,bigdecimal:string,date:string,optional:int32,bytes:string,byteArray:string")); // Include an optional fields and fields with defaults to validate their values are passed through properly SchemaBuilder builder = SchemaBuilder.struct(); @@ -442,6 +447,9 @@ public void castFieldsWithSchema() { builder.field("date", org.apache.kafka.connect.data.Date.SCHEMA); builder.field("optional", Schema.OPTIONAL_FLOAT32_SCHEMA); builder.field("timestamp", Timestamp.SCHEMA); + builder.field("bytes", Schema.BYTES_SCHEMA); + builder.field("byteArray", Schema.BYTES_SCHEMA); + Schema supportedTypesSchema = builder.build(); Struct recordValue = new Struct(supportedTypesSchema); @@ -456,6 +464,9 @@ public void castFieldsWithSchema() { recordValue.put("date", day); recordValue.put("string", "42"); recordValue.put("timestamp", new Date(0)); + recordValue.put("bytes", byteBuffer); + recordValue.put("byteArray", byteArray); + // optional field intentionally omitted SourceRecord transformed = xformValue.apply(new SourceRecord(null, null, "topic", 0, @@ -475,6 +486,9 @@ public void castFieldsWithSchema() { assertEquals("42", ((Struct) transformed.value()).get("bigdecimal")); assertEquals(Values.dateFormatFor(day).format(day), ((Struct) transformed.value()).get("date")); assertEquals(new Date(0), ((Struct) transformed.value()).get("timestamp")); + assertEquals("/ty6mHZUMhA=", ((Struct) transformed.value()).get("bytes")); + assertEquals("/ty6mHZUMhA=", ((Struct) transformed.value()).get("byteArray")); + assertNull(((Struct) transformed.value()).get("optional")); Schema transformedSchema = ((Struct) transformed.value()).schema(); @@ -489,6 +503,9 @@ public void castFieldsWithSchema() { assertEquals(Schema.STRING_SCHEMA.type(), transformedSchema.field("bigdecimal").schema().type()); assertEquals(Schema.STRING_SCHEMA.type(), transformedSchema.field("date").schema().type()); assertEquals(Schema.OPTIONAL_INT32_SCHEMA.type(), transformedSchema.field("optional").schema().type()); + assertEquals(Schema.STRING_SCHEMA.type(), transformedSchema.field("bytes").schema().type()); + assertEquals(Schema.STRING_SCHEMA.type(), transformedSchema.field("byteArray").schema().type()); + // The following fields are not changed assertEquals(Timestamp.SCHEMA.type(), transformedSchema.field("timestamp").schema().type()); } From 3ef39e13658be2ea0c79d44fba241fdf9257e51c Mon Sep 17 00:00:00 2001 From: David Jacot Date: Thu, 4 Mar 2021 10:31:35 +0100 Subject: [PATCH 102/243] MINOR; Clean up LeaderAndIsrResponse construction in `ReplicaManager#becomeLeaderOrFollower` (#10234) This patch refactors the code, which constructs the `LeaderAndIsrResponse` in `ReplicaManager#becomeLeaderOrFollower`, to improve the readability and to remove unnecessary operations. Reviewers: Chia-Ping Tsai --- .../common/requests/LeaderAndIsrRequest.java | 25 +++++------ .../common/requests/LeaderAndIsrResponse.java | 6 +-- .../common/message/LeaderAndIsrResponse.json | 3 +- .../requests/LeaderAndIsrRequestTest.java | 41 ++++++++++++++---- .../requests/LeaderAndIsrResponseTest.java | 11 ++--- .../common/requests/RequestResponseTest.java | 3 +- .../scala/kafka/server/ReplicaManager.scala | 43 ++++++++----------- .../ControllerChannelManagerTest.scala | 12 +++--- 8 files changed, 80 insertions(+), 64 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java index 939212a16ea30..d738286317616 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java @@ -145,24 +145,21 @@ public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setErrorCode(error.code())); } responseData.setPartitionErrors(partitions); - return new LeaderAndIsrResponse(responseData, version()); - } - - List topics = new ArrayList<>(data.topicStates().size()); - Map topicIds = topicIds(); - for (LeaderAndIsrTopicState topicState : data.topicStates()) { - LeaderAndIsrTopicError topicError = new LeaderAndIsrTopicError(); - topicError.setTopicId(topicIds.get(topicState.topicName())); - List partitions = new ArrayList<>(topicState.partitionStates().size()); - for (LeaderAndIsrPartitionState partition : topicState.partitionStates()) { - partitions.add(new LeaderAndIsrPartitionError() + } else { + for (LeaderAndIsrTopicState topicState : data.topicStates()) { + List partitions = new ArrayList<>( + topicState.partitionStates().size()); + for (LeaderAndIsrPartitionState partition : topicState.partitionStates()) { + partitions.add(new LeaderAndIsrPartitionError() .setPartitionIndex(partition.partitionIndex()) .setErrorCode(error.code())); + } + responseData.topics().add(new LeaderAndIsrTopicError() + .setTopicId(topicState.topicId()) + .setPartitionErrors(partitions)); } - topicError.setPartitionErrors(partitions); - topics.add(topicError); } - responseData.setTopics(topics); + return new LeaderAndIsrResponse(responseData, version()); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index 490983f1b5d21..c7c04e2d99b41 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -20,13 +20,13 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import java.nio.ByteBuffer; import java.util.Collections; -import java.util.List; import java.util.HashMap; import java.util.Map; @@ -39,7 +39,7 @@ public class LeaderAndIsrResponse extends AbstractResponse { * STALE_BROKER_EPOCH (77) */ private final LeaderAndIsrResponseData data; - private short version; + private final short version; public LeaderAndIsrResponse(LeaderAndIsrResponseData data, short version) { super(ApiKeys.LEADER_AND_ISR); @@ -47,7 +47,7 @@ public LeaderAndIsrResponse(LeaderAndIsrResponseData data, short version) { this.version = version; } - public List topics() { + public LeaderAndIsrTopicErrorCollection topics() { return this.data.topics(); } diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json index dc5879b143671..958448be2744b 100644 --- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json +++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json @@ -36,7 +36,8 @@ "about": "Each partition in v0 to v4 message."}, { "name": "Topics", "type": "[]LeaderAndIsrTopicError", "versions": "5+", "about": "Each topic", "fields": [ - { "name": "TopicId", "type": "uuid", "versions": "5+", "about": "The unique topic ID" }, + { "name": "TopicId", "type": "uuid", "versions": "5+", "mapKey": true, + "about": "The unique topic ID" }, { "name": "PartitionErrors", "type": "[]LeaderAndIsrPartitionError", "versions": "5+", "about": "Each partition."} ]} diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java index 9f636d7ef8be3..4fe51a0dccd3c 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java @@ -24,6 +24,8 @@ import org.apache.kafka.common.message.LeaderAndIsrRequestData; import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader; import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.test.TestUtils; @@ -59,18 +61,41 @@ public void testUnsupportedVersion() { @Test public void testGetErrorResponse() { - Uuid id = Uuid.randomUuid(); - for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) { - LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder(version, 0, 0, 0, - Collections.emptyList(), Collections.singletonMap("topic", id), Collections.emptySet()); - LeaderAndIsrRequest request = builder.build(); + Uuid topicId = Uuid.randomUuid(); + String topicName = "topic"; + int partition = 0; + + for (short version = LEADER_AND_ISR.oldestVersion(); version <= LEADER_AND_ISR.latestVersion(); version++) { + LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(version, 0, 0, 0, + Collections.singletonList(new LeaderAndIsrPartitionState() + .setTopicName(topicName) + .setPartitionIndex(partition)), + Collections.singletonMap(topicName, topicId), + Collections.emptySet() + ).build(version); + LeaderAndIsrResponse response = request.getErrorResponse(0, - new ClusterAuthorizationException("Not authorized")); + new ClusterAuthorizationException("Not authorized")); + assertEquals(Errors.CLUSTER_AUTHORIZATION_FAILED, response.error()); + if (version < 5) { - assertEquals(0, response.topics().size()); + assertEquals( + Collections.singletonList(new LeaderAndIsrPartitionError() + .setTopicName(topicName) + .setPartitionIndex(partition) + .setErrorCode(Errors.CLUSTER_AUTHORIZATION_FAILED.code())), + response.data().partitionErrors()); + assertEquals(0, response.data().topics().size()); } else { - assertEquals(id, response.topics().get(0).topicId()); + LeaderAndIsrTopicError topicState = response.topics().find(topicId); + assertEquals(topicId, topicState.topicId()); + assertEquals( + Collections.singletonList(new LeaderAndIsrPartitionError() + .setPartitionIndex(partition) + .setErrorCode(Errors.CLUSTER_AUTHORIZATION_FAILED.code())), + topicState.partitionErrors()); + assertEquals(0, response.data().partitionErrors().size()); } } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java index 3f6a22496ec2b..9ae2fdb4204fc 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.junit.jupiter.api.Test; @@ -80,7 +81,7 @@ public void testErrorCountsWithTopLevelError() { .setPartitionErrors(partitions), version); } else { Uuid id = Uuid.randomUuid(); - List topics = createTopic(id, asList(Errors.NONE, Errors.NOT_LEADER_OR_FOLLOWER)); + LeaderAndIsrTopicErrorCollection topics = createTopic(id, asList(Errors.NONE, Errors.NOT_LEADER_OR_FOLLOWER)); response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData() .setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code()) .setTopics(topics), version); @@ -101,7 +102,7 @@ public void testErrorCountsNoTopLevelError() { .setPartitionErrors(partitions), version); } else { Uuid id = Uuid.randomUuid(); - List topics = createTopic(id, asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED)); + LeaderAndIsrTopicErrorCollection topics = createTopic(id, asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED)); response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData() .setErrorCode(Errors.NONE.code()) .setTopics(topics), version); @@ -130,7 +131,7 @@ public void testToString() { } else { Uuid id = Uuid.randomUuid(); - List topics = createTopic(id, asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED)); + LeaderAndIsrTopicErrorCollection topics = createTopic(id, asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED)); response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData() .setErrorCode(Errors.NONE.code()) .setTopics(topics), version); @@ -155,8 +156,8 @@ private List createPartitions(String topicName, List return partitions; } - private List createTopic(Uuid id, List errors) { - List topics = new ArrayList<>(); + private LeaderAndIsrTopicErrorCollection createTopic(Uuid id, List errors) { + LeaderAndIsrTopicErrorCollection topics = new LeaderAndIsrTopicErrorCollection(); LeaderAndIsrTopicError topic = new LeaderAndIsrTopicError(); topic.setTopicId(id); List partitions = new ArrayList<>(); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index d873d15207374..228f32afcea94 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -126,6 +126,7 @@ import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember; import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState; import org.apache.kafka.common.message.LeaderAndIsrResponseData; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection; import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.ListGroupsRequestData; @@ -1685,7 +1686,7 @@ private LeaderAndIsrResponse createLeaderAndIsrResponse(int version) { new LeaderAndIsrResponseData.LeaderAndIsrPartitionError() .setPartitionIndex(0) .setErrorCode(Errors.NONE.code())); - List topics = new ArrayList<>(); + LeaderAndIsrTopicErrorCollection topics = new LeaderAndIsrTopicErrorCollection(); topics.add(new LeaderAndIsrResponseData.LeaderAndIsrTopicError() .setTopicId(Uuid.randomUuid()) .setPartitionErrors(partition)); diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 820af7b6ece2c..87b446208536c 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -1441,38 +1441,29 @@ class ReplicaManager(val config: KafkaConfig, replicaFetcherManager.shutdownIdleFetcherThreads() replicaAlterLogDirsManager.shutdownIdleFetcherThreads() onLeadershipChange(partitionsBecomeLeader, partitionsBecomeFollower) - if (leaderAndIsrRequest.version() < 5) { - val responsePartitions = responseMap.iterator.map { case (tp, error) => - new LeaderAndIsrPartitionError() + + val data = new LeaderAndIsrResponseData().setErrorCode(Errors.NONE.code) + if (leaderAndIsrRequest.version < 5) { + responseMap.forKeyValue { (tp, error) => + data.partitionErrors.add(new LeaderAndIsrPartitionError() .setTopicName(tp.topic) .setPartitionIndex(tp.partition) - .setErrorCode(error.code) - }.toBuffer - new LeaderAndIsrResponse(new LeaderAndIsrResponseData() - .setErrorCode(Errors.NONE.code) - .setPartitionErrors(responsePartitions.asJava), leaderAndIsrRequest.version()) + .setErrorCode(error.code)) + } } else { - val topics = new mutable.HashMap[String, List[LeaderAndIsrPartitionError]] - responseMap.asJava.forEach { case (tp, error) => - if (!topics.contains(tp.topic)) { - topics.put(tp.topic, List(new LeaderAndIsrPartitionError() - .setPartitionIndex(tp.partition) - .setErrorCode(error.code))) - } else { - topics.put(tp.topic, new LeaderAndIsrPartitionError() - .setPartitionIndex(tp.partition) - .setErrorCode(error.code)::topics(tp.topic)) + responseMap.forKeyValue { (tp, error) => + val topicId = topicIds.get(tp.topic) + var topic = data.topics.find(topicId) + if (topic == null) { + topic = new LeaderAndIsrTopicError().setTopicId(topicId) + data.topics.add(topic) } + topic.partitionErrors.add(new LeaderAndIsrPartitionError() + .setPartitionIndex(tp.partition) + .setErrorCode(error.code)) } - val topicErrors = topics.iterator.map { case (topic, partitionError) => - new LeaderAndIsrTopicError() - .setTopicId(topicIds.get(topic)) - .setPartitionErrors(partitionError.asJava) - }.toBuffer - new LeaderAndIsrResponse(new LeaderAndIsrResponseData() - .setErrorCode(Errors.NONE.code) - .setTopics(topicErrors.asJava), leaderAndIsrRequest.version()) } + new LeaderAndIsrResponse(data, leaderAndIsrRequest.version) } } val endMs = time.milliseconds() diff --git a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala index cb494e62338d0..8a816865b45f4 100644 --- a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala +++ b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala @@ -847,17 +847,17 @@ class ControllerChannelManagerTest { sentRequests.filter(_.request.apiKey == ApiKeys.LEADER_AND_ISR).filter(_.responseCallback != null).foreach { sentRequest => val leaderAndIsrRequest = sentRequest.request.build().asInstanceOf[LeaderAndIsrRequest] val topicIds = leaderAndIsrRequest.topicIds - val topicErrors = leaderAndIsrRequest.data.topicStates.asScala.map(t => - new LeaderAndIsrTopicError() + val data = new LeaderAndIsrResponseData() + .setErrorCode(error.code) + leaderAndIsrRequest.data.topicStates.asScala.foreach { t => + data.topics.add(new LeaderAndIsrTopicError() .setTopicId(topicIds.get(t.topicName)) .setPartitionErrors(t.partitionStates.asScala.map(p => new LeaderAndIsrPartitionError() .setPartitionIndex(p.partitionIndex) .setErrorCode(error.code)).asJava)) - val leaderAndIsrResponse = new LeaderAndIsrResponse( - new LeaderAndIsrResponseData() - .setErrorCode(error.code) - .setTopics(topicErrors.toBuffer.asJava), leaderAndIsrRequest.version()) + } + val leaderAndIsrResponse = new LeaderAndIsrResponse(data, leaderAndIsrRequest.version) sentRequest.responseCallback(leaderAndIsrResponse) } } From 8205051e90e3ea16165f8dc1f5c81af744bb1b9a Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Thu, 4 Mar 2021 18:06:50 +0800 Subject: [PATCH 103/243] =?UTF-8?q?MINOR:=20remove=20FetchResponse.Aborted?= =?UTF-8?q?Transaction=20and=20redundant=20construc=E2=80=A6=20(#9758)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. rename INVALID_HIGHWATERMARK to INVALID_HIGH_WATERMARK 2. replace FetchResponse.AbortedTransaction by FetchResponseData.AbortedTransaction 3. remove redundant constructors from FetchResponse.PartitionData 4. rename recordSet to records 5. add helpers "recordsOrFail" and "recordsSize" to FetchResponse to process record casting Reviewers: Ismael Juma --- .../kafka/clients/FetchSessionHandler.java | 8 +- .../clients/consumer/internals/Fetcher.java | 48 +-- .../kafka/common/requests/FetchRequest.java | 11 +- .../kafka/common/requests/FetchResponse.java | 380 ++++++------------ .../common/message/FetchResponse.json | 6 +- .../clients/FetchSessionHandlerTest.java | 65 ++- .../clients/consumer/KafkaConsumerTest.java | 27 +- .../consumer/internals/FetcherTest.java | 301 +++++++++----- .../common/requests/RequestResponseTest.java | 114 ++++-- core/src/main/scala/kafka/log/Log.scala | 5 +- .../scala/kafka/log/TransactionIndex.scala | 7 +- .../scala/kafka/network/RequestChannel.scala | 2 +- .../kafka/network/RequestConvertToJson.scala | 2 +- .../kafka/server/AbstractFetcherThread.scala | 51 ++- .../scala/kafka/server/ControllerApis.scala | 3 +- .../scala/kafka/server/FetchDataInfo.scala | 4 +- .../scala/kafka/server/FetchSession.scala | 64 +-- .../main/scala/kafka/server/KafkaApis.scala | 123 +++--- .../server/ReplicaAlterLogDirsThread.scala | 32 +- .../kafka/server/ReplicaFetcherThread.scala | 6 +- .../scala/kafka/server/ReplicaManager.scala | 3 +- .../kafka/tools/ReplicaVerificationTool.scala | 45 +-- .../kafka/tools/TestRaftRequestHandler.scala | 3 +- .../kafka/api/AuthorizerIntegrationTest.scala | 44 +- .../tools/ReplicaVerificationToolTest.scala | 10 +- .../test/scala/unit/kafka/log/LogTest.scala | 5 +- .../unit/kafka/log/TransactionIndexTest.scala | 4 +- .../server/AbstractFetcherThreadTest.scala | 22 +- ...FetchRequestDownConversionConfigTest.scala | 23 +- .../server/FetchRequestMaxBytesTest.scala | 9 +- .../unit/kafka/server/FetchRequestTest.scala | 112 +++--- .../unit/kafka/server/FetchSessionTest.scala | 326 ++++++++++----- .../unit/kafka/server/KafkaApisTest.scala | 58 +-- .../unit/kafka/server/LogOffsetTest.scala | 22 +- .../server/ReplicaFetcherThreadTest.scala | 40 +- .../kafka/server/ReplicaManagerTest.scala | 31 +- .../kafka/server/UpdateFeaturesTest.scala | 2 +- .../util/ReplicaFetcherMockBlockingSend.scala | 16 +- .../jmh/common/FetchResponseBenchmark.java | 18 +- .../ReplicaFetcherThreadBenchmark.java | 19 +- .../fetchsession/FetchSessionBenchmark.java | 17 +- .../apache/kafka/raft/KafkaRaftClient.java | 15 +- .../java/org/apache/kafka/raft/RaftUtil.java | 14 +- .../raft/KafkaRaftClientSnapshotTest.java | 12 +- .../kafka/raft/RaftClientTestContext.java | 20 +- 45 files changed, 1102 insertions(+), 1047 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java index 50252698c456a..b516e01923c58 100644 --- a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java @@ -318,7 +318,7 @@ static Set findMissing(Set toFind, Set response) { + String verifyFullFetchResponsePartitions(FetchResponse response) { StringBuilder bld = new StringBuilder(); Set extra = findMissing(response.responseData().keySet(), sessionPartitions.keySet()); @@ -343,7 +343,7 @@ String verifyFullFetchResponsePartitions(FetchResponse response) { * @param response The response. * @return True if the incremental fetch response partitions are valid. */ - String verifyIncrementalFetchResponsePartitions(FetchResponse response) { + String verifyIncrementalFetchResponsePartitions(FetchResponse response) { Set extra = findMissing(response.responseData().keySet(), sessionPartitions.keySet()); if (!extra.isEmpty()) { @@ -362,7 +362,7 @@ String verifyIncrementalFetchResponsePartitions(FetchResponse response) { * @param response The FetchResponse. * @return The string to log. */ - private String responseDataToLogString(FetchResponse response) { + private String responseDataToLogString(FetchResponse response) { if (!log.isTraceEnabled()) { int implied = sessionPartitions.size() - response.responseData().size(); if (implied > 0) { @@ -398,7 +398,7 @@ private String responseDataToLogString(FetchResponse response) { * @return True if the response is well-formed; false if it can't be processed * because of missing or unexpected partitions. */ - public boolean handleResponse(FetchResponse response) { + public boolean handleResponse(FetchResponse response) { if (response.error() != Errors.NONE) { log.info("Node {} was unable to process the fetch request with {}: {}.", node, nextMetadata, response.error()); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 5e8ad3c2454e2..257cff6b206bf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -47,6 +47,7 @@ import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; @@ -66,7 +67,6 @@ import org.apache.kafka.common.record.ControlRecordType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.record.Records; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; @@ -277,7 +277,7 @@ public void onSuccess(ClientResponse resp) { synchronized (Fetcher.this) { try { @SuppressWarnings("unchecked") - FetchResponse response = (FetchResponse) resp.responseBody(); + FetchResponse response = (FetchResponse) resp.responseBody(); FetchSessionHandler handler = sessionHandler(fetchTarget.id()); if (handler == null) { log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.", @@ -291,7 +291,7 @@ public void onSuccess(ClientResponse resp) { Set partitions = new HashSet<>(response.responseData().keySet()); FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions); - for (Map.Entry> entry : response.responseData().entrySet()) { + for (Map.Entry entry : response.responseData().entrySet()) { TopicPartition partition = entry.getKey(); FetchRequest.PartitionData requestData = data.sessionPartitions().get(partition); if (requestData == null) { @@ -310,12 +310,12 @@ public void onSuccess(ClientResponse resp) { throw new IllegalStateException(message); } else { long fetchOffset = requestData.fetchOffset; - FetchResponse.PartitionData partitionData = entry.getValue(); + FetchResponseData.PartitionData partitionData = entry.getValue(); log.debug("Fetch {} at offset {} for partition {} returned fetch data {}", isolationLevel, fetchOffset, partition, partitionData); - Iterator batches = partitionData.records().batches().iterator(); + Iterator batches = FetchResponse.recordsOrFail(partitionData).batches().iterator(); short responseVersion = resp.requestHeader().apiVersion(); completedFetches.add(new CompletedFetch(partition, partitionData, @@ -618,8 +618,8 @@ public Map>> fetchedRecords() { // The first condition ensures that the completedFetches is not stuck with the same completedFetch // in cases such as the TopicAuthorizationException, and the second condition ensures that no // potential data loss due to an exception in a following record. - FetchResponse.PartitionData partition = records.partitionData; - if (fetched.isEmpty() && (partition.records() == null || partition.records().sizeInBytes() == 0)) { + FetchResponseData.PartitionData partition = records.partitionData; + if (fetched.isEmpty() && FetchResponse.recordsOrFail(partition).sizeInBytes() == 0) { completedFetches.poll(); } throw e; @@ -1229,10 +1229,10 @@ private Map> regroupPartitionMapByNode(Map partition = nextCompletedFetch.partitionData; + FetchResponseData.PartitionData partition = nextCompletedFetch.partitionData; long fetchOffset = nextCompletedFetch.nextFetchOffset; CompletedFetch completedFetch = null; - Errors error = partition.error(); + Errors error = Errors.forCode(partition.errorCode()); try { if (!subscriptions.hasValidPosition(tp)) { @@ -1249,11 +1249,11 @@ private CompletedFetch initializeCompletedFetch(CompletedFetch nextCompletedFetc } log.trace("Preparing to read {} bytes of data for partition {} with offset {}", - partition.records().sizeInBytes(), tp, position); - Iterator batches = partition.records().batches().iterator(); + FetchResponse.recordsSize(partition), tp, position); + Iterator batches = FetchResponse.recordsOrFail(partition).batches().iterator(); completedFetch = nextCompletedFetch; - if (!batches.hasNext() && partition.records().sizeInBytes() > 0) { + if (!batches.hasNext() && FetchResponse.recordsSize(partition) > 0) { if (completedFetch.responseVersion < 3) { // Implement the pre KIP-74 behavior of throwing a RecordTooLargeException. Map recordTooLargePartitions = Collections.singletonMap(tp, fetchOffset); @@ -1286,11 +1286,11 @@ private CompletedFetch initializeCompletedFetch(CompletedFetch nextCompletedFetc subscriptions.updateLastStableOffset(tp, partition.lastStableOffset()); } - if (partition.preferredReadReplica().isPresent()) { - subscriptions.updatePreferredReadReplica(completedFetch.partition, partition.preferredReadReplica().get(), () -> { + if (FetchResponse.isPreferredReplica(partition)) { + subscriptions.updatePreferredReadReplica(completedFetch.partition, partition.preferredReadReplica(), () -> { long expireTimeMs = time.milliseconds() + metadata.metadataExpireMs(); log.debug("Updating preferred read replica for partition {} to {}, set to expire at {}", - tp, partition.preferredReadReplica().get(), expireTimeMs); + tp, partition.preferredReadReplica(), expireTimeMs); return expireTimeMs; }); } @@ -1455,8 +1455,8 @@ private class CompletedFetch { private final TopicPartition partition; private final Iterator batches; private final Set abortedProducerIds; - private final PriorityQueue abortedTransactions; - private final FetchResponse.PartitionData partitionData; + private final PriorityQueue abortedTransactions; + private final FetchResponseData.PartitionData partitionData; private final FetchResponseMetricAggregator metricAggregator; private final short responseVersion; @@ -1473,7 +1473,7 @@ private class CompletedFetch { private boolean initialized = false; private CompletedFetch(TopicPartition partition, - FetchResponse.PartitionData partitionData, + FetchResponseData.PartitionData partitionData, FetchResponseMetricAggregator metricAggregator, Iterator batches, Long fetchOffset, @@ -1641,9 +1641,9 @@ private void consumeAbortedTransactionsUpTo(long offset) { if (abortedTransactions == null) return; - while (!abortedTransactions.isEmpty() && abortedTransactions.peek().firstOffset <= offset) { - FetchResponse.AbortedTransaction abortedTransaction = abortedTransactions.poll(); - abortedProducerIds.add(abortedTransaction.producerId); + while (!abortedTransactions.isEmpty() && abortedTransactions.peek().firstOffset() <= offset) { + FetchResponseData.AbortedTransaction abortedTransaction = abortedTransactions.poll(); + abortedProducerIds.add(abortedTransaction.producerId()); } } @@ -1651,12 +1651,12 @@ private boolean isBatchAborted(RecordBatch batch) { return batch.isTransactional() && abortedProducerIds.contains(batch.producerId()); } - private PriorityQueue abortedTransactions(FetchResponse.PartitionData partition) { + private PriorityQueue abortedTransactions(FetchResponseData.PartitionData partition) { if (partition.abortedTransactions() == null || partition.abortedTransactions().isEmpty()) return null; - PriorityQueue abortedTransactions = new PriorityQueue<>( - partition.abortedTransactions().size(), Comparator.comparingLong(o -> o.firstOffset) + PriorityQueue abortedTransactions = new PriorityQueue<>( + partition.abortedTransactions().size(), Comparator.comparingLong(FetchResponseData.AbortedTransaction::firstOffset) ); abortedTransactions.addAll(partition.abortedTransactions()); return abortedTransactions; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java index 591c6e19a1977..05caaa405833f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java @@ -19,10 +19,10 @@ import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.FetchRequestData; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.utils.Utils; @@ -296,14 +296,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { // may not be any partitions at all in the response. For this reason, the top-level error code // is essential for them. Errors error = Errors.forException(e); - LinkedHashMap> responseData = new LinkedHashMap<>(); + LinkedHashMap responseData = new LinkedHashMap<>(); for (Map.Entry entry : fetchData.entrySet()) { - FetchResponse.PartitionData partitionResponse = new FetchResponse.PartitionData<>(error, - FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, - FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, MemoryRecords.EMPTY); - responseData.put(entry.getKey(), partitionResponse); + responseData.put(entry.getKey(), FetchResponse.partitionResponse(entry.getKey().partition(), error)); } - return new FetchResponse<>(error, responseData, throttleTimeMs, data.sessionId()); + return FetchResponse.of(error, throttleTimeMs, data.sessionId(), responseData); } public int replicaId() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index 4e8dce5f32031..5b1f1f7e80eb8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Records; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -33,7 +33,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; @@ -57,238 +56,43 @@ * the fetch offset after the index lookup * - {@link Errors#UNKNOWN_SERVER_ERROR} For any unexpected errors */ -public class FetchResponse extends AbstractResponse { - - public static final long INVALID_HIGHWATERMARK = -1L; +public class FetchResponse extends AbstractResponse { + public static final long INVALID_HIGH_WATERMARK = -1L; public static final long INVALID_LAST_STABLE_OFFSET = -1L; public static final long INVALID_LOG_START_OFFSET = -1L; public static final int INVALID_PREFERRED_REPLICA_ID = -1; private final FetchResponseData data; - private final LinkedHashMap> responseDataMap; + // we build responseData when needed. + private volatile LinkedHashMap responseData = null; @Override public FetchResponseData data() { return data; } - public static final class AbortedTransaction { - public final long producerId; - public final long firstOffset; - - public AbortedTransaction(long producerId, long firstOffset) { - this.producerId = producerId; - this.firstOffset = firstOffset; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - AbortedTransaction that = (AbortedTransaction) o; - - return producerId == that.producerId && firstOffset == that.firstOffset; - } - - @Override - public int hashCode() { - int result = Long.hashCode(producerId); - result = 31 * result + Long.hashCode(firstOffset); - return result; - } - - @Override - public String toString() { - return "(producerId=" + producerId + ", firstOffset=" + firstOffset + ")"; - } - - static AbortedTransaction fromMessage(FetchResponseData.AbortedTransaction abortedTransaction) { - return new AbortedTransaction(abortedTransaction.producerId(), abortedTransaction.firstOffset()); - } - } - - public static final class PartitionData { - private final FetchResponseData.FetchablePartitionResponse partitionResponse; - - // Derived fields - private final Optional preferredReplica; - private final List abortedTransactions; - private final Errors error; - - private PartitionData(FetchResponseData.FetchablePartitionResponse partitionResponse) { - // We partially construct FetchablePartitionResponse since we don't know the partition ID at this point - // When we convert the PartitionData (and other fields) into FetchResponseData down in toMessage, we - // set the partition IDs. - this.partitionResponse = partitionResponse; - this.preferredReplica = Optional.of(partitionResponse.preferredReadReplica()) - .filter(replicaId -> replicaId != INVALID_PREFERRED_REPLICA_ID); - - if (partitionResponse.abortedTransactions() == null) { - this.abortedTransactions = null; - } else { - this.abortedTransactions = partitionResponse.abortedTransactions().stream() - .map(AbortedTransaction::fromMessage) - .collect(Collectors.toList()); - } - - this.error = Errors.forCode(partitionResponse.errorCode()); - } - - public PartitionData(Errors error, - long highWatermark, - long lastStableOffset, - long logStartOffset, - Optional preferredReadReplica, - List abortedTransactions, - Optional divergingEpoch, - T records) { - this.preferredReplica = preferredReadReplica; - this.abortedTransactions = abortedTransactions; - this.error = error; - - FetchResponseData.FetchablePartitionResponse partitionResponse = - new FetchResponseData.FetchablePartitionResponse(); - partitionResponse.setErrorCode(error.code()) - .setHighWatermark(highWatermark) - .setLastStableOffset(lastStableOffset) - .setLogStartOffset(logStartOffset); - if (abortedTransactions != null) { - partitionResponse.setAbortedTransactions(abortedTransactions.stream().map( - aborted -> new FetchResponseData.AbortedTransaction() - .setProducerId(aborted.producerId) - .setFirstOffset(aborted.firstOffset)) - .collect(Collectors.toList())); - } else { - partitionResponse.setAbortedTransactions(null); - } - partitionResponse.setPreferredReadReplica(preferredReadReplica.orElse(INVALID_PREFERRED_REPLICA_ID)); - partitionResponse.setRecordSet(records); - divergingEpoch.ifPresent(partitionResponse::setDivergingEpoch); - - this.partitionResponse = partitionResponse; - } - - public PartitionData(Errors error, - long highWatermark, - long lastStableOffset, - long logStartOffset, - Optional preferredReadReplica, - List abortedTransactions, - T records) { - this(error, highWatermark, lastStableOffset, logStartOffset, preferredReadReplica, - abortedTransactions, Optional.empty(), records); - } - - public PartitionData(Errors error, - long highWatermark, - long lastStableOffset, - long logStartOffset, - List abortedTransactions, - T records) { - this(error, highWatermark, lastStableOffset, logStartOffset, Optional.empty(), abortedTransactions, records); - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - PartitionData that = (PartitionData) o; - - return this.partitionResponse.equals(that.partitionResponse); - } - - @Override - public int hashCode() { - return this.partitionResponse.hashCode(); - } - - @Override - public String toString() { - return "(error=" + error() + - ", highWaterMark=" + highWatermark() + - ", lastStableOffset = " + lastStableOffset() + - ", logStartOffset = " + logStartOffset() + - ", preferredReadReplica = " + preferredReadReplica().map(Object::toString).orElse("absent") + - ", abortedTransactions = " + abortedTransactions() + - ", divergingEpoch =" + divergingEpoch() + - ", recordsSizeInBytes=" + records().sizeInBytes() + ")"; - } - - public Errors error() { - return error; - } - - public long highWatermark() { - return partitionResponse.highWatermark(); - } - - public long lastStableOffset() { - return partitionResponse.lastStableOffset(); - } - - public long logStartOffset() { - return partitionResponse.logStartOffset(); - } - - public Optional preferredReadReplica() { - return preferredReplica; - } - - public List abortedTransactions() { - return abortedTransactions; - } - - public Optional divergingEpoch() { - FetchResponseData.EpochEndOffset epochEndOffset = partitionResponse.divergingEpoch(); - if (epochEndOffset.epoch() < 0) { - return Optional.empty(); - } else { - return Optional.of(epochEndOffset); - } - } - - @SuppressWarnings("unchecked") - public T records() { - return (T) partitionResponse.recordSet(); - } - } - - /** - * From version 3 or later, the entries in `responseData` should be in the same order as the entries in - * `FetchRequest.fetchData`. - * - * @param error The top-level error code. - * @param responseData The fetched data grouped by partition. - * @param throttleTimeMs The time in milliseconds that the response was throttled - * @param sessionId The fetch session id. - */ - public FetchResponse(Errors error, - LinkedHashMap> responseData, - int throttleTimeMs, - int sessionId) { - super(ApiKeys.FETCH); - this.data = toMessage(throttleTimeMs, error, responseData.entrySet().iterator(), sessionId); - this.responseDataMap = responseData; - } - public FetchResponse(FetchResponseData fetchResponseData) { super(ApiKeys.FETCH); this.data = fetchResponseData; - this.responseDataMap = toResponseDataMap(fetchResponseData); } public Errors error() { return Errors.forCode(data.errorCode()); } - public LinkedHashMap> responseData() { - return responseDataMap; + public LinkedHashMap responseData() { + if (responseData == null) { + synchronized (this) { + if (responseData == null) { + responseData = new LinkedHashMap<>(); + data.responses().forEach(topicResponse -> + topicResponse.partitions().forEach(partition -> + responseData.put(new TopicPartition(topicResponse.topic(), partition.partitionIndex()), partition)) + ); + } + } + } + return responseData; } @Override @@ -304,58 +108,15 @@ public int sessionId() { public Map errorCounts() { Map errorCounts = new HashMap<>(); updateErrorCounts(errorCounts, error()); - responseDataMap.values().forEach(response -> - updateErrorCounts(errorCounts, response.error()) + data.responses().forEach(topicResponse -> + topicResponse.partitions().forEach(partition -> + updateErrorCounts(errorCounts, Errors.forCode(partition.errorCode()))) ); return errorCounts; } - public static FetchResponse parse(ByteBuffer buffer, short version) { - return new FetchResponse<>(new FetchResponseData(new ByteBufferAccessor(buffer), version)); - } - - @SuppressWarnings("unchecked") - private static LinkedHashMap> toResponseDataMap( - FetchResponseData message) { - LinkedHashMap> responseMap = new LinkedHashMap<>(); - message.responses().forEach(topicResponse -> { - topicResponse.partitionResponses().forEach(partitionResponse -> { - TopicPartition tp = new TopicPartition(topicResponse.topic(), partitionResponse.partition()); - PartitionData partitionData = new PartitionData<>(partitionResponse); - responseMap.put(tp, partitionData); - }); - }); - return responseMap; - } - - private static FetchResponseData toMessage(int throttleTimeMs, Errors error, - Iterator>> partIterator, - int sessionId) { - List topicResponseList = new ArrayList<>(); - partIterator.forEachRemaining(entry -> { - PartitionData partitionData = entry.getValue(); - // Since PartitionData alone doesn't know the partition ID, we set it here - partitionData.partitionResponse.setPartition(entry.getKey().partition()); - // We have to keep the order of input topic-partition. Hence, we batch the partitions only if the last - // batch is in the same topic group. - FetchResponseData.FetchableTopicResponse previousTopic = topicResponseList.isEmpty() ? null - : topicResponseList.get(topicResponseList.size() - 1); - if (previousTopic != null && previousTopic.topic().equals(entry.getKey().topic())) - previousTopic.partitionResponses().add(partitionData.partitionResponse); - else { - List partitionResponses = new ArrayList<>(); - partitionResponses.add(partitionData.partitionResponse); - topicResponseList.add(new FetchResponseData.FetchableTopicResponse() - .setTopic(entry.getKey().topic()) - .setPartitionResponses(partitionResponses)); - } - }); - - return new FetchResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setErrorCode(error.code()) - .setSessionId(sessionId) - .setResponses(topicResponseList); + public static FetchResponse parse(ByteBuffer buffer, short version) { + return new FetchResponse(new FetchResponseData(new ByteBufferAccessor(buffer), version)); } /** @@ -365,11 +126,11 @@ private static FetchResponseData toMessage(int throttleT * @param partIterator The partition iterator. * @return The response size in bytes. */ - public static int sizeOf(short version, - Iterator>> partIterator) { + public static int sizeOf(short version, + Iterator> partIterator) { // Since the throttleTimeMs and metadata field sizes are constant and fixed, we can // use arbitrary values here without affecting the result. - FetchResponseData data = toMessage(0, Errors.NONE, partIterator, INVALID_SESSION_ID); + FetchResponseData data = toMessage(Errors.NONE, 0, INVALID_SESSION_ID, partIterator); ObjectSerializationCache cache = new ObjectSerializationCache(); return 4 + data.size(cache, version); } @@ -378,4 +139,91 @@ public static int sizeOf(short version, public boolean shouldClientThrottle(short version) { return version >= 8; } -} + + public static Optional divergingEpoch(FetchResponseData.PartitionData partitionResponse) { + return partitionResponse.divergingEpoch().epoch() < 0 ? Optional.empty() + : Optional.of(partitionResponse.divergingEpoch()); + } + + public static boolean isDivergingEpoch(FetchResponseData.PartitionData partitionResponse) { + return partitionResponse.divergingEpoch().epoch() >= 0; + } + + public static Optional preferredReadReplica(FetchResponseData.PartitionData partitionResponse) { + return partitionResponse.preferredReadReplica() == INVALID_PREFERRED_REPLICA_ID ? Optional.empty() + : Optional.of(partitionResponse.preferredReadReplica()); + } + + public static boolean isPreferredReplica(FetchResponseData.PartitionData partitionResponse) { + return partitionResponse.preferredReadReplica() != INVALID_PREFERRED_REPLICA_ID; + } + + public static FetchResponseData.PartitionData partitionResponse(int partition, Errors error) { + return new FetchResponseData.PartitionData() + .setPartitionIndex(partition) + .setErrorCode(error.code()) + .setHighWatermark(FetchResponse.INVALID_HIGH_WATERMARK); + } + + /** + * Returns `partition.records` as `Records` (instead of `BaseRecords`). If `records` is `null`, returns `MemoryRecords.EMPTY`. + * + * If this response was deserialized after a fetch, this method should never fail. An example where this would + * fail is a down-converted response (e.g. LazyDownConversionRecords) on the broker (before it's serialized and + * sent on the wire). + * + * @param partition partition data + * @return Records or empty record if the records in PartitionData is null. + */ + public static Records recordsOrFail(FetchResponseData.PartitionData partition) { + if (partition.records() == null) return MemoryRecords.EMPTY; + if (partition.records() instanceof Records) return (Records) partition.records(); + throw new ClassCastException("The record type is " + partition.records().getClass().getSimpleName() + ", which is not a subtype of " + + Records.class.getSimpleName() + ". This method is only safe to call if the `FetchResponse` was deserialized from bytes."); + } + + /** + * @return The size in bytes of the records. 0 is returned if records of input partition is null. + */ + public static int recordsSize(FetchResponseData.PartitionData partition) { + return partition.records() == null ? 0 : partition.records().sizeInBytes(); + } + + public static FetchResponse of(Errors error, + int throttleTimeMs, + int sessionId, + LinkedHashMap responseData) { + return new FetchResponse(toMessage(error, throttleTimeMs, sessionId, responseData.entrySet().iterator())); + } + + private static FetchResponseData toMessage(Errors error, + int throttleTimeMs, + int sessionId, + Iterator> partIterator) { + List topicResponseList = new ArrayList<>(); + partIterator.forEachRemaining(entry -> { + FetchResponseData.PartitionData partitionData = entry.getValue(); + // Since PartitionData alone doesn't know the partition ID, we set it here + partitionData.setPartitionIndex(entry.getKey().partition()); + // We have to keep the order of input topic-partition. Hence, we batch the partitions only if the last + // batch is in the same topic group. + FetchResponseData.FetchableTopicResponse previousTopic = topicResponseList.isEmpty() ? null + : topicResponseList.get(topicResponseList.size() - 1); + if (previousTopic != null && previousTopic.topic().equals(entry.getKey().topic())) + previousTopic.partitions().add(partitionData); + else { + List partitionResponses = new ArrayList<>(); + partitionResponses.add(partitionData); + topicResponseList.add(new FetchResponseData.FetchableTopicResponse() + .setTopic(entry.getKey().topic()) + .setPartitions(partitionResponses)); + } + }); + + return new FetchResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode(error.code()) + .setSessionId(sessionId) + .setResponses(topicResponseList); + } +} \ No newline at end of file diff --git a/clients/src/main/resources/common/message/FetchResponse.json b/clients/src/main/resources/common/message/FetchResponse.json index 0aa9a3a029e33..22807255bfcf1 100644 --- a/clients/src/main/resources/common/message/FetchResponse.json +++ b/clients/src/main/resources/common/message/FetchResponse.json @@ -53,9 +53,9 @@ "about": "The response topics.", "fields": [ { "name": "Topic", "type": "string", "versions": "0+", "entityType": "topicName", "about": "The topic name." }, - { "name": "PartitionResponses", "type": "[]FetchablePartitionResponse", "versions": "0+", + { "name": "Partitions", "type": "[]PartitionData", "versions": "0+", "about": "The topic partitions.", "fields": [ - { "name": "Partition", "type": "int32", "versions": "0+", + { "name": "PartitionIndex", "type": "int32", "versions": "0+", "about": "The partition index." }, { "name": "ErrorCode", "type": "int16", "versions": "0+", "about": "The error code, or 0 if there was no fetch error." }, @@ -94,7 +94,7 @@ ]}, { "name": "PreferredReadReplica", "type": "int32", "versions": "11+", "default": "-1", "ignorable": false, "about": "The preferred read replica for the consumer to use on its next fetch request"}, - { "name": "RecordSet", "type": "records", "versions": "0+", "nullableVersions": "0+", "about": "The record data."} + { "name": "Records", "type": "records", "versions": "0+", "nullableVersions": "0+", "about": "The record data."} ]} ]} ] diff --git a/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java index 97a1b177ecad0..72cedd0fa0cae 100644 --- a/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java @@ -17,8 +17,8 @@ package org.apache.kafka.clients; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; import org.apache.kafka.common.utils.LogContext; @@ -150,22 +150,21 @@ private static void assertListEquals(List expected, List data; + final FetchResponseData.PartitionData data; RespEntry(String topic, int partition, long highWatermark, long lastStableOffset) { this.part = new TopicPartition(topic, partition); - this.data = new FetchResponse.PartitionData<>( - Errors.NONE, - highWatermark, - lastStableOffset, - 0, - null, - null); + + this.data = new FetchResponseData.PartitionData() + .setPartitionIndex(partition) + .setHighWatermark(highWatermark) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0); } } - private static LinkedHashMap> respMap(RespEntry... entries) { - LinkedHashMap> map = new LinkedHashMap<>(); + private static LinkedHashMap respMap(RespEntry... entries) { + LinkedHashMap map = new LinkedHashMap<>(); for (RespEntry entry : entries) { map.put(entry.part, entry.data); } @@ -191,10 +190,10 @@ public void testSessionless() { assertEquals(INVALID_SESSION_ID, data.metadata().sessionId()); assertEquals(INITIAL_EPOCH, data.metadata().epoch()); - FetchResponse resp = new FetchResponse<>(Errors.NONE, + FetchResponse resp = FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, respMap(new RespEntry("foo", 0, 0, 0), - new RespEntry("foo", 1, 0, 0)), - 0, INVALID_SESSION_ID); + new RespEntry("foo", 1, 0, 0)) + ); handler.handleResponse(resp); FetchSessionHandler.Builder builder2 = handler.newBuilder(); @@ -225,10 +224,9 @@ public void testIncrementals() { assertEquals(INVALID_SESSION_ID, data.metadata().sessionId()); assertEquals(INITIAL_EPOCH, data.metadata().epoch()); - FetchResponse resp = new FetchResponse<>(Errors.NONE, + FetchResponse resp = FetchResponse.of(Errors.NONE, 0, 123, respMap(new RespEntry("foo", 0, 10, 20), - new RespEntry("foo", 1, 10, 20)), - 0, 123); + new RespEntry("foo", 1, 10, 20))); handler.handleResponse(resp); // Test an incremental fetch request which adds one partition and modifies another. @@ -249,15 +247,14 @@ public void testIncrementals() { new ReqEntry("foo", 1, 10, 120, 210)), data2.toSend()); - FetchResponse resp2 = new FetchResponse<>(Errors.NONE, - respMap(new RespEntry("foo", 1, 20, 20)), - 0, 123); + FetchResponse resp2 = FetchResponse.of(Errors.NONE, 0, 123, + respMap(new RespEntry("foo", 1, 20, 20))); handler.handleResponse(resp2); // Skip building a new request. Test that handling an invalid fetch session epoch response results // in a request which closes the session. - FetchResponse resp3 = new FetchResponse<>(Errors.INVALID_FETCH_SESSION_EPOCH, respMap(), - 0, INVALID_SESSION_ID); + FetchResponse resp3 = FetchResponse.of(Errors.INVALID_FETCH_SESSION_EPOCH, + 0, INVALID_SESSION_ID, respMap()); handler.handleResponse(resp3); FetchSessionHandler.Builder builder4 = handler.newBuilder(); @@ -312,11 +309,10 @@ public void testIncrementalPartitionRemoval() { data.toSend(), data.sessionPartitions()); assertTrue(data.metadata().isFull()); - FetchResponse resp = new FetchResponse<>(Errors.NONE, + FetchResponse resp = FetchResponse.of(Errors.NONE, 0, 123, respMap(new RespEntry("foo", 0, 10, 20), new RespEntry("foo", 1, 10, 20), - new RespEntry("bar", 0, 10, 20)), - 0, 123); + new RespEntry("bar", 0, 10, 20))); handler.handleResponse(resp); // Test an incremental fetch request which removes two partitions. @@ -337,8 +333,8 @@ public void testIncrementalPartitionRemoval() { // A FETCH_SESSION_ID_NOT_FOUND response triggers us to close the session. // The next request is a session establishing FULL request. - FetchResponse resp2 = new FetchResponse<>(Errors.FETCH_SESSION_ID_NOT_FOUND, - respMap(), 0, INVALID_SESSION_ID); + FetchResponse resp2 = FetchResponse.of(Errors.FETCH_SESSION_ID_NOT_FOUND, + 0, INVALID_SESSION_ID, respMap()); handler.handleResponse(resp2); FetchSessionHandler.Builder builder3 = handler.newBuilder(); builder3.add(new TopicPartition("foo", 0), @@ -354,11 +350,10 @@ public void testIncrementalPartitionRemoval() { @Test public void testVerifyFullFetchResponsePartitions() throws Exception { FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1); - String issue = handler.verifyFullFetchResponsePartitions(new FetchResponse<>(Errors.NONE, + String issue = handler.verifyFullFetchResponsePartitions(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, respMap(new RespEntry("foo", 0, 10, 20), new RespEntry("foo", 1, 10, 20), - new RespEntry("bar", 0, 10, 20)), - 0, INVALID_SESSION_ID)); + new RespEntry("bar", 0, 10, 20)))); assertTrue(issue.contains("extra")); assertFalse(issue.contains("omitted")); FetchSessionHandler.Builder builder = handler.newBuilder(); @@ -369,16 +364,14 @@ public void testVerifyFullFetchResponsePartitions() throws Exception { builder.add(new TopicPartition("bar", 0), new FetchRequest.PartitionData(20, 120, 220, Optional.empty())); builder.build(); - String issue2 = handler.verifyFullFetchResponsePartitions(new FetchResponse<>(Errors.NONE, + String issue2 = handler.verifyFullFetchResponsePartitions(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, respMap(new RespEntry("foo", 0, 10, 20), new RespEntry("foo", 1, 10, 20), - new RespEntry("bar", 0, 10, 20)), - 0, INVALID_SESSION_ID)); + new RespEntry("bar", 0, 10, 20)))); assertTrue(issue2 == null); - String issue3 = handler.verifyFullFetchResponsePartitions(new FetchResponse<>(Errors.NONE, + String issue3 = handler.verifyFullFetchResponsePartitions(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, respMap(new RespEntry("foo", 0, 10, 20), - new RespEntry("foo", 1, 10, 20)), - 0, INVALID_SESSION_ID)); + new RespEntry("foo", 1, 10, 20)))); assertFalse(issue3.contains("extra")); assertTrue(issue3.contains("omitted")); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index cfdc2bf29de9f..251170747cf9f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -45,18 +45,20 @@ import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.message.JoinGroupRequestData; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData; +import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsResponseData; -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; -import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.SyncGroupResponseData; -import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; @@ -91,8 +93,6 @@ import org.apache.kafka.test.MockConsumerInterceptor; import org.apache.kafka.test.MockMetricsReporter; import org.apache.kafka.test.TestUtils; -import org.apache.kafka.common.metrics.stats.Avg; - import org.junit.jupiter.api.Test; import javax.management.MBeanServer; @@ -2276,8 +2276,8 @@ private ListOffsetsResponse listOffsetsResponse(Map partit return new ListOffsetsResponse(data); } - private FetchResponse fetchResponse(Map fetches) { - LinkedHashMap> tpResponses = new LinkedHashMap<>(); + private FetchResponse fetchResponse(Map fetches) { + LinkedHashMap tpResponses = new LinkedHashMap<>(); for (Map.Entry fetchEntry : fetches.entrySet()) { TopicPartition partition = fetchEntry.getKey(); long fetchOffset = fetchEntry.getValue().offset; @@ -2294,14 +2294,17 @@ private FetchResponse fetchResponse(Map( - Errors.NONE, highWatermark, FetchResponse.INVALID_LAST_STABLE_OFFSET, - logStartOffset, null, records)); + tpResponses.put(partition, + new FetchResponseData.PartitionData() + .setPartitionIndex(partition.partition()) + .setHighWatermark(highWatermark) + .setLogStartOffset(logStartOffset) + .setRecords(records)); } - return new FetchResponse<>(Errors.NONE, tpResponses, 0, INVALID_SESSION_ID); + return FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, tpResponses); } - private FetchResponse fetchResponse(TopicPartition partition, long fetchOffset, int count) { + private FetchResponse fetchResponse(TopicPartition partition, long fetchOffset, int count) { FetchInfo fetchInfo = new FetchInfo(fetchOffset, count); return fetchResponse(Collections.singletonMap(partition, fetchInfo)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 0a7d5cfde49b4..985d95947c236 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -48,6 +48,7 @@ import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic; @@ -1270,13 +1271,18 @@ public void testFetchPositionAfterException() { assertEquals(1, fetcher.sendFetches()); - Map> partitions = new LinkedHashMap<>(); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, records)); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)); - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), - 0, INVALID_SESSION_ID)); + + Map partitions = new LinkedHashMap<>(); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setRecords(records)); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + client.prepareResponse(FetchResponse.of(Errors.NONE, + 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); List> allFetchedRecords = new ArrayList<>(); @@ -1316,17 +1322,29 @@ public void testCompletedFetchRemoval() { assertEquals(1, fetcher.sendFetches()); - Map> partitions = new LinkedHashMap<>(); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.NONE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, - FetchResponse.INVALID_LOG_START_OFFSET, null, records)); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)); - partitions.put(tp2, new FetchResponse.PartitionData<>(Errors.NONE, 100L, 4, - 0L, null, nextRecords)); - partitions.put(tp3, new FetchResponse.PartitionData<>(Errors.NONE, 100L, 4, - 0L, null, partialRecords)); - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), - 0, INVALID_SESSION_ID)); + Map partitions = new LinkedHashMap<>(); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setRecords(records)); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + partitions.put(tp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(nextRecords)); + partitions.put(tp3, new FetchResponseData.PartitionData() + .setPartitionIndex(tp3.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(partialRecords)); + client.prepareResponse(FetchResponse.of(Errors.NONE, + 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); List> fetchedRecords = new ArrayList<>(); @@ -1384,9 +1402,11 @@ public void testSeekBeforeException() { assignFromUser(Utils.mkSet(tp0)); subscriptions.seek(tp0, 1); assertEquals(1, fetcher.sendFetches()); - Map> partitions = new HashMap<>(); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, records)); + Map partitions = new HashMap<>(); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setRecords(records)); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); @@ -1397,9 +1417,11 @@ public void testSeekBeforeException() { assertEquals(1, fetcher.sendFetches()); partitions = new HashMap<>(); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, MemoryRecords.EMPTY)); - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), 0, INVALID_SESSION_ID)); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); assertEquals(1, fetcher.fetchedRecords().get(tp0).size()); @@ -2094,7 +2116,7 @@ public void testQuotaMetrics() { ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true); client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); - FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); + FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); buffer = RequestTestUtils.serializeResponseWithHeader(response, ApiKeys.FETCH.latestVersion(), request.correlationId()); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); @@ -2256,7 +2278,7 @@ public void testFetchResponseMetrics() { client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, partitionCounts, tp -> validLeaderEpoch)); int expectedBytes = 0; - LinkedHashMap> fetchPartitionData = new LinkedHashMap<>(); + LinkedHashMap fetchPartitionData = new LinkedHashMap<>(); for (TopicPartition tp : Utils.mkSet(tp1, tp2)) { subscriptions.seek(tp, 0); @@ -2269,12 +2291,15 @@ public void testFetchResponseMetrics() { for (Record record : records.records()) expectedBytes += record.sizeInBytes(); - fetchPartitionData.put(tp, new FetchResponse.PartitionData<>(Errors.NONE, 15L, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); + fetchPartitionData.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setHighWatermark(15) + .setLogStartOffset(0) + .setRecords(records)); } assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(new FetchResponse<>(Errors.NONE, fetchPartitionData, 0, INVALID_SESSION_ID)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData)); consumerClient.poll(time.timer(0)); Map>> fetchedRecords = fetchedRecords(); @@ -2333,15 +2358,21 @@ public void testFetchResponseMetricsWithOnePartitionError() { builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); MemoryRecords records = builder.build(); - Map> partitions = new HashMap<>(); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, MemoryRecords.EMPTY)); + Map partitions = new HashMap<>(); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(records)); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100) + .setLogStartOffset(0)); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), - 0, INVALID_SESSION_ID)); + client.prepareResponse(FetchResponse.of(Errors.NONE, + 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); fetcher.fetchedRecords(); @@ -2375,15 +2406,19 @@ public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); MemoryRecords records = builder.build(); - Map> partitions = new HashMap<>(); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, - MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes())))); - - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), - 0, INVALID_SESSION_ID)); + Map partitions = new HashMap<>(); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(records)); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes())))); + + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); fetcher.fetchedRecords(); @@ -2778,8 +2813,8 @@ public void testSkippingAbortedTransactions() { buffer.flip(); - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -2811,7 +2846,6 @@ public void testReturnCommittedTransactions() { commitTransaction(buffer, 1L, currentOffset); buffer.flip(); - List abortedTransactions = new ArrayList<>(); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -2824,7 +2858,7 @@ public void testReturnCommittedTransactions() { FetchRequest request = (FetchRequest) body; assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); return true; - }, fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -2840,7 +2874,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024); - List abortedTransactions = new ArrayList<>(); + List abortedTransactions = new ArrayList<>(); long pid1 = 1L; long pid2 = 2L; @@ -2863,7 +2897,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { // abort producer 2 abortTransaction(buffer, pid2, 5L); - abortedTransactions.add(new FetchResponse.AbortedTransaction(pid2, 2L)); + abortedTransactions.add(new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(2L)); // New transaction for producer 1 (eventually aborted) appendTransactionalRecords(buffer, pid1, 6L, @@ -2879,7 +2913,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { // abort producer 1 abortTransaction(buffer, pid1, 9L); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 6)); + abortedTransactions.add(new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(6)); // commit producer 2 commitTransaction(buffer, pid2, 10L); @@ -2931,8 +2965,9 @@ public void testMultipleAbortMarkers() { commitTransaction(buffer, 1L, currentOffset); buffer.flip(); - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0) + ); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -2983,8 +3018,8 @@ public void testReadCommittedAbortMarkerWithNoData() { assertEquals(1, fetcher.sendFetches()); // prepare the response. the aborted transactions begin at offsets which are no longer in the log - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(producerId, 0L)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(producerId).setFirstOffset(0L)); client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); @@ -3120,9 +3155,10 @@ public void testReadCommittedWithCompactedTopic() { assertEquals(1, fetcher.sendFetches()); // prepare the response. the aborted transactions begin at offsets which are no longer in the log - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(pid2, 6L)); - abortedTransactions.add(new FetchResponse.AbortedTransaction(pid1, 0L)); + List abortedTransactions = Arrays.asList( + new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(6), + new FetchResponseData.AbortedTransaction().setProducerId(pid1).setFirstOffset(0) + ); client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); @@ -3151,8 +3187,8 @@ public void testReturnAbortedTransactionsinUncommittedMode() { buffer.flip(); - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -3184,8 +3220,8 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { currentOffset += abortTransaction(buffer, 1L, currentOffset); buffer.flip(); - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -3216,12 +3252,19 @@ public void testConsumingViaIncrementalFetchRequests() { subscriptions.seekValidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); // Fetch some records and establish an incremental fetch session. - LinkedHashMap> partitions1 = new LinkedHashMap<>(); - partitions1.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 2L, - 2, 0L, null, this.records)); - partitions1.put(tp1, new FetchResponse.PartitionData<>(Errors.NONE, 100L, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, emptyRecords)); - FetchResponse resp1 = new FetchResponse<>(Errors.NONE, partitions1, 0, 123); + LinkedHashMap partitions1 = new LinkedHashMap<>(); + partitions1.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(2) + .setLastStableOffset(2) + .setLogStartOffset(0) + .setRecords(this.records)); + partitions1.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(emptyRecords)); + FetchResponse resp1 = FetchResponse.of(Errors.NONE, 0, 123, partitions1); client.prepareResponse(resp1); assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); @@ -3246,8 +3289,8 @@ public void testConsumingViaIncrementalFetchRequests() { assertEquals(4L, subscriptions.position(tp0).offset); // The second response contains no new records. - LinkedHashMap> partitions2 = new LinkedHashMap<>(); - FetchResponse resp2 = new FetchResponse<>(Errors.NONE, partitions2, 0, 123); + LinkedHashMap partitions2 = new LinkedHashMap<>(); + FetchResponse resp2 = FetchResponse.of(Errors.NONE, 0, 123, partitions2); client.prepareResponse(resp2); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(time.timer(0)); @@ -3257,10 +3300,14 @@ public void testConsumingViaIncrementalFetchRequests() { assertEquals(1L, subscriptions.position(tp1).offset); // The third response contains some new records for tp0. - LinkedHashMap> partitions3 = new LinkedHashMap<>(); - partitions3.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100L, - 4, 0L, null, this.nextRecords)); - FetchResponse resp3 = new FetchResponse<>(Errors.NONE, partitions3, 0, 123); + LinkedHashMap partitions3 = new LinkedHashMap<>(); + partitions3.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(this.nextRecords)); + FetchResponse resp3 = FetchResponse.of(Errors.NONE, 0, 123, partitions3); client.prepareResponse(resp3); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(time.timer(0)); @@ -3319,7 +3366,7 @@ public Builder newBuilder() { } @Override - public boolean handleResponse(FetchResponse response) { + public boolean handleResponse(FetchResponse response) { verifySessionPartitions(); return handler.handleResponse(response); } @@ -3367,14 +3414,18 @@ private void verifySessionPartitions() { if (!client.requests().isEmpty()) { ClientRequest request = client.requests().peek(); FetchRequest fetchRequest = (FetchRequest) request.requestBuilder().build(); - LinkedHashMap> responseMap = new LinkedHashMap<>(); + LinkedHashMap responseMap = new LinkedHashMap<>(); for (Map.Entry entry : fetchRequest.fetchData().entrySet()) { TopicPartition tp = entry.getKey(); long offset = entry.getValue().fetchOffset; - responseMap.put(tp, new FetchResponse.PartitionData<>(Errors.NONE, offset + 2L, offset + 2, - 0L, null, buildRecords(offset, 2, offset))); + responseMap.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setHighWatermark(offset + 2) + .setLastStableOffset(offset + 2) + .setLogStartOffset(0) + .setRecords(buildRecords(offset, 2, offset))); } - client.respondToRequest(request, new FetchResponse<>(Errors.NONE, responseMap, 0, 123)); + client.respondToRequest(request, FetchResponse.of(Errors.NONE, 0, 123, responseMap)); consumerClient.poll(time.timer(0)); } } @@ -3429,11 +3480,15 @@ public void testFetcherSessionEpochUpdate() throws Exception { assertTrue(epoch == 0 || epoch == nextEpoch, String.format("Unexpected epoch expected %d got %d", nextEpoch, epoch)); nextEpoch++; - LinkedHashMap> responseMap = new LinkedHashMap<>(); - responseMap.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, nextOffset + 2L, nextOffset + 2, - 0L, null, buildRecords(nextOffset, 2, nextOffset))); + LinkedHashMap responseMap = new LinkedHashMap<>(); + responseMap.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(nextOffset + 2) + .setLastStableOffset(nextOffset + 2) + .setLogStartOffset(0) + .setRecords(buildRecords(nextOffset, 2, nextOffset))); nextOffset += 2; - client.respondToRequest(request, new FetchResponse<>(Errors.NONE, responseMap, 0, 123)); + client.respondToRequest(request, FetchResponse.of(Errors.NONE, 0, 123, responseMap)); consumerClient.poll(time.timer(0)); } } @@ -3483,7 +3538,6 @@ public void testEmptyControlBatch() { commitTransaction(buffer, 1L, currentOffset); buffer.flip(); - List abortedTransactions = new ArrayList<>(); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -3496,7 +3550,7 @@ public void testEmptyControlBatch() { FetchRequest request = (FetchRequest) body; assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); return true; - }, fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -4463,41 +4517,66 @@ private ListOffsetsResponse listOffsetResponse(Map offsets return new ListOffsetsResponse(data); } - private FetchResponse fullFetchResponseWithAbortedTransactions(MemoryRecords records, - List abortedTransactions, + private FetchResponse fullFetchResponseWithAbortedTransactions(MemoryRecords records, + List abortedTransactions, Errors error, long lastStableOffset, long hw, int throttleTime) { - Map> partitions = Collections.singletonMap(tp0, - new FetchResponse.PartitionData<>(error, hw, lastStableOffset, 0L, abortedTransactions, records)); - return new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); - } - - private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + Map partitions = Collections.singletonMap(tp0, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setAbortedTransactions(abortedTransactions) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { return fullFetchResponse(tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); } - private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, + private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime) { - Map> partitions = Collections.singletonMap(tp, - new FetchResponse.PartitionData<>(error, hw, lastStableOffset, 0L, null, records)); - return new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); - } - - private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime, Optional preferredReplicaId) { - Map> partitions = Collections.singletonMap(tp, - new FetchResponse.PartitionData<>(error, hw, lastStableOffset, 0L, - preferredReplicaId, null, records)); - return new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); - } - - private FetchResponse fetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setRecords(records) + .setPreferredReadReplica(preferredReplicaId.orElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID))); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, long logStartOffset, int throttleTime) { - Map> partitions = Collections.singletonMap(tp, - new FetchResponse.PartitionData<>(error, hw, lastStableOffset, logStartOffset, null, records)); - return new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(logStartOffset) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); } private MetadataResponse newMetadataResponse(String topic, Errors error) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 228f32afcea94..cebb99d17ef22 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -803,15 +803,18 @@ public void produceRequestGetErrorResponseTest() { @Test public void fetchResponseVersionTest() { - LinkedHashMap> responseData = new LinkedHashMap<>(); + LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10)); - responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>( - Errors.NONE, 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, - 0L, Optional.empty(), Collections.emptyList(), records)); - - FetchResponse v0Response = new FetchResponse<>(Errors.NONE, responseData, 0, INVALID_SESSION_ID); - FetchResponse v1Response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); + responseData.put(new TopicPartition("test", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setRecords(records)); + + FetchResponse v0Response = FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, responseData); + FetchResponse v1Response = FetchResponse.of(Errors.NONE, 10, INVALID_SESSION_ID, responseData); assertEquals(0, v0Response.throttleTimeMs(), "Throttle time must be zero"); assertEquals(10, v1Response.throttleTimeMs(), "Throttle time must be 10"); assertEquals(responseData, v0Response.responseData(), "Response data does not match"); @@ -820,22 +823,34 @@ public void fetchResponseVersionTest() { @Test public void testFetchResponseV4() { - LinkedHashMap> responseData = new LinkedHashMap<>(); + LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10)); - List abortedTransactions = asList( - new FetchResponse.AbortedTransaction(10, 100), - new FetchResponse.AbortedTransaction(15, 50) + List abortedTransactions = asList( + new FetchResponseData.AbortedTransaction().setProducerId(10).setFirstOffset(100), + new FetchResponseData.AbortedTransaction().setProducerId(15).setFirstOffset(50) ); - responseData.put(new TopicPartition("bar", 0), new FetchResponse.PartitionData<>(Errors.NONE, 100000, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), abortedTransactions, records)); - responseData.put(new TopicPartition("bar", 1), new FetchResponse.PartitionData<>(Errors.NONE, 900000, - 5, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, records)); - responseData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData<>(Errors.NONE, 70000, - 6, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), emptyList(), records)); - - FetchResponse response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); - FetchResponse deserialized = FetchResponse.parse(response.serialize((short) 4), (short) 4); + responseData.put(new TopicPartition("bar", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(1000000) + .setAbortedTransactions(abortedTransactions) + .setRecords(records)); + responseData.put(new TopicPartition("bar", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(900000) + .setLastStableOffset(5) + .setRecords(records)); + responseData.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(70000) + .setLastStableOffset(6) + .setRecords(records)); + + FetchResponse response = FetchResponse.of(Errors.NONE, 10, INVALID_SESSION_ID, responseData); + FetchResponse deserialized = FetchResponse.parse(response.serialize((short) 4), (short) 4); assertEquals(responseData, deserialized.responseData()); } @@ -849,7 +864,7 @@ public void verifyFetchResponseFullWrites() throws Exception { } } - private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchResponse) throws Exception { + private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchResponse) throws Exception { int correlationId = 15; short responseHeaderVersion = FETCH.responseHeaderVersion(apiVersion); @@ -1158,38 +1173,49 @@ private FetchRequest createFetchRequest(int version) { return FetchRequest.Builder.forConsumer(100, 100000, fetchData).setMaxBytes(1000).build((short) version); } - private FetchResponse createFetchResponse(Errors error, int sessionId) { - return new FetchResponse<>(error, new LinkedHashMap<>(), 25, sessionId); + private FetchResponse createFetchResponse(Errors error, int sessionId) { + return FetchResponse.of(error, 25, sessionId, new LinkedHashMap<>()); } - private FetchResponse createFetchResponse(int sessionId) { - LinkedHashMap> responseData = new LinkedHashMap<>(); + private FetchResponse createFetchResponse(int sessionId) { + LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("blah".getBytes())); - responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>(Errors.NONE, - 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, Optional.empty(), Collections.emptyList(), records)); - List abortedTransactions = Collections.singletonList( - new FetchResponse.AbortedTransaction(234L, 999L)); - responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData<>(Errors.NONE, - 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, Optional.empty(), abortedTransactions, MemoryRecords.EMPTY)); - return new FetchResponse<>(Errors.NONE, responseData, 25, sessionId); - } - - private FetchResponse createFetchResponse(boolean includeAborted) { - LinkedHashMap> responseData = new LinkedHashMap<>(); + responseData.put(new TopicPartition("test", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setRecords(records)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(234L).setFirstOffset(999L)); + responseData.put(new TopicPartition("test", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setAbortedTransactions(abortedTransactions)); + return FetchResponse.of(Errors.NONE, 25, sessionId, responseData); + } + + private FetchResponse createFetchResponse(boolean includeAborted) { + LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("blah".getBytes())); + responseData.put(new TopicPartition("test", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setRecords(records)); - responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>(Errors.NONE, - 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, Optional.empty(), Collections.emptyList(), records)); - - List abortedTransactions = Collections.emptyList(); + List abortedTransactions = Collections.emptyList(); if (includeAborted) { abortedTransactions = Collections.singletonList( - new FetchResponse.AbortedTransaction(234L, 999L)); + new FetchResponseData.AbortedTransaction().setProducerId(234L).setFirstOffset(999L)); } - responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData<>(Errors.NONE, - 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, Optional.empty(), abortedTransactions, MemoryRecords.EMPTY)); + responseData.put(new TopicPartition("test", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setAbortedTransactions(abortedTransactions)); - return new FetchResponse<>(Errors.NONE, responseData, 25, INVALID_SESSION_ID); + return FetchResponse.of(Errors.NONE, 25, INVALID_SESSION_ID, responseData); } private HeartbeatRequest createHeartBeatRequest() { diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index 5ab2fac3e75b2..0ad82f46b9d4b 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -40,7 +40,6 @@ import org.apache.kafka.common.errors._ import org.apache.kafka.common.message.{DescribeProducersResponseData, FetchResponseData} import org.apache.kafka.common.record.FileRecords.TimestampAndOffset import org.apache.kafka.common.record._ -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction import org.apache.kafka.common.requests.ListOffsetsRequest import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.UNDEFINED_EPOCH_OFFSET import org.apache.kafka.common.requests.ProduceResponse.RecordError @@ -1572,7 +1571,7 @@ class Log(@volatile private var _dir: File, private def emptyFetchDataInfo(fetchOffsetMetadata: LogOffsetMetadata, includeAbortedTxns: Boolean): FetchDataInfo = { val abortedTransactions = - if (includeAbortedTxns) Some(List.empty[AbortedTransaction]) + if (includeAbortedTxns) Some(List.empty[FetchResponseData.AbortedTransaction]) else None FetchDataInfo(fetchOffsetMetadata, MemoryRecords.EMPTY, @@ -1676,7 +1675,7 @@ class Log(@volatile private var _dir: File, logEndOffset } - val abortedTransactions = ListBuffer.empty[AbortedTransaction] + val abortedTransactions = ListBuffer.empty[FetchResponseData.AbortedTransaction] def accumulator(abortedTxns: List[AbortedTxn]): Unit = abortedTransactions ++= abortedTxns.map(_.asAbortedTransaction) collectAbortedTransactions(startOffset, upperBoundOffset, segmentEntry, accumulator) diff --git a/core/src/main/scala/kafka/log/TransactionIndex.scala b/core/src/main/scala/kafka/log/TransactionIndex.scala index 9152bc41ab353..565c4eb574060 100644 --- a/core/src/main/scala/kafka/log/TransactionIndex.scala +++ b/core/src/main/scala/kafka/log/TransactionIndex.scala @@ -20,10 +20,9 @@ import java.io.{File, IOException} import java.nio.ByteBuffer import java.nio.channels.FileChannel import java.nio.file.{Files, StandardOpenOption} - import kafka.utils.{Logging, nonthreadsafe} import org.apache.kafka.common.KafkaException -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.utils.Utils import scala.collection.mutable.ListBuffer @@ -245,7 +244,9 @@ private[log] class AbortedTxn(val buffer: ByteBuffer) { def lastStableOffset: Long = buffer.getLong(LastStableOffsetOffset) - def asAbortedTransaction: AbortedTransaction = new AbortedTransaction(producerId, firstOffset) + def asAbortedTransaction: FetchResponseData.AbortedTransaction = new FetchResponseData.AbortedTransaction() + .setProducerId(producerId) + .setFirstOffset(firstOffset) override def toString: String = s"AbortedTxn(version=$version, producerId=$producerId, firstOffset=$firstOffset, " + diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 40d0bd62e622d..a2a2144f0128e 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -192,7 +192,7 @@ object RequestChannel extends Logging { resources.add(newResource) } val data = new IncrementalAlterConfigsRequestData() - .setValidateOnly(alterConfigs.data().validateOnly()) + .setValidateOnly(alterConfigs.data.validateOnly()) .setResources(resources) new IncrementalAlterConfigsRequest.Builder(data).build(alterConfigs.version) diff --git a/core/src/main/scala/kafka/network/RequestConvertToJson.scala b/core/src/main/scala/kafka/network/RequestConvertToJson.scala index 2f23dbc2b2f79..c0fbd68e75bf2 100644 --- a/core/src/main/scala/kafka/network/RequestConvertToJson.scala +++ b/core/src/main/scala/kafka/network/RequestConvertToJson.scala @@ -135,7 +135,7 @@ object RequestConvertToJson { case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version) case res: EnvelopeResponse => EnvelopeResponseDataJsonConverter.write(res.data, version) case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version) - case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false) + case res: FetchResponse => FetchResponseDataJsonConverter.write(res.data, version, false) case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version) case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version) case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version) diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala index 5e716e29e94f0..6118c5f85cbcf 100755 --- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala +++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala @@ -17,37 +17,33 @@ package kafka.server -import java.nio.ByteBuffer -import java.util -import java.util.Optional -import java.util.concurrent.locks.ReentrantLock - import kafka.cluster.BrokerEndPoint -import kafka.utils.{DelayedItem, Pool, ShutdownableThread} -import kafka.utils.Implicits._ -import org.apache.kafka.common.errors._ import kafka.common.ClientIdAndBroker +import kafka.log.LogAppendInfo import kafka.metrics.KafkaMetricsGroup +import kafka.server.AbstractFetcherThread.{ReplicaFetch, ResultWithPartitions} import kafka.utils.CoreUtils.inLock -import org.apache.kafka.common.protocol.Errors - -import scala.collection.{Map, Set, mutable} -import scala.compat.java8.OptionConverters._ -import scala.jdk.CollectionConverters._ -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicLong - -import kafka.log.LogAppendInfo -import kafka.server.AbstractFetcherThread.ReplicaFetch -import kafka.server.AbstractFetcherThread.ResultWithPartitions -import org.apache.kafka.common.{InvalidRecordException, TopicPartition} +import kafka.utils.Implicits._ +import kafka.utils.{DelayedItem, Pool, ShutdownableThread} +import org.apache.kafka.common.errors._ import org.apache.kafka.common.internals.PartitionStates -import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset +import org.apache.kafka.common.message.{FetchResponseData, OffsetForLeaderEpochRequestData} +import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.record.{FileRecords, MemoryRecords, Records} -import org.apache.kafka.common.requests._ import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET} +import org.apache.kafka.common.requests._ +import org.apache.kafka.common.{InvalidRecordException, TopicPartition} +import java.nio.ByteBuffer +import java.util +import java.util.Optional +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.locks.ReentrantLock +import scala.collection.{Map, Set, mutable} +import scala.compat.java8.OptionConverters._ +import scala.jdk.CollectionConverters._ import scala.math._ /** @@ -62,7 +58,7 @@ abstract class AbstractFetcherThread(name: String, val brokerTopicStats: BrokerTopicStats) //BrokerTopicStats's lifecycle managed by ReplicaManager extends ShutdownableThread(name, isInterruptible) { - type FetchData = FetchResponse.PartitionData[Records] + type FetchData = FetchResponseData.PartitionData type EpochData = OffsetForLeaderEpochRequestData.OffsetForLeaderPartition private val partitionStates = new PartitionStates[PartitionFetchState] @@ -340,7 +336,7 @@ abstract class AbstractFetcherThread(name: String, // the current offset is the same as the offset requested. val fetchPartitionData = sessionPartitions.get(topicPartition) if (fetchPartitionData != null && fetchPartitionData.fetchOffset == currentFetchState.fetchOffset && currentFetchState.isReadyForFetch) { - partitionData.error match { + Errors.forCode(partitionData.errorCode) match { case Errors.NONE => try { // Once we hand off the partition data to the subclass, we can't mess with it any more in this thread @@ -364,7 +360,7 @@ abstract class AbstractFetcherThread(name: String, } } if (isTruncationOnFetchSupported) { - partitionData.divergingEpoch.ifPresent { divergingEpoch => + FetchResponse.divergingEpoch(partitionData).ifPresent { divergingEpoch => divergingEndOffsets += topicPartition -> new EpochEndOffset() .setPartition(topicPartition.partition) .setErrorCode(Errors.NONE.code) @@ -416,9 +412,8 @@ abstract class AbstractFetcherThread(name: String, "expected to persist.") partitionsWithError += topicPartition - case _ => - error(s"Error for partition $topicPartition at offset ${currentFetchState.fetchOffset}", - partitionData.error.exception) + case partitionError => + error(s"Error for partition $topicPartition at offset ${currentFetchState.fetchOffset}", partitionError.exception) partitionsWithError += topicPartition } } diff --git a/core/src/main/scala/kafka/server/ControllerApis.scala b/core/src/main/scala/kafka/server/ControllerApis.scala index 53a41dc919a1a..5670448765b29 100644 --- a/core/src/main/scala/kafka/server/ControllerApis.scala +++ b/core/src/main/scala/kafka/server/ControllerApis.scala @@ -34,7 +34,6 @@ import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicRe import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseBroker import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, BrokerHeartbeatResponseData, BrokerRegistrationResponseData, CreateTopicsResponseData, DescribeQuorumResponseData, EndQuorumEpochResponseData, FetchResponseData, MetadataResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, UnregisterBrokerResponseData, VoteResponseData} import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage, Errors} -import org.apache.kafka.common.record.BaseRecords import org.apache.kafka.common.requests._ import org.apache.kafka.common.resource.Resource import org.apache.kafka.common.resource.Resource.CLUSTER_NAME @@ -116,7 +115,7 @@ class ControllerApis(val requestChannel: RequestChannel, private def handleFetch(request: RequestChannel.Request): Unit = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) - handleRaftRequest(request, response => new FetchResponse[BaseRecords](response.asInstanceOf[FetchResponseData])) + handleRaftRequest(request, response => new FetchResponse(response.asInstanceOf[FetchResponseData])) } def handleMetadataRequest(request: RequestChannel.Request): Unit = { diff --git a/core/src/main/scala/kafka/server/FetchDataInfo.scala b/core/src/main/scala/kafka/server/FetchDataInfo.scala index a55fe23e1eca6..f6cf725843ef9 100644 --- a/core/src/main/scala/kafka/server/FetchDataInfo.scala +++ b/core/src/main/scala/kafka/server/FetchDataInfo.scala @@ -17,8 +17,8 @@ package kafka.server +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.record.Records -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction sealed trait FetchIsolation case object FetchLogEnd extends FetchIsolation @@ -28,4 +28,4 @@ case object FetchTxnCommitted extends FetchIsolation case class FetchDataInfo(fetchOffsetMetadata: LogOffsetMetadata, records: Records, firstEntryIncomplete: Boolean = false, - abortedTransactions: Option[List[AbortedTransaction]] = None) + abortedTransactions: Option[List[FetchResponseData.AbortedTransaction]] = None) diff --git a/core/src/main/scala/kafka/server/FetchSession.scala b/core/src/main/scala/kafka/server/FetchSession.scala index c6280f384dfd7..a6f6fa8e4f72b 100644 --- a/core/src/main/scala/kafka/server/FetchSession.scala +++ b/core/src/main/scala/kafka/server/FetchSession.scala @@ -17,27 +17,26 @@ package kafka.server -import java.util -import java.util.Optional -import java.util.concurrent.{ThreadLocalRandom, TimeUnit} - import kafka.metrics.KafkaMetricsGroup import kafka.utils.Logging import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.Records import org.apache.kafka.common.requests.FetchMetadata.{FINAL_EPOCH, INITIAL_EPOCH, INVALID_SESSION_ID} import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.utils.{ImplicitLinkedHashCollection, Time, Utils} -import scala.math.Ordered.orderingToOrdered +import java.util +import java.util.Optional +import java.util.concurrent.{ThreadLocalRandom, TimeUnit} import scala.collection.{mutable, _} +import scala.math.Ordered.orderingToOrdered object FetchSession { type REQ_MAP = util.Map[TopicPartition, FetchRequest.PartitionData] - type RESP_MAP = util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + type RESP_MAP = util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] type CACHE_MAP = ImplicitLinkedHashCollection[CachedPartition] - type RESP_MAP_ITER = util.Iterator[util.Map.Entry[TopicPartition, FetchResponse.PartitionData[Records]]] + type RESP_MAP_ITER = util.Iterator[util.Map.Entry[TopicPartition, FetchResponseData.PartitionData]] val NUM_INCREMENTAL_FETCH_SESSISONS = "NumIncrementalFetchSessions" val NUM_INCREMENTAL_FETCH_PARTITIONS_CACHED = "NumIncrementalFetchPartitionsCached" @@ -100,7 +99,7 @@ class CachedPartition(val topic: String, reqData.currentLeaderEpoch, reqData.logStartOffset, -1, reqData.lastFetchedEpoch) def this(part: TopicPartition, reqData: FetchRequest.PartitionData, - respData: FetchResponse.PartitionData[Records]) = + respData: FetchResponseData.PartitionData) = this(part.topic, part.partition, reqData.maxBytes, reqData.fetchOffset, respData.highWatermark, reqData.currentLeaderEpoch, reqData.logStartOffset, respData.logStartOffset, reqData.lastFetchedEpoch) @@ -125,10 +124,10 @@ class CachedPartition(val topic: String, * @param updateResponseData if set to true, update this CachedPartition with new request and response data. * @return True if this partition should be included in the response; false if it can be omitted. */ - def maybeUpdateResponseData(respData: FetchResponse.PartitionData[Records], updateResponseData: Boolean): Boolean = { + def maybeUpdateResponseData(respData: FetchResponseData.PartitionData, updateResponseData: Boolean): Boolean = { // Check the response data. var mustRespond = false - if ((respData.records != null) && (respData.records.sizeInBytes > 0)) { + if (FetchResponse.recordsSize(respData) > 0) { // Partitions with new data are always included in the response. mustRespond = true } @@ -142,11 +141,11 @@ class CachedPartition(val topic: String, if (updateResponseData) localLogStartOffset = respData.logStartOffset } - if (respData.preferredReadReplica.isPresent) { + if (FetchResponse.isPreferredReplica(respData)) { // If the broker computed a preferred read replica, we need to include it in the response mustRespond = true } - if (respData.error.code != 0) { + if (respData.errorCode != Errors.NONE.code) { // Partitions with errors are always included in the response. // We also set the cached highWatermark to an invalid offset, -1. // This ensures that when the error goes away, we re-send the partition. @@ -154,7 +153,8 @@ class CachedPartition(val topic: String, highWatermark = -1 mustRespond = true } - if (respData.divergingEpoch.isPresent) { + + if (FetchResponse.isDivergingEpoch(respData)) { // Partitions with diverging epoch are always included in response to trigger truncation. mustRespond = true } @@ -163,7 +163,7 @@ class CachedPartition(val topic: String, override def hashCode: Int = (31 * partition) + topic.hashCode - def canEqual(that: Any) = that.isInstanceOf[CachedPartition] + def canEqual(that: Any): Boolean = that.isInstanceOf[CachedPartition] override def equals(that: Any): Boolean = that match { @@ -292,7 +292,7 @@ trait FetchContext extends Logging { * Updates the fetch context with new partition information. Generates response data. * The response data may require subsequent down-conversion. */ - def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] + def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse def partitionsToLogString(partitions: util.Collection[TopicPartition]): String = FetchSession.partitionsToLogString(partitions, isTraceEnabled) @@ -300,8 +300,8 @@ trait FetchContext extends Logging { /** * Return an empty throttled response due to quota violation. */ - def getThrottledResponse(throttleTimeMs: Int): FetchResponse[Records] = - new FetchResponse(Errors.NONE, new FetchSession.RESP_MAP, throttleTimeMs, INVALID_SESSION_ID) + def getThrottledResponse(throttleTimeMs: Int): FetchResponse = + FetchResponse.of(Errors.NONE, throttleTimeMs, INVALID_SESSION_ID, new FetchSession.RESP_MAP) } /** @@ -318,9 +318,9 @@ class SessionErrorContext(val error: Errors, } // Because of the fetch session error, we don't know what partitions were supposed to be in this request. - override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] = { + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { debug(s"Session error fetch context returning $error") - new FetchResponse(error, new FetchSession.RESP_MAP, 0, INVALID_SESSION_ID) + FetchResponse.of(error, 0, INVALID_SESSION_ID, new FetchSession.RESP_MAP) } } @@ -341,9 +341,9 @@ class SessionlessFetchContext(val fetchData: util.Map[TopicPartition, FetchReque FetchResponse.sizeOf(versionId, updates.entrySet.iterator) } - override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] = { + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { debug(s"Sessionless fetch context returning ${partitionsToLogString(updates.keySet)}") - new FetchResponse(Errors.NONE, updates, 0, INVALID_SESSION_ID) + FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, updates) } } @@ -372,7 +372,7 @@ class FullFetchContext(private val time: Time, FetchResponse.sizeOf(versionId, updates.entrySet.iterator) } - override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] = { + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { def createNewSession: FetchSession.CACHE_MAP = { val cachedPartitions = new FetchSession.CACHE_MAP(updates.size) updates.forEach { (part, respData) => @@ -385,7 +385,7 @@ class FullFetchContext(private val time: Time, updates.size, () => createNewSession) debug(s"Full fetch context with session id $responseSessionId returning " + s"${partitionsToLogString(updates.keySet)}") - new FetchResponse(Errors.NONE, updates, 0, responseSessionId) + FetchResponse.of(Errors.NONE, 0, responseSessionId, updates) } } @@ -417,7 +417,7 @@ class IncrementalFetchContext(private val time: Time, private class PartitionIterator(val iter: FetchSession.RESP_MAP_ITER, val updateFetchContextAndRemoveUnselected: Boolean) extends FetchSession.RESP_MAP_ITER { - var nextElement: util.Map.Entry[TopicPartition, FetchResponse.PartitionData[Records]] = null + var nextElement: util.Map.Entry[TopicPartition, FetchResponseData.PartitionData] = null override def hasNext: Boolean = { while ((nextElement == null) && iter.hasNext) { @@ -441,7 +441,7 @@ class IncrementalFetchContext(private val time: Time, nextElement != null } - override def next(): util.Map.Entry[TopicPartition, FetchResponse.PartitionData[Records]] = { + override def next(): util.Map.Entry[TopicPartition, FetchResponseData.PartitionData] = { if (!hasNext) throw new NoSuchElementException val element = nextElement nextElement = null @@ -463,7 +463,7 @@ class IncrementalFetchContext(private val time: Time, } } - override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] = { + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { session.synchronized { // Check to make sure that the session epoch didn't change in between // creating this fetch context and generating this response. @@ -471,7 +471,7 @@ class IncrementalFetchContext(private val time: Time, if (session.epoch != expectedEpoch) { info(s"Incremental fetch session ${session.id} expected epoch $expectedEpoch, but " + s"got ${session.epoch}. Possible duplicate request.") - new FetchResponse(Errors.INVALID_FETCH_SESSION_EPOCH, new FetchSession.RESP_MAP, 0, session.id) + FetchResponse.of(Errors.INVALID_FETCH_SESSION_EPOCH, 0, session.id, new FetchSession.RESP_MAP) } else { // Iterate over the update list using PartitionIterator. This will prune updates which don't need to be sent val partitionIter = new PartitionIterator(updates.entrySet.iterator, true) @@ -480,12 +480,12 @@ class IncrementalFetchContext(private val time: Time, } debug(s"Incremental fetch context with session id ${session.id} returning " + s"${partitionsToLogString(updates.keySet)}") - new FetchResponse(Errors.NONE, updates, 0, session.id) + FetchResponse.of(Errors.NONE, 0, session.id, updates) } } } - override def getThrottledResponse(throttleTimeMs: Int): FetchResponse[Records] = { + override def getThrottledResponse(throttleTimeMs: Int): FetchResponse = { session.synchronized { // Check to make sure that the session epoch didn't change in between // creating this fetch context and generating this response. @@ -493,9 +493,9 @@ class IncrementalFetchContext(private val time: Time, if (session.epoch != expectedEpoch) { info(s"Incremental fetch session ${session.id} expected epoch $expectedEpoch, but " + s"got ${session.epoch}. Possible duplicate request.") - new FetchResponse(Errors.INVALID_FETCH_SESSION_EPOCH, new FetchSession.RESP_MAP, throttleTimeMs, session.id) + FetchResponse.of(Errors.INVALID_FETCH_SESSION_EPOCH, throttleTimeMs, session.id, new FetchSession.RESP_MAP) } else { - new FetchResponse(Errors.NONE, new FetchSession.RESP_MAP, throttleTimeMs, session.id) + FetchResponse.of(Errors.NONE, throttleTimeMs, session.id, new FetchSession.RESP_MAP) } } } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 22054a3f57388..de7bb0b58fe10 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -17,24 +17,18 @@ package kafka.server -import java.lang.{Long => JLong} -import java.nio.ByteBuffer -import java.util -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.{Collections, Optional} - import kafka.admin.AdminUtils import kafka.api.{ApiVersion, ElectLeadersRequestOps, KAFKA_0_11_0_IV0, KAFKA_2_3_IV0} import kafka.common.OffsetAndMetadata import kafka.controller.ReplicaAssignment -import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult, LeaveGroupResult, SyncGroupResult} +import kafka.coordinator.group._ import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator} import kafka.log.AppendOrigin import kafka.message.ZStdCompressionCodec import kafka.network.RequestChannel import kafka.security.authorizer.AuthorizerUtils import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota} +import kafka.server.metadata.ConfigRepository import kafka.utils.Implicits._ import kafka.utils.{CoreUtils, Logging} import org.apache.kafka.clients.admin.AlterConfigOp.OpType @@ -61,7 +55,7 @@ import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsParti import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic} import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult, OffsetForLeaderTopicResultCollection} -import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, DescribeTransactionsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} +import org.apache.kafka.common.message._ import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.{ListenerName, Send} import org.apache.kafka.common.protocol.{ApiKeys, Errors} @@ -80,14 +74,18 @@ import org.apache.kafka.common.utils.{ProducerIdAndEpoch, Time} import org.apache.kafka.common.{Node, TopicPartition, Uuid} import org.apache.kafka.server.authorizer._ +import java.lang.{Long => JLong} +import java.nio.ByteBuffer +import java.util +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.{Collections, Optional} import scala.annotation.nowarn import scala.collection.mutable.ArrayBuffer import scala.collection.{Map, Seq, Set, immutable, mutable} import scala.compat.java8.OptionConverters._ import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try} -import kafka.coordinator.group.GroupOverview -import kafka.server.metadata.ConfigRepository /** * Logic to handle the various Kafka requests @@ -681,25 +679,20 @@ class KafkaApis(val requestChannel: RequestChannel, None } - def errorResponse[T >: MemoryRecords <: BaseRecords](error: Errors): FetchResponse.PartitionData[T] = { - new FetchResponse.PartitionData[T](error, FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, - FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) - } - - val erroneous = mutable.ArrayBuffer[(TopicPartition, FetchResponse.PartitionData[Records])]() + val erroneous = mutable.ArrayBuffer[(TopicPartition, FetchResponseData.PartitionData)]() val interesting = mutable.ArrayBuffer[(TopicPartition, FetchRequest.PartitionData)]() if (fetchRequest.isFromFollower) { // The follower must have ClusterAction on ClusterResource in order to fetch partition data. if (authHelper.authorize(request.context, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) { fetchContext.foreachPartition { (topicPartition, data) => if (!metadataCache.contains(topicPartition)) - erroneous += topicPartition -> errorResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION) + erroneous += topicPartition -> FetchResponse.partitionResponse(topicPartition.partition, Errors.UNKNOWN_TOPIC_OR_PARTITION) else interesting += (topicPartition -> data) } } else { fetchContext.foreachPartition { (part, _) => - erroneous += part -> errorResponse(Errors.TOPIC_AUTHORIZATION_FAILED) + erroneous += part -> FetchResponse.partitionResponse(part.partition, Errors.TOPIC_AUTHORIZATION_FAILED) } } } else { @@ -711,9 +704,9 @@ class KafkaApis(val requestChannel: RequestChannel, val authorizedTopics = authHelper.filterByAuthorized(request.context, READ, TOPIC, partitionDatas)(_._1.topic) partitionDatas.foreach { case (topicPartition, data) => if (!authorizedTopics.contains(topicPartition.topic)) - erroneous += topicPartition -> errorResponse(Errors.TOPIC_AUTHORIZATION_FAILED) + erroneous += topicPartition -> FetchResponse.partitionResponse(topicPartition.partition, Errors.TOPIC_AUTHORIZATION_FAILED) else if (!metadataCache.contains(topicPartition)) - erroneous += topicPartition -> errorResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION) + erroneous += topicPartition -> FetchResponse.partitionResponse(topicPartition.partition, Errors.UNKNOWN_TOPIC_OR_PARTITION) else interesting += (topicPartition -> data) } @@ -732,12 +725,12 @@ class KafkaApis(val requestChannel: RequestChannel, } def maybeConvertFetchedData(tp: TopicPartition, - partitionData: FetchResponse.PartitionData[Records]): FetchResponse.PartitionData[BaseRecords] = { + partitionData: FetchResponseData.PartitionData): FetchResponseData.PartitionData = { val logConfig = replicaManager.getLogConfig(tp) if (logConfig.exists(_.compressionType == ZStdCompressionCodec.name) && versionId < 10) { trace(s"Fetching messages is disabled for ZStandard compressed partition $tp. Sending unsupported version response to $clientId.") - errorResponse(Errors.UNSUPPORTED_COMPRESSION_TYPE) + FetchResponse.partitionResponse(tp.partition, Errors.UNSUPPORTED_COMPRESSION_TYPE) } else { // Down-conversion of the fetched records is needed when the stored magic version is // greater than that supported by the client (as indicated by the fetch request version). If the @@ -746,7 +739,7 @@ class KafkaApis(val requestChannel: RequestChannel, // know it must be supported. However, if the magic version is changed from a higher version back to a // lower version, this check will no longer be valid and we will fail to down-convert the messages // which were written in the new format prior to the version downgrade. - val unconvertedRecords = partitionData.records + val unconvertedRecords = FetchResponse.recordsOrFail(partitionData) val downConvertMagic = logConfig.map(_.messageFormatVersion.recordVersion.value).flatMap { magic => if (magic > RecordBatch.MAGIC_VALUE_V0 && versionId <= 1 && !unconvertedRecords.hasCompatibleMagic(RecordBatch.MAGIC_VALUE_V0)) @@ -762,7 +755,7 @@ class KafkaApis(val requestChannel: RequestChannel, // For fetch requests from clients, check if down-conversion is disabled for the particular partition if (!fetchRequest.isFromFollower && !logConfig.forall(_.messageDownConversionEnable)) { trace(s"Conversion to message format ${downConvertMagic.get} is disabled for partition $tp. Sending unsupported version response to $clientId.") - errorResponse(Errors.UNSUPPORTED_VERSION) + FetchResponse.partitionResponse(tp.partition, Errors.UNSUPPORTED_VERSION) } else { try { trace(s"Down converting records from partition $tp to message format version $magic for fetch request from $clientId") @@ -770,71 +763,77 @@ class KafkaApis(val requestChannel: RequestChannel, // as possible. With KIP-283, we have the ability to lazily down-convert in a chunked manner. The lazy, chunked // down-conversion always guarantees that at least one batch of messages is down-converted and sent out to the // client. - val error = maybeDownConvertStorageError(partitionData.error) - new FetchResponse.PartitionData[BaseRecords](error, partitionData.highWatermark, - partitionData.lastStableOffset, partitionData.logStartOffset, - partitionData.preferredReadReplica, partitionData.abortedTransactions, - new LazyDownConversionRecords(tp, unconvertedRecords, magic, fetchContext.getFetchOffset(tp).get, time)) + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setErrorCode(maybeDownConvertStorageError(Errors.forCode(partitionData.errorCode)).code) + .setHighWatermark(partitionData.highWatermark) + .setLastStableOffset(partitionData.lastStableOffset) + .setLogStartOffset(partitionData.logStartOffset) + .setAbortedTransactions(partitionData.abortedTransactions) + .setRecords(new LazyDownConversionRecords(tp, unconvertedRecords, magic, fetchContext.getFetchOffset(tp).get, time)) + .setPreferredReadReplica(partitionData.preferredReadReplica()) } catch { case e: UnsupportedCompressionTypeException => trace("Received unsupported compression type error during down-conversion", e) - errorResponse(Errors.UNSUPPORTED_COMPRESSION_TYPE) + FetchResponse.partitionResponse(tp.partition, Errors.UNSUPPORTED_COMPRESSION_TYPE) } } case None => - val error = maybeDownConvertStorageError(partitionData.error) - new FetchResponse.PartitionData[BaseRecords](error, - partitionData.highWatermark, - partitionData.lastStableOffset, - partitionData.logStartOffset, - partitionData.preferredReadReplica, - partitionData.abortedTransactions, - partitionData.divergingEpoch, - unconvertedRecords) + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setErrorCode(maybeDownConvertStorageError(Errors.forCode(partitionData.errorCode)).code) + .setHighWatermark(partitionData.highWatermark) + .setLastStableOffset(partitionData.lastStableOffset) + .setLogStartOffset(partitionData.logStartOffset) + .setAbortedTransactions(partitionData.abortedTransactions) + .setRecords(unconvertedRecords) + .setPreferredReadReplica(partitionData.preferredReadReplica) + .setDivergingEpoch(partitionData.divergingEpoch) } } } // the callback for process a fetch response, invoked before throttling def processResponseCallback(responsePartitionData: Seq[(TopicPartition, FetchPartitionData)]): Unit = { - val partitions = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + val partitions = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] val reassigningPartitions = mutable.Set[TopicPartition]() responsePartitionData.foreach { case (tp, data) => val abortedTransactions = data.abortedTransactions.map(_.asJava).orNull val lastStableOffset = data.lastStableOffset.getOrElse(FetchResponse.INVALID_LAST_STABLE_OFFSET) - if (data.isReassignmentFetch) - reassigningPartitions.add(tp) - val error = maybeDownConvertStorageError(data.error) - partitions.put(tp, new FetchResponse.PartitionData( - error, - data.highWatermark, - lastStableOffset, - data.logStartOffset, - data.preferredReadReplica.map(int2Integer).asJava, - abortedTransactions, - data.divergingEpoch.asJava, - data.records)) + if (data.isReassignmentFetch) reassigningPartitions.add(tp) + val partitionData = new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setErrorCode(maybeDownConvertStorageError(data.error).code) + .setHighWatermark(data.highWatermark) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(data.logStartOffset) + .setAbortedTransactions(abortedTransactions) + .setRecords(data.records) + .setPreferredReadReplica(data.preferredReadReplica.getOrElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID)) + data.divergingEpoch.foreach(partitionData.setDivergingEpoch) + partitions.put(tp, partitionData) } erroneous.foreach { case (tp, data) => partitions.put(tp, data) } - var unconvertedFetchResponse: FetchResponse[Records] = null + var unconvertedFetchResponse: FetchResponse = null - def createResponse(throttleTimeMs: Int): FetchResponse[BaseRecords] = { + def createResponse(throttleTimeMs: Int): FetchResponse = { // Down-convert messages for each partition if required - val convertedData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[BaseRecords]] + val convertedData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] unconvertedFetchResponse.responseData.forEach { (tp, unconvertedPartitionData) => - if (unconvertedPartitionData.error != Errors.NONE) + val error = Errors.forCode(unconvertedPartitionData.errorCode) + if (error != Errors.NONE) debug(s"Fetch request with correlation id ${request.header.correlationId} from client $clientId " + - s"on partition $tp failed due to ${unconvertedPartitionData.error.exceptionName}") + s"on partition $tp failed due to ${error.exceptionName}") convertedData.put(tp, maybeConvertFetchedData(tp, unconvertedPartitionData)) } // Prepare fetch response from converted data - val response = new FetchResponse(unconvertedFetchResponse.error, convertedData, throttleTimeMs, - unconvertedFetchResponse.sessionId) + val response = FetchResponse.of(unconvertedFetchResponse.error, throttleTimeMs, unconvertedFetchResponse.sessionId, convertedData) // record the bytes out metrics only when the response is being sent response.responseData.forEach { (tp, data) => - brokerTopicStats.updateBytesOut(tp.topic, fetchRequest.isFromFollower, reassigningPartitions.contains(tp), data.records.sizeInBytes) + brokerTopicStats.updateBytesOut(tp.topic, fetchRequest.isFromFollower, + reassigningPartitions.contains(tp), FetchResponse.recordsSize(data)) } response } @@ -3367,7 +3366,7 @@ object KafkaApis { // Traffic from both in-sync and out of sync replicas are accounted for in replication quota to ensure total replication // traffic doesn't exceed quota. private[server] def sizeOfThrottledPartitions(versionId: Short, - unconvertedResponse: FetchResponse[Records], + unconvertedResponse: FetchResponse, quota: ReplicationQuotaManager): Int = { FetchResponse.sizeOf(versionId, unconvertedResponse.responseData.entrySet .iterator.asScala.filter(element => quota.isThrottled(element.getKey)).asJava) diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala index dae3fc226d15c..f12ae9e4ca2f3 100644 --- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala +++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala @@ -17,28 +17,24 @@ package kafka.server -import java.util -import java.util.Optional - import kafka.api.Request import kafka.cluster.BrokerEndPoint import kafka.log.{LeaderOffsetIncremented, LogAppendInfo} -import kafka.server.AbstractFetcherThread.ReplicaFetch -import kafka.server.AbstractFetcherThread.ResultWithPartitions +import kafka.server.AbstractFetcherThread.{ReplicaFetch, ResultWithPartitions} import kafka.server.QuotaFactory.UnboundedQuota import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.errors.KafkaStorageException +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.Records -import org.apache.kafka.common.requests.FetchResponse.PartitionData -import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.UNDEFINED_EPOCH -import org.apache.kafka.common.requests.RequestUtils +import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, RequestUtils} -import scala.jdk.CollectionConverters._ +import java.util +import java.util.Optional import scala.collection.{Map, Seq, Set, mutable} import scala.compat.java8.OptionConverters._ +import scala.jdk.CollectionConverters._ class ReplicaAlterLogDirsThread(name: String, sourceBroker: BrokerEndPoint, @@ -77,15 +73,21 @@ class ReplicaAlterLogDirsThread(name: String, } def fetchFromLeader(fetchRequest: FetchRequest.Builder): Map[TopicPartition, FetchData] = { - var partitionData: Seq[(TopicPartition, FetchResponse.PartitionData[Records])] = null + var partitionData: Seq[(TopicPartition, FetchData)] = null val request = fetchRequest.build() def processResponseCallback(responsePartitionData: Seq[(TopicPartition, FetchPartitionData)]): Unit = { partitionData = responsePartitionData.map { case (tp, data) => val abortedTransactions = data.abortedTransactions.map(_.asJava).orNull val lastStableOffset = data.lastStableOffset.getOrElse(FetchResponse.INVALID_LAST_STABLE_OFFSET) - tp -> new FetchResponse.PartitionData(data.error, data.highWatermark, lastStableOffset, - data.logStartOffset, abortedTransactions, data.records) + tp -> new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setErrorCode(data.error.code) + .setHighWatermark(data.highWatermark) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(data.logStartOffset) + .setAbortedTransactions(abortedTransactions) + .setRecords(data.records) } } @@ -110,10 +112,10 @@ class ReplicaAlterLogDirsThread(name: String, // process fetched data override def processPartitionData(topicPartition: TopicPartition, fetchOffset: Long, - partitionData: PartitionData[Records]): Option[LogAppendInfo] = { + partitionData: FetchData): Option[LogAppendInfo] = { val partition = replicaMgr.getPartitionOrException(topicPartition) val futureLog = partition.futureLocalLogOrException - val records = toMemoryRecords(partitionData.records) + val records = toMemoryRecords(FetchResponse.recordsOrFail(partitionData)) if (fetchOffset != futureLog.logEndOffset) throw new IllegalStateException("Offset mismatch for the future replica %s: fetched offset = %d, log end offset = %d.".format( diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala index 103fdb81eefbc..170b5b9a889d5 100644 --- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala +++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala @@ -35,7 +35,7 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetFor import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.{MemoryRecords, Records} +import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests._ import org.apache.kafka.common.utils.{LogContext, Time} @@ -162,7 +162,7 @@ class ReplicaFetcherThread(name: String, val logTrace = isTraceEnabled val partition = replicaMgr.getPartitionOrException(topicPartition) val log = partition.localLogOrException - val records = toMemoryRecords(partitionData.records) + val records = toMemoryRecords(FetchResponse.recordsOrFail(partitionData)) maybeWarnIfOversizedRecords(records, topicPartition) @@ -215,7 +215,7 @@ class ReplicaFetcherThread(name: String, override protected def fetchFromLeader(fetchRequest: FetchRequest.Builder): Map[TopicPartition, FetchData] = { try { val clientResponse = leaderEndpoint.sendRequest(fetchRequest) - val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse[Records]] + val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse] if (!fetchSessionHandler.handleResponse(fetchResponse)) { Map.empty } else { diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 87b446208536c..069059b335e79 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -57,7 +57,6 @@ import org.apache.kafka.common.replica.PartitionView.DefaultPartitionView import org.apache.kafka.common.replica.ReplicaView.DefaultReplicaView import org.apache.kafka.common.replica.{ClientMetadata, _} import org.apache.kafka.common.requests.FetchRequest.PartitionData -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.common.requests._ import org.apache.kafka.common.utils.Time @@ -150,7 +149,7 @@ case class FetchPartitionData(error: Errors = Errors.NONE, records: Records, divergingEpoch: Option[FetchResponseData.EpochEndOffset], lastStableOffset: Option[Long], - abortedTransactions: Option[List[AbortedTransaction]], + abortedTransactions: Option[List[FetchResponseData.AbortedTransaction]], preferredReadReplica: Option[Int], isReassignmentFetch: Boolean) diff --git a/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala b/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala index a00753eb273f7..9dd1b05b5c1fe 100644 --- a/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala +++ b/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala @@ -17,33 +17,31 @@ package kafka.tools -import java.net.SocketTimeoutException -import java.text.SimpleDateFormat -import java.util -import java.util.concurrent.CountDownLatch -import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} -import java.util.regex.{Pattern, PatternSyntaxException} -import java.util.{Date, Optional, Properties} - import joptsimple.OptionParser import kafka.api._ -import kafka.utils.IncludeList -import kafka.utils._ +import kafka.utils.{IncludeList, _} import org.apache.kafka.clients._ import org.apache.kafka.clients.admin.{Admin, ListTopicsOptions, TopicDescription} import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer} +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.{NetworkReceive, Selectable, Selector} import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.AbstractRequest.Builder import org.apache.kafka.common.requests.{AbstractRequest, FetchResponse, ListOffsetsRequest, FetchRequest => JFetchRequest} import org.apache.kafka.common.serialization.StringDeserializer import org.apache.kafka.common.utils.{LogContext, Time} import org.apache.kafka.common.{Node, TopicPartition} -import scala.jdk.CollectionConverters._ +import java.net.SocketTimeoutException +import java.text.SimpleDateFormat +import java.util +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} +import java.util.regex.{Pattern, PatternSyntaxException} +import java.util.{Date, Optional, Properties} import scala.collection.Seq +import scala.jdk.CollectionConverters._ /** * For verifying the consistency among replicas. @@ -261,7 +259,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: collection.Map[To expectedNumFetchers: Int, reportInterval: Long) extends Logging { private val fetchOffsetMap = new Pool[TopicPartition, Long] - private val recordsCache = new Pool[TopicPartition, Pool[Int, FetchResponse.PartitionData[MemoryRecords]]] + private val recordsCache = new Pool[TopicPartition, Pool[Int, FetchResponseData.PartitionData]] private val fetcherBarrier = new AtomicReference(new CountDownLatch(expectedNumFetchers)) private val verificationBarrier = new AtomicReference(new CountDownLatch(1)) @volatile private var lastReportTime = Time.SYSTEM.milliseconds @@ -284,7 +282,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: collection.Map[To private def initialize(): Unit = { for (topicPartition <- expectedReplicasPerTopicPartition.keySet) - recordsCache.put(topicPartition, new Pool[Int, FetchResponse.PartitionData[MemoryRecords]]) + recordsCache.put(topicPartition, new Pool[Int, FetchResponseData.PartitionData]) setInitialOffsets() } @@ -294,7 +292,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: collection.Map[To fetchOffsetMap.put(tp, offset) } - def addFetchedData(topicAndPartition: TopicPartition, replicaId: Int, partitionData: FetchResponse.PartitionData[MemoryRecords]): Unit = { + def addFetchedData(topicAndPartition: TopicPartition, replicaId: Int, partitionData: FetchResponseData.PartitionData): Unit = { recordsCache.get(topicAndPartition).put(replicaId, partitionData) } @@ -311,7 +309,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: collection.Map[To "fetched " + fetchResponsePerReplica.size + " replicas for " + topicPartition + ", but expected " + expectedReplicasPerTopicPartition(topicPartition) + " replicas") val recordBatchIteratorMap = fetchResponsePerReplica.map { case (replicaId, fetchResponse) => - replicaId -> fetchResponse.records.batches.iterator + replicaId -> FetchResponse.recordsOrFail(fetchResponse).batches.iterator } val maxHw = fetchResponsePerReplica.values.map(_.highWatermark).max @@ -403,10 +401,10 @@ private class ReplicaFetcher(name: String, sourceBroker: Node, topicPartitions: debug("Issuing fetch request ") - var fetchResponse: FetchResponse[MemoryRecords] = null + var fetchResponse: FetchResponse = null try { val clientResponse = fetchEndpoint.sendRequest(fetchRequestBuilder) - fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse[MemoryRecords]] + fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse] } catch { case t: Throwable => if (!isRunning) @@ -414,14 +412,13 @@ private class ReplicaFetcher(name: String, sourceBroker: Node, topicPartitions: } if (fetchResponse != null) { - fetchResponse.responseData.forEach { (tp, partitionData) => - replicaBuffer.addFetchedData(tp, sourceBroker.id, partitionData) - } + fetchResponse.data.responses.forEach(topicResponse => + topicResponse.partitions.forEach(partitionResponse => + replicaBuffer.addFetchedData(new TopicPartition(topicResponse.topic, partitionResponse.partitionIndex), + sourceBroker.id, partitionResponse))) } else { - val emptyResponse = new FetchResponse.PartitionData(Errors.NONE, FetchResponse.INVALID_HIGHWATERMARK, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) for (topicAndPartition <- topicPartitions) - replicaBuffer.addFetchedData(topicAndPartition, sourceBroker.id, emptyResponse) + replicaBuffer.addFetchedData(topicAndPartition, sourceBroker.id, FetchResponse.partitionResponse(topicAndPartition.partition, Errors.NONE)) } fetcherBarrier.countDown() diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index db825ff7d91d2..91d5ecd3997ac 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -24,7 +24,6 @@ import kafka.utils.Logging import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, EndQuorumEpochResponseData, FetchResponseData, FetchSnapshotResponseData, VoteResponseData} import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage} -import org.apache.kafka.common.record.BaseRecords import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, BeginQuorumEpochResponse, EndQuorumEpochResponse, FetchResponse, FetchSnapshotResponse, VoteResponse} import org.apache.kafka.common.utils.Time @@ -81,7 +80,7 @@ class TestRaftRequestHandler( } private def handleFetch(request: RequestChannel.Request): Unit = { - handle(request, response => new FetchResponse[BaseRecords](response.asInstanceOf[FetchResponseData])) + handle(request, response => new FetchResponse(response.asInstanceOf[FetchResponseData])) } private def handleFetchSnapshot(request: RequestChannel.Request): Unit = { diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index bb4266c98a09b..2eefa4419e5bf 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -49,7 +49,7 @@ import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadata import org.apache.kafka.common.message.{AddOffsetsToTxnRequestData, AlterPartitionReassignmentsRequestData, AlterReplicaLogDirsRequestData, ControlledShutdownRequestData, CreateAclsRequestData, CreatePartitionsRequestData, CreateTopicsRequestData, DeleteAclsRequestData, DeleteGroupsRequestData, DeleteRecordsRequestData, DeleteTopicsRequestData, DescribeClusterRequestData, DescribeConfigsRequestData, DescribeGroupsRequestData, DescribeLogDirsRequestData, DescribeProducersRequestData, DescribeTransactionsRequestData, FindCoordinatorRequestData, HeartbeatRequestData, IncrementalAlterConfigsRequestData, JoinGroupRequestData, ListPartitionReassignmentsRequestData, ListTransactionsRequestData, MetadataRequestData, OffsetCommitRequestData, ProduceRequestData, SyncGroupRequestData} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, Records, SimpleRecord} +import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, SimpleRecord} import org.apache.kafka.common.requests._ import org.apache.kafka.common.resource.PatternType.{LITERAL, PREFIXED} import org.apache.kafka.common.resource.ResourceType._ @@ -159,7 +159,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { val requestKeyToError = (topicNames: Map[Uuid, String]) => Map[ApiKeys, Nothing => Errors]( ApiKeys.METADATA -> ((resp: requests.MetadataResponse) => resp.errors.asScala.find(_._1 == topic).getOrElse(("test", Errors.NONE))._2), ApiKeys.PRODUCE -> ((resp: requests.ProduceResponse) => resp.responses.asScala.find(_._1 == tp).get._2.error), - ApiKeys.FETCH -> ((resp: requests.FetchResponse[Records]) => resp.responseData.asScala.find(_._1 == tp).get._2.error), + ApiKeys.FETCH -> ((resp: requests.FetchResponse) => Errors.forCode(resp.responseData.asScala.find(_._1 == tp).get._2.errorCode)), ApiKeys.LIST_OFFSETS -> ((resp: ListOffsetsResponse) => { Errors.forCode( resp.data @@ -169,12 +169,12 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ) }), ApiKeys.OFFSET_COMMIT -> ((resp: requests.OffsetCommitResponse) => Errors.forCode( - resp.data().topics().get(0).partitions().get(0).errorCode())), + resp.data.topics().get(0).partitions().get(0).errorCode)), ApiKeys.OFFSET_FETCH -> ((resp: requests.OffsetFetchResponse) => resp.error), ApiKeys.FIND_COORDINATOR -> ((resp: FindCoordinatorResponse) => resp.error), ApiKeys.UPDATE_METADATA -> ((resp: requests.UpdateMetadataResponse) => resp.error), ApiKeys.JOIN_GROUP -> ((resp: JoinGroupResponse) => resp.error), - ApiKeys.SYNC_GROUP -> ((resp: SyncGroupResponse) => Errors.forCode(resp.data.errorCode())), + ApiKeys.SYNC_GROUP -> ((resp: SyncGroupResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.DESCRIBE_GROUPS -> ((resp: DescribeGroupsResponse) => { Errors.forCode(resp.data.groups.asScala.find(g => group == g.groupId).head.errorCode) }), @@ -187,8 +187,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.STOP_REPLICA -> ((resp: requests.StopReplicaResponse) => Errors.forCode( resp.partitionErrors.asScala.find(pe => pe.topicName == tp.topic && pe.partitionIndex == tp.partition).get.errorCode)), ApiKeys.CONTROLLED_SHUTDOWN -> ((resp: requests.ControlledShutdownResponse) => resp.error), - ApiKeys.CREATE_TOPICS -> ((resp: CreateTopicsResponse) => Errors.forCode(resp.data.topics.find(topic).errorCode())), - ApiKeys.DELETE_TOPICS -> ((resp: requests.DeleteTopicsResponse) => Errors.forCode(resp.data.responses.find(topic).errorCode())), + ApiKeys.CREATE_TOPICS -> ((resp: CreateTopicsResponse) => Errors.forCode(resp.data.topics.find(topic).errorCode)), + ApiKeys.DELETE_TOPICS -> ((resp: requests.DeleteTopicsResponse) => Errors.forCode(resp.data.responses.find(topic).errorCode)), ApiKeys.DELETE_RECORDS -> ((resp: requests.DeleteRecordsResponse) => Errors.forCode( resp.data.topics.find(tp.topic).partitions.find(tp.partition).errorCode)), ApiKeys.OFFSET_FOR_LEADER_EPOCH -> ((resp: OffsetsForLeaderEpochResponse) => Errors.forCode( @@ -211,17 +211,17 @@ class AuthorizerIntegrationTest extends BaseRequestTest { .find(p => p.partitionIndex == tp.partition).get.errorCode)), ApiKeys.DESCRIBE_LOG_DIRS -> ((resp: DescribeLogDirsResponse) => if (resp.data.results.size() > 0) Errors.forCode(resp.data.results.get(0).errorCode) else Errors.CLUSTER_AUTHORIZATION_FAILED), - ApiKeys.CREATE_PARTITIONS -> ((resp: CreatePartitionsResponse) => Errors.forCode(resp.data.results.asScala.head.errorCode())), - ApiKeys.ELECT_LEADERS -> ((resp: ElectLeadersResponse) => Errors.forCode(resp.data().errorCode())), + ApiKeys.CREATE_PARTITIONS -> ((resp: CreatePartitionsResponse) => Errors.forCode(resp.data.results.asScala.head.errorCode)), + ApiKeys.ELECT_LEADERS -> ((resp: ElectLeadersResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.INCREMENTAL_ALTER_CONFIGS -> ((resp: IncrementalAlterConfigsResponse) => { - val topicResourceError = IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)) + val topicResourceError = IncrementalAlterConfigsResponse.fromResponseData(resp.data).get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)) if (topicResourceError == null) - IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.BROKER_LOGGER, brokerId.toString)).error + IncrementalAlterConfigsResponse.fromResponseData(resp.data).get(new ConfigResource(ConfigResource.Type.BROKER_LOGGER, brokerId.toString)).error else topicResourceError.error() }), - ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> ((resp: AlterPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode())), - ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode())), + ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> ((resp: AlterPartitionReassignmentsResponse) => Errors.forCode(resp.data.errorCode)), + ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.OFFSET_DELETE -> ((resp: OffsetDeleteResponse) => { Errors.forCode( resp.data @@ -326,9 +326,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest { requests.ProduceRequest.forCurrentMagic(new ProduceRequestData() .setTopicData(new ProduceRequestData.TopicProduceDataCollection( Collections.singletonList(new ProduceRequestData.TopicProduceData() - .setName(tp.topic()).setPartitionData(Collections.singletonList( + .setName(tp.topic).setPartitionData(Collections.singletonList( new ProduceRequestData.PartitionProduceData() - .setIndex(tp.partition()) + .setIndex(tp.partition) .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("test".getBytes)))))) .iterator)) .setAcks(1.toShort) @@ -363,9 +363,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest { private def offsetsForLeaderEpochRequest: OffsetsForLeaderEpochRequest = { val epochs = new OffsetForLeaderTopicCollection() epochs.add(new OffsetForLeaderTopic() - .setTopic(tp.topic()) + .setTopic(tp.topic) .setPartitions(List(new OffsetForLeaderPartition() - .setPartition(tp.partition()) + .setPartition(tp.partition) .setLeaderEpoch(7) .setCurrentLeaderEpoch(27)).asJava)) OffsetsForLeaderEpochRequest.Builder.forConsumer(epochs).build() @@ -509,9 +509,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest { private def stopReplicaRequest: StopReplicaRequest = { val topicStates = Seq( new StopReplicaTopicState() - .setTopicName(tp.topic()) + .setTopicName(tp.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(tp.partition()) + .setPartitionIndex(tp.partition) .setLeaderEpoch(LeaderAndIsr.initialLeaderEpoch + 2) .setDeletePartition(true)).asJava) ).asJava @@ -658,7 +658,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { List(new AlterPartitionReassignmentsRequestData.ReassignableTopic() .setName(topic) .setPartitions( - List(new AlterPartitionReassignmentsRequestData.ReassignablePartition().setPartitionIndex(tp.partition())).asJava + List(new AlterPartitionReassignmentsRequestData.ReassignablePartition().setPartitionIndex(tp.partition)).asJava )).asJava ) ).build() @@ -1625,7 +1625,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { @Test def testUnauthorizedCreatePartitions(): Unit = { val createPartitionsResponse = connectAndReceive[CreatePartitionsResponse](createPartitionsRequest) - assertEquals(Errors.TOPIC_AUTHORIZATION_FAILED.code(), createPartitionsResponse.data.results.asScala.head.errorCode()) + assertEquals(Errors.TOPIC_AUTHORIZATION_FAILED.code, createPartitionsResponse.data.results.asScala.head.errorCode) } @Test @@ -1633,7 +1633,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { createTopic(topic) addAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, ALTER, ALLOW)), new ResourcePattern(TOPIC, "*", LITERAL)) val createPartitionsResponse = connectAndReceive[CreatePartitionsResponse](createPartitionsRequest) - assertEquals(Errors.NONE.code(), createPartitionsResponse.data.results.asScala.head.errorCode()) + assertEquals(Errors.NONE.code, createPartitionsResponse.data.results.asScala.head.errorCode) } @Test @@ -2133,7 +2133,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { numRecords: Int, tp: TopicPartition): Unit = { val futures = (0 until numRecords).map { i => - producer.send(new ProducerRecord(tp.topic(), tp.partition(), i.toString.getBytes, i.toString.getBytes)) + producer.send(new ProducerRecord(tp.topic, tp.partition, i.toString.getBytes, i.toString.getBytes)) } try { futures.foreach(_.get) diff --git a/core/src/test/scala/kafka/tools/ReplicaVerificationToolTest.scala b/core/src/test/scala/kafka/tools/ReplicaVerificationToolTest.scala index cb98116f0a0c0..217260403df4e 100644 --- a/core/src/test/scala/kafka/tools/ReplicaVerificationToolTest.scala +++ b/core/src/test/scala/kafka/tools/ReplicaVerificationToolTest.scala @@ -18,9 +18,8 @@ package kafka.tools import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.record.{CompressionType, MemoryRecords, SimpleRecord} -import org.apache.kafka.common.requests.FetchResponse import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions.assertTrue @@ -44,7 +43,12 @@ class ReplicaVerificationToolTest { } val initialOffset = 4 val memoryRecords = MemoryRecords.withRecords(initialOffset, CompressionType.NONE, records: _*) - val partitionData = new FetchResponse.PartitionData(Errors.NONE, 20, 20, 0L, null, memoryRecords) + val partitionData = new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setHighWatermark(20) + .setLastStableOffset(20) + .setLogStartOffset(0) + .setRecords(memoryRecords) replicaBuffer.addFetchedData(tp, replicaId, partitionData) } diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index a76345fd730e5..1e5257df49853 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -34,11 +34,11 @@ import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchI import kafka.utils._ import org.apache.kafka.common.{InvalidRecordException, KafkaException, TopicPartition, Uuid} import org.apache.kafka.common.errors._ +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.record.FileRecords.TimestampAndOffset import org.apache.kafka.common.record.MemoryRecords.RecordFilter import org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention import org.apache.kafka.common.record._ -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction import org.apache.kafka.common.requests.{ListOffsetsRequest, ListOffsetsResponse} import org.apache.kafka.common.utils.{BufferSupplier, Time, Utils} import org.easymock.EasyMock @@ -4725,7 +4725,8 @@ class LogTest { assertEquals(1, fetchDataInfo.abortedTransactions.size) assertTrue(fetchDataInfo.abortedTransactions.isDefined) - assertEquals(new AbortedTransaction(pid, 0), fetchDataInfo.abortedTransactions.get.head) + assertEquals(new FetchResponseData.AbortedTransaction().setProducerId(pid).setFirstOffset(0), + fetchDataInfo.abortedTransactions.get.head) } @Test diff --git a/core/src/test/scala/unit/kafka/log/TransactionIndexTest.scala b/core/src/test/scala/unit/kafka/log/TransactionIndexTest.scala index 05f23cfbe3c83..790bcd88a9a09 100644 --- a/core/src/test/scala/unit/kafka/log/TransactionIndexTest.scala +++ b/core/src/test/scala/unit/kafka/log/TransactionIndexTest.scala @@ -17,7 +17,7 @@ package kafka.log import kafka.utils.TestUtils -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction +import org.apache.kafka.common.message.FetchResponseData import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} @@ -136,7 +136,7 @@ class TransactionIndexTest { assertEquals(abortedTransactions.take(3), index.collectAbortedTxns(0L, 100L).abortedTransactions) index.reset() - assertEquals(List.empty[AbortedTransaction], index.collectAbortedTxns(0L, 100L).abortedTransactions) + assertEquals(List.empty[FetchResponseData.AbortedTransaction], index.collectAbortedTxns(0L, 100L).abortedTransactions) } @Test diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala index 9b9fe8e12fa43..394897f41e2f3 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala @@ -20,7 +20,6 @@ package kafka.server import java.nio.ByteBuffer import java.util.Optional import java.util.concurrent.atomic.AtomicInteger - import kafka.cluster.BrokerEndPoint import kafka.log.LogAppendInfo import kafka.message.NoCompressionCodec @@ -37,7 +36,7 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEnd import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record._ import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET} -import org.apache.kafka.common.requests.FetchRequest +import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} import org.apache.kafka.common.utils.Time import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{BeforeEach, Test} @@ -907,8 +906,8 @@ class AbstractFetcherThreadTest { partitionData: FetchData): Option[LogAppendInfo] = { val state = replicaPartitionState(topicPartition) - if (isTruncationOnFetchSupported && partitionData.divergingEpoch.isPresent) { - val divergingEpoch = partitionData.divergingEpoch.get + if (isTruncationOnFetchSupported && FetchResponse.isDivergingEpoch(partitionData)) { + val divergingEpoch = partitionData.divergingEpoch truncateOnFetchResponse(Map(topicPartition -> new EpochEndOffset() .setPartition(topicPartition.partition) .setErrorCode(Errors.NONE.code) @@ -923,7 +922,7 @@ class AbstractFetcherThreadTest { s"fetched offset = $fetchOffset, log end offset = ${state.logEndOffset}.") // Now check message's crc - val batches = partitionData.records.batches.asScala + val batches = FetchResponse.recordsOrFail(partitionData).batches.asScala var maxTimestamp = RecordBatch.NO_TIMESTAMP var offsetOfMaxTimestamp = -1L var lastOffset = state.logEndOffset @@ -955,7 +954,7 @@ class AbstractFetcherThreadTest { sourceCodec = NoCompressionCodec, targetCodec = NoCompressionCodec, shallowCount = batches.size, - validBytes = partitionData.records.sizeInBytes, + validBytes = FetchResponse.recordsSize(partitionData), offsetsMonotonic = true, lastOffsetOfFirstBatch = batches.headOption.map(_.lastOffset).getOrElse(-1))) } @@ -1143,9 +1142,16 @@ class AbstractFetcherThreadTest { (Errors.NONE, records) } + val partitionData = new FetchData() + .setPartitionIndex(partition.partition) + .setErrorCode(error.code) + .setHighWatermark(leaderState.highWatermark) + .setLastStableOffset(leaderState.highWatermark) + .setLogStartOffset(leaderState.logStartOffset) + .setRecords(records) + divergingEpoch.foreach(partitionData.setDivergingEpoch) - (partition, new FetchData(error, leaderState.highWatermark, leaderState.highWatermark, leaderState.logStartOffset, - Optional.empty[Integer], List.empty.asJava, divergingEpoch.asJava, records)) + (partition, partitionData) }.toMap } diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala index 1971fbef4d4d2..03ed0181fd216 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala @@ -23,7 +23,6 @@ import kafka.utils.TestUtils import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} import org.apache.kafka.common.serialization.StringSerializer import org.junit.jupiter.api.Assertions._ @@ -79,8 +78,8 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { partitionMap } - private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse[MemoryRecords] = { - connectAndReceive[FetchResponse[MemoryRecords]](request, destination = brokerSocketServer(leaderId)) + private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse = { + connectAndReceive[FetchResponse](request, destination = brokerSocketServer(leaderId)) } /** @@ -90,11 +89,11 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { def testV1FetchWithDownConversionDisabled(): Unit = { val topicMap = createTopics(numTopics = 5, numPartitions = 1) val topicPartitions = topicMap.keySet.toSeq - topicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic(), "key", "value")).get()) + topicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic, "key", "value")).get()) val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(1024, topicPartitions)).build(1) val fetchResponse = sendFetchRequest(topicMap.head._2, fetchRequest) - topicPartitions.foreach(tp => assertEquals(Errors.UNSUPPORTED_VERSION, fetchResponse.responseData().get(tp).error)) + topicPartitions.foreach(tp => assertEquals(Errors.UNSUPPORTED_VERSION, Errors.forCode(fetchResponse.responseData.get(tp).errorCode))) } /** @@ -104,11 +103,11 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { def testLatestFetchWithDownConversionDisabled(): Unit = { val topicMap = createTopics(numTopics = 5, numPartitions = 1) val topicPartitions = topicMap.keySet.toSeq - topicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic(), "key", "value")).get()) + topicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic, "key", "value")).get()) val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(1024, topicPartitions)).build() val fetchResponse = sendFetchRequest(topicMap.head._2, fetchRequest) - topicPartitions.foreach(tp => assertEquals(Errors.NONE, fetchResponse.responseData().get(tp).error)) + topicPartitions.foreach(tp => assertEquals(Errors.NONE, Errors.forCode(fetchResponse.responseData.get(tp).errorCode))) } /** @@ -129,13 +128,13 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { val allTopics = conversionDisabledTopicPartitions ++ conversionEnabledTopicPartitions val leaderId = conversionDisabledTopicsMap.head._2 - allTopics.foreach(tp => producer.send(new ProducerRecord(tp.topic(), "key", "value")).get()) + allTopics.foreach(tp => producer.send(new ProducerRecord(tp.topic, "key", "value")).get()) val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(1024, allTopics)).build(1) val fetchResponse = sendFetchRequest(leaderId, fetchRequest) - conversionDisabledTopicPartitions.foreach(tp => assertEquals(Errors.UNSUPPORTED_VERSION, fetchResponse.responseData().get(tp).error)) - conversionEnabledTopicPartitions.foreach(tp => assertEquals(Errors.NONE, fetchResponse.responseData().get(tp).error)) + conversionDisabledTopicPartitions.foreach(tp => assertEquals(Errors.UNSUPPORTED_VERSION.code, fetchResponse.responseData.get(tp).errorCode)) + conversionEnabledTopicPartitions.foreach(tp => assertEquals(Errors.NONE.code, fetchResponse.responseData.get(tp).errorCode)) } /** @@ -155,11 +154,11 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { val allTopicPartitions = conversionDisabledTopicPartitions ++ conversionEnabledTopicPartitions val leaderId = conversionDisabledTopicsMap.head._2 - allTopicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic(), "key", "value")).get()) + allTopicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic, "key", "value")).get()) val fetchRequest = FetchRequest.Builder.forReplica(1, 1, Int.MaxValue, 0, createPartitionMap(1024, allTopicPartitions)).build() val fetchResponse = sendFetchRequest(leaderId, fetchRequest) - allTopicPartitions.foreach(tp => assertEquals(Errors.NONE, fetchResponse.responseData().get(tp).error)) + allTopicPartitions.foreach(tp => assertEquals(Errors.NONE.code, fetchResponse.responseData.get(tp).errorCode)) } } diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestMaxBytesTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestMaxBytesTest.scala index 5889cbb33eda8..c919f7f6714cf 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestMaxBytesTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestMaxBytesTest.scala @@ -17,17 +17,16 @@ package kafka.server -import java.util.{Optional, Properties} import kafka.log.LogConfig import kafka.utils.TestUtils import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.FetchRequest.PartitionData import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} +import java.util.{Optional, Properties} import scala.jdk.CollectionConverters._ /** @@ -92,8 +91,8 @@ class FetchRequestMaxBytesTest extends BaseRequestTest { }) } - private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse[MemoryRecords] = { - connectAndReceive[FetchResponse[MemoryRecords]](request, destination = brokerSocketServer(leaderId)) + private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse = { + connectAndReceive[FetchResponse](request, destination = brokerSocketServer(leaderId)) } /** @@ -117,7 +116,7 @@ class FetchRequestMaxBytesTest extends BaseRequestTest { FetchRequest.Builder.forConsumer(Int.MaxValue, 0, Map(testTopicPartition -> new PartitionData(fetchOffset, 0, Integer.MAX_VALUE, Optional.empty())).asJava).build(3)) - val records = response.responseData().get(testTopicPartition).records.records() + val records = FetchResponse.recordsOrFail(response.responseData.get(testTopicPartition)).records() assertNotNull(records) val recordsList = records.asScala.toList assertEquals(expected.size, recordsList.size) diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala index c4b3a2841c150..b7dde352d6bab 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala @@ -16,24 +16,25 @@ */ package kafka.server -import java.io.DataInputStream -import java.util -import java.util.{Optional, Properties} import kafka.api.KAFKA_0_11_0_IV2 import kafka.log.LogConfig import kafka.message.{GZIPCompressionCodec, ProducerCompressionCodec, ZStdCompressionCodec} import kafka.utils.TestUtils import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord, RecordMetadata} +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.{MemoryRecords, Record, RecordBatch} +import org.apache.kafka.common.record.{Record, RecordBatch} import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.serialization.{ByteArraySerializer, StringSerializer} import org.apache.kafka.common.{IsolationLevel, TopicPartition} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} -import scala.jdk.CollectionConverters._ +import java.io.DataInputStream +import java.util +import java.util.{Optional, Properties} import scala.collection.Seq +import scala.jdk.CollectionConverters._ import scala.util.Random /** @@ -70,8 +71,8 @@ class FetchRequestTest extends BaseRequestTest { partitionMap } - private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse[MemoryRecords] = { - connectAndReceive[FetchResponse[MemoryRecords]](request, destination = brokerSocketServer(leaderId)) + private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse = { + connectAndReceive[FetchResponse](request, destination = brokerSocketServer(leaderId)) } private def initProducer(): Unit = { @@ -133,12 +134,12 @@ class FetchRequestTest extends BaseRequestTest { }.sum assertTrue(responseSize3 <= maxResponseBytes) val partitionData3 = fetchResponse3.responseData.get(partitionWithLargeMessage1) - assertEquals(Errors.NONE, partitionData3.error) + assertEquals(Errors.NONE.code, partitionData3.errorCode) assertTrue(partitionData3.highWatermark > 0) val size3 = records(partitionData3).map(_.sizeInBytes).sum assertTrue(size3 <= maxResponseBytes, s"Expected $size3 to be smaller than $maxResponseBytes") assertTrue(size3 > maxPartitionBytes, s"Expected $size3 to be larger than $maxPartitionBytes") - assertTrue(maxPartitionBytes < partitionData3.records.sizeInBytes) + assertTrue(maxPartitionBytes < FetchResponse.recordsSize(partitionData3)) // 4. Partition with message larger than the response limit at the start of the list val shuffledTopicPartitions4 = Seq(partitionWithLargeMessage2, partitionWithLargeMessage1) ++ @@ -151,11 +152,11 @@ class FetchRequestTest extends BaseRequestTest { } assertEquals(Seq(partitionWithLargeMessage2), nonEmptyPartitions4) val partitionData4 = fetchResponse4.responseData.get(partitionWithLargeMessage2) - assertEquals(Errors.NONE, partitionData4.error) + assertEquals(Errors.NONE.code, partitionData4.errorCode) assertTrue(partitionData4.highWatermark > 0) val size4 = records(partitionData4).map(_.sizeInBytes).sum assertTrue(size4 > maxResponseBytes, s"Expected $size4 to be larger than $maxResponseBytes") - assertTrue(maxResponseBytes < partitionData4.records.sizeInBytes) + assertTrue(maxResponseBytes < FetchResponse.recordsSize(partitionData4)) } @Test @@ -169,9 +170,9 @@ class FetchRequestTest extends BaseRequestTest { Seq(topicPartition))).build(2) val fetchResponse = sendFetchRequest(leaderId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertTrue(partitionData.highWatermark > 0) - assertEquals(maxPartitionBytes, partitionData.records.sizeInBytes) + assertEquals(maxPartitionBytes, FetchResponse.recordsSize(partitionData)) assertEquals(0, records(partitionData).map(_.sizeInBytes).sum) } @@ -186,7 +187,7 @@ class FetchRequestTest extends BaseRequestTest { Seq(topicPartition))).isolationLevel(IsolationLevel.READ_COMMITTED).build(4) val fetchResponse = sendFetchRequest(leaderId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertTrue(partitionData.lastStableOffset > 0) assertTrue(records(partitionData).map(_.sizeInBytes).sum > 0) } @@ -209,7 +210,7 @@ class FetchRequestTest extends BaseRequestTest { Seq(topicPartition))).build() val fetchResponse = sendFetchRequest(nonReplicaId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, partitionData.error) + assertEquals(Errors.NOT_LEADER_OR_FOLLOWER.code, partitionData.errorCode) } @Test @@ -243,11 +244,11 @@ class FetchRequestTest extends BaseRequestTest { // Validate the expected truncation val fetchResponse = sendFetchRequest(secondLeaderId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) - assertEquals(0L, partitionData.records.sizeInBytes()) - assertTrue(partitionData.divergingEpoch.isPresent) + assertEquals(Errors.NONE.code, partitionData.errorCode) + assertEquals(0L, FetchResponse.recordsSize(partitionData)) + assertTrue(FetchResponse.isDivergingEpoch(partitionData)) - val divergingEpoch = partitionData.divergingEpoch.get() + val divergingEpoch = partitionData.divergingEpoch assertEquals(firstLeaderEpoch, divergingEpoch.epoch) assertEquals(firstEpochEndOffset, divergingEpoch.endOffset) } @@ -265,7 +266,7 @@ class FetchRequestTest extends BaseRequestTest { val fetchRequest = FetchRequest.Builder.forConsumer(0, 1, partitionMap).build() val fetchResponse = sendFetchRequest(brokerId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(error, partitionData.error) + assertEquals(error.code, partitionData.errorCode) } // We need a leader change in order to check epoch fencing since the first epoch is 0 and @@ -329,7 +330,7 @@ class FetchRequestTest extends BaseRequestTest { .build() val fetchResponse = sendFetchRequest(destinationBrokerId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(expectedError, partitionData.error) + assertEquals(expectedError.code, partitionData.errorCode) } // We only check errors because we do not expect the partition in the response otherwise @@ -366,7 +367,7 @@ class FetchRequestTest extends BaseRequestTest { // batch is not complete, but sent when the producer is closed futures.foreach(_.get) - def fetch(version: Short, maxPartitionBytes: Int, closeAfterPartialResponse: Boolean): Option[FetchResponse[MemoryRecords]] = { + def fetch(version: Short, maxPartitionBytes: Int, closeAfterPartialResponse: Boolean): Option[FetchResponse] = { val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(maxPartitionBytes, Seq(topicPartition))).build(version) @@ -383,7 +384,7 @@ class FetchRequestTest extends BaseRequestTest { s"Fetch size too small $size, broker may have run out of memory") None } else { - Some(receive[FetchResponse[MemoryRecords]](socket, ApiKeys.FETCH, version)) + Some(receive[FetchResponse](socket, ApiKeys.FETCH, version)) } } finally { socket.close() @@ -396,8 +397,8 @@ class FetchRequestTest extends BaseRequestTest { val response = fetch(version, maxPartitionBytes = batchSize, closeAfterPartialResponse = false) val fetchResponse = response.getOrElse(throw new IllegalStateException("No fetch response")) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) - val batches = partitionData.records.batches.asScala.toBuffer + assertEquals(Errors.NONE.code, partitionData.errorCode) + val batches = FetchResponse.recordsOrFail(partitionData).batches.asScala.toBuffer assertEquals(3, batches.size) // size is 3 (not 4) since maxPartitionBytes=msgValueSize*4, excluding key and headers } @@ -442,9 +443,9 @@ class FetchRequestTest extends BaseRequestTest { // validate response val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertTrue(partitionData.highWatermark > 0) - val batches = partitionData.records.batches.asScala.toBuffer + val batches = FetchResponse.recordsOrFail(partitionData).batches.asScala.toBuffer val batch = batches.head assertEquals(expectedMagic, batch.magic) assertEquals(currentExpectedOffset, batch.baseOffset) @@ -504,35 +505,34 @@ class FetchRequestTest extends BaseRequestTest { assertEquals(Errors.NONE, resp1.error()) assertTrue(resp1.sessionId() > 0, "Expected the broker to create a new incremental fetch session") debug(s"Test created an incremental fetch session ${resp1.sessionId}") - assertTrue(resp1.responseData().containsKey(foo0)) - assertTrue(resp1.responseData().containsKey(foo1)) - assertTrue(resp1.responseData().containsKey(bar0)) - assertEquals(Errors.NONE, resp1.responseData().get(foo0).error) - assertEquals(Errors.NONE, resp1.responseData().get(foo1).error) - assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, resp1.responseData().get(bar0).error) + assertTrue(resp1.responseData.containsKey(foo0)) + assertTrue(resp1.responseData.containsKey(foo1)) + assertTrue(resp1.responseData.containsKey(bar0)) + assertEquals(Errors.NONE.code, resp1.responseData.get(foo0).errorCode) + assertEquals(Errors.NONE.code, resp1.responseData.get(foo1).errorCode) + assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION.code, resp1.responseData.get(bar0).errorCode) val req2 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 1), Nil) val resp2 = sendFetchRequest(0, req2) assertEquals(Errors.NONE, resp2.error()) - assertEquals(resp1.sessionId(), - resp2.sessionId(), "Expected the broker to continue the incremental fetch session") + assertEquals(resp1.sessionId(), resp2.sessionId(), "Expected the broker to continue the incremental fetch session") assertFalse(resp2.responseData().containsKey(foo0)) assertFalse(resp2.responseData().containsKey(foo1)) assertTrue(resp2.responseData().containsKey(bar0)) - assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, resp2.responseData().get(bar0).error) + assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION.code(), resp2.responseData().get(bar0).errorCode()) createTopic("bar", Map(0 -> List(0, 1))) val req3 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 2), Nil) val resp3 = sendFetchRequest(0, req3) assertEquals(Errors.NONE, resp3.error()) - assertFalse(resp3.responseData().containsKey(foo0)) - assertFalse(resp3.responseData().containsKey(foo1)) - assertTrue(resp3.responseData().containsKey(bar0)) - assertEquals(Errors.NONE, resp3.responseData().get(bar0).error) + assertFalse(resp3.responseData.containsKey(foo0)) + assertFalse(resp3.responseData.containsKey(foo1)) + assertTrue(resp3.responseData.containsKey(bar0)) + assertEquals(Errors.NONE.code, resp3.responseData.get(bar0).errorCode) val req4 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 3), Nil) val resp4 = sendFetchRequest(0, req4) assertEquals(Errors.NONE, resp4.error()) - assertFalse(resp4.responseData().containsKey(foo0)) - assertFalse(resp4.responseData().containsKey(foo1)) - assertFalse(resp4.responseData().containsKey(bar0)) + assertFalse(resp4.responseData.containsKey(foo0)) + assertFalse(resp4.responseData.containsKey(foo1)) + assertFalse(resp4.responseData.containsKey(bar0)) } @Test @@ -560,7 +560,7 @@ class FetchRequestTest extends BaseRequestTest { val res0 = sendFetchRequest(leaderId, req0) val data0 = res0.responseData.get(topicPartition) - assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE, data0.error) + assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE.code, data0.errorCode) // fetch request with version 10: works fine! val req1= new FetchRequest.Builder(0, 10, -1, Int.MaxValue, 0, @@ -568,14 +568,14 @@ class FetchRequestTest extends BaseRequestTest { .setMaxBytes(800).build() val res1 = sendFetchRequest(leaderId, req1) val data1 = res1.responseData.get(topicPartition) - assertEquals(Errors.NONE, data1.error) + assertEquals(Errors.NONE.code, data1.errorCode) assertEquals(3, records(data1).size) } @Test def testPartitionDataEquals(): Unit = { assertEquals(new FetchRequest.PartitionData(300, 0L, 300, Optional.of(300)), - new FetchRequest.PartitionData(300, 0L, 300, Optional.of(300))); + new FetchRequest.PartitionData(300, 0L, 300, Optional.of(300))) } @Test @@ -614,7 +614,7 @@ class FetchRequestTest extends BaseRequestTest { val res0 = sendFetchRequest(leaderId, req0) val data0 = res0.responseData.get(topicPartition) - assertEquals(Errors.NONE, data0.error) + assertEquals(Errors.NONE.code, data0.errorCode) assertEquals(1, records(data0).size) val req1 = new FetchRequest.Builder(0, 1, -1, Int.MaxValue, 0, @@ -623,7 +623,7 @@ class FetchRequestTest extends BaseRequestTest { val res1 = sendFetchRequest(leaderId, req1) val data1 = res1.responseData.get(topicPartition) - assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE, data1.error) + assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE.code, data1.errorCode) // fetch request with fetch version v3 (magic 1): // gzip compressed record is returned with down-conversion. @@ -634,7 +634,7 @@ class FetchRequestTest extends BaseRequestTest { val res2 = sendFetchRequest(leaderId, req2) val data2 = res2.responseData.get(topicPartition) - assertEquals(Errors.NONE, data2.error) + assertEquals(Errors.NONE.code, data2.errorCode) assertEquals(1, records(data2).size) val req3 = new FetchRequest.Builder(0, 1, -1, Int.MaxValue, 0, @@ -643,7 +643,7 @@ class FetchRequestTest extends BaseRequestTest { val res3 = sendFetchRequest(leaderId, req3) val data3 = res3.responseData.get(topicPartition) - assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE, data3.error) + assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE.code, data3.errorCode) // fetch request with version 10: works fine! val req4= new FetchRequest.Builder(0, 10, -1, Int.MaxValue, 0, @@ -651,15 +651,15 @@ class FetchRequestTest extends BaseRequestTest { .setMaxBytes(800).build() val res4 = sendFetchRequest(leaderId, req4) val data4 = res4.responseData.get(topicPartition) - assertEquals(Errors.NONE, data4.error) + assertEquals(Errors.NONE.code, data4.errorCode) assertEquals(3, records(data4).size) } - private def records(partitionData: FetchResponse.PartitionData[MemoryRecords]): Seq[Record] = { - partitionData.records.records.asScala.toBuffer + private def records(partitionData: FetchResponseData.PartitionData): Seq[Record] = { + FetchResponse.recordsOrFail(partitionData).records.asScala.toBuffer } - private def checkFetchResponse(expectedPartitions: Seq[TopicPartition], fetchResponse: FetchResponse[MemoryRecords], + private def checkFetchResponse(expectedPartitions: Seq[TopicPartition], fetchResponse: FetchResponse, maxPartitionBytes: Int, maxResponseBytes: Int, numMessagesPerPartition: Int): Unit = { assertEquals(expectedPartitions, fetchResponse.responseData.keySet.asScala.toSeq) var emptyResponseSeen = false @@ -668,10 +668,10 @@ class FetchRequestTest extends BaseRequestTest { expectedPartitions.foreach { tp => val partitionData = fetchResponse.responseData.get(tp) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertTrue(partitionData.highWatermark > 0) - val records = partitionData.records + val records = FetchResponse.recordsOrFail(partitionData) responseBufferSize += records.sizeInBytes val batches = records.batches.asScala.toBuffer diff --git a/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala b/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala index e00395a98111e..81844f517d439 100755 --- a/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala @@ -16,26 +16,26 @@ */ package kafka.server -import java.util -import java.util.{Collections, Optional} import kafka.utils.MockTime import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.Records import org.apache.kafka.common.requests.FetchMetadata.{FINAL_EPOCH, INVALID_SESSION_ID} -import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, FetchMetadata => JFetchMetadata} +import org.apache.kafka.common.requests.{FetchRequest, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.utils.Utils import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{Test, Timeout} +import java.util +import java.util.{Collections, Optional} + @Timeout(120) class FetchSessionTest { @Test def testNewSessionId(): Unit = { val cache = new FetchSessionCache(3, 100) - for (i <- 0 to 10000) { + for (_ <- 0 to 10000) { val id = cache.newSessionId() assertTrue(id > 0) } @@ -125,7 +125,7 @@ class FetchSessionTest { assertEquals(3, cache.totalPartitions) } - val EMPTY_PART_LIST = Collections.unmodifiableList(new util.ArrayList[TopicPartition]()) + private val EMPTY_PART_LIST = Collections.unmodifiableList(new util.ArrayList[TopicPartition]()) @Test @@ -155,13 +155,22 @@ class FetchSessionTest { assertEquals(Optional.of(1), epochs1(tp1)) assertEquals(Optional.of(2), epochs1(tp2)) - val response = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - response.put(tp0, new FetchResponse.PartitionData(Errors.NONE, 100, 100, - 100, null, null)) - response.put(tp1, new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) - response.put(tp2, new FetchResponse.PartitionData( - Errors.NONE, 5, 5, 5, null, null)) + val response = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + response.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + response.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) + response.put(tp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition) + .setHighWatermark(5) + .setLastStableOffset(5) + .setLogStartOffset(5)) val sessionId = context1.updateAndGenerateResponseData(response).sessionId() @@ -220,10 +229,22 @@ class FetchSessionTest { assertEquals(Map(tp0 -> Optional.empty, tp1 -> Optional.empty, tp2 -> Optional.of(1)), cachedLastFetchedEpochs(context1)) - val response = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - response.put(tp0, new FetchResponse.PartitionData(Errors.NONE, 100, 100, 100, null, null)) - response.put(tp1, new FetchResponse.PartitionData(Errors.NONE, 10, 10, 10, null, null)) - response.put(tp2, new FetchResponse.PartitionData(Errors.NONE, 5, 5, 5, null, null)) + val response = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + response.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + response.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) + response.put(tp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition) + .setHighWatermark(5) + .setLastStableOffset(5) + .setLogStartOffset(5)) val sessionId = context1.updateAndGenerateResponseData(response).sessionId() @@ -275,15 +296,23 @@ class FetchSessionTest { }) assertEquals(0, context2.getFetchOffset(new TopicPartition("foo", 0)).get) assertEquals(10, context2.getFetchOffset(new TopicPartition("foo", 1)).get) - val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData2.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData2.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData2.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData2.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val resp2 = context2.updateAndGenerateResponseData(respData2) assertEquals(Errors.NONE, resp2.error()) assertTrue(resp2.sessionId() != INVALID_SESSION_ID) - assertEquals(respData2, resp2.responseData()) + assertEquals(respData2, resp2.responseData) // Test trying to create a new session with an invalid epoch val context3 = fetchManager.newContext( @@ -314,7 +343,7 @@ class FetchSessionTest { val resp5 = context5.updateAndGenerateResponseData(respData2) assertEquals(Errors.NONE, resp5.error()) assertEquals(resp2.sessionId(), resp5.sessionId()) - assertEquals(0, resp5.responseData().size()) + assertEquals(0, resp5.responseData.size()) // Test setting an invalid fetch session epoch. val context6 = fetchManager.newContext( @@ -345,11 +374,19 @@ class FetchSessionTest { new JFetchMetadata(prevSessionId, FINAL_EPOCH), reqData8, EMPTY_PART_LIST, false) assertEquals(classOf[SessionlessFetchContext], context8.getClass) assertEquals(0, cache.size) - val respData8 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + val respData8 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] respData8.put(new TopicPartition("bar", 0), - new FetchResponse.PartitionData(Errors.NONE, 100, 100, 100, null, null)) + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) respData8.put(new TopicPartition("bar", 1), - new FetchResponse.PartitionData(Errors.NONE, 100, 100, 100, null, null)) + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) val resp8 = context8.updateAndGenerateResponseData(respData8) assertEquals(Errors.NONE, resp8.error) nextSessionId = resp8.sessionId @@ -370,15 +407,21 @@ class FetchSessionTest { Optional.empty())) val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData1, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], context1.getClass) - val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData1.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val resp1 = context1.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, resp1.error()) assertTrue(resp1.sessionId() != INVALID_SESSION_ID) - assertEquals(2, resp1.responseData().size()) + assertEquals(2, resp1.responseData.size()) // Create an incremental fetch request that removes foo-0 and adds bar-0 val reqData2 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] @@ -391,18 +434,26 @@ class FetchSessionTest { assertEquals(classOf[IncrementalFetchContext], context2.getClass) val parts2 = Set(new TopicPartition("foo", 1), new TopicPartition("bar", 0)) val reqData2Iter = parts2.iterator - context2.foreachPartition((topicPart, data) => { + context2.foreachPartition((topicPart, _) => { assertEquals(reqData2Iter.next(), topicPart) }) assertEquals(None, context2.getFetchOffset(new TopicPartition("foo", 0))) assertEquals(10, context2.getFetchOffset(new TopicPartition("foo", 1)).get) assertEquals(15, context2.getFetchOffset(new TopicPartition("bar", 0)).get) assertEquals(None, context2.getFetchOffset(new TopicPartition("bar", 2))) - val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData2.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) - respData2.put(new TopicPartition("bar", 0), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData2.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) + respData2.put(new TopicPartition("bar", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val resp2 = context2.updateAndGenerateResponseData(respData2) assertEquals(Errors.NONE, resp2.error) assertEquals(1, resp2.responseData.size) @@ -424,15 +475,21 @@ class FetchSessionTest { Optional.empty())) val session1context1 = fetchManager.newContext(JFetchMetadata.INITIAL, session1req, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], session1context1.getClass) - val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData1.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session1resp = session1context1.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, session1resp.error()) assertTrue(session1resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session1resp.responseData().size()) + assertEquals(2, session1resp.responseData.size) // check session entered into case assertTrue(cache.get(session1resp.sessionId()).isDefined) @@ -446,15 +503,22 @@ class FetchSessionTest { Optional.empty())) val session2context = fetchManager.newContext(JFetchMetadata.INITIAL, session1req, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], session2context.getClass) - val session2RespData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - session2RespData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - session2RespData.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val session2RespData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + session2RespData.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + session2RespData.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session2resp = session2context.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, session2resp.error()) assertTrue(session2resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session2resp.responseData().size()) + assertEquals(2, session2resp.responseData.size) // both newly created entries are present in cache assertTrue(cache.get(session1resp.sessionId()).isDefined) @@ -481,19 +545,25 @@ class FetchSessionTest { Optional.empty())) val session3context = fetchManager.newContext(JFetchMetadata.INITIAL, session3req, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], session3context.getClass) - val respData3 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData3.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData3.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData3 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData3.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData3.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session3resp = session3context.updateAndGenerateResponseData(respData3) assertEquals(Errors.NONE, session3resp.error()) assertTrue(session3resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session3resp.responseData().size()) + assertEquals(2, session3resp.responseData.size) assertTrue(cache.get(session1resp.sessionId()).isDefined) - assertFalse(cache.get(session2resp.sessionId()).isDefined, - "session 2 should have been evicted by latest session, as session 1 was used more recently") + assertFalse(cache.get(session2resp.sessionId()).isDefined, "session 2 should have been evicted by latest session, as session 1 was used more recently") assertTrue(cache.get(session3resp.sessionId()).isDefined) } @@ -512,15 +582,21 @@ class FetchSessionTest { Optional.empty())) val session1context = fetchManager.newContext(JFetchMetadata.INITIAL, session1req, EMPTY_PART_LIST, true) assertEquals(classOf[FullFetchContext], session1context.getClass) - val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData1.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session1resp = session1context.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, session1resp.error()) assertTrue(session1resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session1resp.responseData().size()) + assertEquals(2, session1resp.responseData.size) assertEquals(1, cache.size) // move time forward to age session 1 a little compared to session 2 @@ -534,15 +610,23 @@ class FetchSessionTest { Optional.empty())) val session2context = fetchManager.newContext(JFetchMetadata.INITIAL, session1req, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], session2context.getClass) - val session2RespData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - session2RespData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - session2RespData.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val session2RespData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + session2RespData.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + session2RespData.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session2resp = session2context.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, session2resp.error()) assertTrue(session2resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session2resp.responseData().size()) + assertEquals(2, session2resp.responseData.size) // both newly created entries are present in cache assertTrue(cache.get(session1resp.sessionId()).isDefined) @@ -558,21 +642,28 @@ class FetchSessionTest { Optional.empty())) val session3context = fetchManager.newContext(JFetchMetadata.INITIAL, session3req, EMPTY_PART_LIST, true) assertEquals(classOf[FullFetchContext], session3context.getClass) - val respData3 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData3.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData3.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData3 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData3.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData3.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session3resp = session3context.updateAndGenerateResponseData(respData3) assertEquals(Errors.NONE, session3resp.error()) assertTrue(session3resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session3resp.responseData().size()) + assertEquals(2, session3resp.responseData.size) assertTrue(cache.get(session1resp.sessionId()).isDefined) // even though session 2 is more recent than session 1, and has not reached expiry time, it is less // privileged than session 2, and thus session 3 should be entered and session 2 evicted. - assertFalse(cache.get(session2resp.sessionId()).isDefined, - "session 2 should have been evicted by session 3") + assertFalse(cache.get(session2resp.sessionId()).isDefined, "session 2 should have been evicted by session 3") assertTrue(cache.get(session3resp.sessionId()).isDefined) assertEquals(2, cache.size) @@ -586,18 +677,25 @@ class FetchSessionTest { Optional.empty())) val session4context = fetchManager.newContext(JFetchMetadata.INITIAL, session4req, EMPTY_PART_LIST, true) assertEquals(classOf[FullFetchContext], session4context.getClass) - val respData4 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData4.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData4.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData4 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData4.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData4.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session4resp = session3context.updateAndGenerateResponseData(respData4) assertEquals(Errors.NONE, session4resp.error()) assertTrue(session4resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session4resp.responseData().size()) + assertEquals(2, session4resp.responseData.size) - assertFalse(cache.get(session1resp.sessionId()).isDefined, - "session 1 should have been evicted by session 4 even though it is privileged as it has hit eviction time") + assertFalse(cache.get(session1resp.sessionId()).isDefined, "session 1 should have been evicted by session 4 even though it is privileged as it has hit eviction time") assertTrue(cache.get(session3resp.sessionId()).isDefined) assertTrue(cache.get(session4resp.sessionId()).isDefined) assertEquals(2, cache.size) @@ -617,11 +715,17 @@ class FetchSessionTest { Optional.empty())) val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData1, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], context1.getClass) - val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData1.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val resp1 = context1.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, resp1.error) assertTrue(resp1.sessionId() != INVALID_SESSION_ID) @@ -636,10 +740,10 @@ class FetchSessionTest { val context2 = fetchManager.newContext( new JFetchMetadata(resp1.sessionId, 1), reqData2, removed2, false) assertEquals(classOf[SessionlessFetchContext], context2.getClass) - val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] val resp2 = context2.updateAndGenerateResponseData(respData2) assertEquals(INVALID_SESSION_ID, resp2.sessionId) - assertTrue(resp2.responseData().isEmpty) + assertTrue(resp2.responseData.isEmpty) assertEquals(0, cache.size) } @@ -658,12 +762,19 @@ class FetchSessionTest { // Full fetch context returns all partitions in the response val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData, EMPTY_PART_LIST, isFollower = false) assertEquals(classOf[FullFetchContext], context1.getClass) - val respData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData.put(tp1, new FetchResponse.PartitionData(Errors.NONE, - 105, 105, 0, Optional.empty(), Collections.emptyList(), Optional.empty(), null)) - val divergingEpoch = Optional.of(new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(90)) - respData.put(tp2, new FetchResponse.PartitionData(Errors.NONE, - 105, 105, 0, Optional.empty(), Collections.emptyList(), divergingEpoch, null)) + val respData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(105) + .setLastStableOffset(105) + .setLogStartOffset(0)) + val divergingEpoch = new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(90) + respData.put(tp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition) + .setHighWatermark(105) + .setLastStableOffset(105) + .setLogStartOffset(0) + .setDivergingEpoch(divergingEpoch)) val resp1 = context1.updateAndGenerateResponseData(respData) assertEquals(Errors.NONE, resp1.error) assertNotEquals(INVALID_SESSION_ID, resp1.sessionId) @@ -679,8 +790,12 @@ class FetchSessionTest { assertEquals(Collections.singleton(tp2), resp2.responseData.keySet) // All partitions with divergent epoch should be returned. - respData.put(tp1, new FetchResponse.PartitionData(Errors.NONE, - 105, 105, 0, Optional.empty(), Collections.emptyList(), divergingEpoch, null)) + respData.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(105) + .setLastStableOffset(105) + .setLogStartOffset(0) + .setDivergingEpoch(divergingEpoch)) val resp3 = context2.updateAndGenerateResponseData(respData) assertEquals(Errors.NONE, resp3.error) assertEquals(resp1.sessionId, resp3.sessionId) @@ -688,8 +803,11 @@ class FetchSessionTest { // Partitions that meet other conditions should be returned regardless of whether // divergingEpoch is set or not. - respData.put(tp1, new FetchResponse.PartitionData(Errors.NONE, - 110, 110, 0, Optional.empty(), Collections.emptyList(), Optional.empty(), null)) + respData.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(110) + .setLastStableOffset(110) + .setLogStartOffset(0)) val resp4 = context2.updateAndGenerateResponseData(respData) assertEquals(Errors.NONE, resp4.error) assertEquals(resp1.sessionId, resp4.sessionId) diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index e995c69302ce8..4d48fecf99a2b 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -1072,7 +1072,7 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[OffsetCommitResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, - Errors.forCode(response.data().topics().get(0).partitions().get(0).errorCode())) + Errors.forCode(response.data.topics().get(0).partitions().get(0).errorCode)) } checkInvalidPartition(-1) @@ -1425,9 +1425,9 @@ class KafkaApisTest { val produceRequest = ProduceRequest.forCurrentMagic(new ProduceRequestData() .setTopicData(new ProduceRequestData.TopicProduceDataCollection( Collections.singletonList(new ProduceRequestData.TopicProduceData() - .setName(tp.topic()).setPartitionData(Collections.singletonList( + .setName(tp.topic).setPartitionData(Collections.singletonList( new ProduceRequestData.PartitionProduceData() - .setIndex(tp.partition()) + .setIndex(tp.partition) .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("test".getBytes)))))) .iterator)) .setAcks(1.toShort) @@ -1632,21 +1632,21 @@ class KafkaApisTest { val topicStates = Seq( new StopReplicaTopicState() - .setTopicName(groupMetadataPartition.topic()) + .setTopicName(groupMetadataPartition.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(groupMetadataPartition.partition()) + .setPartitionIndex(groupMetadataPartition.partition) .setLeaderEpoch(leaderEpoch) .setDeletePartition(deletePartition)).asJava), new StopReplicaTopicState() - .setTopicName(txnStatePartition.topic()) + .setTopicName(txnStatePartition.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(txnStatePartition.partition()) + .setPartitionIndex(txnStatePartition.partition) .setLeaderEpoch(leaderEpoch) .setDeletePartition(deletePartition)).asJava), new StopReplicaTopicState() - .setTopicName(fooPartition.topic()) + .setTopicName(fooPartition.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(fooPartition.partition()) + .setPartitionIndex(fooPartition.partition) .setLeaderEpoch(leaderEpoch) .setDeletePartition(deletePartition)).asJava) ).asJava @@ -1806,8 +1806,8 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[DescribeGroupsResponse] - val group = response.data().groups().get(0) - assertEquals(Errors.NONE, Errors.forCode(group.errorCode())) + val group = response.data.groups().get(0) + assertEquals(Errors.NONE, Errors.forCode(group.errorCode)) assertEquals(groupId, group.groupId()) assertEquals(groupSummary.state, group.groupState()) assertEquals(groupSummary.protocolType, group.protocolType()) @@ -1873,7 +1873,7 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] def errorForPartition(topic: String, partition: Int): Errors = { - Errors.forCode(response.data.topics.find(topic).partitions.find(partition).errorCode()) + Errors.forCode(response.data.topics.find(topic).partitions.find(partition).errorCode) } assertEquals(2, response.data.topics.size) @@ -1914,7 +1914,7 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, - Errors.forCode(response.data.topics.find(topic).partitions.find(invalidPartitionId).errorCode())) + Errors.forCode(response.data.topics.find(topic).partitions.find(invalidPartitionId).errorCode)) } checkInvalidPartition(-1) @@ -1942,7 +1942,7 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] - assertEquals(Errors.GROUP_ID_NOT_FOUND, Errors.forCode(response.data.errorCode())) + assertEquals(Errors.GROUP_ID_NOT_FOUND, Errors.forCode(response.data.errorCode)) } private def testListOffsetFailedGetLeaderReplica(error: Errors): Unit = { @@ -2130,16 +2130,15 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, fetchManager) createKafkaApis().handleFetchRequest(request) - val response = capturedResponse.getValue.asInstanceOf[FetchResponse[BaseRecords]] + val response = capturedResponse.getValue.asInstanceOf[FetchResponse] assertTrue(response.responseData.containsKey(tp)) val partitionData = response.responseData.get(tp) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertEquals(hw, partitionData.highWatermark) assertEquals(-1, partitionData.lastStableOffset) assertEquals(0, partitionData.logStartOffset) - assertEquals(timestamp, - partitionData.records.asInstanceOf[MemoryRecords].batches.iterator.next.maxTimestamp) + assertEquals(timestamp, FetchResponse.recordsOrFail(partitionData).batches.iterator.next.maxTimestamp) assertNull(partitionData.abortedTransactions) } @@ -2563,7 +2562,7 @@ class KafkaApisTest { .setPartitions(Collections.singletonList( new OffsetCommitResponseData.OffsetCommitResponsePartition() .setPartitionIndex(0) - .setErrorCode(Errors.UNSUPPORTED_VERSION.code()) + .setErrorCode(Errors.UNSUPPORTED_VERSION.code) )) ) val response = capturedResponse.getValue.asInstanceOf[OffsetCommitResponse] @@ -2871,9 +2870,9 @@ class KafkaApisTest { val fooPartition = new TopicPartition("foo", 0) val topicStates = Seq( new StopReplicaTopicState() - .setTopicName(fooPartition.topic()) + .setTopicName(fooPartition.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(fooPartition.partition()) + .setPartitionIndex(fooPartition.partition) .setLeaderEpoch(1) .setDeletePartition(false)).asJava) ).asJava @@ -3246,15 +3245,18 @@ class KafkaApisTest { @Test def testSizeOfThrottledPartitions(): Unit = { - def fetchResponse(data: Map[TopicPartition, String]): FetchResponse[Records] = { - val responseData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]]( + + def fetchResponse(data: Map[TopicPartition, String]): FetchResponse = { + val responseData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData]( data.map { case (tp, raw) => - tp -> new FetchResponse.PartitionData(Errors.NONE, - 105, 105, 0, Optional.empty(), Collections.emptyList(), Optional.empty(), - MemoryRecords.withRecords(CompressionType.NONE, - new SimpleRecord(100, raw.getBytes(StandardCharsets.UTF_8))).asInstanceOf[Records]) + tp -> new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setHighWatermark(105) + .setLastStableOffset(105) + .setLogStartOffset(0) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord(100, raw.getBytes(StandardCharsets.UTF_8)))) }.toMap.asJava) - new FetchResponse(Errors.NONE, responseData, 100, 100) + FetchResponse.of(Errors.NONE, 100, 100, responseData) } val throttledPartition = new TopicPartition("throttledData", 0) diff --git a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala index 1ce8006c63090..746374583434d 100755 --- a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala +++ b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala @@ -17,26 +17,22 @@ package kafka.server -import java.io.File -import java.util.concurrent.atomic.AtomicInteger -import java.util.{Optional, Properties, Random} - import kafka.log.{ClientRecordDeletion, Log, LogSegment} import kafka.utils.{MockTime, TestUtils} -import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic -import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse +import org.apache.kafka.common.message.ListOffsetsRequestData.{ListOffsetsPartition, ListOffsetsTopic} +import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsPartitionResponse, ListOffsetsTopicResponse} import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, ListOffsetsRequest, ListOffsetsResponse} import org.apache.kafka.common.{IsolationLevel, TopicPartition} import org.easymock.{EasyMock, IAnswer} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test -import scala.jdk.CollectionConverters._ +import java.io.File +import java.util.concurrent.atomic.AtomicInteger +import java.util.{Optional, Properties, Random} import scala.collection.mutable.Buffer +import scala.jdk.CollectionConverters._ class LogOffsetTest extends BaseRequestTest { @@ -127,7 +123,7 @@ class LogOffsetTest extends BaseRequestTest { Map(topicPartition -> new FetchRequest.PartitionData(consumerOffsets.head, FetchRequest.INVALID_LOG_START_OFFSET, 300 * 1024, Optional.empty())).asJava).build() val fetchResponse = sendFetchRequest(fetchRequest) - assertFalse(fetchResponse.responseData.get(topicPartition).records.batches.iterator.hasNext) + assertFalse(FetchResponse.recordsOrFail(fetchResponse.responseData.get(topicPartition)).batches.iterator.hasNext) } @Test @@ -251,8 +247,8 @@ class LogOffsetTest extends BaseRequestTest { connectAndReceive[ListOffsetsResponse](request) } - private def sendFetchRequest(request: FetchRequest): FetchResponse[MemoryRecords] = { - connectAndReceive[FetchResponse[MemoryRecords]](request) + private def sendFetchRequest(request: FetchRequest): FetchResponse = { + connectAndReceive[FetchResponse](request) } private def buildTargetTimes(tp: TopicPartition, timestamp: Long, maxNumOffsets: Int): List[ListOffsetsTopic] = { diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala index fda8b64a53d2b..af3989cd827ba 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala @@ -16,9 +16,6 @@ */ package kafka.server -import java.nio.charset.StandardCharsets -import java.util.{Collections, Optional} - import kafka.api.{ApiVersion, KAFKA_2_6_IV0} import kafka.cluster.{BrokerEndPoint, Partition} import kafka.log.{Log, LogAppendInfo, LogManager} @@ -32,17 +29,18 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEnd import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.Errors._ import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.{CompressionType, MemoryRecords, Records, SimpleRecord} +import org.apache.kafka.common.record.{CompressionType, MemoryRecords, SimpleRecord} import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET} -import org.apache.kafka.common.requests.FetchResponse import org.apache.kafka.common.utils.SystemTime import org.easymock.EasyMock._ import org.easymock.{Capture, CaptureType} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} -import scala.jdk.CollectionConverters._ +import java.nio.charset.StandardCharsets +import java.util.Collections import scala.collection.{Map, mutable} +import scala.jdk.CollectionConverters._ class ReplicaFetcherThreadTest { @@ -531,16 +529,18 @@ class ReplicaFetcherThreadTest { assertEquals(1, mockNetwork.fetchCount) partitions.foreach { tp => assertEquals(Fetching, thread.fetchState(tp).get.state) } - def partitionData(divergingEpoch: FetchResponseData.EpochEndOffset): FetchResponse.PartitionData[Records] = { - new FetchResponse.PartitionData[Records]( - Errors.NONE, 0, 0, 0, Optional.empty(), Collections.emptyList(), - Optional.of(divergingEpoch), MemoryRecords.EMPTY) + def partitionData(partition: Int, divergingEpoch: FetchResponseData.EpochEndOffset): FetchResponseData.PartitionData = { + new FetchResponseData.PartitionData() + .setPartitionIndex(partition) + .setLastStableOffset(0) + .setLogStartOffset(0) + .setDivergingEpoch(divergingEpoch) } // Loop 2 should truncate based on diverging epoch and continue to send fetch requests. mockNetwork.setFetchPartitionDataForNextResponse(Map( - t1p0 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(4).setEndOffset(140)), - t1p1 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(4).setEndOffset(141)) + t1p0 -> partitionData(t1p0.partition, new FetchResponseData.EpochEndOffset().setEpoch(4).setEndOffset(140)), + t1p1 -> partitionData(t1p1.partition, new FetchResponseData.EpochEndOffset().setEpoch(4).setEndOffset(141)) )) latestLogEpoch = Some(4) thread.doWork() @@ -555,8 +555,8 @@ class ReplicaFetcherThreadTest { // Loop 3 should truncate because of diverging epoch. Offset truncation is not complete // because divergent epoch is not known to follower. We truncate and stay in Fetching state. mockNetwork.setFetchPartitionDataForNextResponse(Map( - t1p0 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(130)), - t1p1 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(131)) + t1p0 -> partitionData(t1p0.partition, new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(130)), + t1p1 -> partitionData(t1p1.partition, new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(131)) )) thread.doWork() assertEquals(0, mockNetwork.epochFetchCount) @@ -569,8 +569,8 @@ class ReplicaFetcherThreadTest { // because divergent epoch is not known to follower. Last fetched epoch cannot be determined // from the log. We truncate and stay in Fetching state. mockNetwork.setFetchPartitionDataForNextResponse(Map( - t1p0 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(2).setEndOffset(120)), - t1p1 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(2).setEndOffset(121)) + t1p0 -> partitionData(t1p0.partition, new FetchResponseData.EpochEndOffset().setEpoch(2).setEndOffset(120)), + t1p1 -> partitionData(t1p1.partition, new FetchResponseData.EpochEndOffset().setEpoch(2).setEndOffset(121)) )) latestLogEpoch = None thread.doWork() @@ -963,9 +963,11 @@ class ReplicaFetcherThreadTest { val records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord(1000, "foo".getBytes(StandardCharsets.UTF_8))) - - val partitionData: thread.FetchData = new FetchResponse.PartitionData[Records]( - Errors.NONE, 0, 0, 0, Optional.empty(), Collections.emptyList(), records) + val partitionData: thread.FetchData = new FetchResponseData.PartitionData() + .setPartitionIndex(t1p0.partition) + .setLastStableOffset(0) + .setLogStartOffset(0) + .setRecords(records) thread.processPartitionData(t1p0, 0, partitionData) if (isReassigning) diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 9b289e578173d..76ce6c7a5f145 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -17,24 +17,17 @@ package kafka.server -import java.io.File -import java.net.InetAddress -import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} -import java.util.concurrent.{CountDownLatch, TimeUnit} -import java.util.{Collections, Optional, Properties} - import kafka.api._ -import kafka.log.{AppendOrigin, Log, LogConfig, LogManager, ProducerStateManager} import kafka.cluster.{BrokerEndPoint, Partition} -import kafka.log.LeaderOffsetIncremented +import kafka.log._ import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota} -import kafka.server.checkpoints.LazyOffsetCheckpoints -import kafka.server.checkpoints.OffsetCheckpointFile +import kafka.server.checkpoints.{LazyOffsetCheckpoints, OffsetCheckpointFile} import kafka.server.epoch.util.ReplicaFetcherMockBlockingSend import kafka.server.metadata.CachedConfigRepository import kafka.utils.TestUtils.createBroker import kafka.utils.timer.MockTimer import kafka.utils.{MockScheduler, MockTime, TestUtils} +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.message.LeaderAndIsrRequestData import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset @@ -45,21 +38,23 @@ import org.apache.kafka.common.record._ import org.apache.kafka.common.replica.ClientMetadata import org.apache.kafka.common.replica.ClientMetadata.DefaultClientMetadata import org.apache.kafka.common.requests.FetchRequest.PartitionData -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.KafkaPrincipal -import org.apache.kafka.common.utils.Time -import org.apache.kafka.common.utils.Utils +import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.{IsolationLevel, Node, TopicPartition, Uuid} import org.easymock.EasyMock import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.mockito.Mockito -import scala.collection.mutable +import java.io.File +import java.net.InetAddress +import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} +import java.util.concurrent.{CountDownLatch, TimeUnit} +import java.util.{Collections, Optional, Properties} +import scala.collection.{Map, Seq, mutable} import scala.jdk.CollectionConverters._ -import scala.collection.{Map, Seq} class ReplicaManagerTest { @@ -403,7 +398,7 @@ class ReplicaManagerTest { assertEquals(Errors.NONE, fetchData.error) assertTrue(fetchData.records.batches.asScala.isEmpty) assertEquals(Some(0), fetchData.lastStableOffset) - assertEquals(Some(List.empty[AbortedTransaction]), fetchData.abortedTransactions) + assertEquals(Some(List.empty[FetchResponseData.AbortedTransaction]), fetchData.abortedTransactions) // delayed fetch should timeout and return nothing consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0), @@ -416,7 +411,7 @@ class ReplicaManagerTest { assertEquals(Errors.NONE, fetchData.error) assertTrue(fetchData.records.batches.asScala.isEmpty) assertEquals(Some(0), fetchData.lastStableOffset) - assertEquals(Some(List.empty[AbortedTransaction]), fetchData.abortedTransactions) + assertEquals(Some(List.empty[FetchResponseData.AbortedTransaction]), fetchData.abortedTransactions) // now commit the transaction val endTxnMarker = new EndTransactionMarker(ControlRecordType.COMMIT, 0) @@ -448,7 +443,7 @@ class ReplicaManagerTest { fetchData = consumerFetchResult.assertFired assertEquals(Errors.NONE, fetchData.error) assertEquals(Some(numRecords + 1), fetchData.lastStableOffset) - assertEquals(Some(List.empty[AbortedTransaction]), fetchData.abortedTransactions) + assertEquals(Some(List.empty[FetchResponseData.AbortedTransaction]), fetchData.abortedTransactions) assertEquals(numRecords + 1, fetchData.records.batches.asScala.size) } finally { replicaManager.shutdown(checkpointHW = false) diff --git a/core/src/test/scala/unit/kafka/server/UpdateFeaturesTest.scala b/core/src/test/scala/unit/kafka/server/UpdateFeaturesTest.scala index 3ede6af097f38..92ba0425dcb27 100644 --- a/core/src/test/scala/unit/kafka/server/UpdateFeaturesTest.scala +++ b/core/src/test/scala/unit/kafka/server/UpdateFeaturesTest.scala @@ -193,7 +193,7 @@ class UpdateFeaturesTest extends BaseRequestTest { new UpdateFeaturesRequest.Builder(new UpdateFeaturesRequestData().setFeatureUpdates(validUpdates)).build(), notControllerSocketServer) - assertEquals(Errors.NOT_CONTROLLER, Errors.forCode(response.data.errorCode())) + assertEquals(Errors.NOT_CONTROLLER, Errors.forCode(response.data.errorCode)) assertNotNull(response.data.errorMessage()) assertEquals(0, response.data.results.size) checkFeatures( diff --git a/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala b/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala index 384eacca568d1..b367887c6edb2 100644 --- a/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala +++ b/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala @@ -18,14 +18,12 @@ package kafka.server.epoch.util import java.net.SocketTimeoutException import java.util - import kafka.cluster.BrokerEndPoint import kafka.server.BlockingSend import org.apache.kafka.clients.{ClientRequest, ClientResponse, MockClient, NetworkClientUtils} -import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData -import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{OffsetForLeaderTopicResult, EpochEndOffset} +import org.apache.kafka.common.message.{FetchResponseData, OffsetForLeaderEpochResponseData} +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult} import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.Records import org.apache.kafka.common.requests.AbstractRequest.Builder import org.apache.kafka.common.requests.{AbstractRequest, FetchResponse, OffsetsForLeaderEpochResponse, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.utils.{SystemTime, Time} @@ -52,7 +50,7 @@ class ReplicaFetcherMockBlockingSend(offsets: java.util.Map[TopicPartition, Epoc var lastUsedOffsetForLeaderEpochVersion = -1 var callback: Option[() => Unit] = None var currentOffsets: util.Map[TopicPartition, EpochEndOffset] = offsets - var fetchPartitionData: Map[TopicPartition, FetchResponse.PartitionData[Records]] = Map.empty + var fetchPartitionData: Map[TopicPartition, FetchResponseData.PartitionData] = Map.empty private val sourceNode = new Node(sourceBroker.id, sourceBroker.host, sourceBroker.port) def setEpochRequestCallback(postEpochFunction: () => Unit): Unit = { @@ -63,7 +61,7 @@ class ReplicaFetcherMockBlockingSend(offsets: java.util.Map[TopicPartition, Epoc currentOffsets = newOffsets } - def setFetchPartitionDataForNextResponse(partitionData: Map[TopicPartition, FetchResponse.PartitionData[Records]]): Unit = { + def setFetchPartitionDataForNextResponse(partitionData: Map[TopicPartition, FetchResponseData.PartitionData]): Unit = { fetchPartitionData = partitionData } @@ -97,11 +95,11 @@ class ReplicaFetcherMockBlockingSend(offsets: java.util.Map[TopicPartition, Epoc case ApiKeys.FETCH => fetchCount += 1 - val partitionData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + val partitionData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] fetchPartitionData.foreach { case (tp, data) => partitionData.put(tp, data) } fetchPartitionData = Map.empty - new FetchResponse(Errors.NONE, partitionData, 0, - if (partitionData.isEmpty) JFetchMetadata.INVALID_SESSION_ID else 1) + FetchResponse.of(Errors.NONE, 0, + if (partitionData.isEmpty) JFetchMetadata.INVALID_SESSION_ID else 1, partitionData) case _ => throw new UnsupportedOperationException diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchResponseBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchResponseBenchmark.java index 8785ecf6b0273..c2d0d7305a383 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchResponseBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchResponseBenchmark.java @@ -18,6 +18,7 @@ package org.apache.kafka.jmh.common; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; @@ -42,9 +43,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.LinkedHashMap; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -61,11 +60,11 @@ public class FetchResponseBenchmark { @Param({"3", "10", "20"}) private int partitionCount; - LinkedHashMap> responseData; + LinkedHashMap responseData; ResponseHeader header; - FetchResponse fetchResponse; + FetchResponse fetchResponse; @Setup(Level.Trial) public void setup() { @@ -78,19 +77,22 @@ public void setup() { for (int topicIdx = 0; topicIdx < topicCount; topicIdx++) { String topic = UUID.randomUUID().toString(); for (int partitionId = 0; partitionId < partitionCount; partitionId++) { - FetchResponse.PartitionData partitionData = new FetchResponse.PartitionData<>( - Errors.NONE, 0, 0, 0, Optional.empty(), Collections.emptyList(), records); + FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData() + .setPartitionIndex(partitionId) + .setLastStableOffset(0) + .setLogStartOffset(0) + .setRecords(records); responseData.put(new TopicPartition(topic, partitionId), partitionData); } } this.header = new ResponseHeader(100, ApiKeys.FETCH.responseHeaderVersion(ApiKeys.FETCH.latestVersion())); - this.fetchResponse = new FetchResponse<>(Errors.NONE, responseData, 0, 0); + this.fetchResponse = FetchResponse.of(Errors.NONE, 0, 0, responseData); } @Benchmark public int testConstructFetchResponse() { - FetchResponse fetchResponse = new FetchResponse<>(Errors.NONE, responseData, 0, 0); + FetchResponse fetchResponse = FetchResponse.of(Errors.NONE, 0, 0, responseData); return fetchResponse.responseData().size(); } diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java index 424b8df7e762f..453fa40096b2f 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java @@ -44,13 +44,13 @@ import kafka.utils.KafkaScheduler; import kafka.utils.Pool; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.LeaderAndIsrRequestData; import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderPartition; import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.BaseRecords; -import org.apache.kafka.common.record.Records; import org.apache.kafka.common.record.RecordsSend; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; @@ -82,7 +82,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -137,7 +136,7 @@ public void setup() throws IOException { Time.SYSTEM, true); - LinkedHashMap> initialFetched = new LinkedHashMap<>(); + LinkedHashMap initialFetched = new LinkedHashMap<>(); scala.collection.mutable.Map initialFetchStates = new scala.collection.mutable.HashMap<>(); for (int i = 0; i < partitionCount; i++) { TopicPartition tp = new TopicPartition("topic", i); @@ -174,8 +173,11 @@ public RecordsSend toSend() { return null; } }; - initialFetched.put(tp, new FetchResponse.PartitionData<>(Errors.NONE, 0, 0, 0, - new LinkedList<>(), fetched)); + initialFetched.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setLastStableOffset(0) + .setLogStartOffset(0) + .setRecords(fetched)); } ReplicaManager replicaManager = Mockito.mock(ReplicaManager.class); @@ -186,7 +188,7 @@ public RecordsSend toSend() { // so that we do not measure this time as part of the steady state work fetcher.doWork(); // handle response to engage the incremental fetch session handler - fetcher.fetchSessionHandler().handleResponse(new FetchResponse<>(Errors.NONE, initialFetched, 0, 999)); + fetcher.fetchSessionHandler().handleResponse(FetchResponse.of(Errors.NONE, 0, 999, initialFetched)); } @TearDown(Level.Trial) @@ -292,7 +294,8 @@ public Option endOffsetForEpoch(TopicPartition topicPartition, i } @Override - public Option processPartitionData(TopicPartition topicPartition, long fetchOffset, FetchResponse.PartitionData partitionData) { + public Option processPartitionData(TopicPartition topicPartition, long fetchOffset, + FetchResponseData.PartitionData partitionData) { return Option.empty(); } @@ -317,7 +320,7 @@ public Map fetchEpochEndOffsets(Map> fetchFromLeader(FetchRequest.Builder fetchRequest) { + public Map fetchFromLeader(FetchRequest.Builder fetchRequest) { return new scala.collection.mutable.HashMap<>(); } } diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetchsession/FetchSessionBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetchsession/FetchSessionBenchmark.java index 9fa25139909b7..01d83a7b4d04e 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetchsession/FetchSessionBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetchsession/FetchSessionBenchmark.java @@ -19,8 +19,8 @@ import org.apache.kafka.clients.FetchSessionHandler; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; import org.apache.kafka.common.utils.LogContext; @@ -70,24 +70,21 @@ public void setUp() { handler = new FetchSessionHandler(LOG_CONTEXT, 1); FetchSessionHandler.Builder builder = handler.newBuilder(); - LinkedHashMap> respMap = new LinkedHashMap<>(); + LinkedHashMap respMap = new LinkedHashMap<>(); for (int i = 0; i < partitionCount; i++) { TopicPartition tp = new TopicPartition("foo", i); FetchRequest.PartitionData partitionData = new FetchRequest.PartitionData(0, 0, 200, Optional.empty()); fetches.put(tp, partitionData); builder.add(tp, partitionData); - respMap.put(tp, new FetchResponse.PartitionData<>( - Errors.NONE, - 0L, - 0L, - 0, - null, - null)); + respMap.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setLastStableOffset(0) + .setLogStartOffset(0)); } builder.build(); // build and handle an initial response so that the next fetch will be incremental - handler.handleResponse(new FetchResponse<>(Errors.NONE, respMap, 0, 1)); + handler.handleResponse(FetchResponse.of(Errors.NONE, 0, 1, respMap)); int counter = 0; for (TopicPartition topicPartition: new ArrayList<>(fetches.keySet())) { diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index f99ffb6d02298..d252244cf5d3d 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -40,6 +40,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ApiMessage; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.FetchResponse; import org.apache.kafka.common.utils.BufferSupplier; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.MemoryRecords; @@ -906,7 +907,7 @@ private FetchResponseData buildFetchResponse( ) { return RaftUtil.singletonFetchResponse(log.topicPartition(), Errors.NONE, partitionData -> { partitionData - .setRecordSet(records) + .setRecords(records) .setErrorCode(error.code()) .setLogStartOffset(log.startOffset()) .setHighWatermark(highWatermark @@ -991,11 +992,11 @@ private CompletableFuture handleFetchRequest( } FetchResponseData response = tryCompleteFetchRequest(request.replicaId(), fetchPartition, currentTimeMs); - FetchResponseData.FetchablePartitionResponse partitionResponse = - response.responses().get(0).partitionResponses().get(0); + FetchResponseData.PartitionData partitionResponse = + response.responses().get(0).partitions().get(0); if (partitionResponse.errorCode() != Errors.NONE.code() - || partitionResponse.recordSet().sizeInBytes() > 0 + || FetchResponse.recordsSize(partitionResponse) > 0 || request.maxWaitMs() == 0) { return completedFuture(response); } @@ -1084,8 +1085,8 @@ private boolean handleFetchResponse( return false; } - FetchResponseData.FetchablePartitionResponse partitionResponse = - response.responses().get(0).partitionResponses().get(0); + FetchResponseData.PartitionData partitionResponse = + response.responses().get(0).partitions().get(0); FetchResponseData.LeaderIdAndEpoch currentLeaderIdAndEpoch = partitionResponse.currentLeader(); OptionalInt responseLeaderId = optionalLeaderId(currentLeaderIdAndEpoch.leaderId()); @@ -1143,7 +1144,7 @@ private boolean handleFetchResponse( state.setFetchingSnapshot(Optional.of(log.createSnapshot(snapshotId))); } } else { - Records records = (Records) partitionResponse.recordSet(); + Records records = FetchResponse.recordsOrFail(partitionResponse); if (records.sizeInBytes() > 0) { appendAsFollower(records); } diff --git a/raft/src/main/java/org/apache/kafka/raft/RaftUtil.java b/raft/src/main/java/org/apache/kafka/raft/RaftUtil.java index 7462fde041b50..e7acd0800ec3d 100644 --- a/raft/src/main/java/org/apache/kafka/raft/RaftUtil.java +++ b/raft/src/main/java/org/apache/kafka/raft/RaftUtil.java @@ -73,19 +73,19 @@ public static FetchRequestData singletonFetchRequest( public static FetchResponseData singletonFetchResponse( TopicPartition topicPartition, Errors topLevelError, - Consumer partitionConsumer + Consumer partitionConsumer ) { - FetchResponseData.FetchablePartitionResponse fetchablePartition = - new FetchResponseData.FetchablePartitionResponse(); + FetchResponseData.PartitionData fetchablePartition = + new FetchResponseData.PartitionData(); - fetchablePartition.setPartition(topicPartition.partition()); + fetchablePartition.setPartitionIndex(topicPartition.partition()); partitionConsumer.accept(fetchablePartition); FetchResponseData.FetchableTopicResponse fetchableTopic = new FetchResponseData.FetchableTopicResponse() .setTopic(topicPartition.topic()) - .setPartitionResponses(Collections.singletonList(fetchablePartition)); + .setPartitions(Collections.singletonList(fetchablePartition)); return new FetchResponseData() .setErrorCode(topLevelError.code()) @@ -102,8 +102,8 @@ static boolean hasValidTopicPartition(FetchRequestData data, TopicPartition topi static boolean hasValidTopicPartition(FetchResponseData data, TopicPartition topicPartition) { return data.responses().size() == 1 && data.responses().get(0).topic().equals(topicPartition.topic()) && - data.responses().get(0).partitionResponses().size() == 1 && - data.responses().get(0).partitionResponses().get(0).partition() == topicPartition.partition(); + data.responses().get(0).partitions().size() == 1 && + data.responses().get(0).partitions().get(0).partitionIndex() == topicPartition.partition(); } static boolean hasValidTopicPartition(VoteResponseData data, TopicPartition topicPartition) { diff --git a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java index 2b7cea5554151..c66b9bdf31e18 100644 --- a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java @@ -89,7 +89,7 @@ public void testFetchRequestOffsetLessThanLogStart() throws Exception { // Send Fetch request less than start offset context.deliverRequest(context.fetchRequest(epoch, otherNodeId, 0, epoch, 0)); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -176,7 +176,7 @@ public void testFetchRequestTruncateToLogStart() throws Exception { context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset + 1, oldestSnapshotId.epoch + 1, 0) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -265,7 +265,7 @@ public void testFetchRequestAtLogStartOffsetWithInvalidEpoch() throws Exception context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset, oldestSnapshotId.epoch + 1, 0) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -318,7 +318,7 @@ public void testFetchRequestWithLastFetchedEpochLessThanOldestSnapshot() throws ) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -1329,9 +1329,7 @@ private static FetchResponseData snapshotFetchResponse( long highWatermark ) { return RaftUtil.singletonFetchResponse(topicPartition, Errors.NONE, partitionData -> { - partitionData - .setErrorCode(Errors.NONE.code()) - .setHighWatermark(highWatermark); + partitionData.setHighWatermark(highWatermark); partitionData.currentLeader() .setLeaderEpoch(epoch) diff --git a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java index e57995c2cd303..15e550f5f954c 100644 --- a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java +++ b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java @@ -606,7 +606,7 @@ int assertSentFetchRequest( return raftMessage.correlationId(); } - FetchResponseData.FetchablePartitionResponse assertSentFetchPartitionResponse() { + FetchResponseData.PartitionData assertSentFetchPartitionResponse() { List sentMessages = drainSentResponses(ApiKeys.FETCH); assertEquals( 1, sentMessages.size(), "Found unexpected sent messages " + sentMessages); @@ -617,8 +617,8 @@ FetchResponseData.FetchablePartitionResponse assertSentFetchPartitionResponse() assertEquals(1, response.responses().size()); assertEquals(metadataPartition.topic(), response.responses().get(0).topic()); - assertEquals(1, response.responses().get(0).partitionResponses().size()); - return response.responses().get(0).partitionResponses().get(0); + assertEquals(1, response.responses().get(0).partitions().size()); + return response.responses().get(0).partitions().get(0); } void assertSentFetchPartitionResponse(Errors error) { @@ -637,7 +637,7 @@ MemoryRecords assertSentFetchPartitionResponse( int epoch, OptionalInt leaderId ) { - FetchResponseData.FetchablePartitionResponse partitionResponse = assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = assertSentFetchPartitionResponse(); assertEquals(error, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(leaderId.orElse(-1), partitionResponse.currentLeader().leaderId()); @@ -645,14 +645,14 @@ MemoryRecords assertSentFetchPartitionResponse( assertEquals(-1, partitionResponse.divergingEpoch().epoch()); assertEquals(-1, partitionResponse.snapshotId().endOffset()); assertEquals(-1, partitionResponse.snapshotId().epoch()); - return (MemoryRecords) partitionResponse.recordSet(); + return (MemoryRecords) partitionResponse.records(); } MemoryRecords assertSentFetchPartitionResponse( long highWatermark, int leaderEpoch ) { - FetchResponseData.FetchablePartitionResponse partitionResponse = assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(leaderEpoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(highWatermark, partitionResponse.highWatermark()); @@ -660,7 +660,7 @@ MemoryRecords assertSentFetchPartitionResponse( assertEquals(-1, partitionResponse.divergingEpoch().epoch()); assertEquals(-1, partitionResponse.snapshotId().endOffset()); assertEquals(-1, partitionResponse.snapshotId().epoch()); - return (MemoryRecords) partitionResponse.recordSet(); + return (MemoryRecords) partitionResponse.records(); } RaftRequest.Outbound assertSentFetchSnapshotRequest() { @@ -928,7 +928,7 @@ FetchResponseData fetchResponse( ) { return RaftUtil.singletonFetchResponse(metadataPartition, Errors.NONE, partitionData -> { partitionData - .setRecordSet(records) + .setRecords(records) .setErrorCode(error.code()) .setHighWatermark(highWatermark); @@ -946,9 +946,7 @@ FetchResponseData divergingFetchResponse( long highWatermark ) { return RaftUtil.singletonFetchResponse(metadataPartition, Errors.NONE, partitionData -> { - partitionData - .setErrorCode(Errors.NONE.code()) - .setHighWatermark(highWatermark); + partitionData.setHighWatermark(highWatermark); partitionData.currentLeader() .setLeaderEpoch(epoch) From ea005cc700b1b9e3cb001da19d73798ea8520d63 Mon Sep 17 00:00:00 2001 From: Lee Dongjin Date: Thu, 4 Mar 2021 23:24:50 +0900 Subject: [PATCH 104/243] KAFKA-12407: Document Controller Health Metrics (#10257) Reviewers: Luke Chen , Dong Lin --- docs/ops.html | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/ops.html b/docs/ops.html index 1bef09aed1d90..14f967e0b8202 100644 --- a/docs/ops.html +++ b/docs/ops.html @@ -1271,6 +1271,24 @@

    Date: Thu, 4 Mar 2021 16:47:48 +0100 Subject: [PATCH 105/243] KAFKA-12393: Document multi-tenancy considerations (#334) (#10263) KAFKA-12393: Document multi-tenancy considerations Addressed review feedback by @dajac and @rajinisivaram Ported from apache/kafka-site#334 Reviewers: Bill Bejeck --- docs/ops.html | 168 ++++++++++++++++++++++++++++++++++++++++++++++++-- docs/toc.html | 21 +++++-- 2 files changed, 179 insertions(+), 10 deletions(-) diff --git a/docs/ops.html b/docs/ops.html index 14f967e0b8202..402d6f1c76392 100644 --- a/docs/ops.html +++ b/docs/ops.html @@ -1089,7 +1089,165 @@

    -

    6.4 Kafka Configuration

    +

    6.4 Multi-Tenancy

    + +

    Multi-Tenancy Overview

    + +

    + As a highly scalable event streaming platform, Kafka is used by many users as their central nervous system, connecting in real-time a wide range of different systems and applications from various teams and lines of businesses. Such multi-tenant cluster environments command proper control and management to ensure the peaceful coexistence of these different needs. This section highlights features and best practices to set up such shared environments, which should help you operate clusters that meet SLAs/OLAs and that minimize potential collateral damage caused by "noisy neighbors". +

    + +

    + Multi-tenancy is a many-sided subject, including but not limited to: +

    + +
      +
    • Creating user spaces for tenants (sometimes called namespaces)
      • +
    • Configuring topics with data retention policies and more
      • +
    • Securing topics and clusters with encryption, authentication, and authorization
      • +
    • Isolating tenants with quotas and rate limits
      • +
    • Monitoring and metering
      • +
    • Inter-cluster data sharing (cf. geo-replication)
      • +
    + +

    Creating User Spaces (Namespaces) For Tenants With Topic Naming

    + +

    + Kafka administrators operating a multi-tenant cluster typically need to define user spaces for each tenant. For the purpose of this section, "user spaces" are a collection of topics, which are grouped together under the management of a single entity or user. +

    + +

    + In Kafka, the main unit of data is the topic. Users can create and name each topic. They can also delete them, but it is not possible to rename a topic directly. Instead, to rename a topic, the user must create a new topic, move the messages from the original topic to the new, and then delete the original. With this in mind, it is recommended to define logical spaces, based on an hierarchical topic naming structure. This setup can then be combined with security features, such as prefixed ACLs, to isolate different spaces and tenants, while also minimizing the administrative overhead for securing the data in the cluster. +

    + +

    + These logical user spaces can be grouped in different ways, and the concrete choice depends on how your organization prefers to use your Kafka clusters. The most common groupings are as follows. +

    + +

    + By team or organizational unit: Here, the team is the main aggregator. In an organization where teams are the main user of the Kafka infrastructure, this might be the best grouping. +

    + +

    + Example topic naming structure: +

    + +
      +
    • <organization>.<team>.<dataset>.<event-name>
      (e.g., "acme.infosec.telemetry.logins")
      • +
    + +

    + By project or product: Here, a team manages more than one project. Their credentials will be different for each project, so all the controls and settings will always be project related. +

    + +

    + Example topic naming structure: +

    + +
      +
    • <project>.<product>.<event-name>
      (e.g., "mobility.payments.suspicious")
      • +
    + +

    + Certain information should normally not be put in a topic name, such as information that is likely to change over time (e.g., the name of the intended consumer) or that is a technical detail or metadata that is available elsewhere (e.g., the topic's partition count and other configuration settings). +

    + +

    + To enforce a topic naming structure, several options are available: +

    + +
      +
    • Use prefix ACLs (cf. KIP-290) to enforce a common prefix for topic names. For example, team A may only be permitted to create topics whose names start with payments.teamA..
      • +
    • Define a custom CreateTopicPolicy (cf. KIP-108 and the setting create.topic.policy.class.name) to enforce strict naming patterns. These policies provide the most flexibility and can cover complex patterns and rules to match an organization's needs.
      • +
    • Disable topic creation for normal users by denying it with an ACL, and then rely on an external process to create topics on behalf of users (e.g., scripting or your favorite automation toolkit).
      • +
    • It may also be useful to disable the Kafka feature to auto-create topics on demand by setting auto.create.topics.enable=false in the broker configuration. Note that you should not rely solely on this option.
      • +
    + + +

    Configuring Topics: Data Retention And More

    + +

    + Kafka's configuration is very flexible due to its fine granularity, and it supports a plethora of per-topic configuration settings to help administrators set up multi-tenant clusters. For example, administrators often need to define data retention policies to control how much and/or for how long data will be stored in a topic, with settings such as retention.bytes (size) and retention.ms (time). This limits storage consumption within the cluster, and helps complying with legal requirements such as GDPR. +

    + +

    Securing Clusters and Topics: Authentication, Authorization, Encryption

    + +

    + Because the documentation has a dedicated chapter on security that applies to any Kafka deployment, this section focuses on additional considerations for multi-tenant environments. +

    + +

    +Security settings for Kafka fall into three main categories, which are similar to how administrators would secure other client-server data systems, like relational databases and traditional messaging systems. +

    + +
      +
    1. Encryption of data transferred between Kafka brokers and Kafka clients, between brokers, between brokers and ZooKeeper nodes, and between brokers and other, optional tools.
      2. +
    2. Authentication of connections from Kafka clients and applications to Kafka brokers, as well as connections from Kafka brokers to ZooKeeper nodes.
      3. +
    3. Authorization of client operations such as creating, deleting, and altering the configuration of topics; writing events to or reading events from a topic; creating and deleting ACLs. Administrators can also define custom policies to put in place additional restrictions, such as a CreateTopicPolicy and AlterConfigPolicy (see KIP-108 and the settings create.topic.policy.class.name, alter.config.policy.class.name).
      4. +
    + +

    + When securing a multi-tenant Kafka environment, the most common administrative task is the third category (authorization), i.e., managing the user/client permissions that grant or deny access to certain topics and thus to the data stored by users within a cluster. This task is performed predominantly through the setting of access control lists (ACLs). Here, administrators of multi-tenant environments in particular benefit from putting a hierarchical topic naming structure in place as described in a previous section, because they can conveniently control access to topics through prefixed ACLs (--resource-pattern-type Prefixed). This significantly minimizes the administrative overhead of securing topics in multi-tenant environments: administrators can make their own trade-offs between higher developer convenience (more lenient permissions, using fewer and broader ACLs) vs. tighter security (more stringent permissions, using more and narrower ACLs). +

    + +

    + In the following example, user Alice—a new member of ACME corporation's InfoSec team—is granted write permissions to all topics whose names start with "acme.infosec.", such as "acme.infosec.telemetry.logins" and "acme.infosec.syslogs.events". +

    + +
    # Grant permissions to user Alice
+$ bin/kafka-acls.sh \
+    --bootstrap-server broker1:9092 \
+    --add --allow-principal User:Alice \
+    --producer \
+    --resource-pattern-type prefixed --topic acme.infosec.
+
    + +

    + You can similarly use this approach to isolate different customers on the same shared cluster. +

    + +

    Isolating Tenants: Quotas, Rate Limiting, Throttling

    + +

    + Multi-tenant clusters should generally be configured with quotas, which protect against users (tenants) eating up too many cluster resources, such as when they attempt to write or read very high volumes of data, or create requests to brokers at an excessively high rate. This may cause network saturation, monopolize broker resources, and impact other clients—all of which you want to avoid in a shared environment. +

    + +

    + Client quotas: Kafka supports different types of (per-user principal) client quotas. Because a client's quotas apply irrespective of which topics the client is writing to or reading from, they are a convenient and effective tool to allocate resources in a multi-tenant cluster. Request rate quotas, for example, help to limit a user's impact on broker CPU usage by limiting the time a broker spends on the request handling path for that user, after which throttling kicks in. In many situations, isolating users with request rate quotas has a bigger impact in multi-tenant clusters than setting incoming/outgoing network bandwidth quotas, because excessive broker CPU usage for processing requests reduces the effective bandwidth the broker can serve. Furthermore, administrators can also define quotas on topic operations—such as create, delete, and alter—to prevent Kafka clusters from being overwhelmed by highly concurrent topic operations (see KIP-599 and the quota type controller_mutations_rate). +

    + +

    + Server quotas: Kafka also supports different types of broker-side quotas. For example, administrators can set a limit on the rate with which the broker accepts new connections, set the maximum number of connections per broker, or set the maximum number of connections allowed from a specific IP address. +

    + +

    + For more information, please refer to the quota overview and how to set quotas. +

    + +

    Monitoring and Metering

    + +

    + Monitoring is a broader subject that is covered elsewhere in the documentation. Administrators of any Kafka environment, but especially multi-tenant ones, should set up monitoring according to these instructions. Kafka supports a wide range of metrics, such as the rate of failed authentication attempts, request latency, consumer lag, total number of consumer groups, metrics on the quotas described in the previous section, and many more. +

    + +

    + For example, monitoring can be configured to track the size of topic-partitions (with the JMX metric kafka.log.Log.Size.<TOPIC-NAME>), and thus the total size of data stored in a topic. You can then define alerts when tenants on shared clusters are getting close to using too much storage space. +

    + +

    Multi-Tenancy and Geo-Replication

    + +

    + Kafka lets you share data across different clusters, which may be located in different geographical regions, data centers, and so on. Apart from use cases such as disaster recovery, this functionality is useful when a multi-tenant setup requires inter-cluster data sharing. See the section Geo-Replication (Cross-Cluster Data Mirroring) for more information. +

    + +

    Further considerations

    + +

    + Data contracts: You may need to define data contracts between the producers and the consumers of data in a cluster, using event schemas. This ensures that events written to Kafka can always be read properly again, and prevents malformed or corrupt events being written. The best way to achieve this is to deploy a so-called schema registry alongside the cluster. (Kafka does not include a schema registry, but there are third-party implementations available.) A schema registry manages the event schemas and maps the schemas to topics, so that producers know which topics are accepting which types (schemas) of events, and consumers know how to read and parse events in a topic. Some registry implementations provide further functionality, such as schema evolution, storing a history of all schemas, and schema compatibility settings. +

    + + +

    6.5 Kafka Configuration

    Important Client Configurations

    @@ -1122,7 +1280,7 @@

    6.5 Java Version

    +

    6.6 Java Version

    Java 8 and Java 11 are supported. Java 11 performs significantly better if TLS is enabled, so it is highly recommended (it also includes a number of other performance improvements: G1GC, CRC32C, Compact Strings, Thread-Local Handshakes and more). @@ -1145,7 +1303,7 @@

    All of the brokers in that cluster have a 90% GC pause time of about 21ms with less than 1 young GC per second. -

    6.6 Hardware and OS

    +

    6.7 Hardware and OS

    We are using dual quad-core Intel Xeon machines with 24GB of memory.

    You need sufficient memory to buffer active readers and writers. You can do a back-of-the-envelope estimate of memory needs by assuming you want to be able to buffer for 30 seconds and compute your memory need as write_throughput*30. @@ -1230,7 +1388,7 @@

  • delalloc: Delayed allocation means that the filesystem avoid allocating any blocks until the physical write occurs. This allows ext4 to allocate a large extent instead of smaller pages and helps ensure the data is written sequentially. This feature is great for throughput. It does seem to involve some locking in the filesystem which adds a bit of latency variance. -

    6.7 Monitoring

    +

    6.8 Monitoring

    Kafka uses Yammer Metrics for metrics reporting in the server. The Java clients use Kafka Metrics, a built-in metrics registry that minimizes transitive dependencies pulled into client applications. Both expose metrics via JMX and can be configured to report stats using pluggable stats reporters to hook up to your monitoring system.

    @@ -2866,7 +3024,7 @@

    6.8 ZooKeeper

    +

    6.9 ZooKeeper

    Stable version

    The current stable branch is 3.5. Kafka is regularly updated to include the latest release in the 3.5 series. diff --git a/docs/toc.html b/docs/toc.html index 8d15b2b9ea19c..fa11c813d3935 100644 --- a/docs/toc.html +++ b/docs/toc.html @@ -96,13 +96,24 @@
  • Applying Configuration Changes
  • Monitoring Geo-Replication
    • -
  • 6.4 Important Configs +
  • 6.4 Multi-Tenancy
    • + +
  • 6.5 Important Configs -
  • 6.5 Java Version -
  • 6.6 Hardware and OS +
  • 6.6 Java Version +
  • 6.7 Hardware and OS -
  • 6.7 Monitoring +
  • 6.8 Monitoring -
  • 6.8 ZooKeeper +
  • 6.9 ZooKeeper
    • Stable Version
    • Operationalization From be1476869fc93553b3099d387d26bfd0092a9d65 Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Fri, 5 Mar 2021 00:22:57 +0800 Subject: [PATCH 106/243] MINOR: make sure all generated data tests cover all versions (#10078) Reviewers: David Jacot --- .../apache/kafka/common/protocol/ApiKeys.java | 2 +- .../kafka/common/message/MessageTest.java | 24 +-- .../AddPartitionsToTxnRequestTest.java | 2 +- .../AddPartitionsToTxnResponseTest.java | 2 +- .../ControlledShutdownRequestTest.java | 2 +- .../common/requests/EndTxnRequestTest.java | 2 +- .../common/requests/EndTxnResponseTest.java | 2 +- .../common/requests/EnvelopeRequestTest.java | 2 +- .../common/requests/EnvelopeResponseTest.java | 2 +- .../requests/LeaderAndIsrRequestTest.java | 5 +- .../requests/LeaderAndIsrResponseTest.java | 6 +- .../requests/LeaveGroupRequestTest.java | 2 +- .../requests/LeaveGroupResponseTest.java | 10 +- .../requests/OffsetCommitRequestTest.java | 4 +- .../requests/OffsetCommitResponseTest.java | 2 +- .../requests/OffsetFetchRequestTest.java | 6 +- .../requests/OffsetFetchResponseTest.java | 4 +- .../OffsetsForLeaderEpochRequestTest.java | 6 +- .../common/requests/ProduceRequestTest.java | 32 ++-- .../common/requests/ProduceResponseTest.java | 6 +- .../common/requests/RequestResponseTest.java | 171 +++++++++--------- .../requests/StopReplicaRequestTest.java | 8 +- .../requests/StopReplicaResponseTest.java | 2 +- .../requests/TxnOffsetCommitRequestTest.java | 2 +- .../requests/TxnOffsetCommitResponseTest.java | 2 +- .../requests/UpdateMetadataRequestTest.java | 4 +- .../requests/WriteTxnMarkersRequestTest.java | 4 +- 27 files changed, 155 insertions(+), 161 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 3297dc5734aa6..8ec2d02ea3de6 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -201,7 +201,7 @@ public short oldestVersion() { public List allVersions() { List versions = new ArrayList<>(latestVersion() - oldestVersion() + 1); - for (short version = oldestVersion(); version < latestVersion(); version++) { + for (short version = oldestVersion(); version <= latestVersion(); version++) { versions.add(version); } return versions; diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java index 5dc379e0e7fee..e191ad6526ace 100644 --- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java @@ -186,7 +186,7 @@ public void testListOffsetsResponseVersions() throws Exception { .setPartitions(Collections.singletonList(partition))); Supplier response = () -> new ListOffsetsResponseData() .setTopics(topics); - for (short version = 0; version <= ApiKeys.LIST_OFFSETS.latestVersion(); version++) { + for (short version : ApiKeys.LIST_OFFSETS.allVersions()) { ListOffsetsResponseData responseData = response.get(); if (version > 0) { responseData.topics().get(0).partitions().get(0) @@ -459,7 +459,7 @@ public void testOffsetCommitRequestVersions() throws Exception { )))) .setRetentionTimeMs(20); - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_COMMIT.allVersions()) { OffsetCommitRequestData requestData = request.get(); if (version < 1) { requestData.setMemberId(""); @@ -485,7 +485,7 @@ public void testOffsetCommitRequestVersions() throws Exception { if (version == 1) { testEquivalentMessageRoundTrip(version, requestData); } else if (version >= 2 && version <= 4) { - testAllMessageRoundTripsBetweenVersions(version, (short) 4, requestData, requestData); + testAllMessageRoundTripsBetweenVersions(version, (short) 5, requestData, requestData); } else { testAllMessageRoundTripsFromVersion(version, requestData); } @@ -509,7 +509,7 @@ public void testOffsetCommitResponseVersions() throws Exception { ) .setThrottleTimeMs(20); - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_COMMIT.allVersions()) { OffsetCommitResponseData responseData = response.get(); if (version < 3) { responseData.setThrottleTimeMs(0); @@ -568,7 +568,7 @@ public void testTxnOffsetCommitRequestVersions() throws Exception { .setCommittedOffset(offset) )))); - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.TXN_OFFSET_COMMIT.allVersions()) { TxnOffsetCommitRequestData requestData = request.get(); if (version < 2) { requestData.topics().get(0).partitions().get(0).setCommittedLeaderEpoch(-1); @@ -632,7 +632,7 @@ public void testOffsetFetchVersions() throws Exception { .setTopics(topics) .setRequireStable(true); - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { final short finalVersion = version; if (version < 2) { assertThrows(NullPointerException.class, () -> testAllMessageRoundTripsFromVersion(finalVersion, allPartitionData)); @@ -661,7 +661,7 @@ public void testOffsetFetchVersions() throws Exception { .setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()))))) .setErrorCode(Errors.NOT_COORDINATOR.code()) .setThrottleTimeMs(10); - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { OffsetFetchResponseData responseData = response.get(); if (version <= 1) { responseData.setErrorCode(Errors.NONE.code()); @@ -720,7 +720,7 @@ public void testProduceResponseVersions() throws Exception { .setErrorMessage(errorMessage)))).iterator())) .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.PRODUCE.latestVersion(); version++) { + for (short version : ApiKeys.PRODUCE.allVersions()) { ProduceResponseData responseData = response.get(); if (version < 8) { @@ -741,9 +741,9 @@ public void testProduceResponseVersions() throws Exception { } if (version >= 3 && version <= 4) { - testAllMessageRoundTripsBetweenVersions(version, (short) 4, responseData, responseData); + testAllMessageRoundTripsBetweenVersions(version, (short) 5, responseData, responseData); } else if (version >= 6 && version <= 7) { - testAllMessageRoundTripsBetweenVersions(version, (short) 7, responseData, responseData); + testAllMessageRoundTripsBetweenVersions(version, (short) 8, responseData, responseData); } else { testEquivalentMessageRoundTrip(version, responseData); } @@ -924,8 +924,8 @@ public void testNonIgnorableFieldWithDefaultNull() { @Test public void testWriteNullForNonNullableFieldRaisesException() { CreateTopicsRequestData createTopics = new CreateTopicsRequestData().setTopics(null); - for (short i = (short) 0; i <= createTopics.highestSupportedVersion(); i++) { - verifyWriteRaisesNpe(i, createTopics); + for (short version : ApiKeys.CREATE_TOPICS.allVersions()) { + verifyWriteRaisesNpe(version, createTopics); } MetadataRequestData metadata = new MetadataRequestData().setTopics(null); verifyWriteRaisesNpe((short) 0, metadata); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequestTest.java index 24cb9a5d70251..acba8be966a75 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequestTest.java @@ -43,7 +43,7 @@ public void testConstructor() { AddPartitionsToTxnRequest.Builder builder = new AddPartitionsToTxnRequest.Builder(transactionalId, producerId, producerEpoch, partitions); - for (short version = 0; version <= ApiKeys.ADD_PARTITIONS_TO_TXN.latestVersion(); version++) { + for (short version : ApiKeys.ADD_PARTITIONS_TO_TXN.allVersions()) { AddPartitionsToTxnRequest request = builder.build(version); assertEquals(transactionalId, request.data().transactionalId()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java index b7901880bf48f..5b67bd47a01f6 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java @@ -89,7 +89,7 @@ public void testParse() { .setThrottleTimeMs(throttleTimeMs); AddPartitionsToTxnResponse response = new AddPartitionsToTxnResponse(data); - for (short version = 0; version <= ApiKeys.ADD_PARTITIONS_TO_TXN.latestVersion(); version++) { + for (short version : ApiKeys.ADD_PARTITIONS_TO_TXN.allVersions()) { AddPartitionsToTxnResponse parsedResponse = AddPartitionsToTxnResponse.parse(response.serialize(version), version); assertEquals(expectedErrorCounts, parsedResponse.errorCounts()); assertEquals(throttleTimeMs, parsedResponse.throttleTimeMs()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java index 04f294c718263..867be713ba2e2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java @@ -38,7 +38,7 @@ public void testUnsupportedVersion() { @Test public void testGetErrorResponse() { - for (short version = CONTROLLED_SHUTDOWN.oldestVersion(); version < CONTROLLED_SHUTDOWN.latestVersion(); version++) { + for (short version : CONTROLLED_SHUTDOWN.allVersions()) { ControlledShutdownRequest.Builder builder = new ControlledShutdownRequest.Builder( new ControlledShutdownRequestData().setBrokerId(1), version); ControlledShutdownRequest request = builder.build(); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnRequestTest.java index 722e9a5a589a9..f14bf66161980 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnRequestTest.java @@ -41,7 +41,7 @@ public void testConstructor() { .setProducerId(producerId) .setTransactionalId(transactionId)); - for (short version = 0; version <= ApiKeys.END_TXN.latestVersion(); version++) { + for (short version : ApiKeys.END_TXN.allVersions()) { EndTxnRequest request = builder.build(version); EndTxnResponse response = request.getErrorResponse(throttleTimeMs, Errors.NOT_COORDINATOR.exception()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java index 34646dc0b4b5d..39c4bc04104ae 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java @@ -38,7 +38,7 @@ public void testConstructor() { Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); - for (short version = 0; version <= ApiKeys.END_TXN.latestVersion(); version++) { + for (short version : ApiKeys.END_TXN.allVersions()) { EndTxnResponse response = new EndTxnResponse(data); assertEquals(expectedErrorCounts, response.errorCounts()); assertEquals(throttleTimeMs, response.throttleTimeMs()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeRequestTest.java index a7bccd56cf10f..36b8618f49df2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeRequestTest.java @@ -46,7 +46,7 @@ public void testGetPrincipal() { @Test public void testToSend() throws IOException { - for (short version = ApiKeys.ENVELOPE.oldestVersion(); version <= ApiKeys.ENVELOPE.latestVersion(); version++) { + for (short version : ApiKeys.ENVELOPE.allVersions()) { ByteBuffer requestData = ByteBuffer.wrap("foobar".getBytes()); RequestHeader header = new RequestHeader(ApiKeys.ENVELOPE, version, "clientId", 15); EnvelopeRequest request = new EnvelopeRequest.Builder( diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeResponseTest.java index 0a384cdd1661a..e0fa2fdbcddb0 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeResponseTest.java @@ -32,7 +32,7 @@ class EnvelopeResponseTest { @Test public void testToSend() { - for (short version = ApiKeys.ENVELOPE.oldestVersion(); version <= ApiKeys.ENVELOPE.latestVersion(); version++) { + for (short version : ApiKeys.ENVELOPE.allVersions()) { ByteBuffer responseData = ByteBuffer.wrap("foobar".getBytes()); EnvelopeResponse response = new EnvelopeResponse(responseData, Errors.NONE); short headerVersion = ApiKeys.ENVELOPE.responseHeaderVersion(version); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java index 4fe51a0dccd3c..de9914c575e31 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java @@ -64,8 +64,7 @@ public void testGetErrorResponse() { Uuid topicId = Uuid.randomUuid(); String topicName = "topic"; int partition = 0; - - for (short version = LEADER_AND_ISR.oldestVersion(); version <= LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(version, 0, 0, 0, Collections.singletonList(new LeaderAndIsrPartitionState() .setTopicName(topicName) @@ -108,7 +107,7 @@ public void testGetErrorResponse() { */ @Test public void testVersionLogic() { - for (short version = LEADER_AND_ISR.oldestVersion(); version <= LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { List partitionStates = asList( new LeaderAndIsrPartitionState() .setTopicName("topic0") diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java index 9ae2fdb4204fc..9f46304a4de31 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java @@ -71,7 +71,7 @@ public void testErrorCountsFromGetErrorResponse() { @Test public void testErrorCountsWithTopLevelError() { - for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { LeaderAndIsrResponse response; if (version < 5) { List partitions = createPartitions("foo", @@ -92,7 +92,7 @@ public void testErrorCountsWithTopLevelError() { @Test public void testErrorCountsNoTopLevelError() { - for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { LeaderAndIsrResponse response; if (version < 5) { List partitions = createPartitions("foo", @@ -116,7 +116,7 @@ public void testErrorCountsNoTopLevelError() { @Test public void testToString() { - for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { LeaderAndIsrResponse response; if (version < 5) { List partitions = createPartitions("foo", diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java index 411efef71f546..1694ef5fdf938 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java @@ -67,7 +67,7 @@ public void testMultiLeaveConstructor() { .setGroupId(groupId) .setMembers(members); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { try { LeaveGroupRequest request = builder.build(version); if (version <= 2) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index cc9819133f466..d5132182ceee2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -67,7 +67,7 @@ public void testConstructorWithMemberResponses() { expectedErrorCounts.put(Errors.UNKNOWN_MEMBER_ID, 1); expectedErrorCounts.put(Errors.FENCED_INSTANCE_ID, 1); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(memberResponses, Errors.NONE, throttleTimeMs, @@ -95,7 +95,7 @@ public void testConstructorWithMemberResponses() { @Test public void testShouldThrottle() { LeaveGroupResponse response = new LeaveGroupResponse(new LeaveGroupResponseData()); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { if (version >= 2) { assertTrue(response.shouldClientThrottle(version)); } else { @@ -109,7 +109,7 @@ public void testEqualityWithSerialization() { LeaveGroupResponseData responseData = new LeaveGroupResponseData() .setErrorCode(Errors.NONE.code()) .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { LeaveGroupResponse primaryResponse = LeaveGroupResponse.parse( MessageUtil.toByteBuffer(responseData, version), version); LeaveGroupResponse secondaryResponse = LeaveGroupResponse.parse( @@ -129,7 +129,7 @@ public void testParse() { .setErrorCode(Errors.NOT_COORDINATOR.code()) .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { ByteBuffer buffer = MessageUtil.toByteBuffer(data, version); LeaveGroupResponse leaveGroupResponse = LeaveGroupResponse.parse(buffer, version); assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); @@ -146,7 +146,7 @@ public void testParse() { @Test public void testEqualityWithMemberResponses() { - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { List localResponses = version > 2 ? memberResponses : memberResponses.subList(0, 1); LeaveGroupResponse primaryResponse = new LeaveGroupResponse(localResponses, Errors.NONE, diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java index cb853162209be..08ae7a3fbd572 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java @@ -91,7 +91,7 @@ public void testConstructor() { OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(data); - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.TXN_OFFSET_COMMIT.allVersions()) { OffsetCommitRequest request = builder.build(version); assertEquals(expectedOffsets, request.offsets()); @@ -130,7 +130,7 @@ public void testVersionSupportForGroupInstanceId() { .setGroupInstanceId(groupInstanceId) ); - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_COMMIT.allVersions()) { if (version >= 7) { builder.build(version); } else { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java index c6791e64f4f74..b9ce03f1e3ad4 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java @@ -85,7 +85,7 @@ public void testParse() { )) .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_COMMIT.allVersions()) { ByteBuffer buffer = MessageUtil.toByteBuffer(data, version); OffsetCommitResponse response = OffsetCommitResponse.parse(buffer, version); assertEquals(expectedErrorCounts, response.errorCounts()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java index 51db93d629d40..ddb2cd9a943ad 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java @@ -75,7 +75,7 @@ public void testConstructor() { )); } - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { OffsetFetchRequest request = builder.build(version); assertFalse(request.isAllPartitions()); assertEquals(groupId, request.groupId()); @@ -101,7 +101,7 @@ public void testConstructor() { @Test public void testConstructorFailForUnsupportedRequireStable() { - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { // The builder needs to be initialized every cycle as the internal data `requireStable` flag is flipped. builder = new OffsetFetchRequest.Builder(groupId, true, null, false); final short finalVersion = version; @@ -123,7 +123,7 @@ public void testConstructorFailForUnsupportedRequireStable() { @Test public void testBuildThrowForUnsupportedRequireStable() { - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { builder = new OffsetFetchRequest.Builder(groupId, true, null, true); if (version < 7) { final short finalVersion = version; diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java index 62cf3a9535399..e202fc200f7a8 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java @@ -103,7 +103,7 @@ public void testStructBuild() { OffsetFetchResponse latestResponse = new OffsetFetchResponse(throttleTimeMs, Errors.NONE, partitionDataMap); - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { OffsetFetchResponseData data = new OffsetFetchResponseData( new ByteBufferAccessor(latestResponse.serialize(version)), version); @@ -154,7 +154,7 @@ public void testStructBuild() { @Test public void testShouldThrottle() { OffsetFetchResponse response = new OffsetFetchResponse(throttleTimeMs, Errors.NONE, partitionDataMap); - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { if (version >= 4) { assertTrue(response.shouldClientThrottle(version)); } else { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java index 3d4b41b52e435..e5dacd5b3d799 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java @@ -34,15 +34,15 @@ public void testForConsumerRequiresVersion3() { assertThrows(UnsupportedVersionException.class, () -> builder.build(v)); } - for (short version = 3; version < ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion(); version++) { - OffsetsForLeaderEpochRequest request = builder.build((short) 3); + for (short version = 3; version <= ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion(); version++) { + OffsetsForLeaderEpochRequest request = builder.build(version); assertEquals(OffsetsForLeaderEpochRequest.CONSUMER_REPLICA_ID, request.replicaId()); } } @Test public void testDefaultReplicaId() { - for (short version = 0; version < ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FOR_LEADER_EPOCH.allVersions()) { int replicaId = 1; OffsetsForLeaderEpochRequest.Builder builder = OffsetsForLeaderEpochRequest.Builder.forFollower( version, new OffsetForLeaderTopicCollection(), replicaId); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java index a2367e4904285..fee026e7481cf 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java @@ -18,6 +18,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.InvalidRecordException; +import org.apache.kafka.common.errors.UnsupportedCompressionTypeException; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.record.CompressionType; @@ -32,11 +33,12 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.stream.IntStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; public class ProduceRequestTest { @@ -151,7 +153,7 @@ public void testV3AndAboveShouldContainOnlyOneRecordBatch() { .setRecords(MemoryRecords.readableRecords(buffer))))).iterator())) .setAcks((short) 1) .setTimeoutMs(5000)); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, InvalidRecordException.class); } @Test @@ -166,7 +168,7 @@ public void testV3AndAboveCannotHaveNoRecordBatches() { .setRecords(MemoryRecords.EMPTY)))).iterator())) .setAcks((short) 1) .setTimeoutMs(5000)); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, InvalidRecordException.class); } @Test @@ -186,7 +188,7 @@ public void testV3AndAboveCannotUseMagicV0() { .setRecords(builder.build())))).iterator())) .setAcks((short) 1) .setTimeoutMs(5000)); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, InvalidRecordException.class); } @Test @@ -206,7 +208,7 @@ public void testV3AndAboveCannotUseMagicV1() { .iterator())) .setAcks((short) 1) .setTimeoutMs(5000)); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, InvalidRecordException.class); } @Test @@ -230,7 +232,7 @@ public void testV6AndBelowCannotUseZStdCompression() { for (short version = 3; version < 7; version++) { ProduceRequest.Builder requestBuilder = new ProduceRequest.Builder(version, version, produceData); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, UnsupportedCompressionTypeException.class); } // Works fine with current version (>= 7) @@ -291,20 +293,10 @@ public void testMixedIdempotentData() { assertTrue(RequestTestUtils.hasIdempotentRecords(request)); } - private void assertThrowsInvalidRecordExceptionForAllVersions(ProduceRequest.Builder builder) { - for (short version = builder.oldestAllowedVersion(); version < builder.latestAllowedVersion(); version++) { - assertThrowsInvalidRecordException(builder, version); - } - } - - private void assertThrowsInvalidRecordException(ProduceRequest.Builder builder, short version) { - try { - builder.build(version).serialize(); - fail("Builder did not raise " + InvalidRecordException.class.getName() + " as expected"); - } catch (RuntimeException e) { - assertTrue(InvalidRecordException.class.isAssignableFrom(e.getClass()), - "Unexpected exception type " + e.getClass().getName()); - } + private static void assertThrowsForAllVersions(ProduceRequest.Builder builder, + Class expectedType) { + IntStream.range(builder.oldestAllowedVersion(), builder.latestAllowedVersion() + 1) + .forEach(version -> assertThrows(expectedType, () -> builder.build((short) version).serialize())); } private ProduceRequest createNonIdempotentNonTransactionalRecords() { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java index bfadb8c3bbd70..771ff0da7b44e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java @@ -89,10 +89,10 @@ public void produceResponseRecordErrorsTest() { "Produce failed"); responseData.put(tp, partResponse); - for (short ver = 0; ver <= PRODUCE.latestVersion(); ver++) { + for (short version : PRODUCE.allVersions()) { ProduceResponse response = new ProduceResponse(responseData); - ProduceResponse.PartitionResponse deserialized = ProduceResponse.parse(response.serialize(ver), ver).responses().get(tp); - if (ver >= 8) { + ProduceResponse.PartitionResponse deserialized = ProduceResponse.parse(response.serialize(version), version).responses().get(tp); + if (version >= 8) { assertEquals(1, deserialized.recordErrors.size()); assertEquals(3, deserialized.recordErrors.get(0).batchIndex); assertEquals("Record error", deserialized.recordErrors.get(0).message); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index cebb99d17ef22..d3d965d3910ec 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -221,11 +221,16 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.apache.kafka.common.protocol.ApiKeys.CREATE_PARTITIONS; +import static org.apache.kafka.common.protocol.ApiKeys.CREATE_TOPICS; +import static org.apache.kafka.common.protocol.ApiKeys.DELETE_TOPICS; import static org.apache.kafka.common.protocol.ApiKeys.DESCRIBE_CONFIGS; import static org.apache.kafka.common.protocol.ApiKeys.FETCH; import static org.apache.kafka.common.protocol.ApiKeys.JOIN_GROUP; +import static org.apache.kafka.common.protocol.ApiKeys.LEADER_AND_ISR; import static org.apache.kafka.common.protocol.ApiKeys.LIST_GROUPS; import static org.apache.kafka.common.protocol.ApiKeys.LIST_OFFSETS; +import static org.apache.kafka.common.protocol.ApiKeys.STOP_REPLICA; import static org.apache.kafka.common.protocol.ApiKeys.SYNC_GROUP; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -267,26 +272,26 @@ public void testSerialization() throws Exception { checkErrorResponse(createHeartBeatRequest(), unknownServerException, true); checkResponse(createHeartBeatResponse(), 0, true); - for (int v = ApiKeys.JOIN_GROUP.oldestVersion(); v <= ApiKeys.JOIN_GROUP.latestVersion(); v++) { - checkRequest(createJoinGroupRequest(v), true); - checkErrorResponse(createJoinGroupRequest(v), unknownServerException, true); - checkResponse(createJoinGroupResponse(v), v, true); + for (short version : JOIN_GROUP.allVersions()) { + checkRequest(createJoinGroupRequest(version), true); + checkErrorResponse(createJoinGroupRequest(version), unknownServerException, true); + checkResponse(createJoinGroupResponse(version), version, true); } - for (int v = ApiKeys.SYNC_GROUP.oldestVersion(); v <= ApiKeys.SYNC_GROUP.latestVersion(); v++) { - checkRequest(createSyncGroupRequest(v), true); - checkErrorResponse(createSyncGroupRequest(v), unknownServerException, true); - checkResponse(createSyncGroupResponse(v), v, true); + for (short version : SYNC_GROUP.allVersions()) { + checkRequest(createSyncGroupRequest(version), true); + checkErrorResponse(createSyncGroupRequest(version), unknownServerException, true); + checkResponse(createSyncGroupResponse(version), version, true); } checkRequest(createLeaveGroupRequest(), true); checkErrorResponse(createLeaveGroupRequest(), unknownServerException, true); checkResponse(createLeaveGroupResponse(), 0, true); - for (short v = ApiKeys.LIST_GROUPS.oldestVersion(); v <= ApiKeys.LIST_GROUPS.latestVersion(); v++) { - checkRequest(createListGroupsRequest(v), false); - checkErrorResponse(createListGroupsRequest(v), unknownServerException, true); - checkResponse(createListGroupsResponse(v), v, true); + for (short version : ApiKeys.LIST_GROUPS.allVersions()) { + checkRequest(createListGroupsRequest(version), false); + checkErrorResponse(createListGroupsRequest(version), unknownServerException, true); + checkResponse(createListGroupsResponse(version), version, true); } checkRequest(createDescribeGroupRequest(), true); @@ -295,10 +300,10 @@ public void testSerialization() throws Exception { checkRequest(createDeleteGroupsRequest(), true); checkErrorResponse(createDeleteGroupsRequest(), unknownServerException, true); checkResponse(createDeleteGroupsResponse(), 0, true); - for (int i = 0; i < ApiKeys.LIST_OFFSETS.latestVersion(); i++) { - checkRequest(createListOffsetRequest(i), true); - checkErrorResponse(createListOffsetRequest(i), unknownServerException, true); - checkResponse(createListOffsetResponse(i), i, true); + for (short version : LIST_OFFSETS.allVersions()) { + checkRequest(createListOffsetRequest(version), true); + checkErrorResponse(createListOffsetRequest(version), unknownServerException, true); + checkResponse(createListOffsetResponse(version), version, true); } checkRequest(MetadataRequest.Builder.allTopics().build((short) 2), true); checkRequest(createMetadataRequest(1, Collections.singletonList("topic1")), true); @@ -331,18 +336,18 @@ public void testSerialization() throws Exception { checkResponse(createProduceResponse(), 2, true); checkResponse(createProduceResponseWithErrorMessage(), 8, true); - for (int v = ApiKeys.STOP_REPLICA.oldestVersion(); v <= ApiKeys.STOP_REPLICA.latestVersion(); v++) { - checkRequest(createStopReplicaRequest(v, true), true); - checkRequest(createStopReplicaRequest(v, false), true); - checkErrorResponse(createStopReplicaRequest(v, true), unknownServerException, true); - checkErrorResponse(createStopReplicaRequest(v, false), unknownServerException, true); - checkResponse(createStopReplicaResponse(), v, true); + for (short version : STOP_REPLICA.allVersions()) { + checkRequest(createStopReplicaRequest(version, true), true); + checkRequest(createStopReplicaRequest(version, false), true); + checkErrorResponse(createStopReplicaRequest(version, true), unknownServerException, true); + checkErrorResponse(createStopReplicaRequest(version, false), unknownServerException, true); + checkResponse(createStopReplicaResponse(), version, true); } - for (int v = ApiKeys.LEADER_AND_ISR.oldestVersion(); v <= ApiKeys.LEADER_AND_ISR.latestVersion(); v++) { - checkRequest(createLeaderAndIsrRequest(v), true); - checkErrorResponse(createLeaderAndIsrRequest(v), unknownServerException, false); - checkResponse(createLeaderAndIsrResponse(v), v, true); + for (short version : LEADER_AND_ISR.allVersions()) { + checkRequest(createLeaderAndIsrRequest(version), true); + checkErrorResponse(createLeaderAndIsrRequest(version), unknownServerException, false); + checkResponse(createLeaderAndIsrResponse(version), version, true); } checkRequest(createSaslHandshakeRequest(), true); @@ -353,23 +358,23 @@ public void testSerialization() throws Exception { checkResponse(createSaslAuthenticateResponse(), 0, true); checkResponse(createSaslAuthenticateResponse(), 1, true); - for (int v = ApiKeys.CREATE_TOPICS.oldestVersion(); v <= ApiKeys.CREATE_TOPICS.latestVersion(); v++) { - checkRequest(createCreateTopicRequest(v), true); - checkErrorResponse(createCreateTopicRequest(v), unknownServerException, true); - checkResponse(createCreateTopicResponse(), v, true); + for (short version : CREATE_TOPICS.allVersions()) { + checkRequest(createCreateTopicRequest(version), true); + checkErrorResponse(createCreateTopicRequest(version), unknownServerException, true); + checkResponse(createCreateTopicResponse(), version, true); } - for (int v = ApiKeys.DELETE_TOPICS.oldestVersion(); v <= ApiKeys.DELETE_TOPICS.latestVersion(); v++) { - checkRequest(createDeleteTopicsRequest(v), true); - checkErrorResponse(createDeleteTopicsRequest(v), unknownServerException, true); - checkResponse(createDeleteTopicsResponse(), v, true); + for (short version : DELETE_TOPICS.allVersions()) { + checkRequest(createDeleteTopicsRequest(version), true); + checkErrorResponse(createDeleteTopicsRequest(version), unknownServerException, true); + checkResponse(createDeleteTopicsResponse(), version, true); } - for (int v = ApiKeys.CREATE_PARTITIONS.oldestVersion(); v <= ApiKeys.CREATE_PARTITIONS.latestVersion(); v++) { - checkRequest(createCreatePartitionsRequest(v), true); - checkRequest(createCreatePartitionsRequestWithAssignments(v), false); - checkErrorResponse(createCreatePartitionsRequest(v), unknownServerException, true); - checkResponse(createCreatePartitionsResponse(), v, true); + for (short version : CREATE_PARTITIONS.allVersions()) { + checkRequest(createCreatePartitionsRequest(version), true); + checkRequest(createCreatePartitionsRequestWithAssignments(version), false); + checkErrorResponse(createCreatePartitionsRequest(version), unknownServerException, true); + checkResponse(createCreatePartitionsResponse(), version, true); } checkRequest(createInitPidRequest(), true); @@ -518,75 +523,75 @@ public void testSerialization() throws Exception { @Test public void testApiVersionsSerialization() { - for (short v : ApiKeys.API_VERSIONS.allVersions()) { - checkRequest(createApiVersionRequest(v), true); - checkErrorResponse(createApiVersionRequest(v), unknownServerException, true); - checkErrorResponse(createApiVersionRequest(v), new UnsupportedVersionException("Not Supported"), true); - checkResponse(createApiVersionResponse(), v, true); - checkResponse(ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER), v, true); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + checkRequest(createApiVersionRequest(version), true); + checkErrorResponse(createApiVersionRequest(version), unknownServerException, true); + checkErrorResponse(createApiVersionRequest(version), new UnsupportedVersionException("Not Supported"), true); + checkResponse(createApiVersionResponse(), version, true); + checkResponse(ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER), version, true); } } @Test public void testBrokerHeartbeatSerialization() { - for (short v : ApiKeys.BROKER_HEARTBEAT.allVersions()) { - checkRequest(createBrokerHeartbeatRequest(v), true); - checkErrorResponse(createBrokerHeartbeatRequest(v), unknownServerException, true); - checkResponse(createBrokerHeartbeatResponse(), v, true); + for (short version : ApiKeys.BROKER_HEARTBEAT.allVersions()) { + checkRequest(createBrokerHeartbeatRequest(version), true); + checkErrorResponse(createBrokerHeartbeatRequest(version), unknownServerException, true); + checkResponse(createBrokerHeartbeatResponse(), version, true); } } @Test public void testBrokerRegistrationSerialization() { - for (short v : ApiKeys.BROKER_REGISTRATION.allVersions()) { - checkRequest(createBrokerRegistrationRequest(v), true); - checkErrorResponse(createBrokerRegistrationRequest(v), unknownServerException, true); + for (short version : ApiKeys.BROKER_REGISTRATION.allVersions()) { + checkRequest(createBrokerRegistrationRequest(version), true); + checkErrorResponse(createBrokerRegistrationRequest(version), unknownServerException, true); checkResponse(createBrokerRegistrationResponse(), 0, true); } } @Test public void testDescribeProducersSerialization() { - for (short v : ApiKeys.DESCRIBE_PRODUCERS.allVersions()) { - checkRequest(createDescribeProducersRequest(v), true); - checkErrorResponse(createDescribeProducersRequest(v), unknownServerException, true); - checkResponse(createDescribeProducersResponse(), v, true); + for (short version : ApiKeys.DESCRIBE_PRODUCERS.allVersions()) { + checkRequest(createDescribeProducersRequest(version), true); + checkErrorResponse(createDescribeProducersRequest(version), unknownServerException, true); + checkResponse(createDescribeProducersResponse(), version, true); } } @Test public void testDescribeTransactionsSerialization() { - for (short v : ApiKeys.DESCRIBE_TRANSACTIONS.allVersions()) { - checkRequest(createDescribeTransactionsRequest(v), true); - checkErrorResponse(createDescribeTransactionsRequest(v), unknownServerException, true); - checkResponse(createDescribeTransactionsResponse(), v, true); + for (short version : ApiKeys.DESCRIBE_TRANSACTIONS.allVersions()) { + checkRequest(createDescribeTransactionsRequest(version), true); + checkErrorResponse(createDescribeTransactionsRequest(version), unknownServerException, true); + checkResponse(createDescribeTransactionsResponse(), version, true); } } @Test public void testListTransactionsSerialization() { - for (short v : ApiKeys.LIST_TRANSACTIONS.allVersions()) { - checkRequest(createListTransactionsRequest(v), true); - checkErrorResponse(createListTransactionsRequest(v), unknownServerException, true); - checkResponse(createListTransactionsResponse(), v, true); + for (short version : ApiKeys.LIST_TRANSACTIONS.allVersions()) { + checkRequest(createListTransactionsRequest(version), true); + checkErrorResponse(createListTransactionsRequest(version), unknownServerException, true); + checkResponse(createListTransactionsResponse(), version, true); } } @Test public void testDescribeClusterSerialization() { - for (short v : ApiKeys.DESCRIBE_CLUSTER.allVersions()) { - checkRequest(createDescribeClusterRequest(v), true); - checkErrorResponse(createDescribeClusterRequest(v), unknownServerException, true); - checkResponse(createDescribeClusterResponse(), v, true); + for (short version : ApiKeys.DESCRIBE_CLUSTER.allVersions()) { + checkRequest(createDescribeClusterRequest(version), true); + checkErrorResponse(createDescribeClusterRequest(version), unknownServerException, true); + checkResponse(createDescribeClusterResponse(), version, true); } } @Test public void testUnregisterBrokerSerialization() { - for (short v : ApiKeys.UNREGISTER_BROKER.allVersions()) { - checkRequest(createUnregisterBrokerRequest(v), true); - checkErrorResponse(createUnregisterBrokerRequest(v), unknownServerException, true); - checkResponse(createUnregisterBrokerResponse(), v, true); + for (short version : ApiKeys.UNREGISTER_BROKER.allVersions()) { + checkRequest(createUnregisterBrokerRequest(version), true); + checkErrorResponse(createUnregisterBrokerRequest(version), unknownServerException, true); + checkResponse(createUnregisterBrokerResponse(), version, true); } } @@ -623,13 +628,12 @@ public void testResponseHeader() { } private void checkOlderFetchVersions() { - int latestVersion = FETCH.latestVersion(); - for (int i = 0; i < latestVersion; ++i) { - if (i > 7) { - checkErrorResponse(createFetchRequest(i), unknownServerException, true); + for (short version : FETCH.allVersions()) { + if (version > 7) { + checkErrorResponse(createFetchRequest(version), unknownServerException, true); } - checkRequest(createFetchRequest(i), true); - checkResponse(createFetchResponse(i >= 4), i, true); + checkRequest(createFetchRequest(version), true); + checkResponse(createFetchResponse(version >= 4), version, true); } } @@ -668,12 +672,11 @@ private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, Des } private void checkDescribeConfigsResponseVersions() { - for (int version = ApiKeys.DESCRIBE_CONFIGS.oldestVersion(); version < ApiKeys.DESCRIBE_CONFIGS.latestVersion(); ++version) { - short apiVersion = (short) version; - DescribeConfigsResponse response = createDescribeConfigsResponse(apiVersion); + for (short version : ApiKeys.DESCRIBE_CONFIGS.allVersions()) { + DescribeConfigsResponse response = createDescribeConfigsResponse(version); DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) AbstractResponse.parseResponse(ApiKeys.DESCRIBE_CONFIGS, - response.serialize(apiVersion), apiVersion); - verifyDescribeConfigsResponse(response, deserialized0, apiVersion); + response.serialize(version), version); + verifyDescribeConfigsResponse(response, deserialized0, version); } } @@ -859,7 +862,7 @@ public void verifyFetchResponseFullWrites() throws Exception { verifyFetchResponseFullWrite(FETCH.latestVersion(), createFetchResponse(123)); verifyFetchResponseFullWrite(FETCH.latestVersion(), createFetchResponse(Errors.FETCH_SESSION_ID_NOT_FOUND, 123)); - for (short version = 0; version <= FETCH.latestVersion(); version++) { + for (short version : FETCH.allVersions()) { verifyFetchResponseFullWrite(version, createFetchResponse(version >= 4)); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java index 50c6974613951..3446d498c6aac 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java @@ -66,7 +66,7 @@ public void testGetErrorResponse() { } } - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder(version, 0, 0, 0L, false, topicStates); StopReplicaRequest request = builder.build(); @@ -93,7 +93,7 @@ private void testBuilderNormalization(boolean deletePartitions) { Map expectedPartitionStates = StopReplicaRequestTest.partitionStates(topicStates); - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { StopReplicaRequest request = new StopReplicaRequest.Builder(version, 0, 1, 0, deletePartitions, topicStates).build(version); StopReplicaRequestData data = request.data(); @@ -128,7 +128,7 @@ private void testBuilderNormalization(boolean deletePartitions) { public void testTopicStatesNormalization() { List topicStates = topicStates(true); - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); @@ -163,7 +163,7 @@ public void testTopicStatesNormalization() { public void testPartitionStatesNormalization() { List topicStates = topicStates(true); - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java index c3d049d482cc9..a0a5eda01e822 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java @@ -44,7 +44,7 @@ public void testErrorCountsFromGetErrorResponse() { new StopReplicaPartitionState().setPartitionIndex(0), new StopReplicaPartitionState().setPartitionIndex(1)))); - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { StopReplicaRequest request = new StopReplicaRequest.Builder(version, 15, 20, 0, false, topicStates).build(version); StopReplicaResponse response = request diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java index ec31e587c81e1..037066e9a036d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java @@ -116,7 +116,7 @@ public void testConstructor() { )) ); - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.TXN_OFFSET_COMMIT.allVersions()) { final TxnOffsetCommitRequest request; if (version < 3) { request = builder.build(version); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java index f3510f1725159..1f19ff2c93714 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java @@ -54,7 +54,7 @@ public void testParse() { .setErrorCode(errorTwo.code()))) )); - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.TXN_OFFSET_COMMIT.allVersions()) { TxnOffsetCommitResponse response = TxnOffsetCommitResponse.parse( MessageUtil.toByteBuffer(data, version), version); assertEquals(expectedErrorCounts, response.errorCounts()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java index 86178ca8521e3..6f9d5c2454606 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java @@ -61,7 +61,7 @@ public void testUnsupportedVersion() { @Test public void testGetErrorResponse() { - for (short version = UPDATE_METADATA.oldestVersion(); version < UPDATE_METADATA.latestVersion(); version++) { + for (short version : UPDATE_METADATA.allVersions()) { UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder( version, 0, 0, 0, Collections.emptyList(), Collections.emptyList(), Collections.emptyMap()); UpdateMetadataRequest request = builder.build(); @@ -81,7 +81,7 @@ public void testGetErrorResponse() { public void testVersionLogic() { String topic0 = "topic0"; String topic1 = "topic1"; - for (short version = UPDATE_METADATA.oldestVersion(); version <= UPDATE_METADATA.latestVersion(); version++) { + for (short version : UPDATE_METADATA.allVersions()) { List partitionStates = asList( new UpdateMetadataPartitionState() .setTopicName(topic0) diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java index 6435845b766c8..13e8c8cd94035 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java @@ -51,7 +51,7 @@ public void setUp() { @Test public void testConstructor() { WriteTxnMarkersRequest.Builder builder = new WriteTxnMarkersRequest.Builder(ApiKeys.WRITE_TXN_MARKERS.latestVersion(), markers); - for (short version = 0; version <= ApiKeys.WRITE_TXN_MARKERS.latestVersion(); version++) { + for (short version : ApiKeys.WRITE_TXN_MARKERS.allVersions()) { WriteTxnMarkersRequest request = builder.build(version); assertEquals(1, request.markers().size()); WriteTxnMarkersRequest.TxnMarkerEntry marker = request.markers().get(0); @@ -66,7 +66,7 @@ public void testConstructor() { @Test public void testGetErrorResponse() { WriteTxnMarkersRequest.Builder builder = new WriteTxnMarkersRequest.Builder(ApiKeys.WRITE_TXN_MARKERS.latestVersion(), markers); - for (short version = 0; version <= ApiKeys.WRITE_TXN_MARKERS.latestVersion(); version++) { + for (short version : ApiKeys.WRITE_TXN_MARKERS.allVersions()) { WriteTxnMarkersRequest request = builder.build(version); WriteTxnMarkersResponse errorResponse = request.getErrorResponse(throttleTimeMs, Errors.UNKNOWN_PRODUCER_ID.exception()); From 96a2b7aac4f1c4a9e020ccc08dadc7a71b460abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?= Date: Thu, 4 Mar 2021 10:55:43 -0800 Subject: [PATCH 107/243] KAFKA-12376: Apply atomic append to the log (#10253) --- .../main/scala/kafka/raft/RaftManager.scala | 29 ++++++- .../controller/ClientQuotaControlManager.java | 2 +- .../controller/ClusterControlManager.java | 2 +- .../ConfigurationControlManager.java | 4 +- .../kafka/controller/ControllerResult.java | 38 ++++++--- .../controller/ControllerResultAndOffset.java | 32 +++---- .../controller/FeatureControlManager.java | 3 +- .../kafka/controller/QuorumController.java | 23 ++--- .../controller/ReplicationControlManager.java | 12 +-- .../apache/kafka/metalog/LocalLogManager.java | 17 +++- .../apache/kafka/metalog/MetaLogManager.java | 23 ++++- .../ConfigurationControlManagerTest.java | 83 ++++++++++++++----- .../controller/FeatureControlManagerTest.java | 58 +++++++++---- .../apache/kafka/metalog/LocalLogManager.java | 17 +++- .../kafka/raft/metadata/MetaLogRaftShim.java | 28 ++++++- 15 files changed, 268 insertions(+), 103 deletions(-) diff --git a/core/src/main/scala/kafka/raft/RaftManager.scala b/core/src/main/scala/kafka/raft/RaftManager.scala index 1881a1db9a70d..3b714f3786862 100644 --- a/core/src/main/scala/kafka/raft/RaftManager.scala +++ b/core/src/main/scala/kafka/raft/RaftManager.scala @@ -92,6 +92,11 @@ trait RaftManager[T] { listener: RaftClient.Listener[T] ): Unit + def scheduleAtomicAppend( + epoch: Int, + records: Seq[T] + ): Option[Long] + def scheduleAppend( epoch: Int, records: Seq[T] @@ -157,16 +162,32 @@ class KafkaRaftManager[T]( raftClient.register(listener) } + override def scheduleAtomicAppend( + epoch: Int, + records: Seq[T] + ): Option[Long] = { + append(epoch, records, true) + } + override def scheduleAppend( epoch: Int, records: Seq[T] ): Option[Long] = { - val offset: java.lang.Long = raftClient.scheduleAppend(epoch, records.asJava) - if (offset == null) { - None + append(epoch, records, false) + } + + private def append( + epoch: Int, + records: Seq[T], + isAtomic: Boolean + ): Option[Long] = { + val offset = if (isAtomic) { + raftClient.scheduleAtomicAppend(epoch, records.asJava) } else { - Some(Long.unbox(offset)) + raftClient.scheduleAppend(epoch, records.asJava) } + + Option(offset).map(Long.unbox) } override def handleRequest( diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java index 4aac9e4882f46..9b8e2d683b650 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java @@ -86,7 +86,7 @@ ControllerResult> alterClientQuotas( } }); - return new ControllerResult<>(outputRecords, outputResults); + return ControllerResult.atomicOf(outputRecords, outputResults); } /** diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java index 6e329c72a0e3f..4748d195986ab 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java @@ -213,7 +213,7 @@ public ControllerResult registerBroker( List records = new ArrayList<>(); records.add(new ApiMessageAndVersion(record, (short) 0)); - return new ControllerResult<>(records, new BrokerRegistrationReply(brokerEpoch)); + return ControllerResult.of(records, new BrokerRegistrationReply(brokerEpoch)); } public void replay(RegisterBrokerRecord record) { diff --git a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java index 4402b3a117d83..5bc82ecafa477 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java @@ -83,7 +83,7 @@ ControllerResult> incrementalAlterConfigs( outputRecords, outputResults); } - return new ControllerResult<>(outputRecords, outputResults); + return ControllerResult.atomicOf(outputRecords, outputResults); } private void incrementalAlterConfigResource(ConfigResource configResource, @@ -171,7 +171,7 @@ ControllerResult> legacyAlterConfigs( outputRecords, outputResults); } - return new ControllerResult<>(outputRecords, outputResults); + return ControllerResult.atomicOf(outputRecords, outputResults); } private void legacyAlterConfigResource(ConfigResource configResource, diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java index 4906c8b0972a8..e6ae031b9b3b8 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java @@ -19,7 +19,7 @@ import org.apache.kafka.metadata.ApiMessageAndVersion; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -28,15 +28,13 @@ class ControllerResult { private final List records; private final T response; + private final boolean isAtomic; - public ControllerResult(T response) { - this(new ArrayList<>(), response); - } - - public ControllerResult(List records, T response) { + protected ControllerResult(List records, T response, boolean isAtomic) { Objects.requireNonNull(records); this.records = records; this.response = response; + this.isAtomic = isAtomic; } public List records() { @@ -47,6 +45,10 @@ public T response() { return response; } + public boolean isAtomic() { + return isAtomic; + } + @Override public boolean equals(Object o) { if (o == null || (!o.getClass().equals(getClass()))) { @@ -54,22 +56,34 @@ public boolean equals(Object o) { } ControllerResult other = (ControllerResult) o; return records.equals(other.records) && - Objects.equals(response, other.response); + Objects.equals(response, other.response) && + Objects.equals(isAtomic, other.isAtomic); } @Override public int hashCode() { - return Objects.hash(records, response); + return Objects.hash(records, response, isAtomic); } @Override public String toString() { - return "ControllerResult(records=" + String.join(",", - records.stream().map(r -> r.toString()).collect(Collectors.toList())) + - ", response=" + response + ")"; + return String.format( + "ControllerResult(records=%s, response=%s, isAtomic=%s)", + String.join(",", records.stream().map(ApiMessageAndVersion::toString).collect(Collectors.toList())), + response, + isAtomic + ); } public ControllerResult withoutRecords() { - return new ControllerResult<>(new ArrayList<>(), response); + return new ControllerResult<>(Collections.emptyList(), response, false); + } + + public static ControllerResult atomicOf(List records, T response) { + return new ControllerResult<>(records, response, true); + } + + public static ControllerResult of(List records, T response) { + return new ControllerResult<>(records, response, false); } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java index 5e483f773d5e2..8b8ca8dea80da 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java @@ -19,24 +19,15 @@ import org.apache.kafka.metadata.ApiMessageAndVersion; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -class ControllerResultAndOffset extends ControllerResult { +final class ControllerResultAndOffset extends ControllerResult { private final long offset; - public ControllerResultAndOffset(T response) { - super(new ArrayList<>(), response); - this.offset = -1; - } - - public ControllerResultAndOffset(long offset, - List records, - T response) { - super(records, response); + private ControllerResultAndOffset(long offset, ControllerResult result) { + super(result.records(), result.response(), result.isAtomic()); this.offset = offset; } @@ -52,18 +43,27 @@ public boolean equals(Object o) { ControllerResultAndOffset other = (ControllerResultAndOffset) o; return records().equals(other.records()) && response().equals(other.response()) && + isAtomic() == other.isAtomic() && offset == other.offset; } @Override public int hashCode() { - return Objects.hash(records(), response(), offset); + return Objects.hash(records(), response(), isAtomic(), offset); } @Override public String toString() { - return "ControllerResultAndOffset(records=" + String.join(",", - records().stream().map(r -> r.toString()).collect(Collectors.toList())) + - ", response=" + response() + ", offset=" + offset + ")"; + return String.format( + "ControllerResultAndOffset(records=%s, response=%s, isAtomic=%s, offset=%s)", + String.join(",", records().stream().map(ApiMessageAndVersion::toString).collect(Collectors.toList())), + response(), + isAtomic(), + offset + ); + } + + public static ControllerResultAndOffset of(long offset, ControllerResult result) { + return new ControllerResultAndOffset<>(offset, result); } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index 25ff3fdcdd80e..99874ac3c5ef7 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -69,7 +69,8 @@ ControllerResult> updateFeatures( results.put(entry.getKey(), updateFeature(entry.getKey(), entry.getValue(), downgradeables.contains(entry.getKey()), brokerFeatures, records)); } - return new ControllerResult<>(records, results); + + return ControllerResult.atomicOf(records, results); } private ApiError updateFeature(String featureName, diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 198097538985a..759db1efac790 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -17,7 +17,6 @@ package org.apache.kafka.controller; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -265,7 +264,7 @@ private Throwable handleEventException(String name, class ControlEvent implements EventQueue.Event { private final String name; private final Runnable handler; - private long eventCreatedTimeNs = time.nanoseconds(); + private final long eventCreatedTimeNs = time.nanoseconds(); private Optional startProcessingTimeNs = Optional.empty(); ControlEvent(String name, Runnable handler) { @@ -307,7 +306,7 @@ class ControllerReadEvent implements EventQueue.Event { private final String name; private final CompletableFuture future; private final Supplier handler; - private long eventCreatedTimeNs = time.nanoseconds(); + private final long eventCreatedTimeNs = time.nanoseconds(); private Optional startProcessingTimeNs = Optional.empty(); ControllerReadEvent(String name, Supplier handler) { @@ -389,7 +388,7 @@ class ControllerWriteEvent implements EventQueue.Event, DeferredEvent { private final String name; private final CompletableFuture future; private final ControllerWriteOperation op; - private long eventCreatedTimeNs = time.nanoseconds(); + private final long eventCreatedTimeNs = time.nanoseconds(); private Optional startProcessingTimeNs = Optional.empty(); private ControllerResultAndOffset resultAndOffset; @@ -423,8 +422,7 @@ public void run() throws Exception { if (!maybeOffset.isPresent()) { // If the purgatory is empty, there are no pending operations and no // uncommitted state. We can return immediately. - resultAndOffset = new ControllerResultAndOffset<>(-1, - new ArrayList<>(), result.response()); + resultAndOffset = ControllerResultAndOffset.of(-1, result); log.debug("Completing read-only operation {} immediately because " + "the purgatory is empty.", this); complete(null); @@ -432,8 +430,7 @@ public void run() throws Exception { } // If there are operations in the purgatory, we want to wait for the latest // one to complete before returning our result to the user. - resultAndOffset = new ControllerResultAndOffset<>(maybeOffset.get(), - result.records(), result.response()); + resultAndOffset = ControllerResultAndOffset.of(maybeOffset.get(), result); log.debug("Read-only operation {} will be completed when the log " + "reaches offset {}", this, resultAndOffset.offset()); } else { @@ -441,11 +438,15 @@ public void run() throws Exception { // written before we can return our result to the user. Here, we hand off // the batch of records to the metadata log manager. They will be written // out asynchronously. - long offset = logManager.scheduleWrite(controllerEpoch, result.records()); + final long offset; + if (result.isAtomic()) { + offset = logManager.scheduleAtomicWrite(controllerEpoch, result.records()); + } else { + offset = logManager.scheduleWrite(controllerEpoch, result.records()); + } op.processBatchEndOffset(offset); writeOffset = offset; - resultAndOffset = new ControllerResultAndOffset<>(offset, - result.records(), result.response()); + resultAndOffset = ControllerResultAndOffset.of(offset, result); for (ApiMessageAndVersion message : result.records()) { replay(message.message()); } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java index 9fd172f486ae6..ca571058265b7 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @@ -407,7 +407,7 @@ public void replay(PartitionChangeRecord record) { resultsPrefix = ", "; } log.info("createTopics result(s): {}", resultsBuilder.toString()); - return new ControllerResult<>(records, data); + return ControllerResult.atomicOf(records, data); } private ApiError createTopic(CreatableTopic topic, @@ -626,7 +626,7 @@ ControllerResult alterIsr(AlterIsrRequestData request) { setIsr(partitionData.newIsr())); } } - return new ControllerResult<>(records, response); + return ControllerResult.of(records, response); } /** @@ -780,7 +780,7 @@ ControllerResult electLeaders(ElectLeadersRequestData setErrorMessage(error.message())); } } - return new ControllerResult<>(records, response); + return ControllerResult.of(records, response); } static boolean electionIsUnclean(byte electionType) { @@ -875,7 +875,7 @@ ControllerResult processBrokerHeartbeat( states.next().fenced(), states.next().inControlledShutdown(), states.next().shouldShutDown()); - return new ControllerResult<>(records, reply); + return ControllerResult.of(records, reply); } int bestLeader(int[] replicas, int[] isr, boolean unclean) { @@ -904,7 +904,7 @@ public ControllerResult unregisterBroker(int brokerId) { } List records = new ArrayList<>(); handleBrokerUnregistered(brokerId, registration.epoch(), records); - return new ControllerResult<>(records, null); + return ControllerResult.of(records, null); } ControllerResult maybeFenceStaleBrokers() { @@ -916,6 +916,6 @@ ControllerResult maybeFenceStaleBrokers() { handleBrokerFenced(brokerId, records); heartbeatManager.fence(brokerId); } - return new ControllerResult<>(records, null); + return ControllerResult.of(records, null); } } diff --git a/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java b/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java index ef85314e0ef2f..99ae3a7e9baa8 100644 --- a/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java +++ b/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java @@ -328,8 +328,21 @@ public void register(MetaLogListener listener) throws Exception { @Override public long scheduleWrite(long epoch, List batch) { - return shared.tryAppend(nodeId, leader.epoch(), new LocalRecordBatch( - batch.stream().map(r -> r.message()).collect(Collectors.toList()))); + return scheduleAtomicWrite(epoch, batch); + } + + @Override + public long scheduleAtomicWrite(long epoch, List batch) { + return shared.tryAppend( + nodeId, + leader.epoch(), + new LocalRecordBatch( + batch + .stream() + .map(ApiMessageAndVersion::message) + .collect(Collectors.toList()) + ) + ); } @Override diff --git a/metadata/src/main/java/org/apache/kafka/metalog/MetaLogManager.java b/metadata/src/main/java/org/apache/kafka/metalog/MetaLogManager.java index 67a6ca5385f75..9126245ef3855 100644 --- a/metadata/src/main/java/org/apache/kafka/metalog/MetaLogManager.java +++ b/metadata/src/main/java/org/apache/kafka/metalog/MetaLogManager.java @@ -50,13 +50,30 @@ public interface MetaLogManager { * offset before renouncing its leadership. The listener should determine this by * monitoring the committed offsets. * - * @param epoch The controller epoch. - * @param batch The batch of messages to write. + * @param epoch the controller epoch + * @param batch the batch of messages to write * - * @return The offset of the message. + * @return the offset of the last message in the batch + * @throws IllegalArgumentException if buffer allocatio failed and the client should backoff */ long scheduleWrite(long epoch, List batch); + /** + * Schedule a atomic write to the log. + * + * The write will be scheduled to happen at some time in the future. All of the messages in batch + * will be appended atomically in one batch. The listener may regard the write as successful + * if and only if the MetaLogManager reaches the given offset before renouncing its leadership. + * The listener should determine this by monitoring the committed offsets. + * + * @param epoch the controller epoch + * @param batch the batch of messages to write + * + * @return the offset of the last message in the batch + * @throws IllegalArgumentException if buffer allocatio failed and the client should backoff + */ + long scheduleAtomicWrite(long epoch, List batch); + /** * Renounce the leadership. * diff --git a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java index 49a55338309b5..561a25b2d0ee2 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java @@ -135,18 +135,42 @@ public void testIncrementalAlterConfigs() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); ConfigurationControlManager manager = new ConfigurationControlManager(new LogContext(), snapshotRegistry, CONFIGS); - assertEquals(new ControllerResult>(Collections.singletonList( - new ApiMessageAndVersion(new ConfigRecord(). - setResourceType(TOPIC.id()).setResourceName("mytopic"). - setName("abc").setValue("123"), (short) 0)), - toMap(entry(BROKER0, new ApiError( - Errors.INVALID_REQUEST, "A DELETE op was given with a non-null value.")), - entry(MYTOPIC, ApiError.NONE))), - manager.incrementalAlterConfigs(toMap(entry(BROKER0, toMap( - entry("foo.bar", entry(DELETE, "abc")), - entry("quux", entry(SET, "abc")))), - entry(MYTOPIC, toMap( - entry("abc", entry(APPEND, "123"))))))); + assertEquals( + ControllerResult.atomicOf( + Collections.singletonList( + new ApiMessageAndVersion( + new ConfigRecord() + .setResourceType(TOPIC.id()) + .setResourceName("mytopic") + .setName("abc") + .setValue("123"), + (short) 0 + ) + ), + toMap( + entry( + BROKER0, + new ApiError( + Errors.INVALID_REQUEST, + "A DELETE op was given with a non-null value." + ) + ), + entry(MYTOPIC, ApiError.NONE) + ) + ), + manager.incrementalAlterConfigs( + toMap( + entry( + BROKER0, + toMap( + entry("foo.bar", entry(DELETE, "abc")), + entry("quux", entry(SET, "abc")) + ) + ), + entry(MYTOPIC, toMap(entry("abc", entry(APPEND, "123")))) + ) + ) + ); } @Test @@ -184,20 +208,33 @@ public void testLegacyAlterConfigs() { new ApiMessageAndVersion(new ConfigRecord(). setResourceType(TOPIC.id()).setResourceName("mytopic"). setName("def").setValue("901"), (short) 0)); - assertEquals(new ControllerResult>( + assertEquals( + ControllerResult.atomicOf( expectedRecords1, - toMap(entry(MYTOPIC, ApiError.NONE))), - manager.legacyAlterConfigs(toMap(entry(MYTOPIC, toMap( - entry("abc", "456"), entry("def", "901")))))); + toMap(entry(MYTOPIC, ApiError.NONE)) + ), + manager.legacyAlterConfigs( + toMap(entry(MYTOPIC, toMap(entry("abc", "456"), entry("def", "901")))) + ) + ); for (ApiMessageAndVersion message : expectedRecords1) { manager.replay((ConfigRecord) message.message()); } - assertEquals(new ControllerResult>(Arrays.asList( - new ApiMessageAndVersion(new ConfigRecord(). - setResourceType(TOPIC.id()).setResourceName("mytopic"). - setName("abc").setValue(null), (short) 0)), - toMap(entry(MYTOPIC, ApiError.NONE))), - manager.legacyAlterConfigs(toMap(entry(MYTOPIC, toMap( - entry("def", "901")))))); + assertEquals( + ControllerResult.atomicOf( + Arrays.asList( + new ApiMessageAndVersion( + new ConfigRecord() + .setResourceType(TOPIC.id()) + .setResourceName("mytopic") + .setName("abc") + .setValue(null), + (short) 0 + ) + ), + toMap(entry(MYTOPIC, ApiError.NONE)) + ), + manager.legacyAlterConfigs(toMap(entry(MYTOPIC, toMap(entry("def", "901"))))) + ); } } diff --git a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java index 8687cc8f562d8..0670984e52876 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java @@ -18,10 +18,8 @@ package org.apache.kafka.controller; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import org.apache.kafka.common.metadata.FeatureLevelRecord; @@ -61,11 +59,11 @@ public void testUpdateFeatures() { rangeMap("foo", 1, 2), snapshotRegistry); assertEquals(new FeatureMapAndEpoch(new FeatureMap(Collections.emptyMap()), -1), manager.finalizedFeatures(-1)); - assertEquals(new ControllerResult<>(Collections. + assertEquals(ControllerResult.atomicOf(Collections.emptyList(), Collections. singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, "The controller does not support the given feature range."))), manager.updateFeatures(rangeMap("foo", 1, 3), - new HashSet<>(Arrays.asList("foo")), + Collections.singleton("foo"), Collections.emptyMap())); ControllerResult> result = manager.updateFeatures( rangeMap("foo", 1, 2, "bar", 1, 1), Collections.emptySet(), @@ -101,12 +99,24 @@ public void testUpdateFeaturesErrorCases() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); FeatureControlManager manager = new FeatureControlManager( rangeMap("foo", 1, 5, "bar", 1, 2), snapshotRegistry); - assertEquals(new ControllerResult<>(Collections. - singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, - "Broker 5 does not support the given feature range."))), - manager.updateFeatures(rangeMap("foo", 1, 3), - new HashSet<>(Arrays.asList("foo")), - Collections.singletonMap(5, rangeMap()))); + + assertEquals( + ControllerResult.atomicOf( + Collections.emptyList(), + Collections.singletonMap( + "foo", + new ApiError( + Errors.INVALID_UPDATE_VERSION, + "Broker 5 does not support the given feature range." + ) + ) + ), + manager.updateFeatures( + rangeMap("foo", 1, 3), + Collections.singleton("foo"), + Collections.singletonMap(5, rangeMap()) + ) + ); ControllerResult> result = manager.updateFeatures( rangeMap("foo", 1, 3), Collections.emptySet(), Collections.emptyMap()); @@ -114,19 +124,31 @@ public void testUpdateFeaturesErrorCases() { manager.replay((FeatureLevelRecord) result.records().get(0).message(), 3); snapshotRegistry.createSnapshot(3); - assertEquals(new ControllerResult<>(Collections. + assertEquals(ControllerResult.atomicOf(Collections.emptyList(), Collections. singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, "Can't downgrade the maximum version of this feature without " + "setting downgradable to true."))), manager.updateFeatures(rangeMap("foo", 1, 2), Collections.emptySet(), Collections.emptyMap())); - assertEquals(new ControllerResult<>( - Collections.singletonList(new ApiMessageAndVersion(new FeatureLevelRecord(). - setName("foo").setMinFeatureLevel((short) 1).setMaxFeatureLevel((short) 2), - (short) 0)), - Collections.singletonMap("foo", ApiError.NONE)), - manager.updateFeatures(rangeMap("foo", 1, 2), - new HashSet<>(Collections.singletonList("foo")), Collections.emptyMap())); + assertEquals( + ControllerResult.atomicOf( + Collections.singletonList( + new ApiMessageAndVersion( + new FeatureLevelRecord() + .setName("foo") + .setMinFeatureLevel((short) 1) + .setMaxFeatureLevel((short) 2), + (short) 0 + ) + ), + Collections.singletonMap("foo", ApiError.NONE) + ), + manager.updateFeatures( + rangeMap("foo", 1, 2), + Collections.singleton("foo"), + Collections.emptyMap() + ) + ); } } diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java index 7b6cf06212e8e..590f89c3391b3 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java @@ -371,8 +371,21 @@ public void register(MetaLogListener listener) throws Exception { @Override public long scheduleWrite(long epoch, List batch) { - return shared.tryAppend(nodeId, leader.epoch(), new LocalRecordBatch( - batch.stream().map(r -> r.message()).collect(Collectors.toList()))); + return scheduleAtomicWrite(epoch, batch); + } + + @Override + public long scheduleAtomicWrite(long epoch, List batch) { + return shared.tryAppend( + nodeId, + leader.epoch(), + new LocalRecordBatch( + batch + .stream() + .map(ApiMessageAndVersion::message) + .collect(Collectors.toList()) + ) + ); } @Override diff --git a/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java b/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java index bf88e7d8120a1..1ca63f1b9c3cd 100644 --- a/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java +++ b/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java @@ -52,9 +52,35 @@ public void register(MetaLogListener listener) { client.register(new ListenerShim(listener)); } + @Override + public long scheduleAtomicWrite(long epoch, List batch) { + return write(epoch, batch, true); + } + @Override public long scheduleWrite(long epoch, List batch) { - return client.scheduleAppend((int) epoch, batch); + return write(epoch, batch, false); + } + + private long write(long epoch, List batch, boolean isAtomic) { + final Long result; + if (isAtomic) { + result = client.scheduleAtomicAppend((int) epoch, batch); + } else { + result = client.scheduleAppend((int) epoch, batch); + } + + if (result == null) { + throw new IllegalArgumentException( + String.format( + "Unable to alloate a buffer for the schedule write operation: epoch %s, batch %s)", + epoch, + batch + ) + ); + } else { + return result; + } } @Override From 7a3ebbebbc6aed2049f22c650a52f6f685546207 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 4 Mar 2021 11:22:22 -0800 Subject: [PATCH 108/243] KAFKA-12415 Prepare for Gradle 7.0 and restrict transitive scope for non api dependencies (#10203) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gradle 7.0 is required for Java 16 compatibility and it removes a number of deprecated APIs. Fix most issues preventing the upgrade to Gradle 7.0. The remaining ones are more complicated and should be handled in a separate PR. Details of the changes: * Release tarball no longer includes includes test, sources, javadoc and test sources jars (these are still published to the Maven Central repository). * Replace `compile` with `api` or `implementation` - note that `implementation` dependencies appear with `runtime` scope in the pom file so this is a (positive) change in behavior * Add missing dependencies that were uncovered by the usage of `implementation` * Replace `testCompile` with `testImplementation` * Replace `runtime` with `runtimeOnly` and `testRuntime` with `testRuntimeOnly` * Replace `configurations.runtime` with `configurations.runtimeClasspath` * Replace `configurations.testRuntime` with `configurations.testRuntimeClasspath` (except for the usage in the `streams` project as that causes a cyclic dependency error) * Use `java-library` plugin instead of `java` * Use `maven-publish` plugin instead of deprecated `maven` plugin - this changes the commands used to publish and to install locally, but task aliases for `install` and `uploadArchives` were added for backwards compatibility * Removed `-x signArchives` line from the readme since it was wrong (it was a no-op before and it fails now, however) * Replaces `artifacts` block with an approach that works with the `maven-publish` plugin * Don't publish `jmh-benchmark` module - the shadow jar is pretty large and not particularly useful (before this PR, we would publish the non shadow jars) * Replace `version` with `archiveVersion`, `baseName` with `archiveBaseName` and `classifier` with `archiveClassifier` * Update Gradle and plugins to the latest stable version (7.0 is not stable yet) * Use `plugin` DSL to configure plugins * Updated notable changes for 3.0 Reviewers: Chia-Ping Tsai , Randall Hauch --- Jenkinsfile | 4 +- README.md | 16 +- build.gradle | 836 ++++++++++++----------- docs/upgrade.html | 8 + gradle/dependencies.gradle | 9 +- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 2 +- jmh-benchmarks/README.md | 2 +- jmh-benchmarks/jmh.sh | 2 +- release.py | 2 +- 10 files changed, 460 insertions(+), 423 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 557988788cd4d..bc63364fc6a79 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,8 +46,8 @@ def doStreamsArchetype() { echo 'Verify that Kafka Streams archetype compiles' sh ''' - ./gradlew streams:install clients:install connect:json:install connect:api:install \ - || { echo 'Could not install kafka-streams.jar (and dependencies) locally`'; exit 1; } + ./gradlew streams:publishToMavenLocal clients:publishToMavenLocal connect:json:publishToMavenLocal connect:api:publishToMavenLocal \ + || { echo 'Could not publish kafka-streams.jar (and dependencies) locally to Maven'; exit 1; } ''' VERSION = sh(script: 'grep "^version=" gradle.properties | cut -d= -f 2', returnStdout: true).trim() diff --git a/README.md b/README.md index 3c3f013662344..649127ed4aa75 100644 --- a/README.md +++ b/README.md @@ -69,10 +69,6 @@ Generate coverage for a single module, i.e.: ### Building a binary release gzipped tar ball ### ./gradlew clean releaseTarGz -The above command will fail if you haven't set up the signing key. To bypass signing the artifact, you can run: - - ./gradlew clean releaseTarGz -x signArchives - The release file can be found inside `./core/build/distributions/`. ### Building auto generated messages ### @@ -125,6 +121,12 @@ build directory (`${project_dir}/bin`) clashes with Kafka's scripts directory an to avoid known issues with this configuration. ### Publishing the jar for all version of Scala and for all projects to maven ### +The recommended command is: + + ./gradlewAll publish + +For backwards compatibility, the following also works: + ./gradlewAll uploadArchives Please note for this to work you should create/update `${GRADLE_USER_HOME}/gradle.properties` (typically, `~/.gradle/gradle.properties`) and assign the following variables @@ -167,6 +169,12 @@ Please note for this to work you should create/update user maven settings (typic ### Installing the jars to the local Maven repository ### +The recommended command is: + + ./gradlewAll publishToMavenLocal + +For backwards compatibility, the following also works: + ./gradlewAll install ### Building the test jar ### diff --git a/build.gradle b/build.gradle index acd8a84e6377b..be6859bfddf80 100644 --- a/build.gradle +++ b/build.gradle @@ -21,9 +21,6 @@ buildscript { repositories { mavenCentral() jcenter() - maven { - url "https://plugins.gradle.org/m2/" - } } apply from: file('gradle/buildscript.gradle'), to: buildscript apply from: "$rootDir/gradle/dependencies.gradle" @@ -31,17 +28,21 @@ buildscript { dependencies { // For Apache Rat plugin to ignore non-Git files classpath "org.ajoberstar.grgit:grgit-core:$versions.grgit" - classpath "com.github.ben-manes:gradle-versions-plugin:$versions.gradleVersionsPlugin" - classpath "org.scoverage:gradle-scoverage:$versions.scoveragePlugin" - classpath "com.github.jengelman.gradle.plugins:shadow:$versions.shadowPlugin" - classpath "org.owasp:dependency-check-gradle:$versions.owaspDepCheckPlugin" - classpath "com.diffplug.spotless:spotless-plugin-gradle:$versions.spotlessPlugin" - classpath "gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:$versions.spotbugsPlugin" - classpath "org.gradle:test-retry-gradle-plugin:$versions.testRetryPlugin" } } -apply plugin: "com.diffplug.spotless" +plugins { + id 'com.diffplug.spotless' version '5.10.2' + id 'com.github.ben-manes.versions' version '0.36.0' + id 'idea' + id 'org.owasp.dependencycheck' version '6.1.1' + + id "com.github.spotbugs" version '4.6.0' apply false + id 'org.gradle.test-retry' version '1.2.0' apply false + id 'org.scoverage' version '5.0.0' apply false + id 'com.github.johnrengelman.shadow' version '6.1.0' apply false +} + spotless { scala { target 'streams/**/*.scala' @@ -49,17 +50,12 @@ spotless { } } - allprojects { repositories { mavenCentral() } - apply plugin: 'idea' - apply plugin: 'org.owasp.dependencycheck' - apply plugin: 'com.github.ben-manes.versions' - dependencyUpdates { revision="release" resolutionStrategy { @@ -119,7 +115,7 @@ ext { userMaxTestRetryFailures = project.hasProperty('maxTestRetryFailures') ? maxTestRetryFailures.toInteger() : 0 skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean() - shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") && project.gradle.startParameter.taskNames.any { it.contains("upload") } + shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : '' mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : '' @@ -182,17 +178,30 @@ subprojects { // eg: ./gradlew allDepInsight --configuration runtime --dependency com.fasterxml.jackson.core:jackson-databind task allDepInsight(type: DependencyInsightReportTask) doLast {} - apply plugin: 'java' + apply plugin: 'java-library' + apply plugin: 'checkstyle' + apply plugin: "com.github.spotbugs" + apply plugin: 'org.gradle.test-retry' + + // We use the shadow plugin for the jmh-benchmarks module and the `-all` jar can get pretty large, so + // don't publish it + def shouldPublish = !project.name.equals('jmh-benchmarks') + + if (shouldPublish) { + apply plugin: 'maven-publish' + apply plugin: 'signing' + + // Add aliases for the task names used by the maven plugin for backwards compatibility + // The maven plugin was replaced by the maven-publish plugin in Gradle 7.0 + tasks.register('install').configure { dependsOn(publishToMavenLocal) } + tasks.register('uploadArchives').configure { dependsOn(publish) } + } + // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. if (!project.name.equals('connect')) { apply plugin: 'eclipse' fineTuneEclipseClasspathFile(eclipse, project) } - apply plugin: 'maven' - apply plugin: 'signing' - apply plugin: 'checkstyle' - apply plugin: "com.github.spotbugs" - apply plugin: 'org.gradle.test-retry' sourceCompatibility = minJavaVersion targetCompatibility = minJavaVersion @@ -219,34 +228,50 @@ subprojects { options.compilerArgs << "--release" << minJavaVersion } - uploadArchives { - repositories { - signing { - required { shouldSign } - sign configurations.archives - - // To test locally, replace mavenUrl in ~/.gradle/gradle.properties to file://localhost/tmp/myRepo/ - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - repository(url: "${mavenUrl}") { - authentication(userName: "${mavenUsername}", password: "${mavenPassword}") - } - afterEvaluate { - pom.artifactId = "${archivesBaseName}" - pom.project { - name 'Apache Kafka' - packaging 'jar' - url 'https://kafka.apache.org' - licenses { - license { - name 'The Apache Software License, Version 2.0' - url 'https://www.apache.org/licenses/LICENSE-2.0.txt' - distribution 'repo' - } - } - } + if (shouldPublish) { + + publishing { + repositories { + // To test locally, invoke gradlew with `-PmavenUrl=file:///some/local/path` + maven { + url = mavenUrl + credentials { + username = mavenUsername + password = mavenPassword + } + } + } + publications { + mavenJava(MavenPublication) { + from components.java + + afterEvaluate { + ["srcJar", "javadocJar", "scaladocJar", "testJar", "testSrcJar"].forEach { taskName -> + def task = tasks.findByName(taskName) + if (task != null) + artifact task + } + + artifactId = archivesBaseName + pom { + name = 'Apache Kafka' + url = 'https://kafka.apache.org' + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + distribution = 'repo' + } } + } } + } + } + } + + if (shouldSign) { + signing { + sign publishing.publications.mavenJava } } } @@ -427,14 +452,14 @@ subprojects { } task srcJar(type: Jar) { - classifier = 'sources' + archiveClassifier = 'sources' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.main.allSource } task javadocJar(type: Jar, dependsOn: javadoc) { - classifier 'javadoc' + archiveClassifier = 'javadoc' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from javadoc.destinationDir @@ -450,30 +475,21 @@ subprojects { task systemTestLibs(dependsOn: jar) - artifacts { - archives srcJar - archives javadocJar - } - - if(!sourceSets.test.allSource.isEmpty()) { + if (!sourceSets.test.allSource.isEmpty()) { task testJar(type: Jar) { - classifier = 'test' + archiveClassifier = 'test' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.test.output } task testSrcJar(type: Jar, dependsOn: testJar) { - classifier = 'test-sources' + archiveClassifier = 'test-sources' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.test.allSource } - artifacts { - archives testJar - archives testSrcJar - } } plugins.withType(ScalaPlugin) { @@ -483,7 +499,7 @@ subprojects { } task scaladocJar(type:Jar, dependsOn: scaladoc) { - classifier = 'scaladoc' + archiveClassifier = 'scaladoc' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from scaladoc.destinationDir @@ -749,53 +765,57 @@ project(':core') { archivesBaseName = "kafka_${versions.baseScala}" dependencies { - compile project(':clients') - compile project(':metadata') - compile project(':raft') - compile libs.argparse4j - compile libs.jacksonDatabind - compile libs.jacksonModuleScala - compile libs.jacksonDataformatCsv - compile libs.jacksonJDK8Datatypes - compile libs.joptSimple - compile libs.metrics - compile libs.scalaCollectionCompat - compile libs.scalaJava8Compat - compile libs.scalaLibrary + // `core` is often used in users' tests, define the following dependencies as `api` for backwards compatibility + // even though the `core` module doesn't expose any public API + api project(':clients') + api libs.scalaLibrary + + implementation project(':metadata') + implementation project(':raft') + + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonModuleScala + implementation libs.jacksonDataformatCsv + implementation libs.jacksonJDK8Datatypes + implementation libs.joptSimple + implementation libs.metrics + implementation libs.scalaCollectionCompat + implementation libs.scalaJava8Compat // only needed transitively, but set it explicitly to ensure it has the same version as scala-library - compile libs.scalaReflect - compile libs.scalaLogging - compile libs.slf4jApi - compile(libs.zookeeper) { + implementation libs.scalaReflect + implementation libs.scalaLogging + implementation libs.slf4jApi + implementation(libs.zookeeper) { exclude module: 'slf4j-log4j12' exclude module: 'log4j' } // ZooKeeperMain depends on commons-cli but declares the dependency as `provided` - compile libs.commonsCli + implementation libs.commonsCli compileOnly libs.log4j - testCompile project(':clients').sourceSets.test.output - testCompile libs.bcpkix - testCompile libs.mockitoCore - testCompile libs.easymock - testCompile(libs.apacheda) { + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.bcpkix + testImplementation libs.mockitoCore + testImplementation libs.easymock + testImplementation(libs.apacheda) { exclude group: 'xml-apis', module: 'xml-apis' // `mina-core` is a transitive dependency for `apacheds` and `apacheda`. // It is safer to use from `apacheds` since that is the implementation. exclude module: 'mina-core' } - testCompile libs.apachedsCoreApi - testCompile libs.apachedsInterceptorKerberos - testCompile libs.apachedsProtocolShared - testCompile libs.apachedsProtocolKerberos - testCompile libs.apachedsProtocolLdap - testCompile libs.apachedsLdifPartition - testCompile libs.apachedsMavibotPartition - testCompile libs.apachedsJdbmPartition - testCompile libs.junitJupiter - testCompile libs.slf4jlog4j - testCompile(libs.jfreechart) { + testImplementation libs.apachedsCoreApi + testImplementation libs.apachedsInterceptorKerberos + testImplementation libs.apachedsProtocolShared + testImplementation libs.apachedsProtocolKerberos + testImplementation libs.apachedsProtocolLdap + testImplementation libs.apachedsLdifPartition + testImplementation libs.apachedsMavibotPartition + testImplementation libs.apachedsJdbmPartition + testImplementation libs.junitJupiter + testImplementation libs.slf4jlog4j + testImplementation(libs.jfreechart) { exclude group: 'junit', module: 'junit' } } @@ -823,11 +843,11 @@ project(':core') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -933,7 +953,7 @@ project(':core') { ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs', ':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs', ':connect:runtime:genConnectMetricsDocs'], type: Tar) { - classifier = 'site-docs' + archiveClassifier = 'site-docs' compression = Compression.GZIP from project.file("$rootDir/docs") into 'site-docs' @@ -941,41 +961,41 @@ project(':core') { } tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) { - into "kafka_${versions.baseScala}-${version}" + into "kafka_${versions.baseScala}-${archiveVersion.get()}" compression = Compression.GZIP from(project.file("$rootDir/bin")) { into "bin/" } from(project.file("$rootDir/config")) { into "config/" } from "$rootDir/LICENSE" from "$rootDir/NOTICE" - from(configurations.runtime) { into("libs/") } + from(configurations.runtimeClasspath) { into("libs/") } from(configurations.archives.artifacts.files) { into("libs/") } from(project.siteDocsTar) { into("site-docs/") } from(project(':tools').jar) { into("libs/") } - from(project(':tools').configurations.runtime) { into("libs/") } + from(project(':tools').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:api').jar) { into("libs/") } - from(project(':connect:api').configurations.runtime) { into("libs/") } + from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:runtime').jar) { into("libs/") } - from(project(':connect:runtime').configurations.runtime) { into("libs/") } + from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:transforms').jar) { into("libs/") } - from(project(':connect:transforms').configurations.runtime) { into("libs/") } + from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:json').jar) { into("libs/") } - from(project(':connect:json').configurations.runtime) { into("libs/") } + from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:file').jar) { into("libs/") } - from(project(':connect:file').configurations.runtime) { into("libs/") } + from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:basic-auth-extension').jar) { into("libs/") } - from(project(':connect:basic-auth-extension').configurations.runtime) { into("libs/") } + from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:mirror').jar) { into("libs/") } - from(project(':connect:mirror').configurations.runtime) { into("libs/") } + from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:mirror-client').jar) { into("libs/") } - from(project(':connect:mirror-client').configurations.runtime) { into("libs/") } + from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") } from(project(':streams').jar) { into("libs/") } - from(project(':streams').configurations.runtime) { into("libs/") } + from(project(':streams').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:streams-scala').jar) { into("libs/") } - from(project(':streams:streams-scala').configurations.runtime) { into("libs/") } + from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:test-utils').jar) { into("libs/") } - from(project(':streams:test-utils').configurations.runtime) { into("libs/") } + from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:examples').jar) { into("libs/") } - from(project(':streams:examples').configurations.runtime) { into("libs/") } + from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") } duplicatesStrategy 'exclude' } @@ -990,7 +1010,7 @@ project(':core') { } tasks.create(name: "copyDependantTestLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('*.jar') } into "$buildDir/dependant-testlibs" @@ -1027,15 +1047,15 @@ project(':metadata') { archivesBaseName = "kafka-metadata" dependencies { - compile project(':clients') - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.metrics + implementation project(':clients') + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.metrics compileOnly libs.log4j - testCompile libs.junitJupiter - testCompile libs.hamcrest - testCompile libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.hamcrest + testImplementation libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } task processMessages(type:JavaExec) { @@ -1075,7 +1095,8 @@ project(':examples') { archivesBaseName = "kafka-examples" dependencies { - compile project(':core') + implementation project(':clients') + implementation project(':core') } javadoc { @@ -1089,11 +1110,11 @@ project(':examples') { project(':generator') { dependencies { - compile libs.argparse4j - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.jacksonJaxrsJsonProvider - testCompile libs.junitJupiter + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.jacksonJaxrsJsonProvider + testImplementation libs.junitJupiter } javadoc { @@ -1104,32 +1125,23 @@ project(':generator') { project(':clients') { archivesBaseName = "kafka-clients" - configurations { - jacksonDatabindConfig - } - - // add jacksonDatabindConfig as provided scope config with high priority (1000) - conf2ScopeMappings.addMapping(1000, configurations.jacksonDatabindConfig, "provided") - dependencies { - compile libs.zstd - compile libs.lz4 - compile libs.snappy - compile libs.slf4jApi + implementation libs.zstd + implementation libs.lz4 + implementation libs.snappy + implementation libs.slf4jApi compileOnly libs.jacksonDatabind // for SASL/OAUTHBEARER bearer token parsing compileOnly libs.jacksonJDK8Datatypes - jacksonDatabindConfig libs.jacksonDatabind // to publish as provided scope dependency. - - testCompile libs.bcpkix - testCompile libs.junitJupiter - testCompile libs.mockitoCore + testImplementation libs.bcpkix + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore - testRuntime libs.slf4jlog4j - testRuntime libs.jacksonDatabind - testRuntime libs.jacksonJDK8Datatypes - testCompile libs.jacksonJaxrsJsonProvider + testRuntimeOnly libs.slf4jlog4j + testRuntimeOnly libs.jacksonDatabind + testRuntimeOnly libs.jacksonJDK8Datatypes + testImplementation libs.jacksonJaxrsJsonProvider } task createVersionFile(dependsOn: determineCommitId) { @@ -1233,17 +1245,17 @@ project(':raft') { archivesBaseName = "kafka-raft" dependencies { - compile project(':clients') - compile project(':metadata') - compile libs.slf4jApi - compile libs.jacksonDatabind + implementation project(':clients') + implementation project(':metadata') + implementation libs.slf4jApi + implementation libs.jacksonDatabind - testCompile project(':clients') - testCompile project(':clients').sourceSets.test.output - testCompile libs.junitJupiter - testCompile libs.mockitoCore + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } task createVersionFile(dependsOn: determineCommitId) { @@ -1308,28 +1320,29 @@ project(':tools') { archivesBaseName = "kafka-tools" dependencies { - compile project(':clients') - compile project(':log4j-appender') - compile libs.argparse4j - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.slf4jApi - - compile libs.jacksonJaxrsJsonProvider - compile libs.jerseyContainerServlet - compile libs.jerseyHk2 - compile libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 - compile libs.activation // Jersey dependency that was available in the JDK before Java 9 - compile libs.jettyServer - compile libs.jettyServlet - compile libs.jettyServlets - - testCompile project(':clients') - testCompile libs.junitJupiter - testCompile project(':clients').sourceSets.test.output - testCompile libs.mockitoInline // supports mocking static methods, final classes, etc. - - testRuntime libs.slf4jlog4j + implementation project(':clients') + implementation project(':log4j-appender') + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.slf4jApi + implementation libs.log4j + + implementation libs.jacksonJaxrsJsonProvider + implementation libs.jerseyContainerServlet + implementation libs.jerseyHk2 + implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 + implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 + implementation libs.jettyServer + implementation libs.jettyServlet + implementation libs.jettyServlets + + testImplementation project(':clients') + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.mockitoInline // supports mocking static methods, final classes, etc. + + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -1337,11 +1350,11 @@ project(':tools') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1357,23 +1370,23 @@ project(':shell') { archivesBaseName = "kafka-shell" dependencies { - compile libs.argparse4j - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.jline - compile libs.slf4jApi - compile project(':clients') - compile project(':core') - compile project(':log4j-appender') - compile project(':metadata') - compile project(':raft') + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.jline + implementation libs.slf4jApi + implementation project(':clients') + implementation project(':core') + implementation project(':log4j-appender') + implementation project(':metadata') + implementation project(':raft') - compile libs.jacksonJaxrsJsonProvider + implementation libs.jacksonJaxrsJsonProvider - testCompile project(':clients') - testCompile libs.junitJupiter + testImplementation project(':clients') + testImplementation libs.junitJupiter - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -1381,10 +1394,10 @@ project(':shell') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('jline-*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { include('jline-*jar') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1401,33 +1414,37 @@ project(':streams') { ext.buildStreamsVersionFileName = "kafka-streams-version.properties" dependencies { - compile project(':clients') + api project(':clients') + // use `api` dependency for `connect-json` for compatibility (e.g. users who use `JsonSerializer`/`JsonDeserializer` + // at compile-time without an explicit dependency on `connect-json`) // this dependency should be removed after we unify data API - compile(project(':connect:json')) { + api(project(':connect:json')) { // this transitive dependency is not used in Streams, and it breaks SBT builds exclude module: 'javax.ws.rs-api' } - compile libs.slf4jApi - compile libs.rocksDBJni + implementation libs.slf4jApi + implementation libs.rocksDBJni + implementation libs.jacksonAnnotations + implementation libs.jacksonDatabind // testCompileOnly prevents streams from exporting a dependency on test-utils, which would cause a dependency cycle testCompileOnly project(':streams:test-utils') - testCompile project(':clients').sourceSets.test.output - testCompile project(':core') - testCompile project(':core').sourceSets.test.output - testCompile libs.log4j - testCompile libs.junitJupiterApi - testCompile libs.junitVintageEngine - testCompile libs.easymock - testCompile libs.powermockJunit4 - testCompile libs.powermockEasymock - testCompile libs.bcpkix - testCompile libs.hamcrest + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':core').sourceSets.test.output + testImplementation libs.log4j + testImplementation libs.junitJupiterApi + testImplementation libs.junitVintageEngine + testImplementation libs.easymock + testImplementation libs.powermockJunit4 + testImplementation libs.powermockEasymock + testImplementation libs.bcpkix + testImplementation libs.hamcrest testRuntimeOnly project(':streams:test-utils') - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } task processMessages(type:JavaExec) { @@ -1468,7 +1485,7 @@ project(':streams') { include('log4j*jar') include('*hamcrest*') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1539,24 +1556,24 @@ project(':streams:streams-scala') { apply plugin: 'scala' archivesBaseName = "kafka-streams-scala_${versions.baseScala}" dependencies { - compile project(':streams') + api project(':streams') - compile libs.scalaLibrary - compile libs.scalaCollectionCompat + api libs.scalaLibrary + api libs.scalaCollectionCompat - testCompile project(':core') - testCompile project(':core').sourceSets.test.output - testCompile project(':streams').sourceSets.test.output - testCompile project(':clients').sourceSets.test.output - testCompile project(':streams:test-utils') + testImplementation project(':core') + testImplementation project(':core').sourceSets.test.output + testImplementation project(':streams').sourceSets.test.output + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':streams:test-utils') - testCompile libs.junitJupiterApi - testCompile libs.junitVintageEngine - testCompile libs.scalatest - testCompile libs.easymock - testCompile libs.hamcrest + testImplementation libs.junitJupiterApi + testImplementation libs.junitVintageEngine + testImplementation libs.scalatest + testImplementation libs.easymock + testImplementation libs.hamcrest - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -1564,7 +1581,7 @@ project(':streams:streams-scala') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1575,10 +1592,6 @@ project(':streams:streams-scala') { dependsOn 'copyDependantLibs' } - artifacts { - archives scaladocJar - } - test.dependsOn(':spotlessScalaCheck') } @@ -1586,15 +1599,17 @@ project(':streams:test-utils') { archivesBaseName = "kafka-streams-test-utils" dependencies { - compile project(':streams') - compile project(':clients') + api project(':streams') + api project(':clients') + + implementation libs.slf4jApi - testCompile project(':clients').sourceSets.test.output - testCompile libs.junitJupiter - testCompile libs.easymock - testCompile libs.hamcrest + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.easymock + testImplementation libs.hamcrest - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -1603,7 +1618,7 @@ project(':streams:test-utils') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1620,14 +1635,14 @@ project(':streams:examples') { archivesBaseName = "kafka-streams-examples" dependencies { - compile project(':streams') - compile project(':connect:json') // this dependency should be removed after we unify data API - compile libs.slf4jlog4j + implementation project(':streams') + implementation project(':connect:json') // this dependency should be removed after we unify data API + implementation libs.slf4jlog4j - testCompile project(':streams:test-utils') - testCompile project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest - testCompile libs.junitJupiter - testCompile libs.hamcrest + testImplementation project(':streams:test-utils') + testImplementation project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest + testImplementation libs.junitJupiter + testImplementation libs.hamcrest } javadoc { @@ -1635,7 +1650,7 @@ project(':streams:examples') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1651,8 +1666,8 @@ project(':streams:upgrade-system-tests-0100') { archivesBaseName = "kafka-streams-upgrade-system-tests-0100" dependencies { - testCompile libs.kafkaStreams_0100 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_0100 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1664,8 +1679,8 @@ project(':streams:upgrade-system-tests-0101') { archivesBaseName = "kafka-streams-upgrade-system-tests-0101" dependencies { - testCompile libs.kafkaStreams_0101 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_0101 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1677,8 +1692,8 @@ project(':streams:upgrade-system-tests-0102') { archivesBaseName = "kafka-streams-upgrade-system-tests-0102" dependencies { - testCompile libs.kafkaStreams_0102 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_0102 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1690,8 +1705,8 @@ project(':streams:upgrade-system-tests-0110') { archivesBaseName = "kafka-streams-upgrade-system-tests-0110" dependencies { - testCompile libs.kafkaStreams_0110 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_0110 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1703,8 +1718,8 @@ project(':streams:upgrade-system-tests-10') { archivesBaseName = "kafka-streams-upgrade-system-tests-10" dependencies { - testCompile libs.kafkaStreams_10 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_10 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1716,8 +1731,8 @@ project(':streams:upgrade-system-tests-11') { archivesBaseName = "kafka-streams-upgrade-system-tests-11" dependencies { - testCompile libs.kafkaStreams_11 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_11 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1729,8 +1744,8 @@ project(':streams:upgrade-system-tests-20') { archivesBaseName = "kafka-streams-upgrade-system-tests-20" dependencies { - testCompile libs.kafkaStreams_20 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_20 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1742,8 +1757,8 @@ project(':streams:upgrade-system-tests-21') { archivesBaseName = "kafka-streams-upgrade-system-tests-21" dependencies { - testCompile libs.kafkaStreams_21 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_21 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1755,8 +1770,8 @@ project(':streams:upgrade-system-tests-22') { archivesBaseName = "kafka-streams-upgrade-system-tests-22" dependencies { - testCompile libs.kafkaStreams_22 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_22 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1768,8 +1783,8 @@ project(':streams:upgrade-system-tests-23') { archivesBaseName = "kafka-streams-upgrade-system-tests-23" dependencies { - testCompile libs.kafkaStreams_23 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_23 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1781,8 +1796,8 @@ project(':streams:upgrade-system-tests-24') { archivesBaseName = "kafka-streams-upgrade-system-tests-24" dependencies { - testCompile libs.kafkaStreams_24 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_24 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1794,8 +1809,8 @@ project(':streams:upgrade-system-tests-25') { archivesBaseName = "kafka-streams-upgrade-system-tests-25" dependencies { - testCompile libs.kafkaStreams_25 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_25 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1807,8 +1822,8 @@ project(':streams:upgrade-system-tests-26') { archivesBaseName = "kafka-streams-upgrade-system-tests-26" dependencies { - testCompile libs.kafkaStreams_26 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_26 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1820,8 +1835,8 @@ project(':streams:upgrade-system-tests-27') { archivesBaseName = "kafka-streams-upgrade-system-tests-27" dependencies { - testCompile libs.kafkaStreams_27 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_27 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1834,27 +1849,30 @@ project(':jmh-benchmarks') { apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - baseName = 'kafka-jmh-benchmarks-all' - classifier = null - version = null + archiveBaseName = 'kafka-jmh-benchmarks' } dependencies { - compile(project(':core')) { + implementation(project(':core')) { // jmh requires jopt 4.x while `core` depends on 5.0, they are not binary compatible exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' } - compile project(':clients') - compile project(':metadata') - compile project(':streams') - compile project(':core') - compile project(':clients').sourceSets.test.output - compile project(':core').sourceSets.test.output - compile libs.jmhCore + implementation project(':clients') + implementation project(':metadata') + implementation project(':streams') + implementation project(':core') + implementation project(':clients').sourceSets.test.output + implementation project(':core').sourceSets.test.output + + implementation libs.jmhCore annotationProcessor libs.jmhGeneratorAnnProcess - compile libs.jmhCoreBenchmarks - compile libs.mockitoCore - compile libs.slf4jlog4j + implementation libs.jmhCoreBenchmarks + implementation libs.jacksonDatabind + implementation libs.metrics + implementation libs.mockitoCore + implementation libs.slf4jlog4j + implementation libs.scalaLibrary + implementation libs.scalaJava8Compat } tasks.withType(JavaCompile) { @@ -1893,13 +1911,13 @@ project(':log4j-appender') { archivesBaseName = "kafka-log4j-appender" dependencies { - compile project(':clients') - compile libs.slf4jlog4j + implementation project(':clients') + implementation libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output - testCompile libs.junitJupiter - testCompile libs.hamcrest - testCompile libs.easymock + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.hamcrest + testImplementation libs.easymock } javadoc { @@ -1912,13 +1930,13 @@ project(':connect:api') { archivesBaseName = "connect-api" dependencies { - compile project(':clients') - compile libs.slf4jApi - compile libs.jaxrsApi + api project(':clients') + implementation libs.slf4jApi + implementation libs.jaxrsApi - testCompile libs.junitJupiter - testRuntime libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testRuntimeOnly libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } javadoc { @@ -1931,11 +1949,11 @@ project(':connect:api') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -1952,14 +1970,15 @@ project(':connect:transforms') { archivesBaseName = "connect-transforms" dependencies { - compile project(':connect:api') - compile libs.slf4jApi + api project(':connect:api') + + implementation libs.slf4jApi - testCompile libs.easymock - testCompile libs.junitJupiter + testImplementation libs.easymock + testImplementation libs.junitJupiter - testRuntime libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + testRuntimeOnly libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } javadoc { @@ -1967,11 +1986,11 @@ project(':connect:transforms') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -1988,16 +2007,18 @@ project(':connect:json') { archivesBaseName = "connect-json" dependencies { - compile project(':connect:api') - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.slf4jApi + api project(':connect:api') - testCompile libs.easymock - testCompile libs.junitJupiter + api libs.jacksonDatabind + api libs.jacksonJDK8Datatypes - testRuntime libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + implementation libs.slf4jApi + + testImplementation libs.easymock + testImplementation libs.junitJupiter + + testRuntimeOnly libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } javadoc { @@ -2005,11 +2026,11 @@ project(':connect:json') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2026,40 +2047,44 @@ project(':connect:runtime') { archivesBaseName = "connect-runtime" dependencies { - - compile project(':connect:api') - compile project(':clients') - compile project(':tools') - compile project(':connect:json') - compile project(':connect:transforms') - - compile libs.slf4jApi - compile libs.jacksonJaxrsJsonProvider - compile libs.jerseyContainerServlet - compile libs.jerseyHk2 - compile libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 - compile libs.activation // Jersey dependency that was available in the JDK before Java 9 - compile libs.jettyServer - compile libs.jettyServlet - compile libs.jettyServlets - compile libs.jettyClient - compile(libs.reflections) - compile(libs.mavenArtifact) - - testCompile project(':clients').sourceSets.test.output - testCompile libs.easymock - testCompile libs.junitJupiterApi - testCompile libs.junitVintageEngine - testCompile libs.powermockJunit4 - testCompile libs.powermockEasymock - testCompile libs.mockitoCore - testCompile libs.httpclient - - testCompile project(':clients').sourceSets.test.output - testCompile project(':core') - testCompile project(':core').sourceSets.test.output - - testRuntime libs.slf4jlog4j + // connect-runtime is used in tests, use `api` for modules below for backwards compatibility even though + // applications should generally not depend on `connect-runtime` + api project(':connect:api') + api project(':clients') + api project(':connect:json') + api project(':connect:transforms') + + implementation project(':tools') + + implementation libs.slf4jApi + implementation libs.log4j + implementation libs.jacksonAnnotations + implementation libs.jacksonJaxrsJsonProvider + implementation libs.jerseyContainerServlet + implementation libs.jerseyHk2 + implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 + implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 + implementation libs.jettyServer + implementation libs.jettyServlet + implementation libs.jettyServlets + implementation libs.jettyClient + implementation libs.reflections + implementation libs.mavenArtifact + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':metadata') + testImplementation project(':core').sourceSets.test.output + + testImplementation libs.easymock + testImplementation libs.junitJupiterApi + testImplementation libs.junitVintageEngine + testImplementation libs.powermockJunit4 + testImplementation libs.powermockEasymock + testImplementation libs.mockitoCore + testImplementation libs.httpclient + + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -2067,11 +2092,11 @@ project(':connect:runtime') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2131,14 +2156,14 @@ project(':connect:file') { archivesBaseName = "connect-file" dependencies { - compile project(':connect:api') - compile libs.slf4jApi + implementation project(':connect:api') + implementation libs.slf4jApi - testCompile libs.easymock - testCompile libs.junitJupiter + testImplementation libs.easymock + testImplementation libs.junitJupiter - testRuntime libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + testRuntimeOnly libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } javadoc { @@ -2146,11 +2171,11 @@ project(':connect:file') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2167,16 +2192,17 @@ project(':connect:basic-auth-extension') { archivesBaseName = "connect-basic-auth-extension" dependencies { - compile project(':connect:api') - compile libs.slf4jApi + implementation project(':connect:api') + implementation libs.slf4jApi + implementation libs.jaxrsApi - testCompile libs.bcpkix - testCompile libs.easymock - testCompile libs.junitJupiter - testCompile project(':clients').sourceSets.test.output + testImplementation libs.bcpkix + testImplementation libs.easymock + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output - testRuntime libs.slf4jlog4j - testRuntime libs.jerseyContainerServlet + testRuntimeOnly libs.slf4jlog4j + testRuntimeOnly libs.jerseyContainerServlet } javadoc { @@ -2184,11 +2210,11 @@ project(':connect:basic-auth-extension') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2205,23 +2231,25 @@ project(':connect:mirror') { archivesBaseName = "connect-mirror" dependencies { - compile project(':connect:api') - compile project(':connect:runtime') - compile project(':connect:mirror-client') - compile project(':clients') - compile libs.argparse4j - compile libs.slf4jApi + implementation project(':connect:api') + implementation project(':connect:runtime') + implementation project(':connect:mirror-client') + implementation project(':clients') - testCompile libs.junitJupiter - testCompile libs.mockitoCore - testCompile project(':clients').sourceSets.test.output - testCompile project(':connect:runtime').sourceSets.test.output - testCompile project(':core') - testCompile project(':core').sourceSets.test.output + implementation libs.argparse4j + implementation libs.jacksonAnnotations + implementation libs.slf4jApi - testRuntime project(':connect:runtime') - testRuntime libs.slf4jlog4j - testRuntime libs.bcpkix + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':connect:runtime').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':core').sourceSets.test.output + + testRuntimeOnly project(':connect:runtime') + testRuntimeOnly libs.slf4jlog4j + testRuntimeOnly libs.bcpkix } javadoc { @@ -2229,11 +2257,11 @@ project(':connect:mirror') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2250,13 +2278,13 @@ project(':connect:mirror-client') { archivesBaseName = "connect-mirror-client" dependencies { - compile project(':clients') - compile libs.slf4jApi + implementation project(':clients') + implementation libs.slf4jApi - testCompile libs.junitJupiter - testCompile project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -2264,11 +2292,11 @@ project(':connect:mirror-client') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } diff --git a/docs/upgrade.html b/docs/upgrade.html index 9ce10371799e5..30550d755b61d 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -19,6 +19,14 @@
    • NameKey
    "); b.append("" + key.name + ""); @@ -246,10 +260,19 @@ public void visit(Type field) { return hasBuffer.get(); } - public static List brokerApis() { - return Arrays.stream(values()) - .filter(api -> !api.isControllerOnlyApi) + public static EnumSet zkBrokerApis() { + return apisForListener(ApiMessageType.ListenerType.ZK_BROKER); + } + + public static EnumSet apisForListener(ApiMessageType.ListenerType listener) { + return APIS_BY_LISTENER.get(listener); + } + + private static EnumSet filterApisForListener(ApiMessageType.ListenerType listener) { + List controllerApis = Arrays.stream(ApiKeys.values()) + .filter(apiKey -> apiKey.messageType.listeners().contains(listener)) .collect(Collectors.toList()); + return EnumSet.copyOf(controllerApis); } } diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java index f31c613a8c51b..d455b26eb2d87 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java @@ -133,7 +133,7 @@ public static String toHtml() { b.append("\n"); schemaToFieldTableHtml(ResponseHeaderData.SCHEMAS[i], b); } - for (ApiKeys key : ApiKeys.brokerApis()) { + for (ApiKeys key : ApiKeys.zkBrokerApis()) { // Key b.append("
    "); b.append(""); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index 2bf9360f126d6..1190989576380 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.feature.Features; import org.apache.kafka.common.feature.FinalizedVersionRange; import org.apache.kafka.common.feature.SupportedVersionRange; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; @@ -29,11 +30,12 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.RecordVersion; import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; +import java.util.Set; /** * Possible error codes: @@ -44,9 +46,6 @@ public class ApiVersionsResponse extends AbstractResponse { public static final long UNKNOWN_FINALIZED_FEATURES_EPOCH = -1L; - public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE = createApiVersionsResponse( - DEFAULT_THROTTLE_TIME, RecordBatch.CURRENT_MAGIC_VALUE); - private final ApiVersionsResponseData data; public ApiVersionsResponse(ApiVersionsResponseData data) { @@ -96,49 +95,89 @@ public static ApiVersionsResponse parse(ByteBuffer buffer, short version) { } } - public static ApiVersionsResponse createApiVersionsResponse(final int throttleTimeMs, final byte minMagic) { - return createApiVersionsResponse(throttleTimeMs, minMagic, Features.emptySupportedFeatures(), - Features.emptyFinalizedFeatures(), UNKNOWN_FINALIZED_FEATURES_EPOCH); + public static ApiVersionsResponse defaultApiVersionsResponse( + ApiMessageType.ListenerType listenerType + ) { + return defaultApiVersionsResponse(0, listenerType); } - private static ApiVersionsResponse createApiVersionsResponse( - final int throttleTimeMs, - final byte minMagic, - final Features latestSupportedFeatures, - final Features finalizedFeatures, - final long finalizedFeaturesEpoch) { + public static ApiVersionsResponse defaultApiVersionsResponse( + int throttleTimeMs, + ApiMessageType.ListenerType listenerType + ) { + return createApiVersionsResponse(throttleTimeMs, filterApis(RecordVersion.current(), listenerType)); + } + + public static ApiVersionsResponse createApiVersionsResponse( + int throttleTimeMs, + ApiVersionCollection apiVersions + ) { + return createApiVersionsResponse( + throttleTimeMs, + apiVersions, + Features.emptySupportedFeatures(), + Features.emptyFinalizedFeatures(), + UNKNOWN_FINALIZED_FEATURES_EPOCH + ); + } + + public static ApiVersionsResponse createApiVersionsResponse( + int throttleTimeMs, + ApiVersionCollection apiVersions, + Features latestSupportedFeatures, + Features finalizedFeatures, + long finalizedFeaturesEpoch + ) { return new ApiVersionsResponse( createApiVersionsResponseData( throttleTimeMs, Errors.NONE, - defaultApiKeys(minMagic), + apiVersions, latestSupportedFeatures, finalizedFeatures, - finalizedFeaturesEpoch)); + finalizedFeaturesEpoch + ) + ); } - public static ApiVersionCollection defaultApiKeys(final byte minMagic) { + public static ApiVersionCollection filterApis( + RecordVersion minRecordVersion, + ApiMessageType.ListenerType listenerType + ) { ApiVersionCollection apiKeys = new ApiVersionCollection(); - for (ApiKeys apiKey : ApiKeys.brokerApis()) { - if (apiKey.minRequiredInterBrokerMagic <= minMagic) { + for (ApiKeys apiKey : ApiKeys.apisForListener(listenerType)) { + if (apiKey.minRequiredInterBrokerMagic <= minRecordVersion.value) { apiKeys.add(ApiVersionsResponse.toApiVersion(apiKey)); } } return apiKeys; } + public static ApiVersionCollection collectApis(Set apiKeys) { + ApiVersionCollection res = new ApiVersionCollection(); + for (ApiKeys apiKey : apiKeys) { + res.add(ApiVersionsResponse.toApiVersion(apiKey)); + } + return res; + } + /** - * Find the commonly agreed ApiVersions between local software and the controller. + * Find the common range of supported API versions between the locally + * known range and that of another set. * - * @param minMagic min inter broker magic + * @param listenerType the listener type which constrains the set of exposed APIs + * @param minRecordVersion min inter broker magic * @param activeControllerApiVersions controller ApiVersions * @return commonly agreed ApiVersion collection */ - public static ApiVersionCollection intersectControllerApiVersions(final byte minMagic, - final Map activeControllerApiVersions) { + public static ApiVersionCollection intersectForwardableApis( + final ApiMessageType.ListenerType listenerType, + final RecordVersion minRecordVersion, + final Map activeControllerApiVersions + ) { ApiVersionCollection apiKeys = new ApiVersionCollection(); - for (ApiKeys apiKey : ApiKeys.brokerApis()) { - if (apiKey.minRequiredInterBrokerMagic <= minMagic) { + for (ApiKeys apiKey : ApiKeys.apisForListener(listenerType)) { + if (apiKey.minRequiredInterBrokerMagic <= minRecordVersion.value) { ApiVersion brokerApiVersion = toApiVersion(apiKey); final ApiVersion finalApiVersion; @@ -161,7 +200,7 @@ public static ApiVersionCollection intersectControllerApiVersions(final byte min return apiKeys; } - public static ApiVersionsResponseData createApiVersionsResponseData( + private static ApiVersionsResponseData createApiVersionsResponseData( final int throttleTimeMs, final Errors error, final ApiVersionCollection apiKeys, diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index 7fb5687def1d2..243495da9fe7f 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -85,6 +85,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; public class SaslServerAuthenticator implements Authenticator { // GSSAPI limits requests to 64K, but we allow a bit extra for custom SASL mechanisms @@ -127,6 +128,7 @@ private enum SaslState { private final Time time; private final ReauthInfo reauthInfo; private final ChannelMetadataRegistry metadataRegistry; + private final Supplier apiVersionSupplier; // Current SASL state private SaslState saslState = SaslState.INITIAL_REQUEST; @@ -154,7 +156,8 @@ public SaslServerAuthenticator(Map configs, TransportLayer transportLayer, Map connectionsMaxReauthMsByMechanism, ChannelMetadataRegistry metadataRegistry, - Time time) { + Time time, + Supplier apiVersionSupplier) { this.callbackHandlers = callbackHandlers; this.connectionId = connectionId; this.subjects = subjects; @@ -166,6 +169,7 @@ public SaslServerAuthenticator(Map configs, this.time = time; this.reauthInfo = new ReauthInfo(); this.metadataRegistry = metadataRegistry; + this.apiVersionSupplier = apiVersionSupplier; this.configs = configs; @SuppressWarnings("unchecked") @@ -562,11 +566,6 @@ private String handleHandshakeRequest(RequestContext context, SaslHandshakeReque } } - // Visible to override for testing - protected ApiVersionsResponse apiVersionsResponse() { - return ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE; - } - // Visible to override for testing protected void enableKafkaSaslAuthenticateHeaders(boolean flag) { this.enableKafkaSaslAuthenticateHeaders = flag; @@ -583,7 +582,7 @@ else if (!apiVersionsRequest.isValid()) else { metadataRegistry.registerClientInformation(new ClientInformation(apiVersionsRequest.data().clientSoftwareName(), apiVersionsRequest.data().clientSoftwareVersion())); - sendKafkaResponse(context, apiVersionsResponse()); + sendKafkaResponse(context, apiVersionSupplier.get()); setSaslState(SaslState.HANDSHAKE_REQUEST); } } diff --git a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json index b2bb9a78f6470..ade3fc72c9a51 100644 --- a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json +++ b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json @@ -16,6 +16,7 @@ { "apiKey": 25, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "AddOffsetsToTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json index 3e1d7207a51dc..4920da176c723 100644 --- a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json +++ b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json @@ -16,6 +16,7 @@ { "apiKey": 24, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "AddPartitionsToTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json index 715a00644e9c0..6bfdc925c2919 100644 --- a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json @@ -16,6 +16,7 @@ { "apiKey": 49, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "AlterClientQuotasRequest", "validVersions": "0-1", // Version 1 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterConfigsRequest.json b/clients/src/main/resources/common/message/AlterConfigsRequest.json index a1d7d1d4ca4d4..31057e3410aaf 100644 --- a/clients/src/main/resources/common/message/AlterConfigsRequest.json +++ b/clients/src/main/resources/common/message/AlterConfigsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 33, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "AlterConfigsRequest", // Version 1 is the same as version 0. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterIsrRequest.json b/clients/src/main/resources/common/message/AlterIsrRequest.json index 3d5084a4094ec..f950cd7005ae8 100644 --- a/clients/src/main/resources/common/message/AlterIsrRequest.json +++ b/clients/src/main/resources/common/message/AlterIsrRequest.json @@ -16,6 +16,7 @@ { "apiKey": 56, "type": "request", + "listeners": ["zkBroker", "controller"], "name": "AlterIsrRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json index ecf7eca6eae43..2e124413a3bbc 100644 --- a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json +++ b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 45, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "AlterPartitionReassignmentsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json b/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json index a6749077f4439..2306caaf98499 100644 --- a/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json +++ b/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 34, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "AlterReplicaLogDirsRequest", // Version 1 is the same as version 0. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json b/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json index 242bcb97dda79..8937394ef6e51 100644 --- a/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json +++ b/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 51, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "AlterUserScramCredentialsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ApiVersionsRequest.json b/clients/src/main/resources/common/message/ApiVersionsRequest.json index 66e4511a92e93..b86edbfaaec12 100644 --- a/clients/src/main/resources/common/message/ApiVersionsRequest.json +++ b/clients/src/main/resources/common/message/ApiVersionsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 18, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "ApiVersionsRequest", // Versions 0 through 2 of ApiVersionsRequest are the same. // diff --git a/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json b/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json index fd986097085f6..9f7969ff889c5 100644 --- a/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json +++ b/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json @@ -16,6 +16,7 @@ { "apiKey": 53, "type": "request", + "listeners": ["controller"], "name": "BeginQuorumEpochRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json b/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json index 105a81873b519..ce08d119b7cc2 100644 --- a/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json +++ b/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json @@ -16,6 +16,7 @@ { "apiKey": 63, "type": "request", + "listeners": ["controller"], "name": "BrokerHeartbeatRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/BrokerRegistrationRequest.json b/clients/src/main/resources/common/message/BrokerRegistrationRequest.json index b123cc1f5ac75..3e27cf127434f 100644 --- a/clients/src/main/resources/common/message/BrokerRegistrationRequest.json +++ b/clients/src/main/resources/common/message/BrokerRegistrationRequest.json @@ -16,6 +16,7 @@ { "apiKey":62, "type": "request", + "listeners": ["controller"], "name": "BrokerRegistrationRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ControlledShutdownRequest.json b/clients/src/main/resources/common/message/ControlledShutdownRequest.json index 5756d1c16b5df..49561f7b6e7d7 100644 --- a/clients/src/main/resources/common/message/ControlledShutdownRequest.json +++ b/clients/src/main/resources/common/message/ControlledShutdownRequest.json @@ -16,6 +16,7 @@ { "apiKey": 7, "type": "request", + "listeners": ["zkBroker", "controller"], "name": "ControlledShutdownRequest", // Version 0 of ControlledShutdownRequest has a non-standard request header // which does not include clientId. Version 1 and later use the standard diff --git a/clients/src/main/resources/common/message/CreateAclsRequest.json b/clients/src/main/resources/common/message/CreateAclsRequest.json index a9bd9c5f60a11..5b3bfed78162c 100644 --- a/clients/src/main/resources/common/message/CreateAclsRequest.json +++ b/clients/src/main/resources/common/message/CreateAclsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 30, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "CreateAclsRequest", // Version 1 adds resource pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json index 8d881355aa9d2..0c31d32fe56b7 100644 --- a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json @@ -16,6 +16,7 @@ { "apiKey": 38, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "CreateDelegationTokenRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/CreatePartitionsRequest.json b/clients/src/main/resources/common/message/CreatePartitionsRequest.json index ba1138801c1ef..6e249498659fa 100644 --- a/clients/src/main/resources/common/message/CreatePartitionsRequest.json +++ b/clients/src/main/resources/common/message/CreatePartitionsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 37, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "CreatePartitionsRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/CreateTopicsRequest.json b/clients/src/main/resources/common/message/CreateTopicsRequest.json index 1a8d57a3f0bea..0882de9fa64d2 100644 --- a/clients/src/main/resources/common/message/CreateTopicsRequest.json +++ b/clients/src/main/resources/common/message/CreateTopicsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 19, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "CreateTopicsRequest", // Version 1 adds validateOnly. // diff --git a/clients/src/main/resources/common/message/DeleteAclsRequest.json b/clients/src/main/resources/common/message/DeleteAclsRequest.json index 664737e5810f7..fd7c1522b43bd 100644 --- a/clients/src/main/resources/common/message/DeleteAclsRequest.json +++ b/clients/src/main/resources/common/message/DeleteAclsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 31, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "DeleteAclsRequest", // Version 1 adds the pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/DeleteGroupsRequest.json b/clients/src/main/resources/common/message/DeleteGroupsRequest.json index 833ed7a4dc6d7..1ac6a053e63b3 100644 --- a/clients/src/main/resources/common/message/DeleteGroupsRequest.json +++ b/clients/src/main/resources/common/message/DeleteGroupsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 42, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "DeleteGroupsRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/DeleteRecordsRequest.json b/clients/src/main/resources/common/message/DeleteRecordsRequest.json index 93cbd56d50a6c..06a12d85c8bb4 100644 --- a/clients/src/main/resources/common/message/DeleteRecordsRequest.json +++ b/clients/src/main/resources/common/message/DeleteRecordsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 21, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "DeleteRecordsRequest", // Version 1 is the same as version 0. diff --git a/clients/src/main/resources/common/message/DeleteTopicsRequest.json b/clients/src/main/resources/common/message/DeleteTopicsRequest.json index 7e11554ad480c..f757ff7755453 100644 --- a/clients/src/main/resources/common/message/DeleteTopicsRequest.json +++ b/clients/src/main/resources/common/message/DeleteTopicsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 20, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "DeleteTopicsRequest", // Versions 0, 1, 2, and 3 are the same. // diff --git a/clients/src/main/resources/common/message/DescribeAclsRequest.json b/clients/src/main/resources/common/message/DescribeAclsRequest.json index a9c36768c1447..58886da654707 100644 --- a/clients/src/main/resources/common/message/DescribeAclsRequest.json +++ b/clients/src/main/resources/common/message/DescribeAclsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 29, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "DescribeAclsRequest", // Version 1 adds resource pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json index e35d3b0c91a01..d14cfc95733d3 100644 --- a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json @@ -16,6 +16,7 @@ { "apiKey": 48, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "DescribeClientQuotasRequest", // Version 1 enables flexible versions. "validVersions": "0-1", diff --git a/clients/src/main/resources/common/message/DescribeClusterRequest.json b/clients/src/main/resources/common/message/DescribeClusterRequest.json index 31eb57394a438..192e4d87d4497 100644 --- a/clients/src/main/resources/common/message/DescribeClusterRequest.json +++ b/clients/src/main/resources/common/message/DescribeClusterRequest.json @@ -16,6 +16,7 @@ { "apiKey": 60, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "DescribeClusterRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeConfigsRequest.json b/clients/src/main/resources/common/message/DescribeConfigsRequest.json index 01a45eb2a5beb..23be19cb0e625 100644 --- a/clients/src/main/resources/common/message/DescribeConfigsRequest.json +++ b/clients/src/main/resources/common/message/DescribeConfigsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 32, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "DescribeConfigsRequest", // Version 1 adds IncludeSynonyms. // Version 2 is the same as version 1. diff --git a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json index 32a4f5c5150c1..da5bbd046d16c 100644 --- a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json @@ -16,6 +16,7 @@ { "apiKey": 41, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "DescribeDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/DescribeGroupsRequest.json b/clients/src/main/resources/common/message/DescribeGroupsRequest.json index 8a5887a5680a8..6b10b0637a205 100644 --- a/clients/src/main/resources/common/message/DescribeGroupsRequest.json +++ b/clients/src/main/resources/common/message/DescribeGroupsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 15, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "DescribeGroupsRequest", // Versions 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json index c498e0f22238a..cfb160f8166bf 100644 --- a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json +++ b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 35, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "DescribeLogDirsRequest", // Version 1 is the same as version 0. "validVersions": "0-2", diff --git a/clients/src/main/resources/common/message/DescribeProducersRequest.json b/clients/src/main/resources/common/message/DescribeProducersRequest.json index bd35f91035bd4..0ffd834e6bde8 100644 --- a/clients/src/main/resources/common/message/DescribeProducersRequest.json +++ b/clients/src/main/resources/common/message/DescribeProducersRequest.json @@ -16,6 +16,7 @@ { "apiKey": 61, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "DescribeProducersRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeQuorumRequest.json b/clients/src/main/resources/common/message/DescribeQuorumRequest.json index f91d679db8b1b..cd4a7f1db5470 100644 --- a/clients/src/main/resources/common/message/DescribeQuorumRequest.json +++ b/clients/src/main/resources/common/message/DescribeQuorumRequest.json @@ -16,6 +16,7 @@ { "apiKey": 55, "type": "request", + "listeners": ["broker", "controller"], "name": "DescribeQuorumRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json b/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json index f7f8c68991357..2f7a1112c4800 100644 --- a/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json +++ b/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 50, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "DescribeUserScramCredentialsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ElectLeadersRequest.json b/clients/src/main/resources/common/message/ElectLeadersRequest.json index a2ba2bdca735c..dd9fa21641585 100644 --- a/clients/src/main/resources/common/message/ElectLeadersRequest.json +++ b/clients/src/main/resources/common/message/ElectLeadersRequest.json @@ -16,6 +16,7 @@ { "apiKey": 43, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "ElectLeadersRequest", // Version 1 implements multiple leader election types, as described by KIP-460. // diff --git a/clients/src/main/resources/common/message/EndQuorumEpochRequest.json b/clients/src/main/resources/common/message/EndQuorumEpochRequest.json index 45dacde5fe99b..3ef7f6320a915 100644 --- a/clients/src/main/resources/common/message/EndQuorumEpochRequest.json +++ b/clients/src/main/resources/common/message/EndQuorumEpochRequest.json @@ -16,6 +16,7 @@ { "apiKey": 54, "type": "request", + "listeners": ["controller"], "name": "EndQuorumEpochRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/EndTxnRequest.json b/clients/src/main/resources/common/message/EndTxnRequest.json index de18b43b7d9d2..f16ef76246d35 100644 --- a/clients/src/main/resources/common/message/EndTxnRequest.json +++ b/clients/src/main/resources/common/message/EndTxnRequest.json @@ -16,6 +16,7 @@ { "apiKey": 26, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "EndTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/EnvelopeRequest.json b/clients/src/main/resources/common/message/EnvelopeRequest.json index a1aa760a29b18..1f6ff62de8d7f 100644 --- a/clients/src/main/resources/common/message/EnvelopeRequest.json +++ b/clients/src/main/resources/common/message/EnvelopeRequest.json @@ -16,6 +16,7 @@ { "apiKey": 58, "type": "request", + "listeners": ["controller"], "name": "EnvelopeRequest", // Request struct for forwarding. "validVersions": "0", diff --git a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json index a990862d2fc8a..c830a93df398b 100644 --- a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json @@ -16,6 +16,7 @@ { "apiKey": 40, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "ExpireDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/FetchRequest.json b/clients/src/main/resources/common/message/FetchRequest.json index 0dcdd7af8bb00..ab4c95fba8264 100644 --- a/clients/src/main/resources/common/message/FetchRequest.json +++ b/clients/src/main/resources/common/message/FetchRequest.json @@ -16,6 +16,7 @@ { "apiKey": 1, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "FetchRequest", // // Version 1 is the same as version 0. diff --git a/clients/src/main/resources/common/message/FetchSnapshotRequest.json b/clients/src/main/resources/common/message/FetchSnapshotRequest.json index c3518f44765c7..accc227731209 100644 --- a/clients/src/main/resources/common/message/FetchSnapshotRequest.json +++ b/clients/src/main/resources/common/message/FetchSnapshotRequest.json @@ -16,6 +16,7 @@ { "apiKey": 59, "type": "request", + "listeners": ["controller"], "name": "FetchSnapshotRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/FindCoordinatorRequest.json b/clients/src/main/resources/common/message/FindCoordinatorRequest.json index 6a90887b1aa01..cd5b77a3f83e9 100644 --- a/clients/src/main/resources/common/message/FindCoordinatorRequest.json +++ b/clients/src/main/resources/common/message/FindCoordinatorRequest.json @@ -16,6 +16,7 @@ { "apiKey": 10, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "FindCoordinatorRequest", // Version 1 adds KeyType. // diff --git a/clients/src/main/resources/common/message/HeartbeatRequest.json b/clients/src/main/resources/common/message/HeartbeatRequest.json index 4d799aa3c14c1..dcf776d8ec4e3 100644 --- a/clients/src/main/resources/common/message/HeartbeatRequest.json +++ b/clients/src/main/resources/common/message/HeartbeatRequest.json @@ -16,6 +16,7 @@ { "apiKey": 12, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "HeartbeatRequest", // Version 1 and version 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json b/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json index b1fb1e9481ac2..d4955c91b85a4 100644 --- a/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json +++ b/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 44, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "IncrementalAlterConfigsRequest", // Version 1 is the first flexible version. "validVersions": "0-1", diff --git a/clients/src/main/resources/common/message/InitProducerIdRequest.json b/clients/src/main/resources/common/message/InitProducerIdRequest.json index dc85063c29643..e8795e6582169 100644 --- a/clients/src/main/resources/common/message/InitProducerIdRequest.json +++ b/clients/src/main/resources/common/message/InitProducerIdRequest.json @@ -16,6 +16,7 @@ { "apiKey": 22, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "InitProducerIdRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/JoinGroupRequest.json b/clients/src/main/resources/common/message/JoinGroupRequest.json index 2650d89e2b4a8..d9113b76a3c6e 100644 --- a/clients/src/main/resources/common/message/JoinGroupRequest.json +++ b/clients/src/main/resources/common/message/JoinGroupRequest.json @@ -16,6 +16,7 @@ { "apiKey": 11, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "JoinGroupRequest", // Version 1 adds RebalanceTimeoutMs. // diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json index 129b7f77f6fac..57e6f21f3cf6e 100644 --- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json +++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json @@ -16,6 +16,7 @@ { "apiKey": 4, "type": "request", + "listeners": ["zkBroker"], "name": "LeaderAndIsrRequest", // Version 1 adds IsNew. // diff --git a/clients/src/main/resources/common/message/LeaveGroupRequest.json b/clients/src/main/resources/common/message/LeaveGroupRequest.json index acc7938c387b1..893c945c20411 100644 --- a/clients/src/main/resources/common/message/LeaveGroupRequest.json +++ b/clients/src/main/resources/common/message/LeaveGroupRequest.json @@ -16,6 +16,7 @@ { "apiKey": 13, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "LeaveGroupRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/ListGroupsRequest.json b/clients/src/main/resources/common/message/ListGroupsRequest.json index dbe6d9b6f123a..3f62e28350956 100644 --- a/clients/src/main/resources/common/message/ListGroupsRequest.json +++ b/clients/src/main/resources/common/message/ListGroupsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 16, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "ListGroupsRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/ListOffsetsRequest.json b/clients/src/main/resources/common/message/ListOffsetsRequest.json index 9855a4bf9058d..a464c9376444f 100644 --- a/clients/src/main/resources/common/message/ListOffsetsRequest.json +++ b/clients/src/main/resources/common/message/ListOffsetsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 2, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "ListOffsetsRequest", // Version 1 removes MaxNumOffsets. From this version forward, only a single // offset can be returned. diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json index 7322f25bc456e..f013e3fe9ffab 100644 --- a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json +++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 46, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "ListPartitionReassignmentsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/MetadataRequest.json b/clients/src/main/resources/common/message/MetadataRequest.json index 02af116a1c210..66908103e99c8 100644 --- a/clients/src/main/resources/common/message/MetadataRequest.json +++ b/clients/src/main/resources/common/message/MetadataRequest.json @@ -16,6 +16,7 @@ { "apiKey": 3, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "MetadataRequest", "validVersions": "0-11", "flexibleVersions": "9+", diff --git a/clients/src/main/resources/common/message/OffsetCommitRequest.json b/clients/src/main/resources/common/message/OffsetCommitRequest.json index 096b61917ae0f..cf112e1ed72cb 100644 --- a/clients/src/main/resources/common/message/OffsetCommitRequest.json +++ b/clients/src/main/resources/common/message/OffsetCommitRequest.json @@ -16,6 +16,7 @@ { "apiKey": 8, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "OffsetCommitRequest", // Version 1 adds timestamp and group membership information, as well as the commit timestamp. // diff --git a/clients/src/main/resources/common/message/OffsetDeleteRequest.json b/clients/src/main/resources/common/message/OffsetDeleteRequest.json index 108ca9f7b9eca..394d1bb64df7b 100644 --- a/clients/src/main/resources/common/message/OffsetDeleteRequest.json +++ b/clients/src/main/resources/common/message/OffsetDeleteRequest.json @@ -16,6 +16,7 @@ { "apiKey": 47, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "OffsetDeleteRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/OffsetFetchRequest.json b/clients/src/main/resources/common/message/OffsetFetchRequest.json index ddd53fc859600..d4a4d5f22df94 100644 --- a/clients/src/main/resources/common/message/OffsetFetchRequest.json +++ b/clients/src/main/resources/common/message/OffsetFetchRequest.json @@ -16,6 +16,7 @@ { "apiKey": 9, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "OffsetFetchRequest", // In version 0, the request read offsets from ZK. // diff --git a/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json b/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json index 75b6c8dc3227c..2440becd9cb05 100644 --- a/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json +++ b/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json @@ -16,6 +16,7 @@ { "apiKey": 23, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "OffsetForLeaderEpochRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/ProduceRequest.json b/clients/src/main/resources/common/message/ProduceRequest.json index 73e72c352f4d4..121cd42f44df8 100644 --- a/clients/src/main/resources/common/message/ProduceRequest.json +++ b/clients/src/main/resources/common/message/ProduceRequest.json @@ -16,6 +16,7 @@ { "apiKey": 0, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "ProduceRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json index 9fbb0fa07a5e8..182682e4c913a 100644 --- a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json @@ -16,6 +16,7 @@ { "apiKey": 39, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "RenewDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/SaslAuthenticateRequest.json b/clients/src/main/resources/common/message/SaslAuthenticateRequest.json index 122cef577076e..3f5558b812042 100644 --- a/clients/src/main/resources/common/message/SaslAuthenticateRequest.json +++ b/clients/src/main/resources/common/message/SaslAuthenticateRequest.json @@ -16,6 +16,7 @@ { "apiKey": 36, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "SaslAuthenticateRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/SaslHandshakeRequest.json b/clients/src/main/resources/common/message/SaslHandshakeRequest.json index f384f414c5c04..3384db862b52e 100644 --- a/clients/src/main/resources/common/message/SaslHandshakeRequest.json +++ b/clients/src/main/resources/common/message/SaslHandshakeRequest.json @@ -16,6 +16,7 @@ { "apiKey": 17, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "SaslHandshakeRequest", // Version 1 supports SASL_AUTHENTICATE. // NOTE: Version cannot be easily bumped due to incorrect diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json index d43356cab89fe..b10154fe1fbf6 100644 --- a/clients/src/main/resources/common/message/StopReplicaRequest.json +++ b/clients/src/main/resources/common/message/StopReplicaRequest.json @@ -16,6 +16,7 @@ { "apiKey": 5, "type": "request", + "listeners": ["zkBroker"], "name": "StopReplicaRequest", // Version 1 adds the broker epoch and reorganizes the partitions to be stored // per topic. diff --git a/clients/src/main/resources/common/message/SyncGroupRequest.json b/clients/src/main/resources/common/message/SyncGroupRequest.json index a0a599150ef53..5525844138366 100644 --- a/clients/src/main/resources/common/message/SyncGroupRequest.json +++ b/clients/src/main/resources/common/message/SyncGroupRequest.json @@ -16,6 +16,7 @@ { "apiKey": 14, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "SyncGroupRequest", // Versions 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json index bd91df35480c6..a832ef7a96832 100644 --- a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json +++ b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json @@ -16,6 +16,7 @@ { "apiKey": 28, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "TxnOffsetCommitRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/UnregisterBrokerRequest.json b/clients/src/main/resources/common/message/UnregisterBrokerRequest.json index 3c43b1628cb03..ef72bfe61feb7 100644 --- a/clients/src/main/resources/common/message/UnregisterBrokerRequest.json +++ b/clients/src/main/resources/common/message/UnregisterBrokerRequest.json @@ -16,6 +16,7 @@ { "apiKey": 64, "type": "request", + "listeners": ["broker", "controller"], "name": "UnregisterBrokerRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/UpdateFeaturesRequest.json b/clients/src/main/resources/common/message/UpdateFeaturesRequest.json index ab882dff1c754..21e1bf663dda5 100644 --- a/clients/src/main/resources/common/message/UpdateFeaturesRequest.json +++ b/clients/src/main/resources/common/message/UpdateFeaturesRequest.json @@ -16,6 +16,7 @@ { "apiKey": 57, "type": "request", + "listeners": ["zkBroker", "broker", "controller"], "name": "UpdateFeaturesRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json index 99d33f192ed4d..5f397a92c04a5 100644 --- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json +++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json @@ -16,6 +16,7 @@ { "apiKey": 6, "type": "request", + "listeners": ["zkBroker"], "name": "UpdateMetadataRequest", // Version 1 allows specifying multiple endpoints for each broker. // diff --git a/clients/src/main/resources/common/message/VoteRequest.json b/clients/src/main/resources/common/message/VoteRequest.json index 3926ba7c7d6af..fcc0017ed2574 100644 --- a/clients/src/main/resources/common/message/VoteRequest.json +++ b/clients/src/main/resources/common/message/VoteRequest.json @@ -16,6 +16,7 @@ { "apiKey": 52, "type": "request", + "listeners": ["controller"], "name": "VoteRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json index 3fdfa05bf9773..9e29fb39f4525 100644 --- a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json +++ b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json @@ -16,6 +16,7 @@ { "apiKey": 27, "type": "request", + "listeners": ["zkBroker", "broker"], "name": "WriteTxnMarkersRequest", // Version 1 enables flexible versions. "validVersions": "0-1", diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index eb130ff934ee9..b13f85422b6db 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.errors.AuthenticationException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; @@ -247,7 +248,8 @@ private void setExpectedApiVersionsResponse(ApiVersionsResponse response) { private void awaitReady(NetworkClient client, Node node) { if (client.discoverBrokerVersions()) { - setExpectedApiVersionsResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE); + setExpectedApiVersionsResponse(ApiVersionsResponse.defaultApiVersionsResponse( + ApiMessageType.ListenerType.ZK_BROKER)); } while (!client.ready(node, time.milliseconds())) client.poll(1, time.milliseconds()); @@ -295,8 +297,7 @@ public void testApiVersionsRequest() { assertTrue(client.hasInFlightRequests(node.idString())); // prepare response - delayedApiVersionsResponse(0, ApiKeys.API_VERSIONS.latestVersion(), - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE); + delayedApiVersionsResponse(0, ApiKeys.API_VERSIONS.latestVersion(), defaultApiVersionsResponse()); // handle completed receives client.poll(0, time.milliseconds()); @@ -367,8 +368,7 @@ public void testUnsupportedApiVersionsRequestWithVersionProvidedByTheBroker() { assertEquals(2, header.apiVersion()); // prepare response - delayedApiVersionsResponse(1, (short) 0, - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE); + delayedApiVersionsResponse(1, (short) 0, defaultApiVersionsResponse()); // handle completed receives client.poll(0, time.milliseconds()); @@ -434,8 +434,7 @@ public void testUnsupportedApiVersionsRequestWithoutVersionProvidedByTheBroker() assertEquals(0, header.apiVersion()); // prepare response - delayedApiVersionsResponse(1, (short) 0, - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE); + delayedApiVersionsResponse(1, (short) 0, defaultApiVersionsResponse()); // handle completed receives client.poll(0, time.milliseconds()); @@ -1079,6 +1078,10 @@ private void awaitInFlightApiVersionRequest() throws Exception { assertFalse(client.isReady(node, time.milliseconds())); } + private ApiVersionsResponse defaultApiVersionsResponse() { + return ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER); + } + private static class TestCallbackHandler implements RequestCompletionHandler { public boolean executed = false; public ClientResponse response; diff --git a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java index 7c19d9f098866..b04d83b47df2f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java @@ -16,18 +16,21 @@ */ package org.apache.kafka.clients; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.requests.ApiVersionsResponse; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -38,7 +41,7 @@ public void testUnsupportedVersionsToString() { NodeApiVersions versions = new NodeApiVersions(new ApiVersionCollection()); StringBuilder bld = new StringBuilder(); String prefix = "("; - for (ApiKeys apiKey : ApiKeys.brokerApis()) { + for (ApiKeys apiKey : ApiKeys.zkBrokerApis()) { bld.append(prefix).append(apiKey.name). append("(").append(apiKey.id).append("): UNSUPPORTED"); prefix = ", "; @@ -133,27 +136,26 @@ public void testLatestUsableVersionOutOfRange() { () -> apiVersions.latestUsableVersion(ApiKeys.PRODUCE)); } - @Test - public void testUsableVersionLatestVersions() { - List versionList = new LinkedList<>(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().apiKeys()); + @ParameterizedTest + @EnumSource(ApiMessageType.ListenerType.class) + public void testUsableVersionLatestVersions(ApiMessageType.ListenerType scope) { + ApiVersionsResponse defaultResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); + List versionList = new LinkedList<>(defaultResponse.data().apiKeys()); // Add an API key that we don't know about. versionList.add(new ApiVersion() .setApiKey((short) 100) .setMinVersion((short) 0) .setMaxVersion((short) 1)); NodeApiVersions versions = new NodeApiVersions(versionList); - for (ApiKeys apiKey: ApiKeys.values()) { - if (apiKey.isControllerOnlyApi) { - assertNull(versions.apiVersion(apiKey)); - } else { - assertEquals(apiKey.latestVersion(), versions.latestUsableVersion(apiKey)); - } + for (ApiKeys apiKey: ApiKeys.apisForListener(scope)) { + assertEquals(apiKey.latestVersion(), versions.latestUsableVersion(apiKey)); } } - @Test - public void testConstructionFromApiVersionsResponse() { - ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE; + @ParameterizedTest + @EnumSource(ApiMessageType.ListenerType.class) + public void testConstructionFromApiVersionsResponse(ApiMessageType.ListenerType scope) { + ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); NodeApiVersions versions = new NodeApiVersions(apiVersionsResponse.data().apiKeys()); for (ApiVersion apiVersionKey : apiVersionsResponse.data().apiKeys()) { diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index ec05d2c002fd1..f7107c886b160 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -74,6 +74,7 @@ import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirPartitionResult; import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirTopicResult; import org.apache.kafka.common.message.AlterUserScramCredentialsResponseData; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.CreateAclsResponseData; @@ -126,6 +127,7 @@ import org.apache.kafka.common.quota.ClientQuotaEntity; import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.quota.ClientQuotaFilterComponent; +import org.apache.kafka.common.record.RecordVersion; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.AlterClientQuotasResponse; import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse; @@ -541,17 +543,17 @@ private static Features c private static ApiVersionsResponse prepareApiVersionsResponseForDescribeFeatures(Errors error) { if (error == Errors.NONE) { - return new ApiVersionsResponse(ApiVersionsResponse.createApiVersionsResponseData( - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.throttleTimeMs(), - error, - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().apiKeys(), + return ApiVersionsResponse.createApiVersionsResponse( + 0, + ApiVersionsResponse.filterApis(RecordVersion.current(), ApiMessageType.ListenerType.ZK_BROKER), convertSupportedFeaturesMap(defaultFeatureMetadata().supportedFeatures()), convertFinalizedFeaturesMap(defaultFeatureMetadata().finalizedFeatures()), - defaultFeatureMetadata().finalizedFeaturesEpoch().get())); + defaultFeatureMetadata().finalizedFeaturesEpoch().get() + ); } return new ApiVersionsResponse( new ApiVersionsResponseData() - .setThrottleTimeMs(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.throttleTimeMs()) + .setThrottleTimeMs(0) .setErrorCode(error.code())); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 9330f9eb51c55..2c13864fb0589 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -48,6 +48,7 @@ import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic; import org.apache.kafka.common.message.ListOffsetsResponseData; @@ -2074,8 +2075,9 @@ public void testQuotaMetrics() { 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, ClientDnsLookup.USE_ALL_DNS_IPS, time, true, new ApiVersions(), throttleTimeSensor, new LogContext()); - ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(ApiVersionsResponse.createApiVersionsResponse( - 400, RecordBatch.CURRENT_MAGIC_VALUE), ApiKeys.API_VERSIONS.latestVersion(), 0); + ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( + 400, ApiMessageType.ListenerType.ZK_BROKER); + ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 439ba3b96b1a4..e118c1110e704 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -24,13 +24,6 @@ import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.NodeApiVersions; import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.common.errors.InvalidRequestException; -import org.apache.kafka.common.errors.TransactionAbortedException; -import org.apache.kafka.common.message.ProduceRequestData; -import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType; -import org.apache.kafka.common.requests.MetadataRequest; -import org.apache.kafka.common.requests.RequestTestUtils; -import org.apache.kafka.common.utils.ProducerIdAndEpoch; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; @@ -39,15 +32,19 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.ClusterAuthorizationException; +import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.NetworkException; import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.errors.TransactionAbortedException; import org.apache.kafka.common.errors.UnsupportedForMessageFormatException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.EndTxnResponseData; import org.apache.kafka.common.message.InitProducerIdResponseData; +import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.metrics.KafkaMetric; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -66,15 +63,19 @@ import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.EndTxnRequest; import org.apache.kafka.common.requests.EndTxnResponse; +import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType; import org.apache.kafka.common.requests.FindCoordinatorResponse; import org.apache.kafka.common.requests.InitProducerIdRequest; import org.apache.kafka.common.requests.InitProducerIdResponse; +import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.ProduceRequest; import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.requests.TransactionResult; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; import org.apache.kafka.common.utils.Time; import org.apache.kafka.test.DelayedReceive; import org.apache.kafka.test.MockSelector; @@ -105,12 +106,12 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.AdditionalMatchers.geq; import static org.mockito.ArgumentMatchers.any; @@ -287,9 +288,9 @@ public void testQuotaMetrics() { 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, ClientDnsLookup.USE_ALL_DNS_IPS, time, true, new ApiVersions(), throttleTimeSensor, logContext); - ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader( - ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE), - ApiKeys.API_VERSIONS.latestVersion(), 0); + ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( + 400, ApiMessageType.ListenerType.ZK_BROKER); + ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java index 15f8079b65fbe..53b46e7d9b650 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java +++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java @@ -18,9 +18,11 @@ import org.apache.kafka.common.MetricName; import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.metrics.KafkaMetric; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.authenticator.CredentialCache; import org.apache.kafka.common.security.scram.ScramCredential; @@ -117,7 +119,8 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco LogContext logContext = new LogContext(); if (channelBuilder == null) channelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, false, - securityProtocol, config, credentialCache, tokenCache, time, logContext); + securityProtocol, config, credentialCache, tokenCache, time, logContext, + () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER)); this.metrics = new Metrics(); this.selector = new Selector(10000, failedAuthenticationDelayMs, metrics, time, "MetricGroup", channelBuilder, logContext); diff --git a/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java index 820500e965a15..1697c627fc499 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java @@ -21,6 +21,8 @@ import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; +import org.apache.kafka.common.message.ApiMessageType; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.TestSecurityConfig; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -48,6 +50,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.Supplier; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -160,9 +163,8 @@ public void testClientChannelBuilderWithBrokerConfigs() throws Exception { private SaslChannelBuilder createGssapiChannelBuilder(Map jaasContexts, GSSManager gssManager) { SaslChannelBuilder channelBuilder = new SaslChannelBuilder(Mode.SERVER, jaasContexts, - SecurityProtocol.SASL_PLAINTEXT, - new ListenerName("GSSAPI"), false, "GSSAPI", - true, null, null, null, Time.SYSTEM, new LogContext()) { + SecurityProtocol.SASL_PLAINTEXT, new ListenerName("GSSAPI"), false, "GSSAPI", + true, null, null, null, Time.SYSTEM, new LogContext(), defaultApiVersionsSupplier()) { @Override protected GSSManager gssManager() { @@ -174,6 +176,10 @@ protected GSSManager gssManager() { return channelBuilder; } + private Supplier defaultApiVersionsSupplier() { + return () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER); + } + private SaslChannelBuilder createChannelBuilder(SecurityProtocol securityProtocol, String saslMechanism) { Class loginModule = null; switch (saslMechanism) { @@ -198,7 +204,7 @@ private SaslChannelBuilder createChannelBuilder(SecurityProtocol securityProtoco Map jaasContexts = Collections.singletonMap(saslMechanism, jaasContext); return new SaslChannelBuilder(Mode.CLIENT, jaasContexts, securityProtocol, new ListenerName(saslMechanism), false, saslMechanism, true, null, - null, null, Time.SYSTEM, new LogContext()); + null, null, Time.SYSTEM, new LogContext(), defaultApiVersionsSupplier()); } public static final class TestGssapiLoginModule implements LoginModule { diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java index 2b3bf566e3fa4..13f763279f783 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java @@ -22,7 +22,9 @@ import org.apache.kafka.common.config.types.Password; import org.apache.kafka.common.errors.InvalidConfigurationException; import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.TestSecurityConfig; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.ssl.DefaultSslEngineFactory; @@ -58,6 +60,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import java.util.stream.Stream; import javax.net.ssl.SSLContext; @@ -1018,7 +1021,8 @@ public void testInterBrokerSslConfigValidation(Args args) throws Exception { TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, - true, securityProtocol, config, null, null, time, new LogContext()); + true, securityProtocol, config, null, null, time, new LogContext(), + defaultApiVersionsSupplier()); server = new NioEchoServer(listenerName, securityProtocol, config, "localhost", serverChannelBuilder, null, time); server.start(); @@ -1040,8 +1044,9 @@ public void testInterBrokerSslConfigValidationFailure(Args args) { args.sslServerConfigs.put(BrokerSecurityConfigs.SSL_CLIENT_AUTH_CONFIG, "required"); TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); - assertThrows(KafkaException.class, () -> ChannelBuilders.serverChannelBuilder(listenerName, true, securityProtocol, config, - null, null, time, new LogContext())); + assertThrows(KafkaException.class, () -> ChannelBuilders.serverChannelBuilder( + listenerName, true, securityProtocol, config, + null, null, time, new LogContext(), defaultApiVersionsSupplier())); } /** @@ -1055,7 +1060,8 @@ public void testServerKeystoreDynamicUpdate(Args args) throws Exception { TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, - false, securityProtocol, config, null, null, time, new LogContext()); + false, securityProtocol, config, null, null, time, new LogContext(), + defaultApiVersionsSupplier()); server = new NioEchoServer(listenerName, securityProtocol, config, "localhost", serverChannelBuilder, null, time); server.start(); @@ -1111,7 +1117,8 @@ public void testServerKeystoreDynamicUpdateWithNewSubjectAltName(Args args) thro TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, - false, securityProtocol, config, null, null, time, new LogContext()); + false, securityProtocol, config, null, null, time, new LogContext(), + defaultApiVersionsSupplier()); server = new NioEchoServer(listenerName, securityProtocol, config, "localhost", serverChannelBuilder, null, time); server.start(); @@ -1176,7 +1183,8 @@ public void testServerTruststoreDynamicUpdate(Args args) throws Exception { TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, - false, securityProtocol, config, null, null, time, new LogContext()); + false, securityProtocol, config, null, null, time, new LogContext(), + defaultApiVersionsSupplier()); server = new NioEchoServer(listenerName, securityProtocol, config, "localhost", serverChannelBuilder, null, time); server.start(); @@ -1334,6 +1342,10 @@ private interface FailureAction { void run() throws IOException; } + private Supplier defaultApiVersionsSupplier() { + return () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER); + } + static class TestSslChannelBuilder extends SslChannelBuilder { private Integer netReadBufSizeOverride; diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java index 52121dea81a5e..17d2e1ce26e43 100644 --- a/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java +++ b/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java @@ -20,9 +20,12 @@ import org.apache.kafka.common.protocol.types.Schema; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashSet; import java.util.Set; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -60,11 +63,7 @@ public void testResponseThrottleTime() { Set authenticationKeys = EnumSet.of(ApiKeys.SASL_HANDSHAKE, ApiKeys.SASL_AUTHENTICATE); // Newer protocol apis include throttle time ms even for cluster actions Set clusterActionsWithThrottleTimeMs = EnumSet.of(ApiKeys.ALTER_ISR); - for (ApiKeys apiKey: ApiKeys.values()) { - // Disable broker-to-controller API throttling test - if (apiKey.isControllerOnlyApi) { - continue; - } + for (ApiKeys apiKey: ApiKeys.zkBrokerApis()) { Schema responseSchema = apiKey.messageType.responseSchemas()[apiKey.latestVersion()]; BoundField throttleTimeField = responseSchema.get("throttle_time_ms"); if ((apiKey.clusterAction && !clusterActionsWithThrottleTimeMs.contains(apiKey)) @@ -74,4 +73,17 @@ public void testResponseThrottleTime() { assertNotNull(throttleTimeField, "Throttle time field missing: " + apiKey); } } + + @Test + public void testApiScope() { + Set apisMissingScope = new HashSet<>(); + for (ApiKeys apiKey : ApiKeys.values()) { + if (apiKey.messageType.listeners().isEmpty()) { + apisMissingScope.add(apiKey); + } + } + assertEquals(Collections.emptySet(), apisMissingScope, + "Found some APIs missing scope definition"); + } + } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java index 38a586917624c..2c9b1e8fad023 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java @@ -17,17 +17,17 @@ package org.apache.kafka.common.requests; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.RecordVersion; import org.apache.kafka.common.utils.Utils; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; -import java.util.Collection; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -38,21 +38,15 @@ public class ApiVersionsResponseTest { - @Test - public void shouldCreateApiResponseThatHasAllApiKeysSupportedByBroker() { - assertEquals(apiKeysInResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE), new HashSet<>(ApiKeys.brokerApis())); - assertTrue(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().supportedFeatures().isEmpty()); - assertTrue(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().finalizedFeatures().isEmpty()); - assertEquals(ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().finalizedFeaturesEpoch()); - } - - @Test - public void shouldHaveCorrectDefaultApiVersionsResponse() { - Collection apiVersions = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().apiKeys(); - assertEquals(apiVersions.size(), ApiKeys.brokerApis().size(), "API versions for all API keys must be maintained."); + @ParameterizedTest + @EnumSource(ApiMessageType.ListenerType.class) + public void shouldHaveCorrectDefaultApiVersionsResponse(ApiMessageType.ListenerType scope) { + ApiVersionsResponse defaultResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); + assertEquals(ApiKeys.apisForListener(scope).size(), defaultResponse.data().apiKeys().size(), + "API versions for all API keys must be maintained."); - for (ApiKeys key : ApiKeys.brokerApis()) { - ApiVersion version = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.apiVersion(key.id); + for (ApiKeys key : ApiKeys.apisForListener(scope)) { + ApiVersion version = defaultResponse.apiVersion(key.id); assertNotNull(version, "Could not find ApiVersion for API " + key.name); assertEquals(version.minVersion(), key.oldestVersion(), "Incorrect min version for Api " + key.name); assertEquals(version.maxVersion(), key.latestVersion(), "Incorrect max version for Api " + key.name); @@ -74,9 +68,9 @@ public void shouldHaveCorrectDefaultApiVersionsResponse() { } } - assertTrue(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().supportedFeatures().isEmpty()); - assertTrue(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().finalizedFeatures().isEmpty()); - assertEquals(ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().finalizedFeaturesEpoch()); + assertTrue(defaultResponse.data().supportedFeatures().isEmpty()); + assertTrue(defaultResponse.data().finalizedFeatures().isEmpty()); + assertEquals(ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, defaultResponse.data().finalizedFeaturesEpoch()); } @Test @@ -96,9 +90,11 @@ public void shouldHaveCommonlyAgreedApiVersionResponseWithControllerOnForwardabl .setMaxVersion(maxVersion)) ); - ApiVersionCollection commonResponse = ApiVersionsResponse.intersectControllerApiVersions( - RecordBatch.CURRENT_MAGIC_VALUE, - activeControllerApiVersions); + ApiVersionCollection commonResponse = ApiVersionsResponse.intersectForwardableApis( + ApiMessageType.ListenerType.ZK_BROKER, + RecordVersion.current(), + activeControllerApiVersions + ); verifyVersions(forwardableAPIKey.id, minVersion, maxVersion, commonResponse); @@ -149,11 +145,4 @@ private void verifyVersions(short forwardableAPIKey, assertEquals(expectedVersionsForForwardableAPI, commonResponse.find(forwardableAPIKey)); } - private Set apiKeysInResponse(final ApiVersionsResponse apiVersions) { - final Set apiKeys = new HashSet<>(); - for (final ApiVersion version : apiVersions.data().apiKeys()) { - apiKeys.add(ApiKeys.forId(version.apiKey())); - } - return apiKeys; - } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index f87d9f91317ab..4c70771cca97c 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -44,6 +44,7 @@ import org.apache.kafka.common.message.AlterReplicaLogDirsRequestData.AlterReplicaLogDirTopic; import org.apache.kafka.common.message.AlterReplicaLogDirsRequestData.AlterReplicaLogDirTopicCollection; import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; @@ -145,11 +146,11 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection; -import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderPartition; import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic; import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopicCollection; import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData; +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.OffsetForLeaderTopicResult; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.RenewDelegationTokenRequestData; @@ -346,17 +347,6 @@ public void testSerialization() throws Exception { checkErrorResponse(createSaslAuthenticateRequest(), unknownServerException, true); checkResponse(createSaslAuthenticateResponse(), 0, true); checkResponse(createSaslAuthenticateResponse(), 1, true); - checkRequest(createApiVersionRequest(), true); - checkErrorResponse(createApiVersionRequest(), unknownServerException, true); - checkErrorResponse(createApiVersionRequest(), new UnsupportedVersionException("Not Supported"), true); - checkResponse(createApiVersionResponse(), 0, true); - checkResponse(createApiVersionResponse(), 1, true); - checkResponse(createApiVersionResponse(), 2, true); - checkResponse(createApiVersionResponse(), 3, true); - checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 0, true); - checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 1, true); - checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 2, true); - checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 3, true); for (int v = ApiKeys.CREATE_TOPICS.oldestVersion(); v <= ApiKeys.CREATE_TOPICS.latestVersion(); v++) { checkRequest(createCreateTopicRequest(v), true); @@ -521,9 +511,20 @@ public void testSerialization() throws Exception { checkResponse(createAlterClientQuotasResponse(), 0, true); } + @Test + public void testApiVersionsSerialization() { + for (short v : ApiKeys.API_VERSIONS.allVersions()) { + checkRequest(createApiVersionRequest(v), true); + checkErrorResponse(createApiVersionRequest(v), unknownServerException, true); + checkErrorResponse(createApiVersionRequest(v), new UnsupportedVersionException("Not Supported"), true); + checkResponse(createApiVersionResponse(), v, true); + checkResponse(ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER), v, true); + } + } + @Test public void testBrokerHeartbeatSerialization() { - for (short v = ApiKeys.BROKER_HEARTBEAT.oldestVersion(); v <= ApiKeys.BROKER_HEARTBEAT.latestVersion(); v++) { + for (short v : ApiKeys.BROKER_HEARTBEAT.allVersions()) { checkRequest(createBrokerHeartbeatRequest(v), true); checkErrorResponse(createBrokerHeartbeatRequest(v), unknownServerException, true); checkResponse(createBrokerHeartbeatResponse(), v, true); @@ -532,7 +533,7 @@ public void testBrokerHeartbeatSerialization() { @Test public void testBrokerRegistrationSerialization() { - for (short v = ApiKeys.BROKER_REGISTRATION.oldestVersion(); v <= ApiKeys.BROKER_REGISTRATION.latestVersion(); v++) { + for (short v : ApiKeys.BROKER_REGISTRATION.allVersions()) { checkRequest(createBrokerRegistrationRequest(v), true); checkErrorResponse(createBrokerRegistrationRequest(v), unknownServerException, true); checkResponse(createBrokerRegistrationResponse(), 0, true); @@ -540,8 +541,8 @@ public void testBrokerRegistrationSerialization() { } @Test - public void testDescribeProducersSerialization() throws Exception { - for (short v = ApiKeys.DESCRIBE_PRODUCERS.oldestVersion(); v <= ApiKeys.DESCRIBE_PRODUCERS.latestVersion(); v++) { + public void testDescribeProducersSerialization() { + for (short v : ApiKeys.DESCRIBE_PRODUCERS.allVersions()) { checkRequest(createDescribeProducersRequest(v), true); checkErrorResponse(createDescribeProducersRequest(v), unknownServerException, true); checkResponse(createDescribeProducersResponse(), v, true); @@ -549,8 +550,8 @@ public void testDescribeProducersSerialization() throws Exception { } @Test - public void testDescribeClusterSerialization() throws Exception { - for (short v = ApiKeys.DESCRIBE_CLUSTER.oldestVersion(); v <= ApiKeys.DESCRIBE_CLUSTER.latestVersion(); v++) { + public void testDescribeClusterSerialization() { + for (short v : ApiKeys.DESCRIBE_CLUSTER.allVersions()) { checkRequest(createDescribeClusterRequest(v), true); checkErrorResponse(createDescribeClusterRequest(v), unknownServerException, true); checkResponse(createDescribeClusterResponse(), v, true); @@ -559,7 +560,7 @@ public void testDescribeClusterSerialization() throws Exception { @Test public void testUnregisterBrokerSerialization() { - for (short v = ApiKeys.UNREGISTER_BROKER.oldestVersion(); v <= ApiKeys.UNREGISTER_BROKER.latestVersion(); v++) { + for (short v : ApiKeys.UNREGISTER_BROKER.allVersions()) { checkRequest(createUnregisterBrokerRequest(v), true); checkErrorResponse(createUnregisterBrokerRequest(v), unknownServerException, true); checkResponse(createUnregisterBrokerResponse(), v, true); @@ -1013,47 +1014,56 @@ private void testInvalidCase(String name, String version) { @Test public void testApiVersionResponseWithUnsupportedError() { - ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(); - ApiVersionsResponse response = request.getErrorResponse(0, Errors.UNSUPPORTED_VERSION.exception()); - - assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data().errorCode()); - - ApiVersion apiVersion = response.data().apiKeys().find(ApiKeys.API_VERSIONS.id); - assertNotNull(apiVersion); - assertEquals(ApiKeys.API_VERSIONS.id, apiVersion.apiKey()); - assertEquals(ApiKeys.API_VERSIONS.oldestVersion(), apiVersion.minVersion()); - assertEquals(ApiKeys.API_VERSIONS.latestVersion(), apiVersion.maxVersion()); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); + ApiVersionsResponse response = request.getErrorResponse(0, Errors.UNSUPPORTED_VERSION.exception()); + assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data().errorCode()); + + ApiVersion apiVersion = response.data().apiKeys().find(ApiKeys.API_VERSIONS.id); + assertNotNull(apiVersion); + assertEquals(ApiKeys.API_VERSIONS.id, apiVersion.apiKey()); + assertEquals(ApiKeys.API_VERSIONS.oldestVersion(), apiVersion.minVersion()); + assertEquals(ApiKeys.API_VERSIONS.latestVersion(), apiVersion.maxVersion()); + } } @Test public void testApiVersionResponseWithNotUnsupportedError() { - ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(); - ApiVersionsResponse response = request.getErrorResponse(0, Errors.INVALID_REQUEST.exception()); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); + ApiVersionsResponse response = request.getErrorResponse(0, Errors.INVALID_REQUEST.exception()); + assertEquals(response.data().errorCode(), Errors.INVALID_REQUEST.code()); + assertTrue(response.data().apiKeys().isEmpty()); + } + } - assertEquals(response.data().errorCode(), Errors.INVALID_REQUEST.code()); - assertTrue(response.data().apiKeys().isEmpty()); + private ApiVersionsResponse defaultApiVersionsResponse() { + return ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER); } @Test public void testApiVersionResponseParsingFallback() { - ByteBuffer buffer = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.serialize((short) 0); - ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, ApiKeys.API_VERSIONS.latestVersion()); - - assertEquals(Errors.NONE.code(), response.data().errorCode()); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + ByteBuffer buffer = defaultApiVersionsResponse().serialize((short) 0); + ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, version); + assertEquals(Errors.NONE.code(), response.data().errorCode()); + } } @Test public void testApiVersionResponseParsingFallbackException() { - short version = 0; - assertThrows(BufferUnderflowException.class, () -> ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); + for (final short version : ApiKeys.API_VERSIONS.allVersions()) { + assertThrows(BufferUnderflowException.class, () -> ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); + } } @Test public void testApiVersionResponseParsing() { - ByteBuffer buffer = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.serialize(ApiKeys.API_VERSIONS.latestVersion()); - ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, ApiKeys.API_VERSIONS.latestVersion()); - - assertEquals(Errors.NONE.code(), response.data().errorCode()); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + ByteBuffer buffer = defaultApiVersionsResponse().serialize(version); + ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, version); + assertEquals(Errors.NONE.code(), response.data().errorCode()); + } } @Test @@ -1773,8 +1783,8 @@ private SaslAuthenticateResponse createSaslAuthenticateResponse() { return new SaslAuthenticateResponse(data); } - private ApiVersionsRequest createApiVersionRequest() { - return new ApiVersionsRequest.Builder().build(); + private ApiVersionsRequest createApiVersionRequest(short version) { + return new ApiVersionsRequest.Builder().build(version); } private ApiVersionsResponse createApiVersionResponse() { diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index cc466f7ada236..6c836f33c57bc 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -16,41 +16,6 @@ */ package org.apache.kafka.common.security.authenticator; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SelectionKey; -import java.nio.charset.StandardCharsets; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Base64.Encoder; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import javax.net.ssl.SSLPeerUnverifiedException; -import javax.security.auth.Subject; -import javax.security.auth.callback.Callback; -import javax.security.auth.callback.CallbackHandler; -import javax.security.auth.callback.NameCallback; -import javax.security.auth.callback.PasswordCallback; -import javax.security.auth.callback.UnsupportedCallbackException; -import javax.security.auth.login.Configuration; -import javax.security.auth.login.AppConfigurationEntry; -import javax.security.auth.login.LoginContext; -import javax.security.auth.login.LoginException; -import javax.security.sasl.SaslClient; -import javax.security.sasl.SaslException; - import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.config.SaslConfigs; @@ -60,13 +25,14 @@ import org.apache.kafka.common.config.types.Password; import org.apache.kafka.common.errors.SaslAuthenticationException; import org.apache.kafka.common.errors.SslAuthenticationException; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; import org.apache.kafka.common.message.ListOffsetsResponseData; -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; +import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.RequestHeaderData; import org.apache.kafka.common.message.SaslAuthenticateRequestData; import org.apache.kafka.common.message.SaslHandshakeRequestData; @@ -87,26 +53,28 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.SchemaException; -import org.apache.kafka.common.requests.ListOffsetsResponse; -import org.apache.kafka.common.requests.RequestTestUtils; -import org.apache.kafka.common.security.auth.AuthenticationContext; -import org.apache.kafka.common.security.auth.KafkaPrincipalBuilder; -import org.apache.kafka.common.security.auth.Login; -import org.apache.kafka.common.security.auth.SaslAuthenticationContext; -import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.ListOffsetsResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.requests.SaslAuthenticateRequest; import org.apache.kafka.common.requests.SaslHandshakeRequest; import org.apache.kafka.common.requests.SaslHandshakeResponse; import org.apache.kafka.common.security.JaasContext; import org.apache.kafka.common.security.TestSecurityConfig; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; +import org.apache.kafka.common.security.auth.AuthenticationContext; import org.apache.kafka.common.security.auth.KafkaPrincipal; +import org.apache.kafka.common.security.auth.KafkaPrincipalBuilder; +import org.apache.kafka.common.security.auth.Login; +import org.apache.kafka.common.security.auth.SaslAuthenticationContext; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.security.authenticator.TestDigestLoginModule.DigestServerCallbackHandler; import org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule; import org.apache.kafka.common.security.oauthbearer.OAuthBearerToken; import org.apache.kafka.common.security.oauthbearer.OAuthBearerTokenCallback; @@ -115,20 +83,17 @@ import org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredJws; import org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredLoginCallbackHandler; import org.apache.kafka.common.security.plain.PlainLoginModule; +import org.apache.kafka.common.security.plain.internals.PlainServerCallbackHandler; import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.scram.ScramLoginModule; import org.apache.kafka.common.security.scram.internals.ScramCredentialUtils; import org.apache.kafka.common.security.scram.internals.ScramFormatter; -import org.apache.kafka.common.security.scram.ScramLoginModule; import org.apache.kafka.common.security.scram.internals.ScramMechanism; import org.apache.kafka.common.security.token.delegation.TokenInformation; import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.SecurityUtils; -import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; -import org.apache.kafka.common.security.authenticator.TestDigestLoginModule.DigestServerCallbackHandler; -import org.apache.kafka.common.security.plain.internals.PlainServerCallbackHandler; - import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestUtils; @@ -137,6 +102,41 @@ import org.junit.jupiter.api.Test; import org.opentest4j.AssertionFailedError; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.security.auth.Subject; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; +import javax.security.sasl.SaslClient; +import javax.security.sasl.SaslException; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Base64.Encoder; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import static org.apache.kafka.common.protocol.ApiKeys.LIST_OFFSETS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -1903,31 +1903,18 @@ private NioEchoServer startServerApiVersionsUnsupportedByClient(final SecurityPr boolean isScram = ScramMechanism.isScram(saslMechanism); if (isScram) ScramCredentialUtils.createCache(credentialCache, Arrays.asList(saslMechanism)); + + Supplier apiVersionSupplier = () -> { + ApiVersionCollection versionCollection = new ApiVersionCollection(2); + versionCollection.add(new ApiVersion().setApiKey(ApiKeys.SASL_HANDSHAKE.id).setMinVersion((short) 0).setMaxVersion((short) 100)); + versionCollection.add(new ApiVersion().setApiKey(ApiKeys.SASL_AUTHENTICATE.id).setMinVersion((short) 0).setMaxVersion((short) 100)); + return new ApiVersionsResponse(new ApiVersionsResponseData().setApiKeys(versionCollection)); + }; + SaslChannelBuilder serverChannelBuilder = new SaslChannelBuilder(Mode.SERVER, jaasContexts, securityProtocol, listenerName, false, saslMechanism, true, - credentialCache, null, null, time, new LogContext()) { - - @Override - protected SaslServerAuthenticator buildServerAuthenticator(Map configs, - Map callbackHandlers, - String id, - TransportLayer transportLayer, - Map subjects, - Map connectionsMaxReauthMsByMechanism, - ChannelMetadataRegistry metadataRegistry) { - return new SaslServerAuthenticator(configs, callbackHandlers, id, subjects, null, listenerName, - securityProtocol, transportLayer, connectionsMaxReauthMsByMechanism, metadataRegistry, time) { + credentialCache, null, null, time, new LogContext(), apiVersionSupplier); - @Override - protected ApiVersionsResponse apiVersionsResponse() { - ApiVersionCollection versionCollection = new ApiVersionCollection(2); - versionCollection.add(new ApiVersion().setApiKey(ApiKeys.SASL_HANDSHAKE.id).setMinVersion((short) 0).setMaxVersion((short) 100)); - versionCollection.add(new ApiVersion().setApiKey(ApiKeys.SASL_AUTHENTICATE.id).setMinVersion((short) 0).setMaxVersion((short) 100)); - return new ApiVersionsResponse(new ApiVersionsResponseData().setApiKeys(versionCollection)); - } - }; - } - }; serverChannelBuilder.configure(saslServerConfigs); server = new NioEchoServer(listenerName, securityProtocol, new TestSecurityConfig(saslServerConfigs), "localhost", serverChannelBuilder, credentialCache, time); @@ -1945,10 +1932,29 @@ private NioEchoServer startServerWithoutSaslAuthenticateHeader(final SecurityPro boolean isScram = ScramMechanism.isScram(saslMechanism); if (isScram) ScramCredentialUtils.createCache(credentialCache, Arrays.asList(saslMechanism)); + + Supplier apiVersionSupplier = () -> { + ApiVersionsResponse defaultApiVersionResponse = ApiVersionsResponse.defaultApiVersionsResponse( + ApiMessageType.ListenerType.ZK_BROKER); + ApiVersionCollection apiVersions = new ApiVersionCollection(); + for (ApiVersion apiVersion : defaultApiVersionResponse.data().apiKeys()) { + if (apiVersion.apiKey() != ApiKeys.SASL_AUTHENTICATE.id) { + // ApiVersion can NOT be reused in second ApiVersionCollection + // due to the internal pointers it contains. + apiVersions.add(apiVersion.duplicate()); + } + + } + ApiVersionsResponseData data = new ApiVersionsResponseData() + .setErrorCode(Errors.NONE.code()) + .setThrottleTimeMs(0) + .setApiKeys(apiVersions); + return new ApiVersionsResponse(data); + }; + SaslChannelBuilder serverChannelBuilder = new SaslChannelBuilder(Mode.SERVER, jaasContexts, securityProtocol, listenerName, false, saslMechanism, true, - credentialCache, null, null, time, new LogContext()) { - + credentialCache, null, null, time, new LogContext(), apiVersionSupplier) { @Override protected SaslServerAuthenticator buildServerAuthenticator(Map configs, Map callbackHandlers, @@ -1958,27 +1964,7 @@ protected SaslServerAuthenticator buildServerAuthenticator(Map config Map connectionsMaxReauthMsByMechanism, ChannelMetadataRegistry metadataRegistry) { return new SaslServerAuthenticator(configs, callbackHandlers, id, subjects, null, listenerName, - securityProtocol, transportLayer, connectionsMaxReauthMsByMechanism, metadataRegistry, time) { - - @Override - protected ApiVersionsResponse apiVersionsResponse() { - ApiVersionsResponse defaultApiVersionResponse = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE; - ApiVersionCollection apiVersions = new ApiVersionCollection(); - for (ApiVersion apiVersion : defaultApiVersionResponse.data().apiKeys()) { - if (apiVersion.apiKey() != ApiKeys.SASL_AUTHENTICATE.id) { - // ApiVersion can NOT be reused in second ApiVersionCollection - // due to the internal pointers it contains. - apiVersions.add(apiVersion.duplicate()); - } - - } - ApiVersionsResponseData data = new ApiVersionsResponseData() - .setErrorCode(Errors.NONE.code()) - .setThrottleTimeMs(0) - .setApiKeys(apiVersions); - return new ApiVersionsResponse(data); - } - + securityProtocol, transportLayer, connectionsMaxReauthMsByMechanism, metadataRegistry, time, apiVersionSupplier) { @Override protected void enableKafkaSaslAuthenticateHeaders(boolean flag) { // Don't enable Kafka SASL_AUTHENTICATE headers @@ -2003,7 +1989,7 @@ private void createClientConnectionWithoutSaslAuthenticateHeader(final SecurityP SaslChannelBuilder clientChannelBuilder = new SaslChannelBuilder(Mode.CLIENT, jaasContexts, securityProtocol, listenerName, false, saslMechanism, true, - null, null, null, time, new LogContext()) { + null, null, null, time, new LogContext(), null) { @Override protected SaslClientAuthenticator buildClientAuthenticator(Map configs, @@ -2545,7 +2531,8 @@ public AlternateSaslChannelBuilder(Mode mode, Map jaasConte String clientSaslMechanism, boolean handshakeRequestEnable, CredentialCache credentialCache, DelegationTokenCache tokenCache, Time time) { super(mode, jaasContexts, securityProtocol, listenerName, isInterBrokerListener, clientSaslMechanism, - handshakeRequestEnable, credentialCache, tokenCache, null, time, new LogContext()); + handshakeRequestEnable, credentialCache, tokenCache, null, time, new LogContext(), + () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER)); } @Override diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index e5c46c9973361..af0fedd4f5ad9 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -19,6 +19,7 @@ import java.net.InetAddress; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.errors.IllegalSaslStateException; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.network.ChannelMetadataRegistry; import org.apache.kafka.common.network.ClientInformation; import org.apache.kafka.common.network.DefaultChannelMetadataRegistry; @@ -27,6 +28,7 @@ import org.apache.kafka.common.network.TransportLayer; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -152,15 +154,17 @@ private void testApiVersionsRequest(short version, String expectedSoftwareName, } private SaslServerAuthenticator setupAuthenticator(Map configs, TransportLayer transportLayer, - String mechanism, ChannelMetadataRegistry metadataRegistry) throws IOException { + String mechanism, ChannelMetadataRegistry metadataRegistry) { TestJaasConfig jaasConfig = new TestJaasConfig(); jaasConfig.addEntry("jaasContext", PlainLoginModule.class.getName(), new HashMap()); Map subjects = Collections.singletonMap(mechanism, new Subject()); Map callbackHandlers = Collections.singletonMap( mechanism, new SaslServerCallbackHandler()); + ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( + ApiMessageType.ListenerType.ZK_BROKER); return new SaslServerAuthenticator(configs, callbackHandlers, "node", subjects, null, new ListenerName("ssl"), SecurityProtocol.SASL_SSL, transportLayer, Collections.emptyMap(), - metadataRegistry, Time.SYSTEM); + metadataRegistry, Time.SYSTEM, () -> apiVersionsResponse); } } diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala index e89f9fb46a555..879373787365f 100644 --- a/core/src/main/scala/kafka/api/ApiVersion.scala +++ b/core/src/main/scala/kafka/api/ApiVersion.scala @@ -21,10 +21,9 @@ import org.apache.kafka.clients.NodeApiVersions import org.apache.kafka.common.config.ConfigDef.Validator import org.apache.kafka.common.config.ConfigException import org.apache.kafka.common.feature.{Features, FinalizedVersionRange, SupportedVersionRange} -import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.{RecordBatch, RecordVersion} -import org.apache.kafka.common.requests.{AbstractResponse, ApiVersionsResponse} -import org.apache.kafka.common.requests.ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE +import org.apache.kafka.common.message.ApiMessageType.ListenerType +import org.apache.kafka.common.record.RecordVersion +import org.apache.kafka.common.requests.ApiVersionsResponse /** * This class contains the different Kafka versions. @@ -147,52 +146,46 @@ object ApiVersion { } } - def apiVersionsResponse(throttleTimeMs: Int, - maxMagic: Byte, - latestSupportedFeatures: Features[SupportedVersionRange], - controllerApiVersions: Option[NodeApiVersions]): ApiVersionsResponse = { + def apiVersionsResponse( + throttleTimeMs: Int, + minRecordVersion: RecordVersion, + latestSupportedFeatures: Features[SupportedVersionRange], + controllerApiVersions: Option[NodeApiVersions], + listenerType: ListenerType + ): ApiVersionsResponse = { apiVersionsResponse( throttleTimeMs, - maxMagic, + minRecordVersion, latestSupportedFeatures, Features.emptyFinalizedFeatures, ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, - controllerApiVersions + controllerApiVersions, + listenerType ) } - def apiVersionsResponse(throttleTimeMs: Int, - maxMagic: Byte, - latestSupportedFeatures: Features[SupportedVersionRange], - finalizedFeatures: Features[FinalizedVersionRange], - finalizedFeaturesEpoch: Long, - controllerApiVersions: Option[NodeApiVersions]): ApiVersionsResponse = { + def apiVersionsResponse( + throttleTimeMs: Int, + minRecordVersion: RecordVersion, + latestSupportedFeatures: Features[SupportedVersionRange], + finalizedFeatures: Features[FinalizedVersionRange], + finalizedFeaturesEpoch: Long, + controllerApiVersions: Option[NodeApiVersions], + listenerType: ListenerType + ): ApiVersionsResponse = { val apiKeys = controllerApiVersions match { - case None => ApiVersionsResponse.defaultApiKeys(maxMagic) - case Some(controllerApiVersion) => ApiVersionsResponse.intersectControllerApiVersions( - maxMagic, controllerApiVersion.allSupportedApiVersions()) + case None => ApiVersionsResponse.filterApis(minRecordVersion, listenerType) + case Some(controllerApiVersion) => ApiVersionsResponse.intersectForwardableApis( + listenerType, minRecordVersion, controllerApiVersion.allSupportedApiVersions()) } - if (maxMagic == RecordBatch.CURRENT_MAGIC_VALUE && - throttleTimeMs == AbstractResponse.DEFAULT_THROTTLE_TIME) { - new ApiVersionsResponse( - ApiVersionsResponse.createApiVersionsResponseData( - DEFAULT_API_VERSIONS_RESPONSE.throttleTimeMs, - Errors.forCode(DEFAULT_API_VERSIONS_RESPONSE.data.errorCode), - apiKeys, - latestSupportedFeatures, - finalizedFeatures, - finalizedFeaturesEpoch)) - } else { - new ApiVersionsResponse( - ApiVersionsResponse.createApiVersionsResponseData( - throttleTimeMs, - Errors.NONE, - apiKeys, - latestSupportedFeatures, - finalizedFeatures, - finalizedFeaturesEpoch)) - } + ApiVersionsResponse.createApiVersionsResponse( + throttleTimeMs, + apiKeys, + latestSupportedFeatures, + finalizedFeatures, + finalizedFeaturesEpoch + ) } } diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 7d3112560d851..48f723f4d3375 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -30,6 +30,7 @@ import kafka.utils.{Logging, NotNothing, Pool} import kafka.utils.Implicits._ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData._ import org.apache.kafka.common.network.Send @@ -59,16 +60,19 @@ object RequestChannel extends Logging { val sanitizedUser: String = Sanitizer.sanitize(principal.getName) } - class Metrics(allowControllerOnlyApis: Boolean = false) { + class Metrics(enabledApis: Iterable[ApiKeys]) { + def this(scope: ListenerType) = { + this(ApiKeys.apisForListener(scope).asScala) + } private val metricsMap = mutable.Map[String, RequestMetrics]() - (ApiKeys.values.toSeq.filter(!_.isControllerOnlyApi || allowControllerOnlyApis).map(_.name) ++ - Seq(RequestMetrics.consumerFetchMetricName, RequestMetrics.followFetchMetricName)).foreach { name => + (enabledApis.map(_.name) ++ + Seq(RequestMetrics.consumerFetchMetricName, RequestMetrics.followFetchMetricName)).foreach { name => metricsMap.put(name, new RequestMetrics(name)) } - def apply(metricName: String) = metricsMap(metricName) + def apply(metricName: String): RequestMetrics = metricsMap(metricName) def close(): Unit = { metricsMap.values.foreach(_.removeMetrics()) @@ -296,8 +300,6 @@ object RequestChannel extends Logging { def responseLog: Option[JsonNode] = None def onComplete: Option[Send => Unit] = None - - override def toString: String } /** responseLogValue should only be defined if request logging is enabled */ @@ -337,9 +339,8 @@ object RequestChannel extends Logging { class RequestChannel(val queueSize: Int, val metricNamePrefix: String, time: Time, - allowControllerOnlyApis: Boolean = false) extends KafkaMetricsGroup { + val metrics: RequestChannel.Metrics) extends KafkaMetricsGroup { import RequestChannel._ - val metrics = new RequestChannel.Metrics(allowControllerOnlyApis) private val requestQueue = new ArrayBlockingQueue[BaseRequest](queueSize) private val processors = new ConcurrentHashMap[Int, Processor]() val requestQueueSizeMetricName = metricNamePrefix.concat(RequestQueueSizeMetric) diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 72c5141445f2a..24df39f6ef2ef 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -33,7 +33,7 @@ import kafka.network.Processor._ import kafka.network.RequestChannel.{CloseConnectionResponse, EndThrottlingResponse, NoOpResponse, SendResponse, StartThrottlingResponse} import kafka.network.SocketServer._ import kafka.security.CredentialProvider -import kafka.server.{BrokerReconfigurable, KafkaConfig} +import kafka.server.{ApiVersionManager, BrokerReconfigurable, KafkaConfig} import kafka.utils.Implicits._ import kafka.utils._ import org.apache.kafka.common.config.ConfigException @@ -78,15 +78,14 @@ class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time, val credentialProvider: CredentialProvider, - allowControllerOnlyApis: Boolean = false, - controllerSocketServer: Boolean = false) + val apiVersionManager: ApiVersionManager) extends Logging with KafkaMetricsGroup with BrokerReconfigurable { private val maxQueuedRequests = config.queuedMaxRequests private val nodeId = config.brokerId - private val logContext = new LogContext(s"[SocketServer ${if (controllerSocketServer) "controller" else "broker"}Id=${nodeId}] ") + private val logContext = new LogContext(s"[SocketServer listenerType=${apiVersionManager.listenerType}, nodeId=$nodeId] ") this.logIdent = logContext.logPrefix @@ -98,12 +97,12 @@ class SocketServer(val config: KafkaConfig, // data-plane private val dataPlaneProcessors = new ConcurrentHashMap[Int, Processor]() private[network] val dataPlaneAcceptors = new ConcurrentHashMap[EndPoint, Acceptor]() - val dataPlaneRequestChannel = new RequestChannel(maxQueuedRequests, DataPlaneMetricPrefix, time, allowControllerOnlyApis) + val dataPlaneRequestChannel = new RequestChannel(maxQueuedRequests, DataPlaneMetricPrefix, time, apiVersionManager.newRequestMetrics) // control-plane private var controlPlaneProcessorOpt : Option[Processor] = None private[network] var controlPlaneAcceptorOpt : Option[Acceptor] = None val controlPlaneRequestChannelOpt: Option[RequestChannel] = config.controlPlaneListenerName.map(_ => - new RequestChannel(20, ControlPlaneMetricPrefix, time, allowControllerOnlyApis)) + new RequestChannel(20, ControlPlaneMetricPrefix, time, apiVersionManager.newRequestMetrics)) private var nextProcessorId = 0 val connectionQuotas = new ConnectionQuotas(config, time, metrics) @@ -438,8 +437,9 @@ class SocketServer(val config: KafkaConfig, credentialProvider, memoryPool, logContext, - isPrivilegedListener = isPrivilegedListener, - allowControllerOnlyApis = allowControllerOnlyApis + Processor.ConnectionQueueSize, + isPrivilegedListener, + apiVersionManager ) } @@ -772,7 +772,6 @@ private[kafka] object Processor { val IdlePercentMetricName = "IdlePercent" val NetworkProcessorMetricTag = "networkProcessor" val ListenerMetricTag = "listener" - val ConnectionQueueSize = 20 } @@ -800,9 +799,9 @@ private[kafka] class Processor(val id: Int, credentialProvider: CredentialProvider, memoryPool: MemoryPool, logContext: LogContext, - connectionQueueSize: Int = ConnectionQueueSize, - isPrivilegedListener: Boolean = false, - allowControllerOnlyApis: Boolean = false) extends AbstractServerThread(connectionQuotas) with KafkaMetricsGroup { + connectionQueueSize: Int, + isPrivilegedListener: Boolean, + apiVersionManager: ApiVersionManager) extends AbstractServerThread(connectionQuotas) with KafkaMetricsGroup { private object ConnectionId { def fromString(s: String): Option[ConnectionId] = s.split("-") match { @@ -842,14 +841,19 @@ private[kafka] class Processor(val id: Int, metrics.addMetric(expiredConnectionsKilledCountMetricName, expiredConnectionsKilledCount) private val selector = createSelector( - ChannelBuilders.serverChannelBuilder(listenerName, + ChannelBuilders.serverChannelBuilder( + listenerName, listenerName == config.interBrokerListenerName, securityProtocol, config, credentialProvider.credentialCache, credentialProvider.tokenCache, time, - logContext)) + logContext, + () => apiVersionManager.apiVersionResponse(throttleTimeMs = 0) + ) + ) + // Visible to override for testing protected[network] def createSelector(channelBuilder: ChannelBuilder): KSelector = { channelBuilder match { @@ -993,10 +997,10 @@ private[kafka] class Processor(val id: Int, protected def parseRequestHeader(buffer: ByteBuffer): RequestHeader = { val header = RequestHeader.parse(buffer) - if (!header.apiKey.isControllerOnlyApi || allowControllerOnlyApis) { + if (apiVersionManager.isApiEnabled(header.apiKey)) { header } else { - throw new InvalidRequestException("Received request for KIP-500 controller-only api key " + header.apiKey) + throw new InvalidRequestException(s"Received request api key ${header.apiKey} which is not enabled") } } diff --git a/core/src/main/scala/kafka/server/ApiVersionManager.scala b/core/src/main/scala/kafka/server/ApiVersionManager.scala new file mode 100644 index 0000000000000..ebf3e74e89291 --- /dev/null +++ b/core/src/main/scala/kafka/server/ApiVersionManager.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.api.ApiVersion +import kafka.network +import kafka.network.RequestChannel +import org.apache.kafka.common.message.ApiMessageType.ListenerType +import org.apache.kafka.common.message.ApiVersionsResponseData +import org.apache.kafka.common.protocol.ApiKeys +import org.apache.kafka.common.requests.ApiVersionsResponse + +import scala.jdk.CollectionConverters._ + +trait ApiVersionManager { + def listenerType: ListenerType + def enabledApis: collection.Set[ApiKeys] + def apiVersionResponse(throttleTimeMs: Int): ApiVersionsResponse + def isApiEnabled(apiKey: ApiKeys): Boolean = enabledApis.contains(apiKey) + def newRequestMetrics: RequestChannel.Metrics = new network.RequestChannel.Metrics(enabledApis) +} + +object ApiVersionManager { + def apply( + listenerType: ListenerType, + config: KafkaConfig, + forwardingManager: Option[ForwardingManager], + features: BrokerFeatures, + featureCache: FinalizedFeatureCache + ): ApiVersionManager = { + new DefaultApiVersionManager( + listenerType, + config.interBrokerProtocolVersion, + forwardingManager, + features, + featureCache + ) + } +} + +class SimpleApiVersionManager( + val listenerType: ListenerType, + val enabledApis: collection.Set[ApiKeys] +) extends ApiVersionManager { + + def this(listenerType: ListenerType) = { + this(listenerType, ApiKeys.apisForListener(listenerType).asScala) + } + + private val apiVersions = ApiVersionsResponse.collectApis(enabledApis.asJava) + + override def apiVersionResponse(requestThrottleMs: Int): ApiVersionsResponse = { + ApiVersionsResponse.createApiVersionsResponse(0, apiVersions) + } +} + +class DefaultApiVersionManager( + val listenerType: ListenerType, + interBrokerProtocolVersion: ApiVersion, + forwardingManager: Option[ForwardingManager], + features: BrokerFeatures, + featureCache: FinalizedFeatureCache +) extends ApiVersionManager { + + override def apiVersionResponse(throttleTimeMs: Int): ApiVersionsResponse = { + val supportedFeatures = features.supportedFeatures + val finalizedFeaturesOpt = featureCache.get + val controllerApiVersions = forwardingManager.flatMap(_.controllerApiVersions) + + val response = finalizedFeaturesOpt match { + case Some(finalizedFeatures) => ApiVersion.apiVersionsResponse( + throttleTimeMs, + interBrokerProtocolVersion.recordVersion, + supportedFeatures, + finalizedFeatures.features, + finalizedFeatures.epoch, + controllerApiVersions, + listenerType) + case None => ApiVersion.apiVersionsResponse( + throttleTimeMs, + interBrokerProtocolVersion.recordVersion, + supportedFeatures, + controllerApiVersions, + listenerType) + } + + // This is a temporary workaround in order to allow testing of forwarding + // in integration tests. We can remove this after the KIP-500 controller + // is available for integration testing. + if (forwardingManager.isDefined) { + response.data.apiKeys.add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(ApiKeys.ENVELOPE.id) + .setMinVersion(ApiKeys.ENVELOPE.oldestVersion) + .setMaxVersion(ApiKeys.ENVELOPE.latestVersion) + ) + } + + response + } + + override def enabledApis: collection.Set[ApiKeys] = { + forwardingManager match { + case Some(_) => ApiKeys.apisForListener(listenerType).asScala ++ Set(ApiKeys.ENVELOPE) + case None => ApiKeys.apisForListener(listenerType).asScala + } + } + + override def isApiEnabled(apiKey: ApiKeys): Boolean = { + apiKey.inScope(listenerType) || (apiKey == ApiKeys.ENVELOPE && forwardingManager.isDefined) + } +} diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index 57ceb46202fbc..19d65ab2e98bd 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -33,6 +33,7 @@ import kafka.server.KafkaBroker.metricsPrefix import kafka.server.metadata.{BrokerMetadataListener, CachedConfigRepository, ClientQuotaCache, ClientQuotaMetadataManager, RaftMetadataCache} import kafka.utils.{CoreUtils, KafkaScheduler} import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.BrokerRegistrationRequestData.{Listener, ListenerCollection} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.ListenerName @@ -167,7 +168,7 @@ class BrokerServer( // Create log manager, but don't start it because we need to delay any potential unclean shutdown log recovery // until we catch up on the metadata log and have up-to-date topic and broker configs. logManager = LogManager(config, initialOfflineDirs, configRepository, kafkaScheduler, time, - brokerTopicStats, logDirFailureChannel, true) + brokerTopicStats, logDirFailureChannel, keepPartitionMetadataFile = true) metadataCache = MetadataCache.raftMetadataCache(config.nodeId) // Enable delegation token cache for all SCRAM mechanisms to simplify dynamic update. @@ -175,17 +176,44 @@ class BrokerServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) + val controllerNodes = RaftConfig.quorumVoterStringsToNodes(controllerQuorumVotersFuture.get()).asScala + val controllerNodeProvider = RaftControllerNodeProvider(metaLogManager, config, controllerNodes) + + val forwardingChannelManager = BrokerToControllerChannelManager( + controllerNodeProvider, + time, + metrics, + config, + channelName = "forwarding", + threadNamePrefix, + retryTimeoutMs = 60000 + ) + forwardingManager = new ForwardingManagerImpl(forwardingChannelManager) + forwardingManager.start() + + val apiVersionManager = ApiVersionManager( + ListenerType.BROKER, + config, + Some(forwardingManager), + brokerFeatures, + featureCache + ) + // Create and start the socket server acceptor threads so that the bound port is known. // Delay starting processors until the end of the initialization sequence to ensure // that credentials have been loaded before processing authentications. - socketServer = new SocketServer(config, metrics, time, credentialProvider, allowControllerOnlyApis = false) + socketServer = new SocketServer(config, metrics, time, credentialProvider, apiVersionManager) socketServer.startup(startProcessingRequests = false) - val controllerNodes = - RaftConfig.quorumVoterStringsToNodes(controllerQuorumVotersFuture.get()).asScala - val controllerNodeProvider = RaftControllerNodeProvider(metaLogManager, config, controllerNodes) - val alterIsrChannelManager = BrokerToControllerChannelManager(controllerNodeProvider, - time, metrics, config, "alterisr", threadNamePrefix, 60000) + val alterIsrChannelManager = BrokerToControllerChannelManager( + controllerNodeProvider, + time, + metrics, + config, + channelName = "alterisr", + threadNamePrefix, + retryTimeoutMs = Long.MaxValue + ) alterIsrManager = new DefaultAlterIsrManager( controllerChannelManager = alterIsrChannelManager, scheduler = kafkaScheduler, @@ -200,11 +228,6 @@ class BrokerServer( brokerTopicStats, metadataCache, logDirFailureChannel, alterIsrManager, configRepository, threadNamePrefix) - val forwardingChannelManager = BrokerToControllerChannelManager(controllerNodeProvider, - time, metrics, config, "forwarding", threadNamePrefix, 60000) - forwardingManager = new ForwardingManagerImpl(forwardingChannelManager) - forwardingManager.start() - /* start token manager */ if (config.tokenAuthEnabled) { throw new UnsupportedOperationException("Delegation tokens are not supported") @@ -306,7 +329,7 @@ class BrokerServer( dataPlaneRequestProcessor = new KafkaApis(socketServer.dataPlaneRequestChannel, raftSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.nodeId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, - fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + fetchManager, brokerTopicStats, clusterId, time, tokenManager, apiVersionManager) dataPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.nodeId, socketServer.dataPlaneRequestChannel, dataPlaneRequestProcessor, time, config.numIoThreads, s"${SocketServer.DataPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.DataPlaneThreadPrefix) @@ -315,7 +338,7 @@ class BrokerServer( controlPlaneRequestProcessor = new KafkaApis(controlPlaneRequestChannel, raftSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.nodeId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, - fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + fetchManager, brokerTopicStats, clusterId, time, tokenManager, apiVersionManager) controlPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.nodeId, socketServer.controlPlaneRequestChannelOpt.get, controlPlaneRequestProcessor, time, 1, s"${SocketServer.ControlPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.ControlPlaneThreadPrefix) diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index efcebb491c3df..625ce5257b8e3 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -28,6 +28,7 @@ import kafka.raft.RaftManager import kafka.security.CredentialProvider import kafka.server.QuotaFactory.QuotaManagers import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache @@ -124,15 +125,16 @@ class ControllerServer( }.toMap } + val apiVersionManager = new SimpleApiVersionManager(ListenerType.CONTROLLER) + tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) socketServer = new SocketServer(config, metrics, time, credentialProvider, - allowControllerOnlyApis = true, - controllerSocketServer = true) - socketServer.startup(false, None, config.controllerListeners) + apiVersionManager) + socketServer.startup(startProcessingRequests = false, controlPlaneListener = None, config.controllerListeners) socketServerFirstBoundPortFuture.complete(socketServer.boundPort( config.controllerListeners.head.listenerName)) diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 5e8340e6b6163..5a926d4ace403 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -61,7 +61,7 @@ import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsParti import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic} import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult, OffsetForLeaderTopicResultCollection} -import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, ApiVersionsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} +import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.{ListenerName, Send} import org.apache.kafka.common.protocol.{ApiKeys, Errors} @@ -110,8 +110,7 @@ class KafkaApis(val requestChannel: RequestChannel, val clusterId: String, time: Time, val tokenManager: DelegationTokenManager, - val brokerFeatures: BrokerFeatures, - val finalizedFeatureCache: FinalizedFeatureCache) extends ApiRequestHandler with Logging { + val apiVersionManager: ApiVersionManager) extends ApiRequestHandler with Logging { metadataSupport.ensureConsistentWith(config) @@ -158,6 +157,12 @@ class KafkaApis(val requestChannel: RequestChannel, trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" + s"securityProtocol:${request.context.securityProtocol},principal:${request.context.principal}") + if (!apiVersionManager.isApiEnabled(request.header.apiKey)) { + // The socket server will reject APIs which are not exposed in this scope and close the connection + // before handing them to the request handler, so this path should not be exercised in practice + throw new IllegalStateException(s"API ${request.header.apiKey} is not enabled") + } + request.header.apiKey match { case ApiKeys.PRODUCE => handleProduceRequest(request) case ApiKeys.FETCH => handleFetchRequest(request) @@ -217,17 +222,7 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.DESCRIBE_CLUSTER => handleDescribeCluster(request) case ApiKeys.DESCRIBE_PRODUCERS => handleDescribeProducersRequest(request) case ApiKeys.UNREGISTER_BROKER => maybeForwardToController(request, handleUnregisterBrokerRequest) - - // Handle requests that should have been sent to the KIP-500 controller. - // Until we are ready to integrate the Raft layer, these APIs are treated as - // unexpected and we just close the connection. - case ApiKeys.VOTE => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.BEGIN_QUORUM_EPOCH => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.END_QUORUM_EPOCH => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.DESCRIBE_QUORUM => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.FETCH_SNAPSHOT => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.BROKER_REGISTRATION => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.BROKER_HEARTBEAT => requestHelper.closeConnection(request, util.Collections.emptyMap()) + case _ => throw new IllegalStateException(s"No handler for request api key ${request.header.apiKey}") } } catch { case e: FatalExitError => throw e @@ -1686,39 +1681,12 @@ class KafkaApis(val requestChannel: RequestChannel, // ApiVersionRequest is not available. def createResponseCallback(requestThrottleMs: Int): ApiVersionsResponse = { val apiVersionRequest = request.body[ApiVersionsRequest] - if (apiVersionRequest.hasUnsupportedRequestVersion) + if (apiVersionRequest.hasUnsupportedRequestVersion) { apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.UNSUPPORTED_VERSION.exception) - else if (!apiVersionRequest.isValid) + } else if (!apiVersionRequest.isValid) { apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.INVALID_REQUEST.exception) - else { - val supportedFeatures = brokerFeatures.supportedFeatures - val finalizedFeaturesOpt = finalizedFeatureCache.get - val controllerApiVersions = metadataSupport.forwardingManager.flatMap(_.controllerApiVersions) - - val apiVersionsResponse = - finalizedFeaturesOpt match { - case Some(finalizedFeatures) => ApiVersion.apiVersionsResponse( - requestThrottleMs, - config.interBrokerProtocolVersion.recordVersion.value, - supportedFeatures, - finalizedFeatures.features, - finalizedFeatures.epoch, - controllerApiVersions) - case None => ApiVersion.apiVersionsResponse( - requestThrottleMs, - config.interBrokerProtocolVersion.recordVersion.value, - supportedFeatures, - controllerApiVersions) - } - if (request.context.fromPrivilegedListener) { - apiVersionsResponse.data.apiKeys().add( - new ApiVersionsResponseData.ApiVersion() - .setApiKey(ApiKeys.ENVELOPE.id) - .setMinVersion(ApiKeys.ENVELOPE.oldestVersion()) - .setMaxVersion(ApiKeys.ENVELOPE.latestVersion()) - ) - } - apiVersionsResponse + } else { + apiVersionManager.apiVersionResponse(requestThrottleMs) } } requestHelper.sendResponseMaybeThrottle(request, createResponseCallback) diff --git a/core/src/main/scala/kafka/server/KafkaRaftServer.scala b/core/src/main/scala/kafka/server/KafkaRaftServer.scala index dc3fd16fed8a6..7ec46220b91f4 100644 --- a/core/src/main/scala/kafka/server/KafkaRaftServer.scala +++ b/core/src/main/scala/kafka/server/KafkaRaftServer.scala @@ -74,8 +74,17 @@ class KafkaRaftServer( private val metaLogShim = new MetaLogRaftShim(raftManager.kafkaRaftClient, config.nodeId) private val broker: Option[BrokerServer] = if (config.processRoles.contains(BrokerRole)) { - Some(new BrokerServer(config, metaProps, metaLogShim, time, metrics, threadNamePrefix, - offlineDirs, controllerQuorumVotersFuture, Server.SUPPORTED_FEATURES)) + Some(new BrokerServer( + config, + metaProps, + metaLogShim, + time, + metrics, + threadNamePrefix, + offlineDirs, + controllerQuorumVotersFuture, + Server.SUPPORTED_FEATURES + )) } else { None } @@ -89,7 +98,7 @@ class KafkaRaftServer( time, metrics, threadNamePrefix, - CompletableFuture.completedFuture(config.quorumVoters) + controllerQuorumVotersFuture )) } else { None diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 3ad36874385e0..4daee0866d8e9 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -37,6 +37,7 @@ import kafka.utils._ import kafka.zk.{AdminZkClient, BrokerInfo, KafkaZkClient} import org.apache.kafka.clients.{ApiVersions, ClientDnsLookup, ManualMetadataUpdater, NetworkClient, NetworkClientUtils} import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.ControlledShutdownRequestData import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network._ @@ -157,7 +158,6 @@ class KafkaServer( private var _featureChangeListener: FinalizedFeatureChangeListener = null val brokerFeatures: BrokerFeatures = BrokerFeatures.createDefault() - val featureCache: FinalizedFeatureCache = new FinalizedFeatureCache(brokerFeatures) def clusterId: String = _clusterId @@ -256,14 +256,32 @@ class KafkaServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) + if (enableForwarding) { + this.forwardingManager = Some(ForwardingManager( + config, + metadataCache, + time, + metrics, + threadNamePrefix + )) + forwardingManager.foreach(_.start()) + } + + val apiVersionManager = ApiVersionManager( + ListenerType.ZK_BROKER, + config, + forwardingManager, + brokerFeatures, + featureCache + ) + // Create and start the socket server acceptor threads so that the bound port is known. // Delay starting processors until the end of the initialization sequence to ensure // that credentials have been loaded before processing authentications. // // Note that we allow the use of KIP-500 controller APIs when forwarding is enabled // so that the Envelope request is exposed. This is only used in testing currently. - socketServer = new SocketServer(config, metrics, time, credentialProvider, - allowControllerOnlyApis = enableForwarding) + socketServer = new SocketServer(config, metrics, time, credentialProvider, apiVersionManager) socketServer.startup(startProcessingRequests = false) /* start replica manager */ @@ -300,18 +318,6 @@ class KafkaServer( kafkaController = new KafkaController(config, zkClient, time, metrics, brokerInfo, brokerEpoch, tokenManager, brokerFeatures, featureCache, threadNamePrefix) kafkaController.startup() - /* start forwarding manager */ - if (enableForwarding) { - this.forwardingManager = Some(ForwardingManager( - config, - metadataCache, - time, - metrics, - threadNamePrefix - )) - forwardingManager.foreach(_.start()) - } - adminManager = new ZkAdminManager(config, metrics, metadataCache, zkClient) /* start group coordinator */ @@ -363,7 +369,7 @@ class KafkaServer( val zkSupport = ZkSupport(adminManager, kafkaController, zkClient, forwardingManager, metadataCache) dataPlaneRequestProcessor = new KafkaApis(socketServer.dataPlaneRequestChannel, zkSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.brokerId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, - fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + fetchManager, brokerTopicStats, clusterId, time, tokenManager, apiVersionManager) dataPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.dataPlaneRequestChannel, dataPlaneRequestProcessor, time, config.numIoThreads, s"${SocketServer.DataPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.DataPlaneThreadPrefix) @@ -371,7 +377,7 @@ class KafkaServer( socketServer.controlPlaneRequestChannelOpt.foreach { controlPlaneRequestChannel => controlPlaneRequestProcessor = new KafkaApis(controlPlaneRequestChannel, zkSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.brokerId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, - fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + fetchManager, brokerTopicStats, clusterId, time, tokenManager, apiVersionManager) controlPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.controlPlaneRequestChannelOpt.get, controlPlaneRequestProcessor, time, 1, s"${SocketServer.ControlPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.ControlPlaneThreadPrefix) diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index ddfa1fa102f03..e4dec2ee66af3 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -20,7 +20,7 @@ package kafka.tools import kafka.network.RequestChannel import kafka.network.RequestConvertToJson import kafka.raft.RaftManager -import kafka.server.ApiRequestHandler +import kafka.server.{ApiRequestHandler, ApiVersionManager} import kafka.utils.Logging import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, EndQuorumEpochResponseData, FetchResponseData, FetchSnapshotResponseData, VoteResponseData} @@ -38,6 +38,7 @@ class TestRaftRequestHandler( raftManager: RaftManager[_], requestChannel: RequestChannel, time: Time, + apiVersionManager: ApiVersionManager ) extends ApiRequestHandler with Logging { override def handle(request: RequestChannel.Request): Unit = { @@ -45,6 +46,7 @@ class TestRaftRequestHandler( trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" + s"securityProtocol:${request.context.securityProtocol},principal:${request.context.principal}") request.header.apiKey match { + case ApiKeys.API_VERSIONS => handleApiVersions(request) case ApiKeys.VOTE => handleVote(request) case ApiKeys.BEGIN_QUORUM_EPOCH => handleBeginQuorumEpoch(request) case ApiKeys.END_QUORUM_EPOCH => handleEndQuorumEpoch(request) @@ -62,6 +64,10 @@ class TestRaftRequestHandler( } } + private def handleApiVersions(request: RequestChannel.Request): Unit = { + sendResponse(request, Some(apiVersionManager.apiVersionResponse(throttleTimeMs = 0))) + } + private def handleVote(request: RequestChannel.Request): Unit = { handle(request, response => new VoteResponse(response.asInstanceOf[VoteResponseData])) } diff --git a/core/src/main/scala/kafka/tools/TestRaftServer.scala b/core/src/main/scala/kafka/tools/TestRaftServer.scala index 2391ca4c380ed..ba6ab4074a73f 100644 --- a/core/src/main/scala/kafka/tools/TestRaftServer.scala +++ b/core/src/main/scala/kafka/tools/TestRaftServer.scala @@ -24,9 +24,10 @@ import joptsimple.OptionException import kafka.network.SocketServer import kafka.raft.{KafkaRaftManager, RaftManager} import kafka.security.CredentialProvider -import kafka.server.{KafkaConfig, KafkaRequestHandlerPool, MetaProperties} +import kafka.server.{KafkaConfig, KafkaRequestHandlerPool, MetaProperties, SimpleApiVersionManager} import kafka.utils.{CommandDefaultOptions, CommandLineUtils, CoreUtils, Exit, Logging, ShutdownableThread} import org.apache.kafka.common.errors.InvalidConfigurationException +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.metrics.stats.Percentiles.BucketSizing import org.apache.kafka.common.metrics.stats.{Meter, Percentile, Percentiles} @@ -68,7 +69,8 @@ class TestRaftServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) - socketServer = new SocketServer(config, metrics, time, credentialProvider, allowControllerOnlyApis = true) + val apiVersionManager = new SimpleApiVersionManager(ListenerType.CONTROLLER) + socketServer = new SocketServer(config, metrics, time, credentialProvider, apiVersionManager) socketServer.startup(startProcessingRequests = false) val metaProperties = MetaProperties( @@ -96,7 +98,8 @@ class TestRaftServer( val requestHandler = new TestRaftRequestHandler( raftManager, socketServer.dataPlaneRequestChannel, - time + time, + apiVersionManager ) dataPlaneRequestHandlerPool = new KafkaRequestHandlerPool( diff --git a/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala b/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala index 6224591addb3e..2db694f21403e 100644 --- a/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala +++ b/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala @@ -55,7 +55,7 @@ class BrokerApiVersionsCommandTest extends KafkaServerTestHarness { assertTrue(lineIter.hasNext) assertEquals(s"$brokerList (id: 0 rack: null) -> (", lineIter.next()) val nodeApiVersions = NodeApiVersions.create - val enabledApis = ApiKeys.brokerApis.asScala + val enabledApis = ApiKeys.zkBrokerApis.asScala for (apiKey <- enabledApis) { val apiVersion = nodeApiVersions.apiVersion(apiKey) assertNotNull(apiVersion) diff --git a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala index 46156f1c4a7c3..90454bbb3814d 100644 --- a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala +++ b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala @@ -29,7 +29,9 @@ import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.errors.SaslAuthenticationException +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.network._ +import org.apache.kafka.common.requests.ApiVersionsResponse import org.apache.kafka.common.security.{JaasContext, TestSecurityConfig} import org.apache.kafka.common.security.auth.{Login, SecurityProtocol} import org.apache.kafka.common.security.kerberos.KerberosLogin @@ -233,7 +235,8 @@ class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup { val config = new TestSecurityConfig(clientConfig) val jaasContexts = Collections.singletonMap("GSSAPI", JaasContext.loadClientContext(config.values())) val channelBuilder = new SaslChannelBuilder(Mode.CLIENT, jaasContexts, securityProtocol, - null, false, kafkaClientSaslMechanism, true, null, null, null, time, new LogContext()) { + null, false, kafkaClientSaslMechanism, true, null, null, null, time, new LogContext(), + () => ApiVersionsResponse.defaultApiVersionsResponse(ListenerType.ZK_BROKER)) { override protected def defaultLoginClass(): Class[_ <: Login] = classOf[TestableKerberosLogin] } channelBuilder.configure(config.values()) diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala index 1e86876057326..6a3eb317c99cf 100644 --- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala +++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala @@ -20,6 +20,7 @@ package kafka.api import java.util import org.apache.kafka.common.feature.{Features, FinalizedVersionRange, SupportedVersionRange} +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.record.{RecordBatch, RecordVersion} import org.apache.kafka.common.requests.{AbstractResponse, ApiVersionsResponse} @@ -179,9 +180,10 @@ class ApiVersionTest { def shouldCreateApiResponseOnlyWithKeysSupportedByMagicValue(): Unit = { val response = ApiVersion.apiVersionsResponse( 10, - RecordBatch.MAGIC_VALUE_V1, + RecordVersion.V1, Features.emptySupportedFeatures, - None + None, + ListenerType.ZK_BROKER ) verifyApiKeysForMagic(response, RecordBatch.MAGIC_VALUE_V1) assertEquals(10, response.throttleTimeMs) @@ -194,13 +196,14 @@ class ApiVersionTest { def shouldReturnFeatureKeysWhenMagicIsCurrentValueAndThrottleMsIsDefaultThrottle(): Unit = { val response = ApiVersion.apiVersionsResponse( 10, - RecordBatch.MAGIC_VALUE_V1, + RecordVersion.V1, Features.supportedFeatures( Utils.mkMap(Utils.mkEntry("feature", new SupportedVersionRange(1.toShort, 4.toShort)))), Features.finalizedFeatures( Utils.mkMap(Utils.mkEntry("feature", new FinalizedVersionRange(2.toShort, 3.toShort)))), 10, - None + None, + ListenerType.ZK_BROKER ) verifyApiKeysForMagic(response, RecordBatch.MAGIC_VALUE_V1) @@ -228,11 +231,12 @@ class ApiVersionTest { def shouldReturnAllKeysWhenMagicIsCurrentValueAndThrottleMsIsDefaultThrottle(): Unit = { val response = ApiVersion.apiVersionsResponse( AbstractResponse.DEFAULT_THROTTLE_TIME, - RecordBatch.CURRENT_MAGIC_VALUE, + RecordVersion.current(), Features.emptySupportedFeatures, - None + None, + ListenerType.ZK_BROKER ) - assertEquals(new util.HashSet[ApiKeys](ApiKeys.brokerApis), apiKeysInResponse(response)) + assertEquals(new util.HashSet[ApiKeys](ApiKeys.zkBrokerApis), apiKeysInResponse(response)) assertEquals(AbstractResponse.DEFAULT_THROTTLE_TIME, response.throttleTimeMs) assertTrue(response.data.supportedFeatures.isEmpty) assertTrue(response.data.finalizedFeatures.isEmpty) @@ -243,9 +247,10 @@ class ApiVersionTest { def testMetadataQuorumApisAreDisabled(): Unit = { val response = ApiVersion.apiVersionsResponse( AbstractResponse.DEFAULT_THROTTLE_TIME, - RecordBatch.CURRENT_MAGIC_VALUE, + RecordVersion.current(), Features.emptySupportedFeatures, - None + None, + ListenerType.ZK_BROKER ) // Ensure that APIs needed for the internal metadata quorum (KIP-500) diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index dd93e43436288..293614432cb76 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -31,10 +31,11 @@ import com.yammer.metrics.core.{Gauge, Meter} import javax.net.ssl._ import kafka.metrics.KafkaYammerMetrics import kafka.security.CredentialProvider -import kafka.server.{KafkaConfig, ThrottledChannel} +import kafka.server.{KafkaConfig, SimpleApiVersionManager, ThrottledChannel} import kafka.utils.Implicits._ import kafka.utils.TestUtils import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.{ProduceRequestData, SaslAuthenticateRequestData, SaslHandshakeRequestData, VoteRequestData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.KafkaChannel.ChannelMuteState @@ -73,7 +74,8 @@ class SocketServerTest { // Clean-up any metrics left around by previous tests TestUtils.clearYammerMetrics() - val server = new SocketServer(config, metrics, Time.SYSTEM, credentialProvider) + private val apiVersionManager = new SimpleApiVersionManager(ListenerType.ZK_BROKER) + val server = new SocketServer(config, metrics, Time.SYSTEM, credentialProvider, apiVersionManager) server.startup() val sockets = new ArrayBuffer[Socket] @@ -452,7 +454,8 @@ class SocketServerTest { val time = new MockTime() props.put(KafkaConfig.ConnectionsMaxIdleMsProp, idleTimeMs.toString) val serverMetrics = new Metrics - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, time, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, + time, credentialProvider, apiVersionManager) try { overrideServer.startup() @@ -504,12 +507,14 @@ class SocketServerTest { val serverMetrics = new Metrics @volatile var selector: TestableSelector = null val overrideConnectionId = "127.0.0.1:1-127.0.0.1:2-0" - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, time, credentialProvider) { + val overrideServer = new SocketServer( + KafkaConfig.fromProps(props), serverMetrics, time, credentialProvider, apiVersionManager + ) { override def newProcessor(id: Int, requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, listenerName: ListenerName, - protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean = false): Processor = { + protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean): Processor = { new Processor(id, time, config.socketRequestMaxBytes, dataPlaneRequestChannel, connectionQuotas, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, - credentialProvider, memoryPool, new LogContext(), isPrivilegedListener = isPrivilegedListener) { + credentialProvider, memoryPool, new LogContext(), Processor.ConnectionQueueSize, isPrivilegedListener, apiVersionManager) { override protected[network] def connectionId(socket: Socket): String = overrideConnectionId override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = { val testableSelector = new TestableSelector(config, channelBuilder, time, metrics) @@ -799,7 +804,8 @@ class SocketServerTest { val newProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0) newProps.setProperty(KafkaConfig.MaxConnectionsPerIpProp, "0") newProps.setProperty(KafkaConfig.MaxConnectionsPerIpOverridesProp, "%s:%s".format("127.0.0.1", "5")) - val server = new SocketServer(KafkaConfig.fromProps(newProps), new Metrics(), Time.SYSTEM, credentialProvider) + val server = new SocketServer(KafkaConfig.fromProps(newProps), new Metrics(), + Time.SYSTEM, credentialProvider, apiVersionManager) try { server.startup() // make the maximum allowable number of connections @@ -837,7 +843,8 @@ class SocketServerTest { val overrideProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0) overrideProps.put(KafkaConfig.MaxConnectionsPerIpOverridesProp, s"localhost:$overrideNum") val serverMetrics = new Metrics() - val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), serverMetrics, Time.SYSTEM, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), serverMetrics, + Time.SYSTEM, credentialProvider, apiVersionManager) try { overrideServer.startup() // make the maximum allowable number of connections @@ -866,7 +873,8 @@ class SocketServerTest { overrideProps.put(KafkaConfig.NumQuotaSamplesProp, String.valueOf(2)) val connectionRate = 5 val time = new MockTime() - val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), new Metrics(), time, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), new Metrics(), + time, credentialProvider, apiVersionManager) // update the connection rate to 5 overrideServer.connectionQuotas.updateIpConnectionRateQuota(None, Some(connectionRate)) try { @@ -916,7 +924,8 @@ class SocketServerTest { overrideProps.put(KafkaConfig.NumQuotaSamplesProp, String.valueOf(2)) val connectionRate = 5 val time = new MockTime() - val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), new Metrics(), time, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), new Metrics(), + time, credentialProvider, apiVersionManager) overrideServer.connectionQuotas.updateIpConnectionRateQuota(None, Some(connectionRate)) overrideServer.startup() // make the maximum allowable number of connections @@ -938,7 +947,8 @@ class SocketServerTest { @Test def testSslSocketServer(): Unit = { val serverMetrics = new Metrics - val overrideServer = new SocketServer(KafkaConfig.fromProps(sslServerProps), serverMetrics, Time.SYSTEM, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(sslServerProps), serverMetrics, + Time.SYSTEM, credentialProvider, apiVersionManager) try { overrideServer.startup() val sslContext = SSLContext.getInstance(TestSslUtils.DEFAULT_TLS_PROTOCOL_FOR_TESTS) @@ -1078,12 +1088,14 @@ class SocketServerTest { val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0) val serverMetrics = new Metrics var conn: Socket = null - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider) { + val overrideServer = new SocketServer( + KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider, apiVersionManager + ) { override def newProcessor(id: Int, requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, listenerName: ListenerName, protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean = false): Processor = { new Processor(id, time, config.socketRequestMaxBytes, dataPlaneRequestChannel, connectionQuotas, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, - credentialProvider, MemoryPool.NONE, new LogContext(), isPrivilegedListener = isPrivilegedListener) { + credentialProvider, MemoryPool.NONE, new LogContext(), Processor.ConnectionQueueSize, isPrivilegedListener, apiVersionManager) { override protected[network] def sendResponse(response: RequestChannel.Response, responseSend: Send): Unit = { conn.close() super.sendResponse(response, responseSend) @@ -1120,12 +1132,14 @@ class SocketServerTest { def testClientDisconnectionWithOutstandingReceivesProcessedUntilFailedSend(): Unit = { val serverMetrics = new Metrics @volatile var selector: TestableSelector = null - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider) { + val overrideServer = new SocketServer( + KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider, apiVersionManager + ) { override def newProcessor(id: Int, requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, listenerName: ListenerName, - protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean = false): Processor = { + protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean): Processor = { new Processor(id, time, config.socketRequestMaxBytes, dataPlaneRequestChannel, connectionQuotas, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, - credentialProvider, memoryPool, new LogContext(), isPrivilegedListener = isPrivilegedListener) { + credentialProvider, memoryPool, new LogContext(), Processor.ConnectionQueueSize, isPrivilegedListener, apiVersionManager) { override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = { val testableSelector = new TestableSelector(config, channelBuilder, time, metrics) selector = testableSelector @@ -1161,7 +1175,8 @@ class SocketServerTest { props.setProperty(KafkaConfig.ConnectionsMaxIdleMsProp, "110") val serverMetrics = new Metrics var conn: Socket = null - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, + Time.SYSTEM, credentialProvider, apiVersionManager) try { overrideServer.startup() conn = connect(overrideServer) @@ -1873,9 +1888,13 @@ class SocketServerTest { } } - class TestableSocketServer(config : KafkaConfig = KafkaConfig.fromProps(props), val connectionQueueSize: Int = 20, - override val time: Time = Time.SYSTEM) extends SocketServer(config, - new Metrics, time, credentialProvider) { + class TestableSocketServer( + config : KafkaConfig = KafkaConfig.fromProps(props), + connectionQueueSize: Int = 20, + time: Time = Time.SYSTEM + ) extends SocketServer( + config, new Metrics, time, credentialProvider, apiVersionManager, + ) { @volatile var selector: Option[TestableSelector] = None @volatile var uncaughtExceptions = 0 @@ -1884,7 +1903,7 @@ class SocketServerTest { protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean = false): Processor = { new Processor(id, time, config.socketRequestMaxBytes, requestChannel, connectionQuotas, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, credentialProvider, - memoryPool, new LogContext(), connectionQueueSize, isPrivilegedListener) { + memoryPool, new LogContext(), connectionQueueSize, isPrivilegedListener, apiVersionManager) { override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = { val testableSelector = new TestableSelector(config, channelBuilder, time, metrics, metricTags.asScala) diff --git a/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala index f7163cad10f88..9c7b4440a32b8 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala @@ -16,15 +16,17 @@ */ package kafka.server +import java.util.Properties + import integration.kafka.server.IntegrationTestUtils import kafka.test.ClusterInstance +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.requests.{ApiVersionsRequest, ApiVersionsResponse} import org.junit.jupiter.api.Assertions._ -import java.util.Properties import scala.jdk.CollectionConverters._ abstract class AbstractApiVersionsRequestTest(cluster: ClusterInstance) { @@ -53,14 +55,13 @@ abstract class AbstractApiVersionsRequestTest(cluster: ClusterInstance) { } finally socket.close() } - def validateApiVersionsResponse(apiVersionsResponse: ApiVersionsResponse, listenerName: ListenerName): Unit = { - val expectedApis = ApiKeys.brokerApis() - if (listenerName == controlPlaneListenerName) { - expectedApis.add(ApiKeys.ENVELOPE) - } + def validateApiVersionsResponse(apiVersionsResponse: ApiVersionsResponse): Unit = { + val expectedApis = ApiKeys.zkBrokerApis() assertEquals(expectedApis.size(), apiVersionsResponse.data.apiKeys().size(), "API keys in ApiVersionsResponse must match API keys supported by broker.") - for (expectedApiVersion: ApiVersion <- ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data.apiKeys().asScala) { + + val defaultApiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(ListenerType.ZK_BROKER) + for (expectedApiVersion: ApiVersion <- defaultApiVersionsResponse.data.apiKeys().asScala) { val actualApiVersion = apiVersionsResponse.apiVersion(expectedApiVersion.apiKey) assertNotNull(actualApiVersion, s"API key ${actualApiVersion.apiKey} is supported by broker, but not received in ApiVersionsResponse.") assertEquals(expectedApiVersion.apiKey, actualApiVersion.apiKey, "API key must be supported by the broker.") diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala new file mode 100644 index 0000000000000..a93cc90e3e2d6 --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.api.ApiVersion +import org.apache.kafka.clients.NodeApiVersions +import org.apache.kafka.common.message.ApiMessageType.ListenerType +import org.apache.kafka.common.protocol.ApiKeys +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource +import org.mockito.Mockito + +import scala.jdk.CollectionConverters._ + +class ApiVersionManagerTest { + private val brokerFeatures = BrokerFeatures.createDefault() + private val featureCache = new FinalizedFeatureCache(brokerFeatures) + + @ParameterizedTest + @EnumSource(classOf[ListenerType]) + def testApiScope(apiScope: ListenerType): Unit = { + val versionManager = new DefaultApiVersionManager( + listenerType = apiScope, + interBrokerProtocolVersion = ApiVersion.latestVersion, + forwardingManager = None, + features = brokerFeatures, + featureCache = featureCache + ) + assertEquals(ApiKeys.apisForListener(apiScope).asScala, versionManager.enabledApis) + assertTrue(ApiKeys.apisForListener(apiScope).asScala.forall(versionManager.isApiEnabled)) + } + + @Test + def testControllerApiIntersection(): Unit = { + val controllerMinVersion: Short = 1 + val controllerMaxVersion: Short = 5 + + val forwardingManager = Mockito.mock(classOf[ForwardingManager]) + + Mockito.when(forwardingManager.controllerApiVersions).thenReturn(Some(NodeApiVersions.create( + ApiKeys.CREATE_TOPICS.id, + controllerMinVersion, + controllerMaxVersion + ))) + + val versionManager = new DefaultApiVersionManager( + listenerType = ListenerType.ZK_BROKER, + interBrokerProtocolVersion = ApiVersion.latestVersion, + forwardingManager = Some(forwardingManager), + features = brokerFeatures, + featureCache = featureCache + ) + + val apiVersionsResponse = versionManager.apiVersionResponse(throttleTimeMs = 0) + val alterConfigVersion = apiVersionsResponse.data.apiKeys.find(ApiKeys.CREATE_TOPICS.id) + assertNotNull(alterConfigVersion) + assertEquals(controllerMinVersion, alterConfigVersion.minVersion) + assertEquals(controllerMaxVersion, alterConfigVersion.maxVersion) + } + + @Test + def testEnvelopeEnabledWhenForwardingManagerPresent(): Unit = { + val forwardingManager = Mockito.mock(classOf[ForwardingManager]) + Mockito.when(forwardingManager.controllerApiVersions).thenReturn(None) + + val versionManager = new DefaultApiVersionManager( + listenerType = ListenerType.ZK_BROKER, + interBrokerProtocolVersion = ApiVersion.latestVersion, + forwardingManager = Some(forwardingManager), + features = brokerFeatures, + featureCache = featureCache + ) + assertTrue(versionManager.isApiEnabled(ApiKeys.ENVELOPE)) + assertTrue(versionManager.enabledApis.contains(ApiKeys.ENVELOPE)) + + val apiVersionsResponse = versionManager.apiVersionResponse(throttleTimeMs = 0) + val envelopeVersion = apiVersionsResponse.data.apiKeys.find(ApiKeys.ENVELOPE.id) + assertNotNull(envelopeVersion) + assertEquals(ApiKeys.ENVELOPE.oldestVersion, envelopeVersion.minVersion) + assertEquals(ApiKeys.ENVELOPE.latestVersion, envelopeVersion.maxVersion) + } + + @Test + def testEnvelopeDisabledWhenForwardingManagerEmpty(): Unit = { + val versionManager = new DefaultApiVersionManager( + listenerType = ListenerType.ZK_BROKER, + interBrokerProtocolVersion = ApiVersion.latestVersion, + forwardingManager = None, + features = brokerFeatures, + featureCache = featureCache + ) + assertFalse(versionManager.isApiEnabled(ApiKeys.ENVELOPE)) + assertFalse(versionManager.enabledApis.contains(ApiKeys.ENVELOPE)) + + val apiVersionsResponse = versionManager.apiVersionResponse(throttleTimeMs = 0) + assertNull(apiVersionsResponse.data.apiKeys.find(ApiKeys.ENVELOPE.id)) + } + +} diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala index dc35bae4ab09c..34ee74a0e8066 100644 --- a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala @@ -40,14 +40,14 @@ class ApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVersio def testApiVersionsRequest(): Unit = { val request = new ApiVersionsRequest.Builder().build() val apiVersionsResponse = sendApiVersionsRequest(request, cluster.clientListener()) - validateApiVersionsResponse(apiVersionsResponse, cluster.clientListener()) + validateApiVersionsResponse(apiVersionsResponse) } @ClusterTest def testApiVersionsRequestThroughControlPlaneListener(): Unit = { val request = new ApiVersionsRequest.Builder().build() val apiVersionsResponse = sendApiVersionsRequest(request, super.controlPlaneListenerName) - validateApiVersionsResponse(apiVersionsResponse, super.controlPlaneListenerName) + validateApiVersionsResponse(apiVersionsResponse) } @ClusterTest @@ -66,14 +66,14 @@ class ApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVersio def testApiVersionsRequestValidationV0(): Unit = { val apiVersionsRequest = new ApiVersionsRequest.Builder().build(0.asInstanceOf[Short]) val apiVersionsResponse = sendApiVersionsRequest(apiVersionsRequest, cluster.clientListener()) - validateApiVersionsResponse(apiVersionsResponse, cluster.clientListener()) + validateApiVersionsResponse(apiVersionsResponse) } @ClusterTest def testApiVersionsRequestValidationV0ThroughControlPlaneListener(): Unit = { val apiVersionsRequest = new ApiVersionsRequest.Builder().build(0.asInstanceOf[Short]) val apiVersionsResponse = sendApiVersionsRequest(apiVersionsRequest, super.controlPlaneListenerName) - validateApiVersionsResponse(apiVersionsResponse, super.controlPlaneListenerName) + validateApiVersionsResponse(apiVersionsResponse) } @ClusterTest diff --git a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala index bb4f57b333d49..9240af6a10857 100644 --- a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala @@ -29,6 +29,7 @@ import org.apache.kafka.clients.MockClient.RequestMatcher import org.apache.kafka.common.Node import org.apache.kafka.common.config.{ConfigResource, TopicConfig} import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.{AlterConfigsResponseData, ApiVersionsResponseData} import org.apache.kafka.common.network.{ClientInformation, ListenerName} import org.apache.kafka.common.protocol.{ApiKeys, Errors} @@ -195,7 +196,7 @@ class ForwardingManagerTest { startTimeNanos = time.nanoseconds(), memoryPool = MemoryPool.NONE, buffer = requestBuffer, - metrics = new RequestChannel.Metrics(allowControllerOnlyApis = true), + metrics = new RequestChannel.Metrics(ListenerType.CONTROLLER), envelope = None ) } diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 5138bf67655fb..e80c6eb3a736c 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -37,7 +37,6 @@ import kafka.server.QuotaFactory.QuotaManagers import kafka.server.metadata.{CachedConfigRepository, ConfigRepository, RaftMetadataCache} import kafka.utils.{MockTime, TestUtils} import kafka.zk.KafkaZkClient -import org.apache.kafka.clients.NodeApiVersions import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin.{AlterConfigOp, ConfigEntry} import org.apache.kafka.common.acl.AclOperation @@ -45,6 +44,7 @@ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.UnsupportedVersionException import org.apache.kafka.common.internals.{KafkaFutureImpl, Topic} import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicCollection} import org.apache.kafka.common.message.DescribeConfigsResponseData.DescribeConfigsResult import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol @@ -129,8 +129,6 @@ class KafkaApisTest { raftSupport: Boolean = false, overrideProperties: Map[String, String] = Map.empty): KafkaApis = { - val brokerFeatures = BrokerFeatures.createDefault() - val cache = new FinalizedFeatureCache(brokerFeatures) val properties = if (raftSupport) { val properties = TestUtils.createBrokerConfig(brokerId, "") properties.put(KafkaConfig.NodeIdProp, brokerId.toString) @@ -163,6 +161,15 @@ class KafkaApisTest { case _ => throw new IllegalStateException("Test must set an instance of ZkMetadataCache") } } + + val listenerType = if (raftSupport) ListenerType.BROKER else ListenerType.ZK_BROKER + val enabledApis = if (enableForwarding) { + ApiKeys.apisForListener(listenerType).asScala ++ Set(ApiKeys.ENVELOPE) + } else { + ApiKeys.apisForListener(listenerType).asScala.toSet + } + val apiVersionManager = new SimpleApiVersionManager(listenerType, enabledApis) + new KafkaApis(requestChannel, metadataSupport, replicaManager, @@ -181,8 +188,7 @@ class KafkaApisTest { clusterId, time, null, - brokerFeatures, - cache) + apiVersionManager) } @Test @@ -652,75 +658,6 @@ class KafkaApisTest { } } - @Test - def testHandleApiVersionsWithControllerApiVersions(): Unit = { - val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) - - val requestHeader = new RequestHeader(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion, clientId, 0) - - val permittedVersion: Short = 0 - EasyMock.expect(forwardingManager.controllerApiVersions).andReturn( - Some(NodeApiVersions.create(ApiKeys.ALTER_CONFIGS.id, permittedVersion, permittedVersion))) - - val capturedResponse = expectNoThrottling() - - val apiVersionsRequest = new ApiVersionsRequest.Builder() - .build(requestHeader.apiVersion) - val request = buildRequest(apiVersionsRequest, - fromPrivilegedListener = true, requestHeader = Option(requestHeader)) - - EasyMock.replay(replicaManager, clientRequestQuotaManager, forwardingManager, - requestChannel, authorizer, adminManager, controller) - - createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handleApiVersionsRequest(request) - - val expectedVersions = new ApiVersionsResponseData.ApiVersion() - .setApiKey(ApiKeys.ALTER_CONFIGS.id) - .setMaxVersion(permittedVersion) - .setMinVersion(permittedVersion) - - val response = readResponse(apiVersionsRequest, capturedResponse) - .asInstanceOf[ApiVersionsResponse] - assertEquals(Errors.NONE, Errors.forCode(response.data().errorCode())) - - val alterConfigVersions = response.data().apiKeys().find(ApiKeys.ALTER_CONFIGS.id) - assertEquals(expectedVersions, alterConfigVersions) - - verify(authorizer, adminManager, forwardingManager) - } - - @Test - def testGetUnsupportedVersionsWhenControllerApiVersionsNotAvailable(): Unit = { - val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) - - val requestHeader = new RequestHeader(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion, clientId, 0) - - EasyMock.expect(forwardingManager.controllerApiVersions).andReturn(None) - - val capturedResponse = expectNoThrottling() - - val apiVersionsRequest = new ApiVersionsRequest.Builder() - .build(requestHeader.apiVersion) - val request = buildRequest(apiVersionsRequest, - fromPrivilegedListener = true, requestHeader = Option(requestHeader)) - - EasyMock.replay(replicaManager, clientRequestQuotaManager, forwardingManager, - requestChannel, authorizer, adminManager, controller) - - createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handleApiVersionsRequest(request) - - val response = readResponse(apiVersionsRequest, capturedResponse) - .asInstanceOf[ApiVersionsResponse] - assertEquals(Errors.NONE, Errors.forCode(response.data().errorCode())) - - val expectedVersions = ApiVersionsResponse.toApiVersion(ApiKeys.ALTER_CONFIGS) - - val alterConfigVersions = response.data().apiKeys().find(ApiKeys.ALTER_CONFIGS.id) - assertEquals(expectedVersions, alterConfigVersions) - - verify(authorizer, adminManager, forwardingManager) - } - @Test def testCreateTopicsWithAuthorizer(): Unit = { val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index 1924034ffdf45..7706c83cca9bf 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -171,7 +171,7 @@ class RequestQuotaTest extends BaseRequestTest { def testUnauthorizedThrottle(): Unit = { RequestQuotaTest.principal = RequestQuotaTest.UnauthorizedPrincipal - for (apiKey <- ApiKeys.brokerApis.asScala) { + for (apiKey <- ApiKeys.zkBrokerApis.asScala) { submitTest(apiKey, () => checkUnauthorizedRequestThrottle(apiKey)) } @@ -754,9 +754,9 @@ class RequestQuotaTest extends BaseRequestTest { } object RequestQuotaTest { - val ClusterActions = ApiKeys.brokerApis.asScala.filter(_.clusterAction).toSet + val ClusterActions = ApiKeys.zkBrokerApis.asScala.filter(_.clusterAction).toSet val SaslActions = Set(ApiKeys.SASL_HANDSHAKE, ApiKeys.SASL_AUTHENTICATE) - val ClientActions = ApiKeys.brokerApis.asScala.toSet -- ClusterActions -- SaslActions + val ClientActions = ApiKeys.zkBrokerApis.asScala.toSet -- ClusterActions -- SaslActions val UnauthorizedPrincipal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "Unauthorized") // Principal used for all client connections. This is modified by tests which diff --git a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala index bbc71cad2679c..05c83d476d9f8 100644 --- a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala @@ -59,7 +59,7 @@ class SaslApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVe try { val apiVersionsResponse = IntegrationTestUtils.sendAndReceive[ApiVersionsResponse]( new ApiVersionsRequest.Builder().build(0), socket) - validateApiVersionsResponse(apiVersionsResponse, cluster.clientListener()) + validateApiVersionsResponse(apiVersionsResponse) sendSaslHandshakeRequestValidateResponse(socket) } finally { socket.close() @@ -88,7 +88,7 @@ class SaslApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVe assertEquals(Errors.UNSUPPORTED_VERSION.code, apiVersionsResponse.data.errorCode) val apiVersionsResponse2 = IntegrationTestUtils.sendAndReceive[ApiVersionsResponse]( new ApiVersionsRequest.Builder().build(0), socket) - validateApiVersionsResponse(apiVersionsResponse2, cluster.clientListener()) + validateApiVersionsResponse(apiVersionsResponse2) sendSaslHandshakeRequestValidateResponse(socket) } finally { socket.close() diff --git a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java index 075ee485550b0..408e1a75ff2c8 100644 --- a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java +++ b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java @@ -19,14 +19,23 @@ import java.io.BufferedWriter; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collectors; public final class ApiMessageTypeGenerator implements TypeClassGenerator { private final HeaderGenerator headerGenerator; private final CodeBuffer buffer; private final TreeMap apis; + private final EnumMap> apisByListener = new EnumMap<>(RequestListenerType.class); private static final class ApiData { short apiKey; @@ -93,6 +102,13 @@ public void registerMessageType(MessageSpec spec) { "API key " + spec.apiKey().get()); } data.requestSpec = spec; + + if (spec.listeners() != null) { + for (RequestListenerType listener : spec.listeners()) { + apisByListener.putIfAbsent(listener, new ArrayList<>()); + apisByListener.get(listener).add(data); + } + } break; } case RESPONSE: { @@ -140,6 +156,8 @@ private void generate() { buffer.printf("%n"); generateAccessor("highestSupportedVersion", "short"); buffer.printf("%n"); + generateAccessor("listeners", "EnumSet"); + buffer.printf("%n"); generateAccessor("apiKey", "short"); buffer.printf("%n"); generateAccessor("requestSchemas", "Schema[]"); @@ -151,18 +169,48 @@ private void generate() { generateHeaderVersion("request"); buffer.printf("%n"); generateHeaderVersion("response"); + buffer.printf("%n"); + generateListenerTypesEnum(); + buffer.printf("%n"); buffer.decrementIndent(); buffer.printf("}%n"); headerGenerator.generate(); } + private String generateListenerTypeEnumSet(Collection values) { + if (values.isEmpty()) { + return "EnumSet.noneOf(ListenerType.class)"; + } + StringBuilder bldr = new StringBuilder("EnumSet.of("); + Iterator iter = values.iterator(); + while (iter.hasNext()) { + bldr.append("ListenerType."); + bldr.append(iter.next()); + if (iter.hasNext()) { + bldr.append(", "); + } + } + bldr.append(")"); + return bldr.toString(); + } + private void generateEnumValues() { int numProcessed = 0; for (Map.Entry entry : apis.entrySet()) { ApiData apiData = entry.getValue(); String name = apiData.name(); numProcessed++; - buffer.printf("%s(\"%s\", (short) %d, %s, %s, (short) %d, (short) %d)%s%n", + + final Collection listeners; + if (apiData.requestSpec.listeners() == null) { + listeners = Collections.emptyList(); + } else { + listeners = apiData.requestSpec.listeners().stream() + .map(RequestListenerType::name) + .collect(Collectors.toList()); + } + + buffer.printf("%s(\"%s\", (short) %d, %s, %s, (short) %d, (short) %d, %s)%s%n", MessageGenerator.toSnakeCase(name).toUpperCase(Locale.ROOT), MessageGenerator.capitalizeFirst(name), entry.getKey(), @@ -170,6 +218,7 @@ private void generateEnumValues() { apiData.responseSchema(), apiData.requestSpec.struct().versions().lowest(), apiData.requestSpec.struct().versions().highest(), + generateListenerTypeEnumSet(listeners), (numProcessed == apis.size()) ? ";" : ","); } } @@ -181,13 +230,16 @@ private void generateInstanceVariables() { buffer.printf("private final Schema[] responseSchemas;%n"); buffer.printf("private final short lowestSupportedVersion;%n"); buffer.printf("private final short highestSupportedVersion;%n"); + buffer.printf("private final EnumSet listeners;%n"); headerGenerator.addImport(MessageGenerator.SCHEMA_CLASS); + headerGenerator.addImport(MessageGenerator.ENUM_SET_CLASS); } private void generateEnumConstructor() { buffer.printf("ApiMessageType(String name, short apiKey, " + "Schema[] requestSchemas, Schema[] responseSchemas, " + - "short lowestSupportedVersion, short highestSupportedVersion) {%n"); + "short lowestSupportedVersion, short highestSupportedVersion, " + + "EnumSet listeners) {%n"); buffer.incrementIndent(); buffer.printf("this.name = name;%n"); buffer.printf("this.apiKey = apiKey;%n"); @@ -195,6 +247,7 @@ private void generateEnumConstructor() { buffer.printf("this.responseSchemas = responseSchemas;%n"); buffer.printf("this.lowestSupportedVersion = lowestSupportedVersion;%n"); buffer.printf("this.highestSupportedVersion = highestSupportedVersion;%n"); + buffer.printf("this.listeners = listeners;%n"); buffer.decrementIndent(); buffer.printf("}%n"); } @@ -338,6 +391,18 @@ private void generateHeaderVersion(String type) { buffer.printf("}%n"); } + private void generateListenerTypesEnum() { + buffer.printf("public enum ListenerType {%n"); + buffer.incrementIndent(); + Iterator listenerIter = Arrays.stream(RequestListenerType.values()).iterator(); + while (listenerIter.hasNext()) { + RequestListenerType scope = listenerIter.next(); + buffer.printf("%s%s%n", scope.name(), listenerIter.hasNext() ? "," : ";"); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + } + private void write(BufferedWriter writer) throws IOException { headerGenerator.buffer().write(writer); buffer.write(writer); diff --git a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java index b6fb0aa4c616d..a1b972867c0c4 100644 --- a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java +++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java @@ -51,6 +51,8 @@ public final class MessageGenerator { static final String API_MESSAGE_TYPE_JAVA = "ApiMessageType.java"; + static final String API_SCOPE_JAVA = "ApiScope.java"; + static final String METADATA_RECORD_TYPE_JAVA = "MetadataRecordType.java"; static final String METADATA_JSON_CONVERTERS_JAVA = "MetadataJsonConverters.java"; @@ -84,6 +86,8 @@ public final class MessageGenerator { static final String ITERATOR_CLASS = "java.util.Iterator"; + static final String ENUM_SET_CLASS = "java.util.EnumSet"; + static final String TYPE_CLASS = "org.apache.kafka.common.protocol.types.Type"; static final String FIELD_CLASS = "org.apache.kafka.common.protocol.types.Field"; @@ -94,8 +98,6 @@ public final class MessageGenerator { static final String COMPACT_ARRAYOF_CLASS = "org.apache.kafka.common.protocol.types.CompactArrayOf"; - static final String STRUCT_CLASS = "org.apache.kafka.common.protocol.types.Struct"; - static final String BYTES_CLASS = "org.apache.kafka.common.utils.Bytes"; static final String UUID_CLASS = "org.apache.kafka.common.Uuid"; diff --git a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java index 0b53cb1db0945..fdcd7cd867f83 100644 --- a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java +++ b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java @@ -37,6 +37,8 @@ public final class MessageSpec { private final Versions flexibleVersions; + private final List listeners; + @JsonCreator public MessageSpec(@JsonProperty("name") String name, @JsonProperty("validVersions") String validVersions, @@ -44,7 +46,8 @@ public MessageSpec(@JsonProperty("name") String name, @JsonProperty("apiKey") Short apiKey, @JsonProperty("type") MessageSpecType type, @JsonProperty("commonStructs") List commonStructs, - @JsonProperty("flexibleVersions") String flexibleVersions) { + @JsonProperty("flexibleVersions") String flexibleVersions, + @JsonProperty("listeners") List listeners) { this.struct = new StructSpec(name, validVersions, fields); this.apiKey = apiKey == null ? Optional.empty() : Optional.of(apiKey); this.type = Objects.requireNonNull(type); @@ -57,6 +60,12 @@ public MessageSpec(@JsonProperty("name") String name, this.flexibleVersions + ", which is not open-ended. flexibleVersions must " + "be either none, or an open-ended range (that ends with a plus sign)."); } + + if (listeners != null && !listeners.isEmpty() && type != MessageSpecType.REQUEST) { + throw new RuntimeException("The `requestScope` property is only valid for " + + "messages with type `request`"); + } + this.listeners = listeners; } public StructSpec struct() { @@ -106,6 +115,11 @@ public String flexibleVersionsString() { return flexibleVersions.toString(); } + @JsonProperty("listeners") + public List listeners() { + return listeners; + } + public String dataClassName() { switch (type) { case HEADER: diff --git a/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java b/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java new file mode 100644 index 0000000000000..cefd40db6ec4a --- /dev/null +++ b/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.message; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public enum RequestListenerType { + @JsonProperty("zkBroker") + ZK_BROKER, + + @JsonProperty("broker") + BROKER, + + @JsonProperty("controller") + CONTROLLER; +} diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java index c71c05878cc5c..e59a6a14a69b3 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java @@ -23,13 +23,11 @@ import kafka.network.RequestChannel; import kafka.network.RequestConvertToJson; import kafka.server.AutoTopicCreationManager; -import kafka.server.BrokerFeatures; import kafka.server.BrokerTopicStats; import kafka.server.ClientQuotaManager; import kafka.server.ClientRequestQuotaManager; import kafka.server.ControllerMutationQuotaManager; import kafka.server.FetchManager; -import kafka.server.FinalizedFeatureCache; import kafka.server.KafkaApis; import kafka.server.KafkaConfig; import kafka.server.KafkaConfig$; @@ -38,11 +36,13 @@ import kafka.server.QuotaFactory; import kafka.server.ReplicaManager; import kafka.server.ReplicationQuotaManager; +import kafka.server.SimpleApiVersionManager; import kafka.server.ZkAdminManager; import kafka.server.ZkSupport; import kafka.server.metadata.CachedConfigRepository; import kafka.zk.KafkaZkClient; import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker; import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint; import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState; @@ -172,7 +172,6 @@ private KafkaApis createKafkaApis() { Properties kafkaProps = new Properties(); kafkaProps.put(KafkaConfig$.MODULE$.ZkConnectProp(), "zk"); kafkaProps.put(KafkaConfig$.MODULE$.BrokerIdProp(), brokerId + ""); - BrokerFeatures brokerFeatures = BrokerFeatures.createDefault(); return new KafkaApis(requestChannel, new ZkSupport(adminManager, kafkaController, kafkaZkClient, Option.empty(), metadataCache), replicaManager, @@ -191,8 +190,7 @@ private KafkaApis createKafkaApis() { "clusterId", new SystemTime(), null, - brokerFeatures, - new FinalizedFeatureCache(brokerFeatures)); + new SimpleApiVersionManager(ApiMessageType.ListenerType.ZK_BROKER)); } @TearDown(Level.Trial) From e29f7a36dbbd316ae03008140a1a0d282a26b82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?= Date: Thu, 18 Feb 2021 16:44:40 -0800 Subject: [PATCH 017/243] KAFKA-12331: Use LEO for the base offset of LeaderChangeMessage batch (#10138) The `KafkaMetadataLog` implementation of `ReplicatedLog` validates that batches appended using `appendAsLeader` and `appendAsFollower` have an offset that matches the LEO. This is enforced by `KafkaRaftClient` and `BatchAccumulator`. When creating control batches for the `LeaderChangeMessage` the default base offset of `0` was being used instead of using the LEO. This is fixed by: 1. Changing the implementation for `MockLog` to validate against this and throw an `RuntimeException` if this invariant is violated. 2. Always create a batch for `LeaderChangeMessage` with an offset equal to the LEO. Reviewers: Jason Gustafson --- .../kafka/common/record/MemoryRecords.java | 15 +++- .../common/record/MemoryRecordsTest.java | 11 ++- .../kafka/raft/KafkaMetadataLogTest.scala | 35 +++++++++ .../apache/kafka/raft/KafkaRaftClient.java | 10 ++- .../org/apache/kafka/raft/ReplicatedLog.java | 2 + .../raft/KafkaRaftClientSnapshotTest.java | 27 ++++--- .../kafka/raft/KafkaRaftClientTest.java | 40 +++++----- .../java/org/apache/kafka/raft/MockLog.java | 49 ++++++------ .../org/apache/kafka/raft/MockLogTest.java | 74 ++++++++++++++++--- .../kafka/raft/RaftClientTestContext.java | 9 ++- 10 files changed, 190 insertions(+), 82 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java index 7d14f67d6e632..ae844bff0330f 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java @@ -637,12 +637,17 @@ public static void writeEndTransactionalMarker(ByteBuffer buffer, long initialOf builder.close(); } - public static MemoryRecords withLeaderChangeMessage(long timestamp, int leaderEpoch, LeaderChangeMessage leaderChangeMessage) { + public static MemoryRecords withLeaderChangeMessage( + long initialOffset, + long timestamp, + int leaderEpoch, + LeaderChangeMessage leaderChangeMessage + ) { // To avoid calling message toStruct multiple times, we supply a fixed message size // for leader change, as it happens rare and the buffer could still grow if not sufficient in // certain edge cases. ByteBuffer buffer = ByteBuffer.allocate(256); - writeLeaderChangeMessage(buffer, 0L, timestamp, leaderEpoch, leaderChangeMessage); + writeLeaderChangeMessage(buffer, initialOffset, timestamp, leaderEpoch, leaderChangeMessage); buffer.flip(); return MemoryRecords.readableRecords(buffer); } @@ -652,10 +657,12 @@ private static void writeLeaderChangeMessage(ByteBuffer buffer, long timestamp, int leaderEpoch, LeaderChangeMessage leaderChangeMessage) { - MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + MemoryRecordsBuilder builder = new MemoryRecordsBuilder( + buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, initialOffset, timestamp, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, - false, true, leaderEpoch, buffer.capacity()); + false, true, leaderEpoch, buffer.capacity() + ); builder.appendLeaderChangeMessage(timestamp, leaderChangeMessage); builder.close(); } diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java index 26b1c485d8d4d..be5b337b7686e 100644 --- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java +++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsTest.java @@ -488,20 +488,25 @@ public void testBuildLeaderChangeMessage() { final int leaderId = 5; final int leaderEpoch = 20; final int voterId = 6; + long initialOffset = 983L; LeaderChangeMessage leaderChangeMessage = new LeaderChangeMessage() .setLeaderId(leaderId) .setVoters(Collections.singletonList( new Voter().setVoterId(voterId))); - MemoryRecords records = MemoryRecords.withLeaderChangeMessage(System.currentTimeMillis(), - leaderEpoch, leaderChangeMessage); + MemoryRecords records = MemoryRecords.withLeaderChangeMessage( + initialOffset, + System.currentTimeMillis(), + leaderEpoch, + leaderChangeMessage + ); List batches = TestUtils.toList(records.batches()); assertEquals(1, batches.size()); RecordBatch batch = batches.get(0); assertTrue(batch.isControlBatch()); - assertEquals(0, batch.baseOffset()); + assertEquals(initialOffset, batch.baseOffset()); assertEquals(leaderEpoch, batch.partitionLeaderEpoch()); assertTrue(batch.isValid()); diff --git a/core/src/test/scala/kafka/raft/KafkaMetadataLogTest.scala b/core/src/test/scala/kafka/raft/KafkaMetadataLogTest.scala index 5229ae7c17ec6..ce55a2ba2160a 100644 --- a/core/src/test/scala/kafka/raft/KafkaMetadataLogTest.scala +++ b/core/src/test/scala/kafka/raft/KafkaMetadataLogTest.scala @@ -63,6 +63,41 @@ final class KafkaMetadataLogTest { Utils.delete(tempDir) } + @Test + def testUnexpectedAppendOffset(): Unit = { + val topicPartition = new TopicPartition("cluster-metadata", 0) + val log = buildMetadataLog(tempDir, mockTime, topicPartition) + + val recordFoo = new SimpleRecord("foo".getBytes()) + val currentEpoch = 3 + val initialOffset = log.endOffset().offset + + log.appendAsLeader( + MemoryRecords.withRecords(initialOffset, CompressionType.NONE, currentEpoch, recordFoo), + currentEpoch + ) + + // Throw exception for out of order records + assertThrows( + classOf[RuntimeException], + () => { + log.appendAsLeader( + MemoryRecords.withRecords(initialOffset, CompressionType.NONE, currentEpoch, recordFoo), + currentEpoch + ) + } + ) + + assertThrows( + classOf[RuntimeException], + () => { + log.appendAsFollower( + MemoryRecords.withRecords(initialOffset, CompressionType.NONE, currentEpoch, recordFoo) + ) + } + ) + } + @Test def testCreateSnapshot(): Unit = { val topicPartition = new TopicPartition("cluster-metadata", 0) diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index 164a9214e24f8..081651a2fcfc0 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -404,7 +404,7 @@ private void onBecomeLeader(long currentTimeMs) { // The high watermark can only be advanced once we have written a record // from the new leader's epoch. Hence we write a control message immediately // to ensure there is no delay committing pending data. - appendLeaderChangeMessage(state, currentTimeMs); + appendLeaderChangeMessage(state, log.endOffset().offset, currentTimeMs); updateLeaderEndOffsetAndTimestamp(state, currentTimeMs); resetConnections(); @@ -429,7 +429,7 @@ private static List convertToVoters(Set voterIds) { .collect(Collectors.toList()); } - private void appendLeaderChangeMessage(LeaderState state, long currentTimeMs) { + private void appendLeaderChangeMessage(LeaderState state, long baseOffset, long currentTimeMs) { List voters = convertToVoters(state.followers()); List grantingVoters = convertToVoters(state.grantingVoters()); @@ -442,7 +442,11 @@ private void appendLeaderChangeMessage(LeaderState state, long currentTimeMs) { .setGrantingVoters(grantingVoters); MemoryRecords records = MemoryRecords.withLeaderChangeMessage( - currentTimeMs, quorum.epoch(), leaderChangeMessage); + baseOffset, + currentTimeMs, + quorum.epoch(), + leaderChangeMessage + ); appendAsLeader(records); flushLeaderLog(state, currentTimeMs); diff --git a/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java b/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java index 417b7690eb4ec..6b4adce074505 100644 --- a/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java +++ b/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java @@ -34,6 +34,7 @@ public interface ReplicatedLog extends Closeable { * * @return the metadata information of the appended batch * @throws IllegalArgumentException if the record set is empty + * @throws RuntimeException if the batch base offset doesn't match the log end offset */ LogAppendInfo appendAsLeader(Records records, int epoch); @@ -44,6 +45,7 @@ public interface ReplicatedLog extends Closeable { * * @return the metadata information of the appended batch * @throws IllegalArgumentException if the record set is empty + * @throws RuntimeException if the batch base offset doesn't match the log end offset */ LogAppendInfo appendAsFollower(Records records); diff --git a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java index 614ff327cffed..9ebb776fe1818 100644 --- a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java @@ -106,7 +106,8 @@ public void testFetchRequestWithLargerLastFetchedEpoch() throws Exception { OffsetAndEpoch oldestSnapshotId = new OffsetAndEpoch(3, 2); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(oldestSnapshotId.offset, oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("d", "e", "f")) .withAppendLingerMs(1) .build(); @@ -115,10 +116,11 @@ public void testFetchRequestWithLargerLastFetchedEpoch() throws Exception { assertEquals(oldestSnapshotId.epoch + 1, epoch); // Advance the highWatermark - context.deliverRequest(context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset, oldestSnapshotId.epoch, 0)); + long localLogEndOffset = context.log.endOffset().offset; + context.deliverRequest(context.fetchRequest(epoch, otherNodeId, localLogEndOffset, epoch, 0)); context.pollUntilResponse(); context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); - assertEquals(oldestSnapshotId.offset, context.client.highWatermark().getAsLong()); + assertEquals(localLogEndOffset, context.client.highWatermark().getAsLong()); // Create a snapshot at the high watermark try (SnapshotWriter snapshot = context.client.createSnapshot(oldestSnapshotId)) { @@ -146,8 +148,8 @@ public void testFetchRequestTruncateToLogStart() throws Exception { OffsetAndEpoch oldestSnapshotId = new OffsetAndEpoch(3, 2); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(oldestSnapshotId.offset, oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) - .appendToLog(oldestSnapshotId.offset + 3, oldestSnapshotId.epoch + 2, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch + 2, Arrays.asList("d", "e", "f")) .withAppendLingerMs(1) .build(); @@ -192,8 +194,9 @@ public void testFetchRequestAtLogStartOffsetWithValidEpoch() throws Exception { OffsetAndEpoch oldestSnapshotId = new OffsetAndEpoch(3, 2); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(oldestSnapshotId.offset, oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) - .appendToLog(oldestSnapshotId.offset + 3, oldestSnapshotId.epoch + 2, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("d", "e", "f")) + .appendToLog(oldestSnapshotId.epoch + 2, Arrays.asList("g", "h", "i")) .withAppendLingerMs(1) .build(); @@ -233,8 +236,9 @@ public void testFetchRequestAtLogStartOffsetWithInvalidEpoch() throws Exception OffsetAndEpoch oldestSnapshotId = new OffsetAndEpoch(3, 2); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(oldestSnapshotId.offset, oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) - .appendToLog(oldestSnapshotId.offset + 3, oldestSnapshotId.epoch + 2, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("d", "e", "f")) + .appendToLog(oldestSnapshotId.epoch + 2, Arrays.asList("g", "h", "i")) .withAppendLingerMs(1) .build(); @@ -279,8 +283,9 @@ public void testFetchRequestWithLastFetchedEpochLessThanOldestSnapshot() throws OffsetAndEpoch oldestSnapshotId = new OffsetAndEpoch(3, 2); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(oldestSnapshotId.offset, oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) - .appendToLog(oldestSnapshotId.offset + 3, oldestSnapshotId.epoch + 2, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("a", "b", "c")) + .appendToLog(oldestSnapshotId.epoch, Arrays.asList("d", "e", "f")) + .appendToLog(oldestSnapshotId.epoch + 2, Arrays.asList("g", "h", "i")) .withAppendLingerMs(1) .build(); diff --git a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientTest.java b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientTest.java index b29f1efaf2ee4..fb188f1eacd3f 100644 --- a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientTest.java @@ -31,7 +31,6 @@ import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.Records; -import org.apache.kafka.common.record.SimpleRecord; import org.apache.kafka.common.requests.DescribeQuorumRequest; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestUtils; @@ -226,7 +225,10 @@ public void testResignWillCompleteFetchPurgatory() throws Exception { context.client.poll(); // append some record, but the fetch in purgatory will still fail - context.log.appendAsLeader(Collections.singleton(new SimpleRecord("raft".getBytes())), epoch); + context.log.appendAsLeader( + context.buildBatch(context.log.endOffset().offset, epoch, Arrays.asList("raft")), + epoch + ); // when transition to resign, all request in fetchPurgatory will fail context.client.shutdown(1000); @@ -449,7 +451,7 @@ public void testEndQuorumStartsNewElectionImmediatelyIfFollowerUnattached() thro RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) .withUnknownLeader(epoch) .build(); - + context.deliverRequest(context.endEpochRequest(epoch, voter2, Arrays.asList(localId, voter3))); @@ -945,7 +947,7 @@ public void testInitializeAsFollowerNonEmptyLog() throws Exception { RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) .withElectedLeader(epoch, otherNodeId) - .appendToLog(0L, lastEpoch, singletonList("foo")) + .appendToLog(lastEpoch, singletonList("foo")) .build(); context.assertElectedLeader(epoch, otherNodeId); @@ -964,7 +966,7 @@ public void testVoterBecomeCandidateAfterFetchTimeout() throws Exception { RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) .withElectedLeader(epoch, otherNodeId) - .appendToLog(0L, lastEpoch, singletonList("foo")) + .appendToLog(lastEpoch, singletonList("foo")) .build(); context.assertElectedLeader(epoch, otherNodeId); @@ -1781,8 +1783,8 @@ public void testFollowerLogReconciliation() throws Exception { RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) .withElectedLeader(epoch, otherNodeId) - .appendToLog(0L, lastEpoch, Arrays.asList("foo", "bar")) - .appendToLog(2L, lastEpoch, Arrays.asList("baz")) + .appendToLog(lastEpoch, Arrays.asList("foo", "bar")) + .appendToLog(lastEpoch, Arrays.asList("baz")) .build(); context.assertElectedLeader(epoch, otherNodeId); @@ -1964,9 +1966,9 @@ public void testHandleClaimCallbackFiresAfterHighWatermarkReachesEpochStartOffse List batch3 = Arrays.asList("7", "8", "9"); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(0L, 1, batch1) - .appendToLog(3L, 1, batch2) - .appendToLog(6L, 2, batch3) + .appendToLog(1, batch1) + .appendToLog(1, batch2) + .appendToLog(2, batch3) .withUnknownLeader(epoch - 1) .build(); @@ -2016,9 +2018,9 @@ public void testLateRegisteredListenerCatchesUp() throws Exception { List batch3 = Arrays.asList("7", "8", "9"); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(0L, 1, batch1) - .appendToLog(3L, 1, batch2) - .appendToLog(6L, 2, batch3) + .appendToLog(1, batch1) + .appendToLog(1, batch2) + .appendToLog(2, batch3) .withUnknownLeader(epoch - 1) .build(); @@ -2105,9 +2107,9 @@ public void testHandleCommitCallbackFiresInVotedState() throws Exception { Set voters = Utils.mkSet(localId, otherNodeId); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(0L, 2, Arrays.asList("a", "b", "c")) - .appendToLog(3L, 4, Arrays.asList("d", "e", "f")) - .appendToLog(6L, 4, Arrays.asList("g", "h", "i")) + .appendToLog(2, Arrays.asList("a", "b", "c")) + .appendToLog(4, Arrays.asList("d", "e", "f")) + .appendToLog(4, Arrays.asList("g", "h", "i")) .withUnknownLeader(epoch - 1) .build(); @@ -2146,9 +2148,9 @@ public void testHandleCommitCallbackFiresInCandidateState() throws Exception { Set voters = Utils.mkSet(localId, otherNodeId); RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters) - .appendToLog(0L, 2, Arrays.asList("a", "b", "c")) - .appendToLog(3L, 4, Arrays.asList("d", "e", "f")) - .appendToLog(6L, 4, Arrays.asList("g", "h", "i")) + .appendToLog(2, Arrays.asList("a", "b", "c")) + .appendToLog(4, Arrays.asList("d", "e", "f")) + .appendToLog(4, Arrays.asList("g", "h", "i")) .withUnknownLeader(epoch - 1) .build(); diff --git a/raft/src/test/java/org/apache/kafka/raft/MockLog.java b/raft/src/test/java/org/apache/kafka/raft/MockLog.java index 3252d54ea01b2..9d8046517b1e9 100644 --- a/raft/src/test/java/org/apache/kafka/raft/MockLog.java +++ b/raft/src/test/java/org/apache/kafka/raft/MockLog.java @@ -35,7 +35,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -250,30 +249,7 @@ private LogEntry buildEntry(Long offset, SimpleRecord record) { @Override public LogAppendInfo appendAsLeader(Records records, int epoch) { - if (records.sizeInBytes() == 0) - throw new IllegalArgumentException("Attempt to append an empty record set"); - - long baseOffset = endOffset().offset; - AtomicLong offsetSupplier = new AtomicLong(baseOffset); - for (RecordBatch batch : records.batches()) { - List entries = buildEntries(batch, record -> offsetSupplier.getAndIncrement()); - appendBatch(new LogBatch(epoch, batch.isControlBatch(), entries)); - } - - return new LogAppendInfo(baseOffset, offsetSupplier.get() - 1); - } - - LogAppendInfo appendAsLeader(Collection records, int epoch) { - long baseOffset = endOffset().offset; - long offset = baseOffset; - - List entries = new ArrayList<>(); - for (SimpleRecord record : records) { - entries.add(buildEntry(offset, record)); - offset += 1; - } - appendBatch(new LogBatch(epoch, false, entries)); - return new LogAppendInfo(baseOffset, offset - 1); + return append(records, OptionalInt.of(epoch)); } private Long appendBatch(LogBatch batch) { @@ -286,6 +262,10 @@ private Long appendBatch(LogBatch batch) { @Override public LogAppendInfo appendAsFollower(Records records) { + return append(records, OptionalInt.empty()); + } + + private LogAppendInfo append(Records records, OptionalInt epoch) { if (records.sizeInBytes() == 0) throw new IllegalArgumentException("Attempt to append an empty record set"); @@ -293,13 +273,26 @@ public LogAppendInfo appendAsFollower(Records records) { long lastOffset = baseOffset; for (RecordBatch batch : records.batches()) { if (batch.baseOffset() != endOffset().offset) { - throw new IllegalArgumentException( - String.format("Illegal append at offset %s with current end offset of %", batch.baseOffset(), endOffset().offset) + /* KafkaMetadataLog throws an kafka.common.UnexpectedAppendOffsetException this is the + * best we can do from this module. + */ + throw new RuntimeException( + String.format( + "Illegal append at offset %s with current end offset of %s", + batch.baseOffset(), + endOffset().offset + ) ); } List entries = buildEntries(batch, Record::offset); - appendBatch(new LogBatch(batch.partitionLeaderEpoch(), batch.isControlBatch(), entries)); + appendBatch( + new LogBatch( + epoch.orElseGet(batch::partitionLeaderEpoch), + batch.isControlBatch(), + entries + ) + ); lastOffset = entries.get(entries.size() - 1).offset; } diff --git a/raft/src/test/java/org/apache/kafka/raft/MockLogTest.java b/raft/src/test/java/org/apache/kafka/raft/MockLogTest.java index 95f098f632712..67231eb1e93d4 100644 --- a/raft/src/test/java/org/apache/kafka/raft/MockLogTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/MockLogTest.java @@ -36,6 +36,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -65,7 +66,7 @@ public void testTopicPartition() { public void testAppendAsLeaderHelper() { int epoch = 2; SimpleRecord recordOne = new SimpleRecord("one".getBytes()); - log.appendAsLeader(Collections.singleton(recordOne), epoch); + appendAsLeader(Collections.singleton(recordOne), epoch); assertEquals(epoch, log.lastFetchedEpoch()); assertEquals(0L, log.startOffset()); assertEquals(1L, log.endOffset().offset); @@ -84,7 +85,7 @@ public void testAppendAsLeaderHelper() { SimpleRecord recordTwo = new SimpleRecord("two".getBytes()); SimpleRecord recordThree = new SimpleRecord("three".getBytes()); - log.appendAsLeader(Arrays.asList(recordTwo, recordThree), epoch); + appendAsLeader(Arrays.asList(recordTwo, recordThree), epoch); assertEquals(0L, log.startOffset()); assertEquals(3L, log.endOffset().offset); @@ -109,10 +110,10 @@ public void testTruncateTo() { int epoch = 2; SimpleRecord recordOne = new SimpleRecord("one".getBytes()); SimpleRecord recordTwo = new SimpleRecord("two".getBytes()); - log.appendAsLeader(Arrays.asList(recordOne, recordTwo), epoch); + appendAsLeader(Arrays.asList(recordOne, recordTwo), epoch); SimpleRecord recordThree = new SimpleRecord("three".getBytes()); - log.appendAsLeader(Collections.singleton(recordThree), epoch); + appendAsLeader(Collections.singleton(recordThree), epoch); assertEquals(0L, log.startOffset()); assertEquals(3L, log.endOffset().offset); @@ -150,11 +151,14 @@ public void testAssignEpochStartOffset() { @Test public void testAppendAsLeader() { - // The record passed-in offsets are not going to affect the eventual offsets. - final long initialOffset = 5L; SimpleRecord recordFoo = new SimpleRecord("foo".getBytes()); final int currentEpoch = 3; - log.appendAsLeader(MemoryRecords.withRecords(initialOffset, CompressionType.NONE, recordFoo), currentEpoch); + final long initialOffset = log.endOffset().offset; + + log.appendAsLeader( + MemoryRecords.withRecords(initialOffset, CompressionType.NONE, recordFoo), + currentEpoch + ); assertEquals(0, log.startOffset()); assertEquals(1, log.endOffset().offset); @@ -171,13 +175,47 @@ public void testAppendAsLeader() { assertEquals(new OffsetAndEpoch(1, currentEpoch), log.endOffsetForEpoch(currentEpoch)); } + @Test + public void testUnexpectedAppendOffset() { + SimpleRecord recordFoo = new SimpleRecord("foo".getBytes()); + final int currentEpoch = 3; + final long initialOffset = log.endOffset().offset; + + log.appendAsLeader( + MemoryRecords.withRecords(initialOffset, CompressionType.NONE, currentEpoch, recordFoo), + currentEpoch + ); + + // Throw exception for out of order records + assertThrows( + RuntimeException.class, + () -> { + log.appendAsLeader( + MemoryRecords.withRecords(initialOffset, CompressionType.NONE, currentEpoch, recordFoo), + currentEpoch + ); + } + ); + + assertThrows( + RuntimeException.class, + () -> { + log.appendAsFollower( + MemoryRecords.withRecords(initialOffset, CompressionType.NONE, currentEpoch, recordFoo) + ); + } + ); + } + @Test public void testAppendControlRecord() { - final long initialOffset = 5L; + final long initialOffset = 0; final int currentEpoch = 3; LeaderChangeMessage messageData = new LeaderChangeMessage().setLeaderId(0); - log.appendAsLeader(MemoryRecords.withLeaderChangeMessage( - initialOffset, 2, messageData), currentEpoch); + log.appendAsLeader( + MemoryRecords.withLeaderChangeMessage(initialOffset, 0L, 2, messageData), + currentEpoch + ); assertEquals(0, log.startOffset()); assertEquals(1, log.endOffset().offset); @@ -239,7 +277,7 @@ public void testReadRecords() { recordTwoBuffer.putInt(2); SimpleRecord recordTwo = new SimpleRecord(recordTwoBuffer); - log.appendAsLeader(Arrays.asList(recordOne, recordTwo), epoch); + appendAsLeader(Arrays.asList(recordOne, recordTwo), epoch); Records records = log.read(0, Isolation.UNCOMMITTED).records; @@ -597,11 +635,23 @@ public int hashCode() { } } + private void appendAsLeader(Collection records, int epoch) { + log.appendAsLeader( + MemoryRecords.withRecords( + log.endOffset().offset, + CompressionType.NONE, + records.toArray(new SimpleRecord[records.size()]) + ), + epoch + ); + } + private void appendBatch(int numRecords, int epoch) { List records = new ArrayList<>(numRecords); for (int i = 0; i < numRecords; i++) { records.add(new SimpleRecord(String.valueOf(i).getBytes())); } - log.appendAsLeader(records, epoch); + + appendAsLeader(records, epoch); } } diff --git a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java index 9d19b8698257c..12fda05cdcec9 100644 --- a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java +++ b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java @@ -171,8 +171,13 @@ Builder withAppendLingerMs(int appendLingerMs) { return this; } - Builder appendToLog(long baseOffset, int epoch, List records) { - MemoryRecords batch = buildBatch(time.milliseconds(), baseOffset, epoch, records); + Builder appendToLog(int epoch, List records) { + MemoryRecords batch = buildBatch( + time.milliseconds(), + log.endOffset().offset, + epoch, + records + ); log.appendAsLeader(batch, epoch); return this; } From c8112b5ecdda6b62d34ad97fcebbf5c7fec3de53 Mon Sep 17 00:00:00 2001 From: Marco Aurelio Lotz Date: Fri, 19 Feb 2021 03:18:53 +0100 Subject: [PATCH 018/243] KAFKA-9524: increase retention time for window and grace periods longer than one day (#10091) Reviewers: Victoria Xia , Matthias J. Sax --- .../org/apache/kafka/streams/kstream/TimeWindows.java | 11 ++++++++--- .../apache/kafka/streams/kstream/TimeWindowsTest.java | 11 ++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java index cd52dd5015281..9fd963af38918 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindows.java @@ -57,6 +57,8 @@ */ public final class TimeWindows extends Windows { + private static final long EMPTY_GRACE_PERIOD = -1; + private final long maintainDurationMs; /** The size of the windows in milliseconds. */ @@ -111,7 +113,7 @@ public static TimeWindows of(final long sizeMs) throws IllegalArgumentException throw new IllegalArgumentException("Window size (sizeMs) must be larger than zero."); } // This is a static factory method, so we initialize grace and retention to the defaults. - return new TimeWindows(sizeMs, sizeMs, -1, DEFAULT_RETENTION_MS); + return new TimeWindows(sizeMs, sizeMs, EMPTY_GRACE_PERIOD, DEFAULT_RETENTION_MS); } /** @@ -214,7 +216,10 @@ public long gracePeriodMs() { // NOTE: in the future, when we remove maintainMs, // we should default the grace period to 24h to maintain the default behavior, // or we can default to (24h - size) if you want to be super accurate. - return graceMs != -1 ? graceMs : maintainMs() - size(); + if (graceMs != EMPTY_GRACE_PERIOD) { + return graceMs; + } + return Math.max(maintainDurationMs - sizeMs, 0); } /** @@ -245,7 +250,7 @@ public TimeWindows until(final long durationMs) throws IllegalArgumentException @Override @Deprecated public long maintainMs() { - return Math.max(maintainDurationMs, sizeMs); + return Math.max(maintainDurationMs, sizeMs + gracePeriodMs()); } @SuppressWarnings("deprecation") // removing segments from Windows will fix this diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/TimeWindowsTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/TimeWindowsTest.java index 69b73c8ee00cc..00e2b4cc55417 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/TimeWindowsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/TimeWindowsTest.java @@ -19,8 +19,10 @@ import org.apache.kafka.streams.kstream.internals.TimeWindow; import org.junit.Test; +import java.time.Duration; import java.util.Map; +import static java.time.Duration.ofDays; import static java.time.Duration.ofMillis; import static org.apache.kafka.streams.EqualityCheck.verifyEquality; import static org.apache.kafka.streams.EqualityCheck.verifyInEquality; @@ -53,11 +55,18 @@ public void shouldSetWindowRetentionTime() { @SuppressWarnings("deprecation") // specifically testing deprecated APIs @Test - public void shouldUseWindowSizeAsRentitionTimeIfWindowSizeIsLargerThanDefaultRetentionTime() { + public void shouldUseWindowSizeAsRetentionTimeIfWindowSizeIsLargerThanDefaultRetentionTime() { final long windowSize = 2 * TimeWindows.of(ofMillis(1)).maintainMs(); assertEquals(windowSize, TimeWindows.of(ofMillis(windowSize)).maintainMs()); } + @Test + public void shouldUseWindowSizeAndGraceAsRetentionTimeIfBothCombinedAreLargerThanDefaultRetentionTime() { + final Duration windowsSize = ofDays(1).minus(ofMillis(1)); + final Duration gracePeriod = ofMillis(2); + assertEquals(windowsSize.toMillis() + gracePeriod.toMillis(), TimeWindows.of(windowsSize).grace(gracePeriod).maintainMs()); + } + @Test public void windowSizeMustNotBeZero() { assertThrows(IllegalArgumentException.class, () -> TimeWindows.of(ofMillis(0))); From 9243c10161eb10631353f34a821a8dfb8cab51ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?= Date: Thu, 18 Feb 2021 19:46:23 -0800 Subject: [PATCH 019/243] KAFKA-12258; Add support for splitting appending records (#10063) 1. Type `BatchAccumulator`. Add support for appending records into one or more batches. 2. Type `RaftClient`. Rename `scheduleAppend` to `scheduleAtomicAppend`. 3. Type `RaftClient`. Add a new method `scheduleAppend` which appends records to the log using as many batches as necessary. 4. Increase the batch size from 1MB to 8MB. Reviewers: David Arthur , Jason Gustafson --- .../apache/kafka/raft/KafkaRaftClient.java | 25 ++- .../org/apache/kafka/raft/RaftClient.java | 48 ++++-- .../raft/internals/BatchAccumulator.java | 124 +++++++++----- .../kafka/raft/internals/BatchBuilder.java | 102 +++++++---- .../raft/internals/BatchAccumulatorTest.java | 159 +++++++++++++----- .../raft/internals/BatchBuilderTest.java | 9 +- 6 files changed, 339 insertions(+), 128 deletions(-) diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index 081651a2fcfc0..9fbbe3182917d 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -141,7 +141,7 @@ public class KafkaRaftClient implements RaftClient { private static final int RETRY_BACKOFF_BASE_MS = 100; private static final int FETCH_MAX_WAIT_MS = 1000; - static final int MAX_BATCH_SIZE = 1024 * 1024; + static final int MAX_BATCH_SIZE = 8 * 1024 * 1024; private final AtomicReference shutdown = new AtomicReference<>(); private final Logger logger; @@ -2188,13 +2188,27 @@ public void poll() throws IOException { @Override public Long scheduleAppend(int epoch, List records) { + return append(epoch, records, false); + } + + @Override + public Long scheduleAtomicAppend(int epoch, List records) { + return append(epoch, records, true); + } + + private Long append(int epoch, List records, boolean isAtomic) { BatchAccumulator accumulator = this.accumulator; if (accumulator == null) { return Long.MAX_VALUE; } boolean isFirstAppend = accumulator.isEmpty(); - Long offset = accumulator.append(epoch, records); + final Long offset; + if (isAtomic) { + offset = accumulator.appendAtomic(epoch, records); + } else { + offset = accumulator.append(epoch, records); + } // Wakeup the network channel if either this is the first append // or the accumulator is ready to drain now. Checking for the first @@ -2351,9 +2365,10 @@ public void fireHandleCommit(long baseOffset, Records records) { /** * This API is used for committed records originating from {@link #scheduleAppend(int, List)} - * on this instance. In this case, we are able to save the original record objects, - * which saves the need to read them back from disk. This is a nice optimization - * for the leader which is typically doing more work than all of the followers. + * or {@link #scheduleAtomicAppend(int, List)} on this instance. In this case, we are able to + * save the original record objects, which saves the need to read them back from disk. This is + * a nice optimization for the leader which is typically doing more work than all of the + * followers. */ public void fireHandleCommit(long baseOffset, int epoch, List records) { BatchReader.Batch batch = new BatchReader.Batch<>(baseOffset, epoch, records); diff --git a/raft/src/main/java/org/apache/kafka/raft/RaftClient.java b/raft/src/main/java/org/apache/kafka/raft/RaftClient.java index e2bec0ed4ee79..74488b450ede1 100644 --- a/raft/src/main/java/org/apache/kafka/raft/RaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/RaftClient.java @@ -32,11 +32,13 @@ interface Listener { * after consuming the reader. * * Note that there is not a one-to-one correspondence between writes through - * {@link #scheduleAppend(int, List)} and this callback. The Raft implementation - * is free to batch together the records from multiple append calls provided - * that batch boundaries are respected. This means that each batch specified - * through {@link #scheduleAppend(int, List)} is guaranteed to be a subset of - * a batch provided by the {@link BatchReader}. + * {@link #scheduleAppend(int, List)} or {@link #scheduleAtomicAppend(int, List)} + * and this callback. The Raft implementation is free to batch together the records + * from multiple append calls provided that batch boundaries are respected. Records + * specified through {@link #scheduleAtomicAppend(int, List)} are guaranteed to be a + * subset of a batch provided by the {@link BatchReader}. Records specified through + * {@link #scheduleAppend(int, List)} are guaranteed to be in the same order but + * they can map to any number of batches provided by the {@link BatchReader}. * * @param reader reader instance which must be iterated and closed */ @@ -48,7 +50,7 @@ interface Listener { * {@link #handleCommit(BatchReader)}. * * After becoming a leader, the client is eligible to write to the log - * using {@link #scheduleAppend(int, List)}. + * using {@link #scheduleAppend(int, List)} or {@link #scheduleAtomicAppend(int, List)}. * * @param epoch the claimed leader epoch */ @@ -84,6 +86,30 @@ default void handleResign(int epoch) {} */ LeaderAndEpoch leaderAndEpoch(); + /** + * Append a list of records to the log. The write will be scheduled for some time + * in the future. There is no guarantee that appended records will be written to + * the log and eventually committed. While the order of the records is preserve, they can + * be appended to the log using one or more batches. Each record may be committed independently. + * If a record is committed, then all records scheduled for append during this epoch + * and prior to this record are also committed. + * + * If the provided current leader epoch does not match the current epoch, which + * is possible when the state machine has yet to observe the epoch change, then + * this method will return {@link Long#MAX_VALUE} to indicate an offset which is + * not possible to become committed. The state machine is expected to discard all + * uncommitted entries after observing an epoch change. + * + * @param epoch the current leader epoch + * @param records the list of records to append + * @return the expected offset of the last record; {@link Long#MAX_VALUE} if the records could + * be committed; null if no memory could be allocated for the batch at this time + * @throws RecordBatchTooLargeException if the size of the records is greater than the maximum + * batch size; if this exception is throw none of the elements in records were + * committed + */ + Long scheduleAppend(int epoch, List records); + /** * Append a list of records to the log. The write will be scheduled for some time * in the future. There is no guarantee that appended records will be written to @@ -98,11 +124,13 @@ default void handleResign(int epoch) {} * * @param epoch the current leader epoch * @param records the list of records to append - * @return the offset within the current epoch that the log entries will be appended, - * or null if the leader was unable to accept the write (e.g. due to memory - * being reached). + * @return the expected offset of the last record; {@link Long#MAX_VALUE} if the records could + * be committed; null if no memory could be allocated for the batch at this time + * @throws RecordBatchTooLargeException if the size of the records is greater than the maximum + * batch size; if this exception is throw none of the elements in records were + * committed */ - Long scheduleAppend(int epoch, List records); + Long scheduleAtomicAppend(int epoch, List records); /** * Attempt a graceful shutdown of the client. This allows the leader to proactively diff --git a/raft/src/main/java/org/apache/kafka/raft/internals/BatchAccumulator.java b/raft/src/main/java/org/apache/kafka/raft/internals/BatchAccumulator.java index 5331e4dd145d8..07d1015c9da92 100644 --- a/raft/src/main/java/org/apache/kafka/raft/internals/BatchAccumulator.java +++ b/raft/src/main/java/org/apache/kafka/raft/internals/BatchAccumulator.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.raft.internals; +import org.apache.kafka.common.errors.RecordBatchTooLargeException; import org.apache.kafka.common.memory.MemoryPool; import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.record.CompressionType; @@ -26,8 +27,10 @@ import java.io.Closeable; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.OptionalInt; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; @@ -79,70 +82,111 @@ public BatchAccumulator( } /** - * Append a list of records into an atomic batch. We guarantee all records - * are included in the same underlying record batch so that either all of - * the records become committed or none of them do. + * Append a list of records into as many batches as necessary. * - * @param epoch the expected leader epoch. If this does not match, then - * {@link Long#MAX_VALUE} will be returned as an offset which - * cannot become committed. - * @param records the list of records to include in a batch - * @return the expected offset of the last record (which will be - * {@link Long#MAX_VALUE} if the epoch does not match), or null if - * no memory could be allocated for the batch at this time + * The order of the elements in the records argument will match the order in the batches. + * This method will use as many batches as necessary to serialize all of the records. Since + * this method can split the records into multiple batches it is possible that some of the + * records will get committed while other will not when the leader fails. + * + * @param epoch the expected leader epoch. If this does not match, then {@link Long#MAX_VALUE} + * will be returned as an offset which cannot become committed + * @param records the list of records to include in the batches + * @return the expected offset of the last record; {@link Long#MAX_VALUE} if the epoch does not + * match; null if no memory could be allocated for the batch at this time + * @throws RecordBatchTooLargeException if the size of one record T is greater than the maximum + * batch size; if this exception is throw some of the elements in records may have + * been committed */ public Long append(int epoch, List records) { + return append(epoch, records, false); + } + + /** + * Append a list of records into an atomic batch. We guarantee all records are included in the + * same underlying record batch so that either all of the records become committed or none of + * them do. + * + * @param epoch the expected leader epoch. If this does not match, then {@link Long#MAX_VALUE} + * will be returned as an offset which cannot become committed + * @param records the list of records to include in a batch + * @return the expected offset of the last record; {@link Long#MAX_VALUE} if the epoch does not + * match; null if no memory could be allocated for the batch at this time + * @throws RecordBatchTooLargeException if the size of the records is greater than the maximum + * batch size; if this exception is throw none of the elements in records were + * committed + */ + public Long appendAtomic(int epoch, List records) { + return append(epoch, records, true); + } + + private Long append(int epoch, List records, boolean isAtomic) { if (epoch != this.epoch) { - // If the epoch does not match, then the state machine probably - // has not gotten the notification about the latest epoch change. - // In this case, ignore the append and return a large offset value - // which will never be committed. return Long.MAX_VALUE; } ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - int batchSize = 0; - for (T record : records) { - batchSize += serde.recordSize(record, serializationCache); - } - - if (batchSize > maxBatchSize) { - throw new IllegalArgumentException("The total size of " + records + " is " + batchSize + - ", which exceeds the maximum allowed batch size of " + maxBatchSize); - } appendLock.lock(); try { maybeCompleteDrain(); - BatchBuilder batch = maybeAllocateBatch(batchSize); - if (batch == null) { - return null; - } - - // Restart the linger timer if necessary - if (!lingerTimer.isRunning()) { - lingerTimer.reset(time.milliseconds() + lingerMs); + BatchBuilder batch = null; + if (isAtomic) { + batch = maybeAllocateBatch(records, serializationCache); } for (T record : records) { + if (!isAtomic) { + batch = maybeAllocateBatch(Collections.singleton(record), serializationCache); + } + + if (batch == null) { + return null; + } + batch.appendRecord(record, serializationCache); nextOffset += 1; } + maybeResetLinger(); + return nextOffset - 1; } finally { appendLock.unlock(); } } - private BatchBuilder maybeAllocateBatch(int batchSize) { + private void maybeResetLinger() { + if (!lingerTimer.isRunning()) { + lingerTimer.reset(time.milliseconds() + lingerMs); + } + } + + private BatchBuilder maybeAllocateBatch( + Collection records, + ObjectSerializationCache serializationCache + ) { if (currentBatch == null) { startNewBatch(); - } else if (!currentBatch.hasRoomFor(batchSize)) { - completeCurrentBatch(); - startNewBatch(); } + + if (currentBatch != null) { + OptionalInt bytesNeeded = currentBatch.bytesNeeded(records, serializationCache); + if (bytesNeeded.isPresent() && bytesNeeded.getAsInt() > maxBatchSize) { + throw new RecordBatchTooLargeException( + String.format( + "The total record(s) size of %s exceeds the maximum allowed batch size of %s", + bytesNeeded.getAsInt(), + maxBatchSize + ) + ); + } else if (bytesNeeded.isPresent()) { + completeCurrentBatch(); + startNewBatch(); + } + } + return currentBatch; } @@ -298,20 +342,22 @@ public static class CompletedBatch { public final List records; public final MemoryRecords data; private final MemoryPool pool; - private final ByteBuffer buffer; + // Buffer that was allocated by the MemoryPool (pool). This may not be the buffer used in + // the MemoryRecords (data) object. + private final ByteBuffer initialBuffer; private CompletedBatch( long baseOffset, List records, MemoryRecords data, MemoryPool pool, - ByteBuffer buffer + ByteBuffer initialBuffer ) { this.baseOffset = baseOffset; this.records = records; this.data = data; this.pool = pool; - this.buffer = buffer; + this.initialBuffer = initialBuffer; } public int sizeInBytes() { @@ -319,7 +365,7 @@ public int sizeInBytes() { } public void release() { - pool.release(buffer); + pool.release(initialBuffer); } } diff --git a/raft/src/main/java/org/apache/kafka/raft/internals/BatchBuilder.java b/raft/src/main/java/org/apache/kafka/raft/internals/BatchBuilder.java index 542bb5197c581..c953b6a6371c7 100644 --- a/raft/src/main/java/org/apache/kafka/raft/internals/BatchBuilder.java +++ b/raft/src/main/java/org/apache/kafka/raft/internals/BatchBuilder.java @@ -33,12 +33,14 @@ import java.io.DataOutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.OptionalInt; /** * Collect a set of records into a single batch. New records are added * through {@link #appendRecord(Object, ObjectSerializationCache)}, but the caller must first - * check whether there is room using {@link #hasRoomFor(int)}. Once the + * check whether there is room using {@link #bytesNeeded(Collection, ObjectSerializationCache)}. Once the * batch is ready, then {@link #build()} should be used to get the resulting * {@link MemoryRecords} instance. * @@ -85,8 +87,8 @@ public BatchBuilder( this.maxBytes = maxBytes; this.records = new ArrayList<>(); - int batchHeaderSizeInBytes = AbstractRecords.recordBatchHeaderSizeInBytes( - RecordBatch.MAGIC_VALUE_V2, compressionType); + // field compressionType must be set before calculating the batch header size + int batchHeaderSizeInBytes = batchHeaderSizeInBytes(); batchOutput.position(initialPosition + batchHeaderSizeInBytes); this.recordOutput = new DataOutputStreamWritable(new DataOutputStream( @@ -95,7 +97,7 @@ public BatchBuilder( /** * Append a record to this patch. The caller must first verify there is room for the batch - * using {@link #hasRoomFor(int)}. + * using {@link #bytesNeeded(Collection, ObjectSerializationCache)}. * * @param record the record to append * @param serializationCache serialization cache for use in {@link RecordSerde#write(Object, ObjectSerializationCache, Writable)} @@ -103,7 +105,7 @@ public BatchBuilder( */ public long appendRecord(T record, ObjectSerializationCache serializationCache) { if (!isOpenForAppends) { - throw new IllegalArgumentException("Cannot append new records after the batch has been built"); + throw new IllegalStateException("Cannot append new records after the batch has been built"); } if (nextOffset - baseOffset > Integer.MAX_VALUE) { @@ -123,39 +125,39 @@ public long appendRecord(T record, ObjectSerializationCache serializationCache) } /** - * Check whether the batch has enough room for a record of the given size in bytes. + * Check whether the batch has enough room for all the record values. * - * @param sizeInBytes the size of the record to be appended - * @return true if there is room for the record to be appended, false otherwise + * Returns an empty {@link OptionalInt} if the batch builder has room for this list of records. + * Otherwise it returns the expected number of bytes needed for a batch to contain these records. + * + * @param records the records to use when checking for room + * @param serializationCache serialization cache for computing sizes + * @return empty {@link OptionalInt} if there is room for the records to be appended, otherwise + * returns the number of bytes needed */ - public boolean hasRoomFor(int sizeInBytes) { - if (!isOpenForAppends) { - return false; - } + public OptionalInt bytesNeeded(Collection records, ObjectSerializationCache serializationCache) { + int bytesNeeded = bytesNeededForRecords( + records, + serializationCache + ); - if (nextOffset - baseOffset >= Integer.MAX_VALUE) { - return false; + if (!isOpenForAppends) { + return OptionalInt.of(batchHeaderSizeInBytes() + bytesNeeded); } - int recordSizeInBytes = DefaultRecord.sizeOfBodyInBytes( - (int) (nextOffset - baseOffset), - 0, - -1, - sizeInBytes, - DefaultRecord.EMPTY_HEADERS - ); - - int unusedSizeInBytes = maxBytes - approximateSizeInBytes(); - if (unusedSizeInBytes >= recordSizeInBytes) { - return true; + int approxUnusedSizeInBytes = maxBytes - approximateSizeInBytes(); + if (approxUnusedSizeInBytes >= bytesNeeded) { + return OptionalInt.empty(); } else if (unflushedBytes > 0) { recordOutput.flush(); unflushedBytes = 0; - unusedSizeInBytes = maxBytes - flushedSizeInBytes(); - return unusedSizeInBytes >= recordSizeInBytes; - } else { - return false; + int unusedSizeInBytes = maxBytes - flushedSizeInBytes(); + if (unusedSizeInBytes >= bytesNeeded) { + return OptionalInt.empty(); + } } + + return OptionalInt.of(batchHeaderSizeInBytes() + bytesNeeded); } private int flushedSizeInBytes() { @@ -307,4 +309,46 @@ public int writeRecord( recordOutput.writeVarint(0); return ByteUtils.sizeOfVarint(sizeInBytes) + sizeInBytes; } + + private int batchHeaderSizeInBytes() { + return AbstractRecords.recordBatchHeaderSizeInBytes( + RecordBatch.MAGIC_VALUE_V2, + compressionType + ); + } + + private int bytesNeededForRecords( + Collection records, + ObjectSerializationCache serializationCache + ) { + long expectedNextOffset = nextOffset; + int bytesNeeded = 0; + for (T record : records) { + if (expectedNextOffset - baseOffset >= Integer.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + "Adding %s records to a batch with base offset of %s and next offset of %s", + records.size(), + baseOffset, + expectedNextOffset + ) + ); + } + + int recordSizeInBytes = DefaultRecord.sizeOfBodyInBytes( + (int) (expectedNextOffset - baseOffset), + 0, + -1, + serde.recordSize(record, serializationCache), + DefaultRecord.EMPTY_HEADERS + ); + + bytesNeeded = Math.addExact(bytesNeeded, ByteUtils.sizeOfVarint(recordSizeInBytes)); + bytesNeeded = Math.addExact(bytesNeeded, recordSizeInBytes); + + expectedNextOffset += 1; + } + + return bytesNeeded; + } } diff --git a/raft/src/test/java/org/apache/kafka/raft/internals/BatchAccumulatorTest.java b/raft/src/test/java/org/apache/kafka/raft/internals/BatchAccumulatorTest.java index 24e289db2656a..b32168ec3101b 100644 --- a/raft/src/test/java/org/apache/kafka/raft/internals/BatchAccumulatorTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/internals/BatchAccumulatorTest.java @@ -19,7 +19,11 @@ import org.apache.kafka.common.memory.MemoryPool; import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.protocol.Writable; +import org.apache.kafka.common.record.AbstractRecords; import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.DefaultRecord; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.utils.ByteUtils; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Utils; import org.junit.jupiter.api.Test; @@ -29,6 +33,8 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; @@ -164,47 +170,85 @@ public void testUnflushedBuffersReleasedByClose() { @Test public void testSingleBatchAccumulation() { - int leaderEpoch = 17; - long baseOffset = 157; - int lingerMs = 50; - int maxBatchSize = 512; - - Mockito.when(memoryPool.tryAllocate(maxBatchSize)) - .thenReturn(ByteBuffer.allocate(maxBatchSize)); - - BatchAccumulator acc = buildAccumulator( - leaderEpoch, - baseOffset, - lingerMs, - maxBatchSize - ); - - List records = asList("a", "b", "c", "d", "e", "f", "g", "h", "i"); - assertEquals(baseOffset, acc.append(leaderEpoch, records.subList(0, 1))); - assertEquals(baseOffset + 2, acc.append(leaderEpoch, records.subList(1, 3))); - assertEquals(baseOffset + 5, acc.append(leaderEpoch, records.subList(3, 6))); - assertEquals(baseOffset + 7, acc.append(leaderEpoch, records.subList(6, 8))); - assertEquals(baseOffset + 8, acc.append(leaderEpoch, records.subList(8, 9))); - - time.sleep(lingerMs); - assertTrue(acc.needsDrain(time.milliseconds())); - - List> batches = acc.drain(); - assertEquals(1, batches.size()); - assertFalse(acc.needsDrain(time.milliseconds())); - assertEquals(Long.MAX_VALUE - time.milliseconds(), acc.timeUntilDrain(time.milliseconds())); - - BatchAccumulator.CompletedBatch batch = batches.get(0); - assertEquals(records, batch.records); - assertEquals(baseOffset, batch.baseOffset); + asList(APPEND, APPEND_ATOMIC).forEach(appender -> { + int leaderEpoch = 17; + long baseOffset = 157; + int lingerMs = 50; + int maxBatchSize = 512; + + Mockito.when(memoryPool.tryAllocate(maxBatchSize)) + .thenReturn(ByteBuffer.allocate(maxBatchSize)); + + BatchAccumulator acc = buildAccumulator( + leaderEpoch, + baseOffset, + lingerMs, + maxBatchSize + ); + + List records = asList("a", "b", "c", "d", "e", "f", "g", "h", "i"); + assertEquals(baseOffset, appender.call(acc, leaderEpoch, records.subList(0, 1))); + assertEquals(baseOffset + 2, appender.call(acc, leaderEpoch, records.subList(1, 3))); + assertEquals(baseOffset + 5, appender.call(acc, leaderEpoch, records.subList(3, 6))); + assertEquals(baseOffset + 7, appender.call(acc, leaderEpoch, records.subList(6, 8))); + assertEquals(baseOffset + 8, appender.call(acc, leaderEpoch, records.subList(8, 9))); + + time.sleep(lingerMs); + assertTrue(acc.needsDrain(time.milliseconds())); + + List> batches = acc.drain(); + assertEquals(1, batches.size()); + assertFalse(acc.needsDrain(time.milliseconds())); + assertEquals(Long.MAX_VALUE - time.milliseconds(), acc.timeUntilDrain(time.milliseconds())); + + BatchAccumulator.CompletedBatch batch = batches.get(0); + assertEquals(records, batch.records); + assertEquals(baseOffset, batch.baseOffset); + }); } @Test public void testMultipleBatchAccumulation() { + asList(APPEND, APPEND_ATOMIC).forEach(appender -> { + int leaderEpoch = 17; + long baseOffset = 157; + int lingerMs = 50; + int maxBatchSize = 256; + + Mockito.when(memoryPool.tryAllocate(maxBatchSize)) + .thenReturn(ByteBuffer.allocate(maxBatchSize)); + + BatchAccumulator acc = buildAccumulator( + leaderEpoch, + baseOffset, + lingerMs, + maxBatchSize + ); + + // Append entries until we have 4 batches to drain (3 completed, 1 building) + while (acc.numCompletedBatches() < 3) { + appender.call(acc, leaderEpoch, singletonList("foo")); + } + + List> batches = acc.drain(); + assertEquals(4, batches.size()); + assertTrue(batches.stream().allMatch(batch -> batch.data.sizeInBytes() <= maxBatchSize)); + }); + } + + @Test + public void testRecordsAreSplit() { int leaderEpoch = 17; long baseOffset = 157; int lingerMs = 50; - int maxBatchSize = 256; + String record = "a"; + int numberOfRecords = 9; + int recordsPerBatch = 2; + int batchHeaderSize = AbstractRecords.recordBatchHeaderSizeInBytes( + RecordBatch.MAGIC_VALUE_V2, + CompressionType.NONE + ); + int maxBatchSize = batchHeaderSize + recordsPerBatch * recordSizeInBytes(record, recordsPerBatch); Mockito.when(memoryPool.tryAllocate(maxBatchSize)) .thenReturn(ByteBuffer.allocate(maxBatchSize)); @@ -216,13 +260,19 @@ public void testMultipleBatchAccumulation() { maxBatchSize ); - // Append entries until we have 4 batches to drain (3 completed, 1 building) - while (acc.numCompletedBatches() < 3) { - acc.append(leaderEpoch, singletonList("foo")); - } + List records = Stream + .generate(() -> record) + .limit(numberOfRecords) + .collect(Collectors.toList()); + assertEquals(baseOffset + numberOfRecords - 1, acc.append(leaderEpoch, records)); + + time.sleep(lingerMs); + assertTrue(acc.needsDrain(time.milliseconds())); List> batches = acc.drain(); - assertEquals(4, batches.size()); + // ceilingDiv(records.size(), recordsPerBatch) + int expectedBatches = (records.size() + recordsPerBatch - 1) / recordsPerBatch; + assertEquals(expectedBatches, batches.size()); assertTrue(batches.stream().allMatch(batch -> batch.data.sizeInBytes() <= maxBatchSize)); } @@ -306,4 +356,35 @@ public void testDrainDoesNotBlockWithConcurrentAppend() throws Exception { }); } + int recordSizeInBytes(String record, int numberOfRecords) { + int serdeSize = serde.recordSize("a", new ObjectSerializationCache()); + + int recordSizeInBytes = DefaultRecord.sizeOfBodyInBytes( + numberOfRecords, + 0, + -1, + serdeSize, + DefaultRecord.EMPTY_HEADERS + ); + + return ByteUtils.sizeOfVarint(recordSizeInBytes) + recordSizeInBytes; + } + + static interface Appender { + Long call(BatchAccumulator acc, int epoch, List records); + } + + static final Appender APPEND_ATOMIC = new Appender() { + @Override + public Long call(BatchAccumulator acc, int epoch, List records) { + return acc.appendAtomic(epoch, records); + } + }; + + static final Appender APPEND = new Appender() { + @Override + public Long call(BatchAccumulator acc, int epoch, List records) { + return acc.append(epoch, records); + } + }; } diff --git a/raft/src/test/java/org/apache/kafka/raft/internals/BatchBuilderTest.java b/raft/src/test/java/org/apache/kafka/raft/internals/BatchBuilderTest.java index f860df7afd1cc..e4611f1b8ca25 100644 --- a/raft/src/test/java/org/apache/kafka/raft/internals/BatchBuilderTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/internals/BatchBuilderTest.java @@ -31,7 +31,6 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -69,8 +68,8 @@ void testBuildBatch(CompressionType compressionType) { records.forEach(record -> builder.appendRecord(record, null)); MemoryRecords builtRecordSet = builder.build(); - assertFalse(builder.hasRoomFor(1)); - assertThrows(IllegalArgumentException.class, () -> builder.appendRecord("a", null)); + assertTrue(builder.bytesNeeded(Arrays.asList("a"), null).isPresent()); + assertThrows(IllegalStateException.class, () -> builder.appendRecord("a", null)); List builtBatches = Utils.toList(builtRecordSet.batchIterator()); assertEquals(1, builtBatches.size()); @@ -112,9 +111,8 @@ public void testHasRoomForUncompressed(int batchSize) { ); String record = "i am a record"; - int recordSize = serde.recordSize(record); - while (builder.hasRoomFor(recordSize)) { + while (!builder.bytesNeeded(Arrays.asList(record), null).isPresent()) { builder.appendRecord(record, null); } @@ -125,5 +123,4 @@ public void testHasRoomForUncompressed(int batchSize) { assertTrue(sizeInBytes <= batchSize, "Built batch size " + sizeInBytes + " is larger than max batch size " + batchSize); } - } From a524a751c1a6b4ac95d470b0baed0616643808ed Mon Sep 17 00:00:00 2001 From: Justine Olshan Date: Thu, 18 Feb 2021 23:09:29 -0500 Subject: [PATCH 020/243] MINOR: Added missing import (KafkaVersion) to kafka.py (#10154) Reviewers: Ron Dagostino , Chia-Ping Tsai --- tests/kafkatest/services/kafka/kafka.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py index 207325239ae52..c83319082ca50 100644 --- a/tests/kafkatest/services/kafka/kafka.py +++ b/tests/kafkatest/services/kafka/kafka.py @@ -31,6 +31,7 @@ from kafkatest.services.security.listener_security_config import ListenerSecurityConfig from kafkatest.services.security.security_config import SecurityConfig from kafkatest.version import DEV_BRANCH +from kafkatest.version import KafkaVersion from kafkatest.services.kafka.util import fix_opts_for_new_jvm From b17f70ed665bc2ab1d1fa28babe6f14902d9d75a Mon Sep 17 00:00:00 2001 From: Luke Chen <43372967+showuon@users.noreply.github.com> Date: Fri, 19 Feb 2021 19:36:12 +0800 Subject: [PATCH 021/243] MINOR: Fix broken link in quickstart.html (#10161) Update the old anchor #intro_topic to #intro_concepts_and_terms Reviewers: Mickael Maison --- docs/quickstart.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/quickstart.html b/docs/quickstart.html index 5e2a1d95bdf3d..e70c63a5b641a 100644 --- a/docs/quickstart.html +++ b/docs/quickstart.html @@ -82,7 +82,7 @@

    Example events are payment transactions, geolocation updates from mobile phones, shipping orders, sensor measurements from IoT devices or medical equipment, and much more. These events are organized and stored in - topics. + topics. Very simplified, a topic is similar to a folder in a filesystem, and the events are the files in that folder.

    @@ -95,7 +95,7 @@

    All of Kafka's command line tools have additional options: run the kafka-topics.sh command without any arguments to display usage information. For example, it can also show you - details such as the partition count + details such as the partition count of the new topic:

    From c75a73862aa4bf752179e83c7fbeab771a2aef57 Mon Sep 17 00:00:00 2001 From: Randall Hauch Date: Fri, 19 Feb 2021 11:49:56 -0600 Subject: [PATCH 022/243] KAFKA-12340: Fix potential resource leak in Kafka*BackingStore (#10153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These Kafka*BackingStore classes used in Connect have a recently-added deprecated constructor, which is not used within AK. However, this commit corrects a AdminClient resource leak if those deprecated constructors are used outside of AK. The fix simply ensures that the AdminClient created by the “default” supplier is always closed when the Kafka*BackingStore is stopped. Author: Randall Hauch Reviewers: Konstantine Karantasis , Chia-Ping Tsai --- .../storage/KafkaConfigBackingStore.java | 19 +++++++++++++++++-- .../storage/KafkaOffsetBackingStore.java | 19 +++++++++++++++++-- .../storage/KafkaStatusBackingStore.java | 19 +++++++++++++++++-- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java index d4e6358e2ea99..dcfc28c5496b9 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java @@ -44,6 +44,7 @@ import org.apache.kafka.connect.util.ConnectUtils; import org.apache.kafka.connect.util.ConnectorTaskId; import org.apache.kafka.connect.util.KafkaBasedLog; +import org.apache.kafka.connect.util.SharedTopicAdmin; import org.apache.kafka.connect.util.TopicAdmin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -226,6 +227,7 @@ public static String COMMIT_TASKS_KEY(String connectorName) { private final Map> connectorConfigs = new HashMap<>(); private final Map> taskConfigs = new HashMap<>(); private final Supplier topicAdminSupplier; + private SharedTopicAdmin ownTopicAdmin; // Set of connectors where we saw a task commit with an incomplete set of task config updates, indicating the data // is in an inconsistent state and we cannot safely use them until they have been refreshed. @@ -291,7 +293,13 @@ public void start() { @Override public void stop() { log.info("Closing KafkaConfigBackingStore"); - configLog.stop(); + try { + configLog.stop(); + } finally { + if (ownTopicAdmin != null) { + ownTopicAdmin.close(); + } + } log.info("Closed KafkaConfigBackingStore"); } @@ -479,7 +487,14 @@ KafkaBasedLog setupAndCreateKafkaBasedLog(String topic, final Wo Map adminProps = new HashMap<>(originals); ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId); - Supplier adminSupplier = topicAdminSupplier != null ? topicAdminSupplier : () -> new TopicAdmin(adminProps); + Supplier adminSupplier; + if (topicAdminSupplier != null) { + adminSupplier = topicAdminSupplier; + } else { + // Create our own topic admin supplier that we'll close when we're stopped + ownTopicAdmin = new SharedTopicAdmin(adminProps); + adminSupplier = ownTopicAdmin; + } Map topicSettings = config instanceof DistributedConfig ? ((DistributedConfig) config).configStorageTopicSettings() : Collections.emptyMap(); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java index 26b47f996b18a..313baf72c58c0 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java @@ -32,6 +32,7 @@ import org.apache.kafka.connect.util.ConnectUtils; import org.apache.kafka.connect.util.ConvertingFutureCallback; import org.apache.kafka.connect.util.KafkaBasedLog; +import org.apache.kafka.connect.util.SharedTopicAdmin; import org.apache.kafka.connect.util.TopicAdmin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +66,7 @@ public class KafkaOffsetBackingStore implements OffsetBackingStore { private KafkaBasedLog offsetLog; private HashMap data; private final Supplier topicAdminSupplier; + private SharedTopicAdmin ownTopicAdmin; @Deprecated public KafkaOffsetBackingStore() { @@ -98,7 +100,14 @@ public void configure(final WorkerConfig config) { Map adminProps = new HashMap<>(originals); ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId); - Supplier adminSupplier = topicAdminSupplier != null ? topicAdminSupplier : () -> new TopicAdmin(adminProps); + Supplier adminSupplier; + if (topicAdminSupplier != null) { + adminSupplier = topicAdminSupplier; + } else { + // Create our own topic admin supplier that we'll close when we're stopped + ownTopicAdmin = new SharedTopicAdmin(adminProps); + adminSupplier = ownTopicAdmin; + } Map topicSettings = config instanceof DistributedConfig ? ((DistributedConfig) config).offsetStorageTopicSettings() : Collections.emptyMap(); @@ -140,7 +149,13 @@ public void start() { @Override public void stop() { log.info("Stopping KafkaOffsetBackingStore"); - offsetLog.stop(); + try { + offsetLog.stop(); + } finally { + if (ownTopicAdmin != null) { + ownTopicAdmin.close(); + } + } log.info("Stopped KafkaOffsetBackingStore"); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaStatusBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaStatusBackingStore.java index efa405f3a4b9b..eadbe18786717 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaStatusBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaStatusBackingStore.java @@ -44,6 +44,7 @@ import org.apache.kafka.connect.util.ConnectUtils; import org.apache.kafka.connect.util.ConnectorTaskId; import org.apache.kafka.connect.util.KafkaBasedLog; +import org.apache.kafka.connect.util.SharedTopicAdmin; import org.apache.kafka.connect.util.Table; import org.apache.kafka.connect.util.TopicAdmin; import org.slf4j.Logger; @@ -134,6 +135,7 @@ public class KafkaStatusBackingStore implements StatusBackingStore { private String statusTopic; private KafkaBasedLog kafkaLog; private int generation; + private SharedTopicAdmin ownTopicAdmin; @Deprecated public KafkaStatusBackingStore(Time time, Converter converter) { @@ -177,7 +179,14 @@ public void configure(final WorkerConfig config) { Map adminProps = new HashMap<>(originals); ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId); - Supplier adminSupplier = topicAdminSupplier != null ? topicAdminSupplier : () -> new TopicAdmin(adminProps); + Supplier adminSupplier; + if (topicAdminSupplier != null) { + adminSupplier = topicAdminSupplier; + } else { + // Create our own topic admin supplier that we'll close when we're stopped + ownTopicAdmin = new SharedTopicAdmin(adminProps); + adminSupplier = ownTopicAdmin; + } Map topicSettings = config instanceof DistributedConfig ? ((DistributedConfig) config).statusStorageTopicSettings() @@ -221,7 +230,13 @@ public void start() { @Override public void stop() { - kafkaLog.stop(); + try { + kafkaLog.stop(); + } finally { + if (ownTopicAdmin != null) { + ownTopicAdmin.close(); + } + } } @Override From 76de61475bf61fdd746ba035cc8410b38462982d Mon Sep 17 00:00:00 2001 From: Ron Dagostino Date: Thu, 18 Feb 2021 14:57:29 -0500 Subject: [PATCH 023/243] MINOR: Fix Raft broker restart issue when offset partitions are deferred #10155 A Raft-based broker is unable to restart if the broker defers partition metadata changes for a __consumer_offsets topic-partition. The issue is that GroupMetadataManager is asked to removeGroupsForPartition() upon the broker becoming a follower, but in order for that code to function it requires that the manager's scheduler be started. There are multiple possible solutions here since removeGroupsForPartition() is a no-op at this point in the broker startup cycle (nothing has been loaded, so there is nothing to unload). We could just not invoke the callback. But it seems more reasonable to not special-case this and instead start ReplicaManager and the coordinators just before applying the deferred partitions states. We also mark deferred partitions for which we are a follower as being online a bit earlier to avoid NotLeaderOrFollowerException that was being thrown upon restart. Fixing this issue exposed the above issue regarding the scheduler not being started. Reviewers: Colin P. McCabe , Ismael Juma --- core/src/main/scala/kafka/server/BrokerServer.scala | 6 ++++-- .../main/scala/kafka/server/RaftReplicaChangeDelegate.scala | 4 ++++ core/src/main/scala/kafka/server/RaftReplicaManager.scala | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index 19d65ab2e98bd..9aae5e30e6659 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -349,14 +349,16 @@ class BrokerServer( // Start log manager, which will perform (potentially lengthy) recovery-from-unclean-shutdown if required. logManager.startup(metadataCache.getAllTopics()) // Start other services that we've delayed starting, in the appropriate order. - replicaManager.endMetadataChangeDeferral( - RequestHandlerHelper.onLeadershipChange(groupCoordinator, transactionCoordinator, _, _)) replicaManager.startup() replicaManager.startHighWatermarkCheckPointThread() groupCoordinator.startup(() => metadataCache.numPartitions(Topic.GROUP_METADATA_TOPIC_NAME). getOrElse(config.offsetsTopicPartitions)) transactionCoordinator.startup(() => metadataCache.numPartitions(Topic.TRANSACTION_STATE_TOPIC_NAME). getOrElse(config.transactionTopicPartitions)) + // Apply deferred partition metadata changes after starting replica manager and coordinators + // so that those services are ready and able to process the changes. + replicaManager.endMetadataChangeDeferral( + RequestHandlerHelper.onLeadershipChange(groupCoordinator, transactionCoordinator, _, _)) socketServer.startProcessingRequests(authorizerFutures) diff --git a/core/src/main/scala/kafka/server/RaftReplicaChangeDelegate.scala b/core/src/main/scala/kafka/server/RaftReplicaChangeDelegate.scala index 0b7ee4264ae26..1bf21e074069d 100644 --- a/core/src/main/scala/kafka/server/RaftReplicaChangeDelegate.scala +++ b/core/src/main/scala/kafka/server/RaftReplicaChangeDelegate.scala @@ -35,6 +35,7 @@ trait RaftReplicaChangeDelegateHelper { def getLogDir(topicPartition: TopicPartition): Option[String] def error(msg: => String, e: => Throwable): Unit def markOffline(topicPartition: TopicPartition): Unit + def markOnline(partition: Partition): Unit def completeDelayedFetchOrProduceRequests(topicPartition: TopicPartition): Unit def isShuttingDown: Boolean def initialFetchOffset(log: Log): Long @@ -216,6 +217,9 @@ class RaftReplicaChangeDelegate(helper: RaftReplicaChangeDelegateHelper) { val leader = allBrokersByIdMap(partition.leaderReplicaIdOpt.get).brokerEndPoint(helper.config.interBrokerListenerName) val log = partition.localLogOrException val fetchOffset = helper.initialFetchOffset(log) + if (deferredBatches) { + helper.markOnline(partition) + } partition.topicPartition -> InitialFetchState(leader, partition.getLeaderEpoch, fetchOffset) }.toMap diff --git a/core/src/main/scala/kafka/server/RaftReplicaManager.scala b/core/src/main/scala/kafka/server/RaftReplicaManager.scala index 255b34974d3b6..143709deefe86 100644 --- a/core/src/main/scala/kafka/server/RaftReplicaManager.scala +++ b/core/src/main/scala/kafka/server/RaftReplicaManager.scala @@ -104,6 +104,8 @@ class RaftReplicaManager(config: KafkaConfig, override def markOffline(topicPartition: TopicPartition): Unit = raftReplicaManager.markPartitionOffline(topicPartition) + override def markOnline(partition: Partition): Unit = raftReplicaManager.allPartitions.put(partition.topicPartition, HostedPartition.Online(partition)) + override def replicaAlterLogDirsManager: ReplicaAlterLogDirsManager = raftReplicaManager.replicaAlterLogDirsManager override def replicaFetcherManager: ReplicaFetcherManager = raftReplicaManager.replicaFetcherManager From d030dc55ab4e94b4bb049b91474d5a8f533fb3bb Mon Sep 17 00:00:00 2001 From: Justine Olshan Date: Fri, 19 Feb 2021 14:08:00 -0500 Subject: [PATCH 024/243] KAFKA-12332; Error partitions from topics with invalid IDs in LISR requests (#10143) Changes how invalid IDs are handled in LeaderAndIsr requests. The ID check now occurs before leader epoch. If the ID exists and is invalid, the partition is ignored and a new `INCONSISTENT_TOPIC_ID` error is returned in the response. Reviewers: Jason Gustafson --- .../errors/InconsistentTopicIdException.java | 27 ++++++++ .../apache/kafka/common/protocol/Errors.java | 4 +- .../main/scala/kafka/cluster/Partition.scala | 40 +++++++++++- .../scala/kafka/server/ReplicaManager.scala | 31 +++------- .../kafka/server/ReplicaManagerTest.scala | 62 +++++++++++++++++-- 5 files changed, 134 insertions(+), 30 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/InconsistentTopicIdException.java diff --git a/clients/src/main/java/org/apache/kafka/common/errors/InconsistentTopicIdException.java b/clients/src/main/java/org/apache/kafka/common/errors/InconsistentTopicIdException.java new file mode 100644 index 0000000000000..1dfe468564bf5 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/errors/InconsistentTopicIdException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.errors; + +public class InconsistentTopicIdException extends InvalidMetadataException { + + private static final long serialVersionUID = 1L; + + public InconsistentTopicIdException(String message) { + super(message); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java index 03c1248055878..34c42064e78ea 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java @@ -47,6 +47,7 @@ import org.apache.kafka.common.errors.IllegalGenerationException; import org.apache.kafka.common.errors.IllegalSaslStateException; import org.apache.kafka.common.errors.InconsistentGroupProtocolException; +import org.apache.kafka.common.errors.InconsistentTopicIdException; import org.apache.kafka.common.errors.InconsistentVoterSetException; import org.apache.kafka.common.errors.InvalidCommitOffsetSizeException; import org.apache.kafka.common.errors.InvalidConfigurationException; @@ -354,7 +355,8 @@ public enum Errors { "Requested position is not greater than or equal to zero, and less than the size of the snapshot.", PositionOutOfRangeException::new), UNKNOWN_TOPIC_ID(100, "This server does not host this topic ID.", UnknownTopicIdException::new), - DUPLICATE_BROKER_REGISTRATION(101, "This broker ID is already in use.", DuplicateBrokerRegistrationException::new); + DUPLICATE_BROKER_REGISTRATION(101, "This broker ID is already in use.", DuplicateBrokerRegistrationException::new), + INCONSISTENT_TOPIC_ID(102, "The log's topic ID did not match the topic ID in the request", InconsistentTopicIdException::new); private static final Logger log = LoggerFactory.getLogger(Errors.class); diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala index 6f6cb88e92230..cfd029b38f55f 100755 --- a/core/src/main/scala/kafka/cluster/Partition.scala +++ b/core/src/main/scala/kafka/cluster/Partition.scala @@ -40,7 +40,7 @@ import org.apache.kafka.common.record.{MemoryRecords, RecordBatch} import org.apache.kafka.common.requests._ import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET} import org.apache.kafka.common.utils.Time -import org.apache.kafka.common.{IsolationLevel, TopicPartition} +import org.apache.kafka.common.{IsolationLevel, TopicPartition, Uuid} import scala.collection.{Map, Seq} import scala.jdk.CollectionConverters._ @@ -428,6 +428,44 @@ class Partition(val topicPartition: TopicPartition, this.log = Some(log) } + /** + * Checks if the topic ID provided in the request is consistent with the topic ID in the log. + * If a valid topic ID is provided, and the log exists but has no ID set, set the log ID to be the request ID. + * + * @param requestTopicId the topic ID from the request + * @return true if the request topic id is consistent, false otherwise + */ + def checkOrSetTopicId(requestTopicId: Uuid): Boolean = { + // If the request had an invalid topic ID, then we assume that topic IDs are not supported. + // The topic ID was not inconsistent, so return true. + // If the log is empty, then we can not say that topic ID is inconsistent, so return true. + if (requestTopicId == null || requestTopicId == Uuid.ZERO_UUID) + true + else { + log match { + case None => true + case Some(log) => { + // Check if topic ID is in memory, if not, it must be new to the broker and does not have a metadata file. + // This is because if the broker previously wrote it to file, it would be recovered on restart after failure. + // Topic ID is consistent since we are just setting it here. + if (log.topicId == Uuid.ZERO_UUID) { + log.partitionMetadataFile.write(requestTopicId) + log.topicId = requestTopicId + true + } else if (log.topicId != requestTopicId) { + stateChangeLogger.error(s"Topic Id in memory: ${log.topicId} does not" + + s" match the topic Id for partition $topicPartition provided in the request: " + + s"$requestTopicId.") + false + } else { + // topic ID in log exists and matches request topic ID + true + } + } + } + } + } + // remoteReplicas will be called in the hot path, and must be inexpensive def remoteReplicas: Iterable[Replica] = remoteReplicasMap.values diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index ba50c86dd4011..820af7b6ece2c 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -37,7 +37,7 @@ import kafka.server.metadata.ConfigRepository import kafka.utils._ import kafka.utils.Implicits._ import kafka.zk.KafkaZkClient -import org.apache.kafka.common.{ElectionType, IsolationLevel, Node, TopicPartition, Uuid} +import org.apache.kafka.common.{ElectionType, IsolationLevel, Node, TopicPartition} import org.apache.kafka.common.errors._ import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState @@ -1364,11 +1364,15 @@ class ReplicaManager(val config: KafkaConfig, Some(partition) } - // Next check partition's leader epoch + // Next check the topic ID and the partition's leader epoch partitionOpt.foreach { partition => val currentLeaderEpoch = partition.getLeaderEpoch val requestLeaderEpoch = partitionState.leaderEpoch - if (requestLeaderEpoch > currentLeaderEpoch) { + val requestTopicId = topicIds.get(topicPartition.topic) + + if (!partition.checkOrSetTopicId(requestTopicId)) { + responseMap.put(topicPartition, Errors.INCONSISTENT_TOPIC_ID) + } else if (requestLeaderEpoch > currentLeaderEpoch) { // If the leader epoch is valid record the epoch of the controller that made the leadership decision. // This is useful while updating the isr to maintain the decision maker controller's epoch in the zookeeper path if (partitionState.replicas.contains(localBrokerId)) @@ -1424,27 +1428,8 @@ class ReplicaManager(val config: KafkaConfig, * In this case ReplicaManager.allPartitions will map this topic-partition to an empty Partition object. * we need to map this topic-partition to OfflinePartition instead. */ - val local = localLog(topicPartition) - if (local.isEmpty) + if (localLog(topicPartition).isEmpty) markPartitionOffline(topicPartition) - else { - val id = topicIds.get(topicPartition.topic()) - // Ensure we have not received a request from an older protocol - if (id != null && !id.equals(Uuid.ZERO_UUID)) { - val log = local.get - // Check if topic ID is in memory, if not, it must be new to the broker and does not have a metadata file. - // This is because if the broker previously wrote it to file, it would be recovered on restart after failure. - if (log.topicId.equals(Uuid.ZERO_UUID)) { - log.partitionMetadataFile.write(id) - log.topicId = id - // Warn if the topic ID in the request does not match the log. - } else if (!log.topicId.equals(id)) { - stateChangeLogger.warn(s"Topic Id in memory: ${log.topicId.toString} does not" + - s" match the topic Id provided in the request: " + - s"${id.toString}.") - } - } - } } // we initialize highwatermark thread after the first leaderisrrequest. This ensures that all the partitions diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index c31bf8efe54fe..9b289e578173d 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -2229,6 +2229,7 @@ class ReplicaManagerTest { .createLogIfNotExists(isNew = false, isFutureReplica = false, new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)) val topicIds = Collections.singletonMap(topic, Uuid.randomUuid()) + val topicNames = topicIds.asScala.map(_.swap).asJava def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, Seq(new LeaderAndIsrPartitionState() @@ -2244,7 +2245,8 @@ class ReplicaManagerTest { topicIds, Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build() - replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0), (_, _) => ()) + val response = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0), (_, _) => ()) + assertEquals(Errors.NONE, response.partitionErrors(topicNames).get(topicPartition)) assertFalse(replicaManager.localLog(topicPartition).isEmpty) val id = topicIds.get(topicPartition.topic()) val log = replicaManager.localLog(topicPartition).get @@ -2257,6 +2259,51 @@ class ReplicaManagerTest { } finally replicaManager.shutdown(checkpointHW = false) } + @Test + def testInvalidIdReturnsError() = { + val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer(time)) + try { + val brokerList = Seq[Integer](0, 1).asJava + val topicPartition = new TopicPartition(topic, 0) + replicaManager.createPartition(topicPartition) + .createLogIfNotExists(isNew = false, isFutureReplica = false, + new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)) + val topicIds = Collections.singletonMap(topic, Uuid.randomUuid()) + val topicNames = topicIds.asScala.map(_.swap).asJava + + val invalidTopicIds = Collections.singletonMap(topic, Uuid.randomUuid()) + val invalidTopicNames = invalidTopicIds.asScala.map(_.swap).asJava + + def leaderAndIsrRequest(epoch: Int, topicIds: java.util.Map[String, Uuid]): LeaderAndIsrRequest = + new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch, + Seq(new LeaderAndIsrPartitionState() + .setTopicName(topic) + .setPartitionIndex(0) + .setControllerEpoch(0) + .setLeader(0) + .setLeaderEpoch(epoch) + .setIsr(brokerList) + .setZkVersion(0) + .setReplicas(brokerList) + .setIsNew(true)).asJava, + topicIds, + Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build() + + val response = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, topicIds), (_, _) => ()) + assertEquals(Errors.NONE, response.partitionErrors(topicNames).get(topicPartition)) + + val response2 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(1, topicIds), (_, _) => ()) + assertEquals(Errors.NONE, response2.partitionErrors(topicNames).get(topicPartition)) + + // Send request with invalid ID. + val response3 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(1, invalidTopicIds), (_, _) => ()) + assertEquals(Errors.INCONSISTENT_TOPIC_ID, response3.partitionErrors(invalidTopicNames).get(topicPartition)) + + val response4 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(2, invalidTopicIds), (_, _) => ()) + assertEquals(Errors.INCONSISTENT_TOPIC_ID, response4.partitionErrors(invalidTopicNames).get(topicPartition)) + } finally replicaManager.shutdown(checkpointHW = false) + } + @Test def testPartitionMetadataFileNotCreated() = { val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer(time)) @@ -2268,6 +2315,7 @@ class ReplicaManagerTest { .createLogIfNotExists(isNew = false, isFutureReplica = false, new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints)) val topicIds = Map(topic -> Uuid.ZERO_UUID, "foo" -> Uuid.randomUuid()).asJava + val topicNames = topicIds.asScala.map(_.swap).asJava def leaderAndIsrRequest(epoch: Int, name: String, version: Short): LeaderAndIsrRequest = LeaderAndIsrRequest.parse( new LeaderAndIsrRequest.Builder(version, 0, 0, brokerEpoch, @@ -2285,28 +2333,32 @@ class ReplicaManagerTest { Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build().serialize(), version) // There is no file if the topic does not have an associated topic ID. - replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, "fakeTopic", ApiKeys.LEADER_AND_ISR.latestVersion), (_, _) => ()) + val response = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, "fakeTopic", ApiKeys.LEADER_AND_ISR.latestVersion), (_, _) => ()) assertFalse(replicaManager.localLog(topicPartition).isEmpty) val log = replicaManager.localLog(topicPartition).get assertFalse(log.partitionMetadataFile.exists()) + assertEquals(Errors.NONE, response.partitionErrors(topicNames).get(topicPartition)) // There is no file if the topic has the default UUID. - replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, topic, ApiKeys.LEADER_AND_ISR.latestVersion), (_, _) => ()) + val response2 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, topic, ApiKeys.LEADER_AND_ISR.latestVersion), (_, _) => ()) assertFalse(replicaManager.localLog(topicPartition).isEmpty) val log2 = replicaManager.localLog(topicPartition).get assertFalse(log2.partitionMetadataFile.exists()) + assertEquals(Errors.NONE, response2.partitionErrors(topicNames).get(topicPartition)) // There is no file if the request an older version - replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, "foo", 0), (_, _) => ()) + val response3 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, "foo", 0), (_, _) => ()) assertFalse(replicaManager.localLog(topicPartitionFoo).isEmpty) val log3 = replicaManager.localLog(topicPartitionFoo).get assertFalse(log3.partitionMetadataFile.exists()) + assertEquals(Errors.NONE, response3.partitionErrors(topicNames).get(topicPartitionFoo)) // There is no file if the request is an older version - replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, "foo", 4), (_, _) => ()) + val response4 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(1, "foo", 4), (_, _) => ()) assertFalse(replicaManager.localLog(topicPartitionFoo).isEmpty) val log4 = replicaManager.localLog(topicPartitionFoo).get assertFalse(log4.partitionMetadataFile.exists()) + assertEquals(Errors.NONE, response4.partitionErrors(topicNames).get(topicPartitionFoo)) } finally replicaManager.shutdown(checkpointHW = false) } From 5b761e66ccf08768215e2f25a335a614c3d9b954 Mon Sep 17 00:00:00 2001 From: Bruno Cadonna Date: Fri, 19 Feb 2021 21:26:12 +0100 Subject: [PATCH 025/243] MINOR: Correct warning about increasing capacity when insufficient nodes to assign standby tasks (#10151) We should only recommend to increase the number of KafkaStreams instances, not the number of threads, since a standby task can never be placed on the same instance as an active task regardless of the thread count Reviewers: Chia-Ping Tsai , Anna Sophie Blee-Goldman --- .../internals/assignment/HighAvailabilityTaskAssignor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/HighAvailabilityTaskAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/HighAvailabilityTaskAssignor.java index 76255a40761d5..f6464f8ac3117 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/HighAvailabilityTaskAssignor.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/HighAvailabilityTaskAssignor.java @@ -146,7 +146,7 @@ private static void assignStandbyReplicaTasks(final TreeMap c if (numRemainingStandbys > 0) { log.warn("Unable to assign {} of {} standby tasks for task [{}]. " + "There is not enough available capacity. You should " + - "increase the number of threads and/or application instances " + + "increase the number of application instances " + "to maintain the requested number of standby replicas.", numRemainingStandbys, numStandbyReplicas, task); } From 1c31176ae1ef49071b0d3e717e5f3037492b5b30 Mon Sep 17 00:00:00 2001 From: Randall Hauch Date: Fri, 19 Feb 2021 14:43:32 -0600 Subject: [PATCH 026/243] KAFKA-12343: Handle exceptions better in TopicAdmin, including UnsupportedVersionException (#10158) Refactored the KafkaBasedLog logic to read end offsets into a separate method to make it easier to test. Also changed the TopicAdmin.endOffsets method to throw the original UnsupportedVersionException, LeaderNotAvailableException, and TimeoutException rather than wrapping, to better conform with the consumer method and how the KafkaBasedLog retries those exceptions. Added new tests to verify various scenarios and errors. Author: Randall Hauch Reviewers: Konstantine Karantasis , Chia-Ping Tsai --- .../kafka/connect/util/KafkaBasedLog.java | 65 ++++++++------ .../apache/kafka/connect/util/TopicAdmin.java | 16 ++-- .../kafka/connect/util/KafkaBasedLogTest.java | 89 ++++++++++++++++++- .../kafka/connect/util/TopicAdminTest.java | 12 ++- 4 files changed, 140 insertions(+), 42 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java index 6a2a787578e84..6e2350fae0894 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java @@ -28,7 +28,9 @@ import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.utils.Time; import org.apache.kafka.connect.errors.ConnectException; @@ -318,32 +320,8 @@ private void poll(long timeoutMs) { } private void readToLogEnd() { - log.trace("Reading to end of offset log"); - Set assignment = consumer.assignment(); - Map endOffsets; - // Note that we'd prefer to not use the consumer to find the end offsets for the assigned topic partitions. - // That is because it's possible that the consumer is already blocked waiting for new records to appear, when - // the consumer is already at the end. In such cases, using 'consumer.endOffsets(...)' will block until at least - // one more record becomes available, meaning we can't even check whether we're at the end offset. - // Since all we're trying to do here is get the end offset, we should use the supplied admin client - // (if available) - // (which prevents 'consumer.endOffsets(...)' - // from - - // Deprecated constructors do not provide an admin supplier, so the admin is potentially null. - if (admin != null) { - // Use the admin client to immediately find the end offsets for the assigned topic partitions. - // Unlike using the consumer - endOffsets = admin.endOffsets(assignment); - } else { - // The admin may be null if older deprecated constructor is used, though AK Connect currently always provides an admin client. - // Using the consumer is not ideal, because when the topic has low volume, the 'poll(...)' method called from the - // work thread may have blocked the consumer while waiting for more records (even when there are none). - // In such cases, this call to the consumer to simply find the end offsets will block even though we might already be - // at the end offset. - endOffsets = consumer.endOffsets(assignment); - } + Map endOffsets = readEndOffsets(assignment); log.trace("Reading to end of log offsets {}", endOffsets); while (!endOffsets.isEmpty()) { @@ -366,6 +344,37 @@ private void readToLogEnd() { } } + // Visible for testing + Map readEndOffsets(Set assignment) { + log.trace("Reading to end of offset log"); + + // Note that we'd prefer to not use the consumer to find the end offsets for the assigned topic partitions. + // That is because it's possible that the consumer is already blocked waiting for new records to appear, when + // the consumer is already at the end. In such cases, using 'consumer.endOffsets(...)' will block until at least + // one more record becomes available, meaning we can't even check whether we're at the end offset. + // Since all we're trying to do here is get the end offset, we should use the supplied admin client + // (if available) to obtain the end offsets for the given topic partitions. + + // Deprecated constructors do not provide an admin supplier, so the admin is potentially null. + if (admin != null) { + // Use the admin client to immediately find the end offsets for the assigned topic partitions. + // Unlike using the consumer + try { + return admin.endOffsets(assignment); + } catch (UnsupportedVersionException e) { + // This may happen with really old brokers that don't support the auto topic creation + // field in metadata requests + log.debug("Reading to end of log offsets with consumer since admin client is unsupported: {}", e.getMessage()); + // Forget the reference to the admin so that we won't even try to use the admin the next time this method is called + admin = null; + // continue and let the consumer handle the read + } + // Other errors, like timeouts and retriable exceptions are intentionally propagated + } + // The admin may be null if older deprecated constructor is used or if the admin client is using a broker that doesn't + // support getting the end offsets (e.g., 0.10.x). In such cases, we should use the consumer, which is not ideal (see above). + return consumer.endOffsets(assignment); + } private class WorkThread extends Thread { public WorkThread() { @@ -390,7 +399,11 @@ public void run() { log.trace("Finished read to end log for topic {}", topic); } catch (TimeoutException e) { log.warn("Timeout while reading log to end for topic '{}'. Retrying automatically. " + - "This may occur when brokers are unavailable or unreachable. Reason: {}", topic, e.getMessage()); + "This may occur when brokers are unavailable or unreachable. Reason: {}", topic, e.getMessage()); + continue; + } catch (RetriableException | org.apache.kafka.connect.errors.RetriableException e) { + log.warn("Retriable error while reading log to end for topic '{}'. Retrying automatically. " + + "Reason: {}", topic, e.getMessage()); continue; } catch (WakeupException e) { // Either received another get() call and need to retry reading to end of log or stop() was diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java index 9a7907bcdafff..9661c69e63fb5 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java @@ -651,8 +651,12 @@ public Map describeTopicConfigs(String... topicNames) { * @param partitions the topic partitions * @return the map of offset for each topic partition, or an empty map if the supplied partitions * are null or empty - * @throws RetriableException if a retriable error occurs, the operation takes too long, or the - * thread is interrupted while attempting to perform this operation + * @throws UnsupportedVersionException if the admin client cannot read end offsets + * @throws TimeoutException if the offset metadata could not be fetched before the amount of time allocated + * by {@code request.timeout.ms} expires, and this call can be retried + * @throws LeaderNotAvailableException if the leader was not available and this call can be retried + * @throws RetriableException if a retriable error occurs, or the thread is interrupted while attempting + * to perform this operation * @throws ConnectException if a non retriable error occurs */ public Map endOffsets(Set partitions) { @@ -677,13 +681,15 @@ public Map endOffsets(Set partitions) { // Should theoretically never happen, because this method is the same as what the consumer uses and therefore // should exist in the broker since before the admin client was added String msg = String.format("API to get the get the end offsets for topic '%s' is unsupported on brokers at %s", topic, bootstrapServers()); - throw new ConnectException(msg, e); + throw new UnsupportedVersionException(msg, e); } else if (cause instanceof TimeoutException) { String msg = String.format("Timed out while waiting to get end offsets for topic '%s' on brokers at %s", topic, bootstrapServers()); - throw new RetriableException(msg, e); + throw new TimeoutException(msg, e); } else if (cause instanceof LeaderNotAvailableException) { String msg = String.format("Unable to get end offsets during leader election for topic '%s' on brokers at %s", topic, bootstrapServers()); - throw new RetriableException(msg, e); + throw new LeaderNotAvailableException(msg, e); + } else if (cause instanceof org.apache.kafka.common.errors.RetriableException) { + throw (org.apache.kafka.common.errors.RetriableException) cause; } else { String msg = String.format("Error while getting end offsets for topic '%s' on brokers at %s", topic, bootstrapServers()); throw new ConnectException(msg, e); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/KafkaBasedLogTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/KafkaBasedLogTest.java index 15bf8ca9b4f2e..e36f2a902facd 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/KafkaBasedLogTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/KafkaBasedLogTest.java @@ -31,6 +31,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.LeaderNotAvailableException; import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.TimestampType; @@ -61,11 +62,13 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @RunWith(PowerMockRunner.class) @@ -117,6 +120,8 @@ public class KafkaBasedLogTest { @Mock private KafkaProducer producer; private MockConsumer consumer; + @Mock + private TopicAdmin admin; private Map>> consumedRecords = new HashMap<>(); private Callback> consumedCallback = (error, record) -> { @@ -463,15 +468,91 @@ public void testProducerError() throws Exception { PowerMock.verifyAll(); } + @Test + public void testReadEndOffsetsUsingAdmin() throws Exception { + // Create a log that uses the admin supplier + setupWithAdmin(); + expectProducerAndConsumerCreate(); + + Set tps = new HashSet<>(Arrays.asList(TP0, TP1)); + Map endOffsets = new HashMap<>(); + endOffsets.put(TP0, 0L); + endOffsets.put(TP1, 0L); + admin.endOffsets(EasyMock.eq(tps)); + PowerMock.expectLastCall().andReturn(endOffsets).times(2); + + PowerMock.replayAll(); + + store.start(); + assertEquals(endOffsets, store.readEndOffsets(tps)); + } + + @Test + public void testReadEndOffsetsUsingAdminThatFailsWithUnsupported() throws Exception { + // Create a log that uses the admin supplier + setupWithAdmin(); + expectProducerAndConsumerCreate(); + + Set tps = new HashSet<>(Arrays.asList(TP0, TP1)); + // Getting end offsets using the admin client should fail with unsupported version + admin.endOffsets(EasyMock.eq(tps)); + PowerMock.expectLastCall().andThrow(new UnsupportedVersionException("too old")); + + // Falls back to the consumer + Map endOffsets = new HashMap<>(); + endOffsets.put(TP0, 0L); + endOffsets.put(TP1, 0L); + consumer.updateEndOffsets(endOffsets); + + PowerMock.replayAll(); + + store.start(); + assertEquals(endOffsets, store.readEndOffsets(tps)); + } + + @Test + public void testReadEndOffsetsUsingAdminThatFailsWithRetriable() throws Exception { + // Create a log that uses the admin supplier + setupWithAdmin(); + expectProducerAndConsumerCreate(); + + Set tps = new HashSet<>(Arrays.asList(TP0, TP1)); + Map endOffsets = new HashMap<>(); + endOffsets.put(TP0, 0L); + endOffsets.put(TP1, 0L); + // Getting end offsets upon startup should work fine + admin.endOffsets(EasyMock.eq(tps)); + PowerMock.expectLastCall().andReturn(endOffsets).times(1); + // Getting end offsets using the admin client should fail with leader not available + admin.endOffsets(EasyMock.eq(tps)); + PowerMock.expectLastCall().andThrow(new LeaderNotAvailableException("retry")); + + PowerMock.replayAll(); + + store.start(); + assertThrows(LeaderNotAvailableException.class, () -> store.readEndOffsets(tps)); + } + + @SuppressWarnings("unchecked") + private void setupWithAdmin() { + Supplier adminSupplier = () -> admin; + java.util.function.Consumer initializer = admin -> { }; + store = PowerMock.createPartialMock(KafkaBasedLog.class, new String[]{"createConsumer", "createProducer"}, + TOPIC, PRODUCER_PROPS, CONSUMER_PROPS, adminSupplier, consumedCallback, time, initializer); + } + + private void expectProducerAndConsumerCreate() throws Exception { + PowerMock.expectPrivate(store, "createProducer") + .andReturn(producer); + PowerMock.expectPrivate(store, "createConsumer") + .andReturn(consumer); + } private void expectStart() throws Exception { initializer.run(); EasyMock.expectLastCall().times(1); - PowerMock.expectPrivate(store, "createProducer") - .andReturn(producer); - PowerMock.expectPrivate(store, "createConsumer") - .andReturn(consumer); + expectProducerAndConsumerCreate(); } private void expectStop() { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java index 9ba0b1d0aabff..edd989125b43e 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java @@ -33,6 +33,7 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.config.TopicConfig; import org.apache.kafka.common.errors.ClusterAuthorizationException; +import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.CreateTopicsResponseData; @@ -50,7 +51,6 @@ import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.errors.RetriableException; import org.junit.Test; import java.util.ArrayList; @@ -485,7 +485,7 @@ public void endOffsetsShouldFailWithNonRetriableWhenAuthorizationFailureOccurs() } @Test - public void endOffsetsShouldFailWithNonRetriableWhenVersionUnsupportedErrorOccurs() { + public void endOffsetsShouldFailWithUnsupportedVersionWhenVersionUnsupportedErrorOccurs() { String topicName = "myTopic"; TopicPartition tp1 = new TopicPartition(topicName, 0); Set tps = Collections.singleton(tp1); @@ -496,15 +496,14 @@ public void endOffsetsShouldFailWithNonRetriableWhenVersionUnsupportedErrorOccur env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE)); env.kafkaClient().prepareResponse(listOffsetsResultWithUnsupportedVersion(tp1, offset)); TopicAdmin admin = new TopicAdmin(null, env.adminClient()); - ConnectException e = assertThrows(ConnectException.class, () -> { + UnsupportedVersionException e = assertThrows(UnsupportedVersionException.class, () -> { admin.endOffsets(tps); }); - assertTrue(e.getMessage().contains("is unsupported on brokers")); } } @Test - public void endOffsetsShouldFailWithRetriableWhenTimeoutErrorOccurs() { + public void endOffsetsShouldFailWithTimeoutExceptionWhenTimeoutErrorOccurs() { String topicName = "myTopic"; TopicPartition tp1 = new TopicPartition(topicName, 0); Set tps = Collections.singleton(tp1); @@ -515,10 +514,9 @@ public void endOffsetsShouldFailWithRetriableWhenTimeoutErrorOccurs() { env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE)); env.kafkaClient().prepareResponse(listOffsetsResultWithTimeout(tp1, offset)); TopicAdmin admin = new TopicAdmin(null, env.adminClient()); - RetriableException e = assertThrows(RetriableException.class, () -> { + TimeoutException e = assertThrows(TimeoutException.class, () -> { admin.endOffsets(tps); }); - assertTrue(e.getMessage().contains("Timed out while waiting")); } } From b35ca4349dabb199411cb6bc4c80ef89f19d9328 Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Fri, 19 Feb 2021 13:36:07 -0800 Subject: [PATCH 027/243] KAFKA-9274: Throw TaskCorruptedException instead of TimeoutException when TX commit times out (#10072) Part of KIP-572: follow up work to PR #9800. It's not save to retry a TX commit after a timeout, because it's unclear if the commit was successful or not, and thus on retry we might get an IllegalStateException. Instead, we will throw a TaskCorruptedException to retry the TX if the commit failed. Reviewers: A. Sophie Blee-Goldman --- docs/streams/upgrade-guide.html | 9 ++ .../errors/TaskCorruptedException.java | 22 ++-- .../streams/errors/TaskTimeoutExceptions.java | 58 --------- .../internals/ProcessorStateManager.java | 10 +- .../internals/RecordCollectorImpl.java | 22 ++-- .../internals/StoreChangelogReader.java | 9 +- .../processor/internals/StreamTask.java | 2 +- .../processor/internals/StreamThread.java | 4 +- .../processor/internals/TaskManager.java | 114 ++++++++++-------- .../internals/ProcessorStateManagerTest.java | 4 +- .../processor/internals/StreamThreadTest.java | 20 ++- .../processor/internals/TaskManagerTest.java | 52 ++++---- 12 files changed, 141 insertions(+), 185 deletions(-) delete mode 100644 streams/src/main/java/org/apache/kafka/streams/errors/TaskTimeoutExceptions.java diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html index 2a6a7604f43e2..38138e134e69c 100644 --- a/docs/streams/upgrade-guide.html +++ b/docs/streams/upgrade-guide.html @@ -128,6 +128,15 @@

    Streams API into the constructor, it is no longer required to set mandatory configuration parameters (cf. KIP-680).

    +

    + Kafka Streams is now handling TimeoutException thrown by the consumer, producer, and admin client. + If a timeout occurs on a task, Kafka Streams moves to the next task and retries to make progress on the failed + task in the next iteration. + To bound how long Kafka Streams retries a task, you can set task.timeout.ms (default is 5 minutes). + If a task does not make progress within the specified task timeout, which is tracked on a per-task basis, + Kafka Streams throws a TimeoutException + (cf. KIP-572). +

    Streams API changes in 2.7.0

    diff --git a/streams/src/main/java/org/apache/kafka/streams/errors/TaskCorruptedException.java b/streams/src/main/java/org/apache/kafka/streams/errors/TaskCorruptedException.java index 52f668b806f1a..bf5bd17fbdbc8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/errors/TaskCorruptedException.java +++ b/streams/src/main/java/org/apache/kafka/streams/errors/TaskCorruptedException.java @@ -17,11 +17,9 @@ package org.apache.kafka.streams.errors; import org.apache.kafka.clients.consumer.InvalidOffsetException; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.streams.processor.TaskId; -import java.util.Collection; -import java.util.Map; +import java.util.Set; /** * Indicates a specific task is corrupted and need to be re-initialized. It can be thrown when @@ -33,20 +31,20 @@ */ public class TaskCorruptedException extends StreamsException { - private final Map> taskWithChangelogs; + private final Set corruptedTasks; - public TaskCorruptedException(final Map> taskWithChangelogs) { - super("Tasks with changelogs " + taskWithChangelogs + " are corrupted and hence needs to be re-initialized"); - this.taskWithChangelogs = taskWithChangelogs; + public TaskCorruptedException(final Set corruptedTasks) { + super("Tasks " + corruptedTasks + " are corrupted and hence needs to be re-initialized"); + this.corruptedTasks = corruptedTasks; } - public TaskCorruptedException(final Map> taskWithChangelogs, + public TaskCorruptedException(final Set corruptedTasks, final InvalidOffsetException e) { - super("Tasks with changelogs " + taskWithChangelogs + " are corrupted and hence needs to be re-initialized", e); - this.taskWithChangelogs = taskWithChangelogs; + super("Tasks " + corruptedTasks + " are corrupted and hence needs to be re-initialized", e); + this.corruptedTasks = corruptedTasks; } - public Map> corruptedTaskWithChangelogs() { - return taskWithChangelogs; + public Set corruptedTasks() { + return corruptedTasks; } } diff --git a/streams/src/main/java/org/apache/kafka/streams/errors/TaskTimeoutExceptions.java b/streams/src/main/java/org/apache/kafka/streams/errors/TaskTimeoutExceptions.java deleted file mode 100644 index 521778d0935b7..0000000000000 --- a/streams/src/main/java/org/apache/kafka/streams/errors/TaskTimeoutExceptions.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.streams.errors; - -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.streams.processor.internals.Task; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -public class TaskTimeoutExceptions extends StreamsException { - private static final long serialVersionUID = 1L; - - private final TimeoutException timeoutException; - private final Map exceptions; - - public TaskTimeoutExceptions() { - super(""); - timeoutException = null; - exceptions = new HashMap<>(); - } - - public TaskTimeoutExceptions(final TimeoutException timeoutException) { - super(""); - this.timeoutException = timeoutException; - exceptions = null; - } - - public void recordException(final Task task, - final TimeoutException timeoutException) { - Objects.requireNonNull(exceptions) - .put(task, timeoutException); - } - - public Map exceptions() { - return exceptions; - } - - public TimeoutException timeoutException() { - return timeoutException; - } - -} diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java index b3460060b6ca1..c455c4bd30761 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.streams.processor.internals; -import java.util.ArrayList; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.FixedOrderMap; @@ -25,12 +24,12 @@ import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.errors.TaskCorruptedException; import org.apache.kafka.streams.errors.TaskMigratedException; -import org.apache.kafka.streams.processor.StateRestoreListener; -import org.apache.kafka.streams.processor.StateStoreContext; -import org.apache.kafka.streams.processor.internals.Task.TaskType; import org.apache.kafka.streams.processor.StateRestoreCallback; +import org.apache.kafka.streams.processor.StateRestoreListener; import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.processor.StateStoreContext; import org.apache.kafka.streams.processor.TaskId; +import org.apache.kafka.streams.processor.internals.Task.TaskType; import org.apache.kafka.streams.state.internals.CachedStateStore; import org.apache.kafka.streams.state.internals.OffsetCheckpoint; import org.apache.kafka.streams.state.internals.RecordConverter; @@ -39,6 +38,7 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -251,7 +251,7 @@ void initializeStoreOffsetsFromCheckpoint(final boolean storeDirIsEmpty) { "treat it as a task corruption error and wipe out the local state of task {} " + "before re-bootstrapping", store.stateStore.name(), taskId); - throw new TaskCorruptedException(Collections.singletonMap(taskId, changelogPartitions())); + throw new TaskCorruptedException(Collections.singleton(taskId)); } else { log.info("State store {} did not find checkpoint offset, hence would " + "default to the starting offset at changelog {}", diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java index 2de9caa4523bc..16a451d0b1550 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java @@ -40,6 +40,7 @@ import org.apache.kafka.streams.errors.ProductionExceptionHandler; import org.apache.kafka.streams.errors.ProductionExceptionHandler.ProductionExceptionHandlerResponse; import org.apache.kafka.streams.errors.StreamsException; +import org.apache.kafka.streams.errors.TaskCorruptedException; import org.apache.kafka.streams.errors.TaskMigratedException; import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.TaskId; @@ -213,27 +214,24 @@ private void recordSendError(final String topic, final Exception exception, fina "indicating the task may be migrated out"; sendException.set(new TaskMigratedException(errorMessage, exception)); } else { - // TODO: KIP-572 handle `TimeoutException extends RetriableException` - // is seems inappropriate to pass `TimeoutException` into the `ProductionExceptionHander` - // -> should we add `TimeoutException` as `isFatalException` above (maybe not) ? - // -> maybe we should try to reset the task by throwing a `TaskCorruptedException` (including triggering `task.timeout.ms`) ? if (exception instanceof RetriableException) { errorMessage += "\nThe broker is either slow or in bad state (like not having enough replicas) in responding the request, " + "or the connection to broker was interrupted sending the request or receiving the response. " + "\nConsider overwriting `max.block.ms` and /or " + "`delivery.timeout.ms` to a larger value to wait longer for such scenarios and avoid timeout errors"; - } - - if (productionExceptionHandler.handle(serializedRecord, exception) == ProductionExceptionHandlerResponse.FAIL) { - errorMessage += "\nException handler choose to FAIL the processing, no more records would be sent."; - sendException.set(new StreamsException(errorMessage, exception)); + sendException.set(new TaskCorruptedException(Collections.singleton(taskId))); } else { - errorMessage += "\nException handler choose to CONTINUE processing in spite of this error but written offsets would not be recorded."; - droppedRecordsSensor.record(); + if (productionExceptionHandler.handle(serializedRecord, exception) == ProductionExceptionHandlerResponse.FAIL) { + errorMessage += "\nException handler choose to FAIL the processing, no more records would be sent."; + sendException.set(new StreamsException(errorMessage, exception)); + } else { + errorMessage += "\nException handler choose to CONTINUE processing in spite of this error but written offsets would not be recorded."; + droppedRecordsSensor.record(); + } } } - log.error(errorMessage); + log.error(errorMessage, exception); } private boolean isFatalException(final Exception exception) { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java index 56394a2a2193e..fdf027f2be776 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java @@ -442,12 +442,9 @@ public void restore(final Map tasks) { "truncated or compacted on the broker, marking the corresponding tasks as corrupted and re-initializing" + " it later.", e); - final Map> taskWithCorruptedChangelogs = new HashMap<>(); - for (final TopicPartition partition : e.partitions()) { - final TaskId taskId = changelogs.get(partition).stateManager.taskId(); - taskWithCorruptedChangelogs.computeIfAbsent(taskId, k -> new HashSet<>()).add(partition); - } - throw new TaskCorruptedException(taskWithCorruptedChangelogs, e); + final Set corruptedTasks = new HashSet<>(); + e.partitions().forEach(partition -> corruptedTasks.add(changelogs.get(partition).stateManager.taskId())); + throw new TaskCorruptedException(corruptedTasks, e); } catch (final KafkaException e) { throw new StreamsException("Restore consumer get unexpected error polling records.", e); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java index fbf010765fe0d..36dc02ac0e886 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java @@ -727,7 +727,7 @@ record = null; throw timeoutException; } else { record = null; - throw new TaskCorruptedException(Collections.singletonMap(id, changelogPartitions())); + throw new TaskCorruptedException(Collections.singleton(id)); } } catch (final StreamsException exception) { record = null; diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java index a9bd6995c9a4f..b967f5af6f7c3 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java @@ -578,10 +578,10 @@ boolean runLoop() { nextProbingRebalanceMs.set(Long.MAX_VALUE); } } catch (final TaskCorruptedException e) { - log.warn("Detected the states of tasks " + e.corruptedTaskWithChangelogs() + " are corrupted. " + + log.warn("Detected the states of tasks " + e.corruptedTasks() + " are corrupted. " + "Will close the task as dirty and re-create and bootstrap from scratch.", e); try { - taskManager.handleCorruption(e.corruptedTaskWithChangelogs()); + taskManager.handleCorruption(e.corruptedTasks()); } catch (final TaskMigratedException taskMigrated) { handleTaskMigrated(taskMigrated); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java index 19e91e2be5372..d74e10c98556f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java @@ -32,9 +32,9 @@ import org.apache.kafka.common.utils.Time; import org.apache.kafka.streams.errors.LockException; import org.apache.kafka.streams.errors.StreamsException; +import org.apache.kafka.streams.errors.TaskCorruptedException; import org.apache.kafka.streams.errors.TaskIdFormatException; import org.apache.kafka.streams.errors.TaskMigratedException; -import org.apache.kafka.streams.errors.TaskTimeoutExceptions; import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.internals.Task.State; import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl; @@ -155,17 +155,16 @@ void handleRebalanceComplete() { /** * @throws TaskMigratedException */ - void handleCorruption(final Map> tasksWithChangelogs) { + void handleCorruption(final Set corruptedTasks) { final Map> corruptedStandbyTasks = new HashMap<>(); final Map> corruptedActiveTasks = new HashMap<>(); - for (final Map.Entry> taskEntry : tasksWithChangelogs.entrySet()) { - final TaskId taskId = taskEntry.getKey(); + for (final TaskId taskId : corruptedTasks) { final Task task = tasks.task(taskId); if (task.isActive()) { - corruptedActiveTasks.put(task, taskEntry.getValue()); + corruptedActiveTasks.put(task, task.changelogPartitions()); } else { - corruptedStandbyTasks.put(task, taskEntry.getValue()); + corruptedStandbyTasks.put(task, task.changelogPartitions()); } } @@ -177,7 +176,7 @@ void handleCorruption(final Map> tasksWithCha .values() .stream() .filter(t -> t.state() == Task.State.RUNNING || t.state() == Task.State.RESTORING) - .filter(t -> !tasksWithChangelogs.containsKey(t.id())) + .filter(t -> !corruptedTasks.contains(t.id())) .collect(Collectors.toSet()) ); @@ -529,11 +528,6 @@ void handleRevocation(final Collection revokedPartitions) { // so we would capture any exception and throw try { commitOffsetsOrTransaction(consumedOffsetsPerTask); - } catch (final TaskTimeoutExceptions taskTimeoutExceptions) { - for (final Map.Entry timeoutException : taskTimeoutExceptions.exceptions().entrySet()) { - log.error("Exception caught while committing revoked task " + timeoutException.getKey(), timeoutException.getValue()); - } - firstException.compareAndSet(null, taskTimeoutExceptions); } catch (final RuntimeException e) { log.error("Exception caught while committing those revoked tasks " + revokedActiveTasks, e); firstException.compareAndSet(null, e); @@ -841,7 +835,7 @@ private Collection tryCloseCleanAllActiveTasks(final boolean clean, } // If any active tasks can't be committed, none of them can be, and all that need a commit must be closed dirty - if (!tasksToCloseDirty.isEmpty()) { + if (processingMode == EXACTLY_ONCE_BETA && !tasksToCloseDirty.isEmpty()) { tasksToCloseClean.removeAll(tasksToCommit); tasksToCloseDirty.addAll(tasksToCommit); } else { @@ -858,15 +852,22 @@ private Collection tryCloseCleanAllActiveTasks(final boolean clean, tasksToCloseClean.remove(task); } } - } catch (final TaskTimeoutExceptions taskTimeoutExceptions) { - for (final Map.Entry timeoutException : taskTimeoutExceptions.exceptions().entrySet()) { - log.error( - "Exception caught while committing task {} during shutdown {}", - timeoutException.getKey(), - timeoutException.getValue() - ); - } - firstException.compareAndSet(null, taskTimeoutExceptions); + } catch (final TimeoutException timeoutException) { + firstException.compareAndSet(null, timeoutException); + + tasksToCloseClean.removeAll(tasksToCommit); + tasksToCloseDirty.addAll(tasksToCommit); + } catch (final TaskCorruptedException taskCorruptedException) { + firstException.compareAndSet(null, taskCorruptedException); + + final Set corruptedTaskIds = taskCorruptedException.corruptedTasks(); + final Set corruptedTasks = tasksToCommit + .stream() + .filter(task -> corruptedTaskIds.contains(task.id())) + .collect(Collectors.toSet()); + + tasksToCloseClean.removeAll(corruptedTasks); + tasksToCloseDirty.addAll(corruptedTasks); } catch (final RuntimeException e) { log.error("Exception caught while committing tasks during shutdown", e); firstException.compareAndSet(null, e); @@ -1004,32 +1005,22 @@ int commit(final Collection tasksToCommit) { } } - final Set uncommittedTasks = new HashSet<>(); try { commitOffsetsOrTransaction(consumedOffsetsAndMetadataPerTask); - } catch (final TaskTimeoutExceptions taskTimeoutExceptions) { - final TimeoutException timeoutException = taskTimeoutExceptions.timeoutException(); - if (timeoutException != null) { - consumedOffsetsAndMetadataPerTask - .keySet() - .forEach(t -> t.maybeInitTaskTimeoutOrThrow(time.milliseconds(), timeoutException)); - uncommittedTasks.addAll(tasksToCommit); - } else { - for (final Map.Entry timeoutExceptions : taskTimeoutExceptions.exceptions().entrySet()) { - final Task task = timeoutExceptions.getKey(); - task.maybeInitTaskTimeoutOrThrow(time.milliseconds(), timeoutExceptions.getValue()); - uncommittedTasks.add(task); + + for (final Task task : tasksToCommit) { + if (task.commitNeeded()) { + task.clearTaskTimeout(); + ++committed; + task.postCommit(false); } } + } catch (final TimeoutException timeoutException) { + consumedOffsetsAndMetadataPerTask + .keySet() + .forEach(t -> t.maybeInitTaskTimeoutOrThrow(time.milliseconds(), timeoutException)); } - for (final Task task : tasksToCommit) { - if (task.commitNeeded() && !uncommittedTasks.contains(task)) { - task.clearTaskTimeout(); - ++committed; - task.postCommit(false); - } - } return committed; } @@ -1055,7 +1046,7 @@ int maybeCommitActiveTasksPerUserRequested() { private void commitOffsetsOrTransaction(final Map> offsetsPerTask) { log.debug("Committing task offsets {}", offsetsPerTask.entrySet().stream().collect(Collectors.toMap(t -> t.getKey().id(), Entry::getValue))); // avoid logging actual Task objects - TaskTimeoutExceptions timeoutExceptions = null; + final Set corruptedTasks = new HashSet<>(); if (!offsetsPerTask.isEmpty()) { if (processingMode == EXACTLY_ONCE_ALPHA) { @@ -1065,10 +1056,11 @@ private void commitOffsetsOrTransaction(final Map t.id().toString()) + .collect(Collectors.joining(", "))), + timeoutException + ); + offsetsPerTask + .keySet() + .forEach(task -> corruptedTasks.add(task.id())); } } else { try { @@ -1088,15 +1091,24 @@ private void commitOffsetsOrTransaction(final Map t.id().toString()) + .collect(Collectors.joining(", "))), + timeoutException + ); + throw timeoutException; } catch (final KafkaException error) { throw new StreamsException("Error encountered committing offsets via consumer", error); } } } - if (timeoutExceptions != null) { - throw timeoutExceptions; + if (!corruptedTasks.isEmpty()) { + throw new TaskCorruptedException(corruptedTasks); } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java index 44609a26d40f4..39788e85aeb87 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java @@ -904,8 +904,8 @@ public void shouldThrowTaskCorruptedWithoutPersistentStoreCheckpointAndNonEmptyD () -> stateMgr.initializeStoreOffsetsFromCheckpoint(false)); assertEquals( - Collections.singletonMap(taskId, stateMgr.changelogPartitions()), - exception.corruptedTaskWithChangelogs() + Collections.singleton(taskId), + exception.corruptedTasks() ); } finally { stateMgr.close(); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java index e4d083ba76d24..bfc32d5340eee 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java @@ -1422,7 +1422,7 @@ public void shouldReinitializeRevivedTasksInAnyState() { "proc", () -> record -> { if (shouldThrow.get()) { - throw new TaskCorruptedException(singletonMap(task1, new HashSet<>(singleton(storeChangelogTopicPartition)))); + throw new TaskCorruptedException(singleton(task1)); } else { processed.set(true); } @@ -1479,7 +1479,7 @@ public void shouldReinitializeRevivedTasksInAnyState() { final TaskCorruptedException taskCorruptedException = assertThrows(TaskCorruptedException.class, thread::runOnce); // Now, we can handle the corruption - thread.taskManager().handleCorruption(taskCorruptedException.corruptedTaskWithChangelogs()); + thread.taskManager().handleCorruption(taskCorruptedException.corruptedTasks()); // again, complete the restoration thread.runOnce(); @@ -2261,16 +2261,14 @@ public void shouldCatchHandleCorruptionOnTaskCorruptedExceptionPath() { final TaskId taskId1 = new TaskId(0, 0); final TaskId taskId2 = new TaskId(0, 2); - final Map> corruptedTasksWithChangelogs = mkMap( - mkEntry(taskId1, emptySet()) - ); + final Set corruptedTasks = singleton(taskId1); expect(task1.state()).andReturn(Task.State.RUNNING).anyTimes(); expect(task1.id()).andReturn(taskId1).anyTimes(); expect(task2.state()).andReturn(Task.State.RUNNING).anyTimes(); expect(task2.id()).andReturn(taskId2).anyTimes(); - taskManager.handleCorruption(corruptedTasksWithChangelogs); + taskManager.handleCorruption(corruptedTasks); EasyMock.replay(task1, task2, taskManager, consumer); @@ -2298,7 +2296,7 @@ public void shouldCatchHandleCorruptionOnTaskCorruptedExceptionPath() { @Override void runOnce() { setState(State.PENDING_SHUTDOWN); - throw new TaskCorruptedException(corruptedTasksWithChangelogs); + throw new TaskCorruptedException(corruptedTasks); } }.updateThreadMetadata(getSharedAdminClientId(CLIENT_ID)); @@ -2326,16 +2324,14 @@ public void shouldCatchTaskMigratedExceptionOnOnTaskCorruptedExceptionPath() { final TaskId taskId1 = new TaskId(0, 0); final TaskId taskId2 = new TaskId(0, 2); - final Map> corruptedTasksWithChangelogs = mkMap( - mkEntry(taskId1, emptySet()) - ); + final Set corruptedTasks = singleton(taskId1); expect(task1.state()).andReturn(Task.State.RUNNING).anyTimes(); expect(task1.id()).andReturn(taskId1).anyTimes(); expect(task2.state()).andReturn(Task.State.RUNNING).anyTimes(); expect(task2.id()).andReturn(taskId2).anyTimes(); - taskManager.handleCorruption(corruptedTasksWithChangelogs); + taskManager.handleCorruption(corruptedTasks); expectLastCall().andThrow(new TaskMigratedException("Task migrated", new RuntimeException("non-corrupted task migrated"))); @@ -2368,7 +2364,7 @@ public void shouldCatchTaskMigratedExceptionOnOnTaskCorruptedExceptionPath() { @Override void runOnce() { setState(State.PENDING_SHUTDOWN); - throw new TaskCorruptedException(corruptedTasksWithChangelogs); + throw new TaskCorruptedException(corruptedTasks); } }.updateThreadMetadata(getSharedAdminClientId(CLIENT_ID)); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java index 36224e0cd5c00..2ca734de5a0ed 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java @@ -40,6 +40,7 @@ import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.errors.LockException; import org.apache.kafka.streams.errors.StreamsException; +import org.apache.kafka.streams.errors.TaskCorruptedException; import org.apache.kafka.streams.errors.TaskMigratedException; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.TaskId; @@ -654,7 +655,8 @@ public void postCommit(final boolean enforceCheckpoint) { assertThat(taskManager.tryToCompleteRestoration(time.milliseconds()), is(true)); assertThat(task00.state(), is(Task.State.RUNNING)); - taskManager.handleCorruption(singletonMap(taskId00, taskId00Partitions)); + task00.setChangelogOffsets(singletonMap(t1p0, 0L)); + taskManager.handleCorruption(singleton(taskId00)); assertThat(task00.commitPrepared, is(true)); assertThat(task00.state(), is(Task.State.CREATED)); @@ -698,7 +700,8 @@ public void suspend() { assertThat(taskManager.tryToCompleteRestoration(time.milliseconds()), is(true)); assertThat(task00.state(), is(Task.State.RUNNING)); - taskManager.handleCorruption(singletonMap(taskId00, taskId00Partitions)); + task00.setChangelogOffsets(singletonMap(t1p0, 0L)); + taskManager.handleCorruption(singleton(taskId00)); assertThat(task00.commitPrepared, is(true)); assertThat(task00.state(), is(Task.State.CREATED)); assertThat(taskManager.activeTaskMap(), is(singletonMap(taskId00, task00))); @@ -743,7 +746,8 @@ public void shouldCommitNonCorruptedTasksOnTaskCorruptedException() { assertThat(nonCorruptedTask.state(), is(Task.State.RUNNING)); nonCorruptedTask.setCommitNeeded(); - taskManager.handleCorruption(singletonMap(taskId00, taskId00Partitions)); + corruptedTask.setChangelogOffsets(singletonMap(t1p0, 0L)); + taskManager.handleCorruption(singleton(taskId00)); assertTrue(nonCorruptedTask.commitPrepared); verify(consumer); @@ -782,7 +786,8 @@ public void shouldNotCommitNonRunningNonCorruptedTasks() { taskManager.handleAssignment(assignment, emptyMap()); assertThat(nonRunningNonCorruptedTask.state(), is(Task.State.CREATED)); - taskManager.handleCorruption(singletonMap(taskId00, taskId00Partitions)); + corruptedTask.setChangelogOffsets(singletonMap(t1p0, 0L)); + taskManager.handleCorruption(singleton(taskId00)); verify(activeTaskCreator); assertFalse(nonRunningNonCorruptedTask.commitPrepared); @@ -822,7 +827,8 @@ public Map prepareCommit() { runningNonCorruptedActive.setCommitNeeded(); - assertThrows(TaskMigratedException.class, () -> taskManager.handleCorruption(singletonMap(taskId00, taskId00Partitions))); + corruptedStandby.setChangelogOffsets(singletonMap(t1p0, 0L)); + assertThrows(TaskMigratedException.class, () -> taskManager.handleCorruption(singleton(taskId00))); assertThat(corruptedStandby.commitPrepared, is(true)); @@ -2718,19 +2724,18 @@ public void shouldNotFailForTimeoutExceptionOnCommitWithEosAlpha() { task00.setCommitNeeded(); task01.setCommitNeeded(); - assertThat(taskManager.commit(mkSet(task00, task01, task02)), equalTo(1)); - assertThat(task00.timeout, equalTo(time.milliseconds())); - assertNull(task01.timeout); - assertNull(task02.timeout); - - assertThat(taskManager.commit(mkSet(task00, task01, task02)), equalTo(1)); - assertNull(task00.timeout); - assertNull(task01.timeout); - assertNull(task02.timeout); + final TaskCorruptedException exception = assertThrows( + TaskCorruptedException.class, + () -> taskManager.commit(mkSet(task00, task01, task02)) + ); + assertThat( + exception.corruptedTasks(), + equalTo(Collections.singleton(taskId00)) + ); } @Test - public void shouldNotFailForTimeoutExceptionOnCommitWithEosBeta() { + public void shouldThrowTaskCorruptedExceptionForTimeoutExceptionOnCommitWithEosBeta() { setUpTaskManager(ProcessingMode.EXACTLY_ONCE_BETA); final StreamsProducer producer = mock(StreamsProducer.class); @@ -2760,15 +2765,14 @@ public void shouldNotFailForTimeoutExceptionOnCommitWithEosBeta() { task00.setCommitNeeded(); task01.setCommitNeeded(); - assertThat(taskManager.commit(mkSet(task00, task01, task02)), equalTo(0)); - assertThat(task00.timeout, equalTo(time.milliseconds())); - assertThat(task01.timeout, equalTo(time.milliseconds())); - assertNull(task02.timeout); - - assertThat(taskManager.commit(mkSet(task00, task01, task02)), equalTo(2)); - assertNull(task00.timeout); - assertNull(task01.timeout); - assertNull(task02.timeout); + final TaskCorruptedException exception = assertThrows( + TaskCorruptedException.class, + () -> taskManager.commit(mkSet(task00, task01, task02)) + ); + assertThat( + exception.corruptedTasks(), + equalTo(mkSet(taskId00, taskId01)) + ); } @Test From bbf145b1b163bada0b20cea42b29d91443161170 Mon Sep 17 00:00:00 2001 From: David Jacot Date: Fri, 19 Feb 2021 23:43:14 +0100 Subject: [PATCH 028/243] KAFKA-10817; Add clusterId validation to raft Fetch handling (#10129) This patch adds clusterId validation in the `Fetch` API as documented in KIP-595. A new error code `INCONSISTENT_CLUSTER_ID` is returned if the request clusterId does not match the value on the server. If no clusterId is provided, the request is treated as valid. Reviewers: Jason Gustafson --- .../InconsistentClusterIdException.java | 28 +++++++++ .../apache/kafka/common/protocol/Errors.java | 4 +- .../common/message/FetchRequest.json | 3 +- .../main/scala/kafka/raft/RaftManager.scala | 1 + .../apache/kafka/raft/KafkaRaftClient.java | 18 ++++++ .../raft/KafkaRaftClientSnapshotTest.java | 24 ++++---- .../kafka/raft/KafkaRaftClientTest.java | 58 ++++++++++++++----- .../kafka/raft/RaftClientTestContext.java | 55 +++++++++++++++--- .../kafka/raft/RaftEventSimulationTest.java | 3 + 9 files changed, 158 insertions(+), 36 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/InconsistentClusterIdException.java diff --git a/clients/src/main/java/org/apache/kafka/common/errors/InconsistentClusterIdException.java b/clients/src/main/java/org/apache/kafka/common/errors/InconsistentClusterIdException.java new file mode 100644 index 0000000000000..62fed41f708b7 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/errors/InconsistentClusterIdException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.errors; + +public class InconsistentClusterIdException extends ApiException { + + public InconsistentClusterIdException(String message) { + super(message); + } + + public InconsistentClusterIdException(String message, Throwable throwable) { + super(message, throwable); + } +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java index 34c42064e78ea..5c2ca7df9a4f1 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java @@ -49,6 +49,7 @@ import org.apache.kafka.common.errors.InconsistentGroupProtocolException; import org.apache.kafka.common.errors.InconsistentTopicIdException; import org.apache.kafka.common.errors.InconsistentVoterSetException; +import org.apache.kafka.common.errors.InconsistentClusterIdException; import org.apache.kafka.common.errors.InvalidCommitOffsetSizeException; import org.apache.kafka.common.errors.InvalidConfigurationException; import org.apache.kafka.common.errors.InvalidFetchSessionEpochException; @@ -356,7 +357,8 @@ public enum Errors { PositionOutOfRangeException::new), UNKNOWN_TOPIC_ID(100, "This server does not host this topic ID.", UnknownTopicIdException::new), DUPLICATE_BROKER_REGISTRATION(101, "This broker ID is already in use.", DuplicateBrokerRegistrationException::new), - INCONSISTENT_TOPIC_ID(102, "The log's topic ID did not match the topic ID in the request", InconsistentTopicIdException::new); + INCONSISTENT_TOPIC_ID(102, "The log's topic ID did not match the topic ID in the request", InconsistentTopicIdException::new), + INCONSISTENT_CLUSTER_ID(103, "The clusterId in the request does not match that found on the server", InconsistentClusterIdException::new); private static final Logger log = LoggerFactory.getLogger(Errors.class); diff --git a/clients/src/main/resources/common/message/FetchRequest.json b/clients/src/main/resources/common/message/FetchRequest.json index ab4c95fba8264..659477320d3f9 100644 --- a/clients/src/main/resources/common/message/FetchRequest.json +++ b/clients/src/main/resources/common/message/FetchRequest.json @@ -50,7 +50,8 @@ "validVersions": "0-12", "flexibleVersions": "12+", "fields": [ - { "name": "ClusterId", "type": "string", "versions": "12+", "nullableVersions": "12+", "default": "null", "taggedVersions": "12+", "tag": 0, + { "name": "ClusterId", "type": "string", "versions": "12+", "nullableVersions": "12+", "default": "null", + "taggedVersions": "12+", "tag": 0, "ignorable": true, "about": "The clusterId if known. This is used to validate metadata fetches prior to broker registration." }, { "name": "ReplicaId", "type": "int32", "versions": "0+", "about": "The broker ID of the follower, of -1 if this request is from a consumer." }, diff --git a/core/src/main/scala/kafka/raft/RaftManager.scala b/core/src/main/scala/kafka/raft/RaftManager.scala index 6a74c27bf06c2..7bf34b091dde1 100644 --- a/core/src/main/scala/kafka/raft/RaftManager.scala +++ b/core/src/main/scala/kafka/raft/RaftManager.scala @@ -200,6 +200,7 @@ class KafkaRaftManager[T]( metrics, expirationService, logContext, + metaProperties.clusterId.toString, OptionalInt.of(config.nodeId), raftConfig ) diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index 9fbbe3182917d..b964a877c803d 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -148,6 +148,7 @@ public class KafkaRaftClient implements RaftClient { private final LogContext logContext; private final Time time; private final int fetchMaxWaitMs; + private final String clusterId; private final OptionalInt nodeId; private final NetworkChannel channel; private final ReplicatedLog log; @@ -184,6 +185,7 @@ public KafkaRaftClient( Metrics metrics, ExpirationService expirationService, LogContext logContext, + String clusterId, OptionalInt nodeId, RaftConfig raftConfig ) { @@ -197,6 +199,7 @@ public KafkaRaftClient( metrics, expirationService, FETCH_MAX_WAIT_MS, + clusterId, nodeId, logContext, new Random(), @@ -214,6 +217,7 @@ public KafkaRaftClient( Metrics metrics, ExpirationService expirationService, int fetchMaxWaitMs, + String clusterId, OptionalInt nodeId, LogContext logContext, Random random, @@ -228,6 +232,7 @@ public KafkaRaftClient( this.fetchPurgatory = new ThresholdPurgatory<>(expirationService); this.appendPurgatory = new ThresholdPurgatory<>(expirationService); this.time = time; + this.clusterId = clusterId; this.nodeId = nodeId; this.metrics = metrics; this.fetchMaxWaitMs = fetchMaxWaitMs; @@ -940,6 +945,14 @@ private FetchResponseData buildEmptyFetchResponse( ); } + private boolean hasValidClusterId(FetchRequestData request) { + // We don't enforce the cluster id if it is not provided. + if (request.clusterId() == null) { + return true; + } + return clusterId.equals(request.clusterId()); + } + /** * Handle a Fetch request. The fetch offset and last fetched epoch are always * validated against the current log. In the case that they do not match, the response will @@ -959,6 +972,10 @@ private CompletableFuture handleFetchRequest( ) { FetchRequestData request = (FetchRequestData) requestMetadata.data; + if (!hasValidClusterId(request)) { + return completedFuture(new FetchResponseData().setErrorCode(Errors.INCONSISTENT_CLUSTER_ID.code())); + } + if (!hasValidTopicPartition(request, log.topicPartition())) { // Until we support multi-raft, we treat topic partition mismatches as invalid requests return completedFuture(new FetchResponseData().setErrorCode(Errors.INVALID_REQUEST.code())); @@ -1766,6 +1783,7 @@ private FetchRequestData buildFetchRequest() { }); return request .setMaxWaitMs(fetchMaxWaitMs) + .setClusterId(clusterId.toString()) .setReplicaId(quorum.localIdOrSentinel()); } diff --git a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java index 9ebb776fe1818..2b7cea5554151 100644 --- a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java @@ -74,7 +74,7 @@ public void testFetchRequestOffsetLessThanLogStart() throws Exception { // Advance the highWatermark context.deliverRequest(context.fetchRequest(epoch, otherNodeId, localLogEndOffset, epoch, 0)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(localLogEndOffset, context.client.highWatermark().getAsLong()); OffsetAndEpoch snapshotId = new OffsetAndEpoch(localLogEndOffset, epoch); @@ -89,7 +89,7 @@ public void testFetchRequestOffsetLessThanLogStart() throws Exception { // Send Fetch request less than start offset context.deliverRequest(context.fetchRequest(epoch, otherNodeId, 0, epoch, 0)); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchResponse(); + FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -119,7 +119,7 @@ public void testFetchRequestWithLargerLastFetchedEpoch() throws Exception { long localLogEndOffset = context.log.endOffset().offset; context.deliverRequest(context.fetchRequest(epoch, otherNodeId, localLogEndOffset, epoch, 0)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(localLogEndOffset, context.client.highWatermark().getAsLong()); // Create a snapshot at the high watermark @@ -135,7 +135,7 @@ public void testFetchRequestWithLargerLastFetchedEpoch() throws Exception { // It is an invalid request to send an last fetched epoch greater than the current epoch context.deliverRequest(context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset + 1, epoch + 1, 0)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); } @Test @@ -162,7 +162,7 @@ public void testFetchRequestTruncateToLogStart() throws Exception { context.fetchRequest(epoch, syncNodeId, context.log.endOffset().offset, epoch, 0) ); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(context.log.endOffset().offset, context.client.highWatermark().getAsLong()); // Create a snapshot at the high watermark @@ -176,7 +176,7 @@ public void testFetchRequestTruncateToLogStart() throws Exception { context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset + 1, oldestSnapshotId.epoch + 1, 0) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchResponse(); + FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -209,7 +209,7 @@ public void testFetchRequestAtLogStartOffsetWithValidEpoch() throws Exception { context.fetchRequest(epoch, syncNodeId, context.log.endOffset().offset, epoch, 0) ); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(context.log.endOffset().offset, context.client.highWatermark().getAsLong()); // Create a snapshot at the high watermark @@ -223,7 +223,7 @@ public void testFetchRequestAtLogStartOffsetWithValidEpoch() throws Exception { context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset, oldestSnapshotId.epoch, 0) ); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); } @Test @@ -251,7 +251,7 @@ public void testFetchRequestAtLogStartOffsetWithInvalidEpoch() throws Exception context.fetchRequest(epoch, syncNodeId, context.log.endOffset().offset, epoch, 0) ); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(context.log.endOffset().offset, context.client.highWatermark().getAsLong()); // Create a snapshot at the high watermark @@ -265,7 +265,7 @@ public void testFetchRequestAtLogStartOffsetWithInvalidEpoch() throws Exception context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset, oldestSnapshotId.epoch + 1, 0) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchResponse(); + FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -298,7 +298,7 @@ public void testFetchRequestWithLastFetchedEpochLessThanOldestSnapshot() throws context.fetchRequest(epoch, syncNodeId, context.log.endOffset().offset, epoch, 0) ); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(context.log.endOffset().offset, context.client.highWatermark().getAsLong()); // Create a snapshot at the high watermark @@ -318,7 +318,7 @@ public void testFetchRequestWithLastFetchedEpochLessThanOldestSnapshot() throws ) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchResponse(); + FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); diff --git a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientTest.java b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientTest.java index fb188f1eacd3f..8093c1f2bacc8 100644 --- a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientTest.java @@ -233,7 +233,7 @@ public void testResignWillCompleteFetchPurgatory() throws Exception { // when transition to resign, all request in fetchPurgatory will fail context.client.shutdown(1000); context.client.poll(); - context.assertSentFetchResponse(Errors.BROKER_NOT_AVAILABLE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.BROKER_NOT_AVAILABLE, epoch, OptionalInt.of(localId)); context.assertResignedLeader(epoch, localId); // shutting down finished @@ -836,7 +836,7 @@ public void testListenerCommitCallbackAfterLeaderWrite() throws Exception { // note the offset 0 would be a control message for becoming the leader context.deliverRequest(context.fetchRequest(epoch, otherNodeId, 0L, epoch, 500)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(OptionalLong.of(0L), context.client.highWatermark()); List records = Arrays.asList("a", "b", "c"); @@ -847,7 +847,7 @@ public void testListenerCommitCallbackAfterLeaderWrite() throws Exception { // Let the follower send a fetch, it should advance the high watermark context.deliverRequest(context.fetchRequest(epoch, otherNodeId, 1L, epoch, 500)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(OptionalLong.of(1L), context.client.highWatermark()); assertEquals(OptionalLong.empty(), context.listener.lastCommitOffset()); @@ -1075,27 +1075,55 @@ public void testInvalidFetchRequest() throws Exception { context.deliverRequest(context.fetchRequest( epoch, otherNodeId, -5L, 0, 0)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); context.deliverRequest(context.fetchRequest( epoch, otherNodeId, 0L, -1, 0)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); context.deliverRequest(context.fetchRequest( epoch, otherNodeId, 0L, epoch + 1, 0)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); context.deliverRequest(context.fetchRequest( epoch + 1, otherNodeId, 0L, 0, 0)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.UNKNOWN_LEADER_EPOCH, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.UNKNOWN_LEADER_EPOCH, epoch, OptionalInt.of(localId)); context.deliverRequest(context.fetchRequest( epoch, otherNodeId, 0L, 0, -1)); context.pollUntilResponse(); - context.assertSentFetchResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); + } + + @Test + public void testFetchRequestClusterIdValidation() throws Exception { + int localId = 0; + int otherNodeId = 1; + int epoch = 5; + Set voters = Utils.mkSet(localId, otherNodeId); + + RaftClientTestContext context = RaftClientTestContext.initializeAsLeader(localId, voters, epoch); + + // null cluster id is accepted + context.deliverRequest(context.fetchRequest( + epoch, null, otherNodeId, -5L, 0, 0)); + context.pollUntilResponse(); + context.assertSentFetchPartitionResponse(Errors.INVALID_REQUEST, epoch, OptionalInt.of(localId)); + + // empty cluster id is rejected + context.deliverRequest(context.fetchRequest( + epoch, "", otherNodeId, -5L, 0, 0)); + context.pollUntilResponse(); + context.assertSentFetchPartitionResponse(Errors.INCONSISTENT_CLUSTER_ID); + + // invalid cluster id is rejected + context.deliverRequest(context.fetchRequest( + epoch, "invalid-uuid", otherNodeId, -5L, 0, 0)); + context.pollUntilResponse(); + context.assertSentFetchPartitionResponse(Errors.INCONSISTENT_CLUSTER_ID); } @Test @@ -1169,7 +1197,7 @@ public void testPurgatoryFetchTimeout() throws Exception { // After expiration of the max wait time, the fetch returns an empty record set context.time.sleep(maxWaitTimeMs); context.client.poll(); - MemoryRecords fetchedRecords = context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + MemoryRecords fetchedRecords = context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); assertEquals(0, fetchedRecords.sizeInBytes()); } @@ -1192,7 +1220,7 @@ public void testPurgatoryFetchSatisfiedByWrite() throws Exception { context.client.scheduleAppend(epoch, Arrays.asList(appendRecords)); context.client.poll(); - MemoryRecords fetchedRecords = context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + MemoryRecords fetchedRecords = context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); RaftClientTestContext.assertMatchingRecords(appendRecords, fetchedRecords); } @@ -1222,7 +1250,7 @@ public void testPurgatoryFetchCompletedByFollowerTransition() throws Exception { context.assertSentBeginQuorumEpochResponse(Errors.NONE, epoch + 1, OptionalInt.of(voter3)); // The fetch should be satisfied immediately and return an error - MemoryRecords fetchedRecords = context.assertSentFetchResponse( + MemoryRecords fetchedRecords = context.assertSentFetchPartitionResponse( Errors.NOT_LEADER_OR_FOLLOWER, epoch + 1, OptionalInt.of(voter3)); assertEquals(0, fetchedRecords.sizeInBytes()); } @@ -1492,7 +1520,7 @@ public void testDescribeQuorum() throws Exception { context.pollUntilResponse(); long highWatermark = 1L; - context.assertSentFetchResponse(highWatermark, epoch); + context.assertSentFetchPartitionResponse(highWatermark, epoch); context.deliverRequest(DescribeQuorumRequest.singletonRequest(context.metadataPartition)); @@ -1710,7 +1738,7 @@ public void testFetchShouldBeTreatedAsLeaderAcknowledgement() throws Exception { context.client.poll(); // The BeginEpoch request eventually times out. We should not send another one. - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); context.time.sleep(context.requestTimeoutMs()); context.client.poll(); @@ -1749,7 +1777,7 @@ public void testLeaderAppendSingleMemberQuorum() throws Exception { context.deliverRequest(context.fetchRequest(1, otherNodeId, 0L, 0, 500)); context.pollUntilResponse(); - MemoryRecords fetchedRecords = context.assertSentFetchResponse(Errors.NONE, 1, OptionalInt.of(localId)); + MemoryRecords fetchedRecords = context.assertSentFetchPartitionResponse(Errors.NONE, 1, OptionalInt.of(localId)); List batches = Utils.toList(fetchedRecords.batchIterator()); assertEquals(2, batches.size()); @@ -2159,7 +2187,7 @@ public void testHandleCommitCallbackFiresInCandidateState() throws Exception { context.deliverRequest(context.fetchRequest(epoch, otherNodeId, 9L, epoch, 500)); context.pollUntilResponse(); assertEquals(OptionalLong.of(9L), context.client.highWatermark()); - context.assertSentFetchResponse(Errors.NONE, epoch, OptionalInt.of(localId)); + context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId)); // Now we receive a vote request which transitions us to the 'unattached' state context.deliverRequest(context.voteRequest(epoch + 1, otherNodeId, epoch, 9L)); diff --git a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java index 12fda05cdcec9..e57995c2cd303 100644 --- a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java +++ b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java @@ -17,6 +17,7 @@ package org.apache.kafka.raft; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; import org.apache.kafka.common.memory.MemoryPool; import org.apache.kafka.common.message.BeginQuorumEpochRequestData; import org.apache.kafka.common.message.BeginQuorumEpochResponseData; @@ -95,6 +96,7 @@ public final class RaftClientTestContext { private int appendLingerMs; private final QuorumStateStore quorumStateStore; + private final Uuid clusterId; private final OptionalInt localId; public final KafkaRaftClient client; final Metrics metrics; @@ -127,6 +129,7 @@ public static final class Builder { private final Set voters; private final OptionalInt localId; + private Uuid clusterId = Uuid.randomUuid(); private int requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS; private int electionTimeoutMs = DEFAULT_ELECTION_TIMEOUT_MS; private int appendLingerMs = DEFAULT_APPEND_LINGER_MS; @@ -192,6 +195,11 @@ Builder withRequestTimeoutMs(int requestTimeoutMs) { return this; } + Builder withClusterId(Uuid clusterId) { + this.clusterId = clusterId; + return this; + } + public RaftClientTestContext build() throws IOException { Metrics metrics = new Metrics(time); MockNetworkChannel channel = new MockNetworkChannel(voters); @@ -213,6 +221,7 @@ public RaftClientTestContext build() throws IOException { metrics, new MockExpirationService(time), FETCH_MAX_WAIT_MS, + clusterId.toString(), localId, logContext, random, @@ -223,6 +232,7 @@ public RaftClientTestContext build() throws IOException { client.initialize(); RaftClientTestContext context = new RaftClientTestContext( + clusterId, localId, client, log, @@ -244,6 +254,7 @@ public RaftClientTestContext build() throws IOException { } private RaftClientTestContext( + Uuid clusterId, OptionalInt localId, KafkaRaftClient client, MockLog log, @@ -255,6 +266,7 @@ private RaftClientTestContext( Metrics metrics, MockListener listener ) { + this.clusterId = clusterId; this.localId = localId; this.client = client; this.log = log; @@ -594,7 +606,7 @@ int assertSentFetchRequest( return raftMessage.correlationId(); } - FetchResponseData.FetchablePartitionResponse assertSentFetchResponse() { + FetchResponseData.FetchablePartitionResponse assertSentFetchPartitionResponse() { List sentMessages = drainSentResponses(ApiKeys.FETCH); assertEquals( 1, sentMessages.size(), "Found unexpected sent messages " + sentMessages); @@ -609,13 +621,23 @@ FetchResponseData.FetchablePartitionResponse assertSentFetchResponse() { return response.responses().get(0).partitionResponses().get(0); } + void assertSentFetchPartitionResponse(Errors error) { + List sentMessages = drainSentResponses(ApiKeys.FETCH); + assertEquals( + 1, sentMessages.size(), "Found unexpected sent messages " + sentMessages); + RaftResponse.Outbound raftMessage = sentMessages.get(0); + assertEquals(ApiKeys.FETCH.id, raftMessage.data.apiKey()); + FetchResponseData response = (FetchResponseData) raftMessage.data(); + assertEquals(error, Errors.forCode(response.errorCode())); + } + - MemoryRecords assertSentFetchResponse( + MemoryRecords assertSentFetchPartitionResponse( Errors error, int epoch, OptionalInt leaderId ) { - FetchResponseData.FetchablePartitionResponse partitionResponse = assertSentFetchResponse(); + FetchResponseData.FetchablePartitionResponse partitionResponse = assertSentFetchPartitionResponse(); assertEquals(error, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(leaderId.orElse(-1), partitionResponse.currentLeader().leaderId()); @@ -626,11 +648,11 @@ MemoryRecords assertSentFetchResponse( return (MemoryRecords) partitionResponse.recordSet(); } - MemoryRecords assertSentFetchResponse( + MemoryRecords assertSentFetchPartitionResponse( long highWatermark, int leaderEpoch ) { - FetchResponseData.FetchablePartitionResponse partitionResponse = assertSentFetchResponse(); + FetchResponseData.FetchablePartitionResponse partitionResponse = assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(leaderEpoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(highWatermark, partitionResponse.highWatermark()); @@ -670,7 +692,7 @@ void buildFollowerSet( deliverRequest(fetchRequest(1, laggingFollower, 0L, 0, 0)); pollUntilResponse(); - assertSentFetchResponse(0L, epoch); + assertSentFetchPartitionResponse(0L, epoch); // Append some records, so that the close follower will be able to advance further. client.scheduleAppend(epoch, Arrays.asList("foo", "bar")); @@ -679,7 +701,7 @@ void buildFollowerSet( deliverRequest(fetchRequest(epoch, closeFollower, 1L, epoch, 0)); pollUntilResponse(); - assertSentFetchResponse(1L, epoch); + assertSentFetchPartitionResponse(1L, epoch); } List collectEndQuorumRequests(int epoch, Set destinationIdSet) { @@ -866,6 +888,24 @@ FetchRequestData fetchRequest( long fetchOffset, int lastFetchedEpoch, int maxWaitTimeMs + ) { + return fetchRequest( + epoch, + clusterId.toString(), + replicaId, + fetchOffset, + lastFetchedEpoch, + maxWaitTimeMs + ); + } + + FetchRequestData fetchRequest( + int epoch, + String clusterId, + int replicaId, + long fetchOffset, + int lastFetchedEpoch, + int maxWaitTimeMs ) { FetchRequestData request = RaftUtil.singletonFetchRequest(metadataPartition, fetchPartition -> { fetchPartition @@ -875,6 +915,7 @@ FetchRequestData fetchRequest( }); return request .setMaxWaitMs(maxWaitTimeMs) + .setClusterId(clusterId) .setReplicaId(replicaId); } diff --git a/raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java b/raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java index f94b85a8e8ba9..a1af9127a1619 100644 --- a/raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.raft; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; import org.apache.kafka.common.memory.MemoryPool; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.ObjectSerializationCache; @@ -562,6 +563,7 @@ private static class Cluster { final Random random; final AtomicInteger correlationIdCounter = new AtomicInteger(); final MockTime time = new MockTime(); + final Uuid clusterId = Uuid.randomUuid(); final Set voters = new HashSet<>(); final Map nodes = new HashMap<>(); final Map running = new HashMap<>(); @@ -758,6 +760,7 @@ void start(int nodeId) { metrics, new MockExpirationService(time), FETCH_MAX_WAIT_MS, + clusterId.toString(), OptionalInt.of(nodeId), logContext, random, From 45b7a2a2ac697636cb4810a8c9357527361babd6 Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Sat, 20 Feb 2021 07:02:09 +0800 Subject: [PATCH 029/243] KAFKA-12339: Add retry to admin client's listOffsets (#10152) `KafkaAdmin.listOffsets` did not handle topic-level errors, hence the UnknownTopicOrPartitionException on topic-level can obstruct a Connect worker from running when the new internal topic is NOT synced to all brokers. The method did handle partition-level retriable errors by retrying, so this changes to handle topic-level retriable errors in the same way. This allows a Connect worker to start up and have the admin client retry when the worker is trying to read to the end of the newly-created internal topics until the internal topic metadata is synced to all brokers. Author: Chia-Ping Tsai Reviewers: Randall Hauch , Konstantine Karantasis --- .../internals/MetadataOperationContext.java | 1 + .../clients/admin/KafkaAdminClientTest.java | 54 ++++++++++++++++--- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/MetadataOperationContext.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/MetadataOperationContext.java index c05e5cfac0f3c..e7f2c07d9de83 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/MetadataOperationContext.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/MetadataOperationContext.java @@ -82,6 +82,7 @@ public Collection topics() { public static void handleMetadataErrors(MetadataResponse response) { for (TopicMetadata tm : response.topicMetadata()) { + if (shouldRefreshMetadata(tm.error())) throw tm.error().exception(); for (PartitionMetadata pm : tm.partitionMetadata()) { if (shouldRefreshMetadata(pm.error)) { throw pm.error.exception(); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index f7107c886b160..66c24824fd45a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -460,12 +460,16 @@ private static FindCoordinatorResponse prepareFindCoordinatorResponse(Errors err } private static MetadataResponse prepareMetadataResponse(Cluster cluster, Errors error) { + return prepareMetadataResponse(cluster, error, error); + } + + private static MetadataResponse prepareMetadataResponse(Cluster cluster, Errors topicError, Errors partitionError) { List metadata = new ArrayList<>(); for (String topic : cluster.topics()) { List pms = new ArrayList<>(); for (PartitionInfo pInfo : cluster.availablePartitionsForTopic(topic)) { MetadataResponsePartition pm = new MetadataResponsePartition() - .setErrorCode(error.code()) + .setErrorCode(partitionError.code()) .setPartitionIndex(pInfo.partition()) .setLeaderId(pInfo.leader().id()) .setLeaderEpoch(234) @@ -475,19 +479,19 @@ private static MetadataResponse prepareMetadataResponse(Cluster cluster, Errors pms.add(pm); } MetadataResponseTopic tm = new MetadataResponseTopic() - .setErrorCode(error.code()) + .setErrorCode(topicError.code()) .setName(topic) .setIsInternal(false) .setPartitions(pms); metadata.add(tm); } return MetadataResponse.prepareResponse(true, - 0, - cluster.nodes(), - cluster.clusterResource().clusterId(), - cluster.controller().id(), - metadata, - MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + 0, + cluster.nodes(), + cluster.clusterResource().clusterId(), + cluster.controller().id(), + metadata, + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); } private static DescribeGroupsResponseData prepareDescribeGroupsResponseData(String groupId, @@ -4060,6 +4064,40 @@ public void testListOffsets() throws Exception { } } + @Test + public void testListOffsetsRetriableErrorOnMetadata() throws Exception { + Node node = new Node(0, "localhost", 8120); + List nodes = Collections.singletonList(node); + final Cluster cluster = new Cluster( + "mockClusterId", + nodes, + Collections.singleton(new PartitionInfo("foo", 0, node, new Node[]{node}, new Node[]{node})), + Collections.emptySet(), + Collections.emptySet(), + node); + final TopicPartition tp0 = new TopicPartition("foo", 0); + + try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) { + env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); + env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.NONE)); + // metadata refresh because of UNKNOWN_TOPIC_OR_PARTITION + env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE)); + // listoffsets response from broker 0 + ListOffsetsResponseData responseData = new ListOffsetsResponseData() + .setThrottleTimeMs(0) + .setTopics(Collections.singletonList(ListOffsetsResponse.singletonListOffsetsTopicResponse(tp0, Errors.NONE, -1L, 123L, 321))); + env.kafkaClient().prepareResponseFrom(new ListOffsetsResponse(responseData), node); + + ListOffsetsResult result = env.adminClient().listOffsets(Collections.singletonMap(tp0, OffsetSpec.latest())); + + Map offsets = result.all().get(3, TimeUnit.SECONDS); + assertEquals(1, offsets.size()); + assertEquals(123L, offsets.get(tp0).offset()); + assertEquals(321, offsets.get(tp0).leaderEpoch().get().intValue()); + assertEquals(-1L, offsets.get(tp0).timestamp()); + } + } + @Test public void testListOffsetsRetriableErrors() throws Exception { From 690f72dd69d31589655c84d3cc1a6eec006bcab5 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Tue, 9 Feb 2021 14:11:35 -0800 Subject: [PATCH 030/243] KAFKA-12334: Add the KIP-500 metadata shell The Kafka Metadata shell is a new command which allows users to interactively examine the metadata stored in a KIP-500 cluster. It can examine snapshot files that are specified via --snapshot. The metadata tool works by replaying the log and storing the state into in-memory nodes. These nodes are presented in a fashion similar to filesystem directories. Reviewers: Jason Gustafson , David Arthur , Igor Soarez --- bin/kafka-metadata-shell.sh | 17 + build.gradle | 43 ++ checkstyle/import-control.xml | 17 + checkstyle/suppressions.xml | 4 + gradle/dependencies.gradle | 2 + .../apache/kafka/metalog/LocalLogManager.java | 378 ++++++++++++++++++ .../common/metadata/IsrChangeRecord.json | 4 +- .../common/metadata/PartitionRecord.json | 4 +- .../kafka/metadata/MetadataParserTest.java | 2 +- .../kafka/metalog/LocalLogManagerTest.java | 2 +- settings.gradle | 1 + .../apache/kafka/shell/CatCommandHandler.java | 120 ++++++ .../apache/kafka/shell/CdCommandHandler.java | 117 ++++++ .../org/apache/kafka/shell/CommandUtils.java | 148 +++++++ .../java/org/apache/kafka/shell/Commands.java | 154 +++++++ .../kafka/shell/ErroneousCommandHandler.java | 58 +++ .../kafka/shell/ExitCommandHandler.java | 88 ++++ .../kafka/shell/FindCommandHandler.java | 121 ++++++ .../org/apache/kafka/shell/GlobComponent.java | 179 +++++++++ .../org/apache/kafka/shell/GlobVisitor.java | 148 +++++++ .../kafka/shell/HelpCommandHandler.java | 88 ++++ .../kafka/shell/HistoryCommandHandler.java | 108 +++++ .../apache/kafka/shell/InteractiveShell.java | 172 ++++++++ .../apache/kafka/shell/LsCommandHandler.java | 299 ++++++++++++++ .../apache/kafka/shell/ManCommandHandler.java | 109 +++++ .../org/apache/kafka/shell/MetadataNode.java | 140 +++++++ .../kafka/shell/MetadataNodeManager.java | 302 ++++++++++++++ .../org/apache/kafka/shell/MetadataShell.java | 174 ++++++++ .../kafka/shell/NoOpCommandHandler.java | 43 ++ .../kafka/shell/NotDirectoryException.java | 30 ++ .../apache/kafka/shell/NotFileException.java | 30 ++ .../apache/kafka/shell/PwdCommandHandler.java | 89 +++++ .../kafka/shell/SnapshotFileReader.java | 194 +++++++++ .../org/apache/kafka/shell/CommandTest.java | 70 ++++ .../apache/kafka/shell/CommandUtilsTest.java | 37 ++ .../apache/kafka/shell/GlobComponentTest.java | 75 ++++ .../apache/kafka/shell/GlobVisitorTest.java | 144 +++++++ .../kafka/shell/LsCommandHandlerTest.java | 99 +++++ .../apache/kafka/shell/MetadataNodeTest.java | 73 ++++ 39 files changed, 3879 insertions(+), 4 deletions(-) create mode 100755 bin/kafka-metadata-shell.sh create mode 100644 metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/CatCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/CdCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/CommandUtils.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/Commands.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/ErroneousCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/ExitCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/FindCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/GlobComponent.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/GlobVisitor.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/HelpCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/HistoryCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/InteractiveShell.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/LsCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/ManCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/MetadataNode.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/MetadataShell.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/NoOpCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/NotDirectoryException.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/NotFileException.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/PwdCommandHandler.java create mode 100644 shell/src/main/java/org/apache/kafka/shell/SnapshotFileReader.java create mode 100644 shell/src/test/java/org/apache/kafka/shell/CommandTest.java create mode 100644 shell/src/test/java/org/apache/kafka/shell/CommandUtilsTest.java create mode 100644 shell/src/test/java/org/apache/kafka/shell/GlobComponentTest.java create mode 100644 shell/src/test/java/org/apache/kafka/shell/GlobVisitorTest.java create mode 100644 shell/src/test/java/org/apache/kafka/shell/LsCommandHandlerTest.java create mode 100644 shell/src/test/java/org/apache/kafka/shell/MetadataNodeTest.java diff --git a/bin/kafka-metadata-shell.sh b/bin/kafka-metadata-shell.sh new file mode 100755 index 0000000000000..289f0c1b51f27 --- /dev/null +++ b/bin/kafka-metadata-shell.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.shell.MetadataShell "$@" diff --git a/build.gradle b/build.gradle index 790cd11613924..d03748848ce02 100644 --- a/build.gradle +++ b/build.gradle @@ -1351,6 +1351,49 @@ project(':tools') { } } +project(':shell') { + archivesBaseName = "kafka-shell" + + dependencies { + compile libs.argparse4j + compile libs.jacksonDatabind + compile libs.jacksonJDK8Datatypes + compile libs.jline + compile libs.slf4jApi + compile project(':clients') + compile project(':core') + compile project(':log4j-appender') + compile project(':metadata') + compile project(':raft') + + compile libs.jacksonJaxrsJsonProvider + + testCompile project(':clients') + testCompile libs.junitJupiter + + testRuntime libs.slf4jlog4j + } + + javadoc { + enabled = false + } + + tasks.create(name: "copyDependantLibs", type: Copy) { + from (configurations.testRuntime) { + include('jline-*jar') + } + from (configurations.runtime) { + include('jline-*jar') + } + into "$buildDir/dependant-libs-${versions.scala}" + duplicatesStrategy 'exclude' + } + + jar { + dependsOn 'copyDependantLibs' + } +} + project(':streams') { archivesBaseName = "kafka-streams" ext.buildStreamsVersionFileName = "kafka-streams-version.properties" diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index aad58b0ddd8cd..63ed7ab238b13 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -269,6 +269,23 @@ + + + + + + + + + + + + + + + + + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 76690bbc829c3..1cfc630c0ea95 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -253,6 +253,10 @@ + + + diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 9340e3c310371..30193bc19f6c6 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -71,6 +71,7 @@ versions += [ jacoco: "0.8.5", jetty: "9.4.33.v20201020", jersey: "2.31", + jline: "3.12.1", jmh: "1.27", hamcrest: "2.2", log4j: "1.2.17", @@ -149,6 +150,7 @@ libs += [ jettyServlets: "org.eclipse.jetty:jetty-servlets:$versions.jetty", jerseyContainerServlet: "org.glassfish.jersey.containers:jersey-container-servlet:$versions.jersey", jerseyHk2: "org.glassfish.jersey.inject:jersey-hk2:$versions.jersey", + jline: "org.jline:jline:$versions.jline", jmhCore: "org.openjdk.jmh:jmh-core:$versions.jmh", jmhCoreBenchmarks: "org.openjdk.jmh:jmh-core-benchmarks:$versions.jmh", jmhGeneratorAnnProcess: "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh", diff --git a/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java b/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java new file mode 100644 index 0000000000000..ef85314e0ef2f --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.metalog; + +import org.apache.kafka.common.protocol.ApiMessage; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.queue.EventQueue; +import org.apache.kafka.queue.KafkaEventQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; + +/** + * The LocalLogManager is a test implementation that relies on the contents of memory. + */ +public final class LocalLogManager implements MetaLogManager, AutoCloseable { + interface LocalBatch { + int size(); + } + + static class LeaderChangeBatch implements LocalBatch { + private final MetaLogLeader newLeader; + + LeaderChangeBatch(MetaLogLeader newLeader) { + this.newLeader = newLeader; + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof LeaderChangeBatch)) return false; + LeaderChangeBatch other = (LeaderChangeBatch) o; + if (!other.newLeader.equals(newLeader)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(newLeader); + } + + @Override + public String toString() { + return "LeaderChangeBatch(newLeader=" + newLeader + ")"; + } + } + + static class LocalRecordBatch implements LocalBatch { + private final List records; + + LocalRecordBatch(List records) { + this.records = records; + } + + @Override + public int size() { + return records.size(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof LocalRecordBatch)) return false; + LocalRecordBatch other = (LocalRecordBatch) o; + if (!other.records.equals(records)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(records); + } + + @Override + public String toString() { + return "LocalRecordBatch(records=" + records + ")"; + } + } + + public static class SharedLogData { + private final Logger log = LoggerFactory.getLogger(SharedLogData.class); + private final HashMap logManagers = new HashMap<>(); + private final TreeMap batches = new TreeMap<>(); + private MetaLogLeader leader = new MetaLogLeader(-1, -1); + private long prevOffset = -1; + + synchronized void registerLogManager(LocalLogManager logManager) { + if (logManagers.put(logManager.nodeId(), logManager) != null) { + throw new RuntimeException("Can't have multiple LocalLogManagers " + + "with id " + logManager.nodeId()); + } + electLeaderIfNeeded(); + } + + synchronized void unregisterLogManager(LocalLogManager logManager) { + if (!logManagers.remove(logManager.nodeId(), logManager)) { + throw new RuntimeException("Log manager " + logManager.nodeId() + + " was not found."); + } + } + + synchronized long tryAppend(int nodeId, long epoch, LocalBatch batch) { + if (epoch != leader.epoch()) { + log.trace("tryAppend(nodeId={}, epoch={}): the provided epoch does not " + + "match the current leader epoch of {}.", nodeId, epoch, leader.epoch()); + return Long.MAX_VALUE; + } + if (nodeId != leader.nodeId()) { + log.trace("tryAppend(nodeId={}, epoch={}): the given node id does not " + + "match the current leader id of {}.", nodeId, leader.nodeId()); + return Long.MAX_VALUE; + } + log.trace("tryAppend(nodeId={}): appending {}.", nodeId, batch); + long offset = append(batch); + electLeaderIfNeeded(); + return offset; + } + + synchronized long append(LocalBatch batch) { + prevOffset += batch.size(); + log.debug("append(batch={}, prevOffset={})", batch, prevOffset); + batches.put(prevOffset, batch); + if (batch instanceof LeaderChangeBatch) { + LeaderChangeBatch leaderChangeBatch = (LeaderChangeBatch) batch; + leader = leaderChangeBatch.newLeader; + } + for (LocalLogManager logManager : logManagers.values()) { + logManager.scheduleLogCheck(); + } + return prevOffset; + } + + synchronized void electLeaderIfNeeded() { + if (leader.nodeId() != -1 || logManagers.isEmpty()) { + return; + } + int nextLeaderIndex = ThreadLocalRandom.current().nextInt(logManagers.size()); + Iterator iter = logManagers.keySet().iterator(); + Integer nextLeaderNode = null; + for (int i = 0; i <= nextLeaderIndex; i++) { + nextLeaderNode = iter.next(); + } + MetaLogLeader newLeader = new MetaLogLeader(nextLeaderNode, leader.epoch() + 1); + log.info("Elected new leader: {}.", newLeader); + append(new LeaderChangeBatch(newLeader)); + } + + synchronized Entry nextBatch(long offset) { + Entry entry = batches.higherEntry(offset); + if (entry == null) { + return null; + } + return new SimpleImmutableEntry<>(entry.getKey(), entry.getValue()); + } + } + + private static class MetaLogListenerData { + private long offset = -1; + private final MetaLogListener listener; + + MetaLogListenerData(MetaLogListener listener) { + this.listener = listener; + } + } + + private final Logger log; + + private final int nodeId; + + private final SharedLogData shared; + + private final EventQueue eventQueue; + + private boolean initialized = false; + + private boolean shutdown = false; + + private long maxReadOffset = Long.MAX_VALUE; + + private final List listeners = new ArrayList<>(); + + private volatile MetaLogLeader leader = new MetaLogLeader(-1, -1); + + public LocalLogManager(LogContext logContext, + int nodeId, + SharedLogData shared, + String threadNamePrefix) { + this.log = logContext.logger(LocalLogManager.class); + this.nodeId = nodeId; + this.shared = shared; + this.eventQueue = new KafkaEventQueue(Time.SYSTEM, logContext, threadNamePrefix); + shared.registerLogManager(this); + } + + private void scheduleLogCheck() { + eventQueue.append(() -> { + try { + log.debug("Node {}: running log check.", nodeId); + int numEntriesFound = 0; + for (MetaLogListenerData listenerData : listeners) { + while (true) { + Entry entry = shared.nextBatch(listenerData.offset); + if (entry == null) { + log.trace("Node {}: reached the end of the log after finding " + + "{} entries.", nodeId, numEntriesFound); + break; + } + long entryOffset = entry.getKey(); + if (entryOffset > maxReadOffset) { + log.trace("Node {}: after {} entries, not reading the next " + + "entry because its offset is {}, and maxReadOffset is {}.", + nodeId, numEntriesFound, entryOffset, maxReadOffset); + break; + } + if (entry.getValue() instanceof LeaderChangeBatch) { + LeaderChangeBatch batch = (LeaderChangeBatch) entry.getValue(); + log.trace("Node {}: handling LeaderChange to {}.", + nodeId, batch.newLeader); + listenerData.listener.handleNewLeader(batch.newLeader); + if (batch.newLeader.epoch() > leader.epoch()) { + leader = batch.newLeader; + } + } else if (entry.getValue() instanceof LocalRecordBatch) { + LocalRecordBatch batch = (LocalRecordBatch) entry.getValue(); + log.trace("Node {}: handling LocalRecordBatch with offset {}.", + nodeId, entryOffset); + listenerData.listener.handleCommits(entryOffset, batch.records); + } + numEntriesFound++; + listenerData.offset = entryOffset; + } + } + log.trace("Completed log check for node " + nodeId); + } catch (Exception e) { + log.error("Exception while handling log check", e); + } + }); + } + + public void beginShutdown() { + eventQueue.beginShutdown("beginShutdown", () -> { + try { + if (initialized && !shutdown) { + log.debug("Node {}: beginning shutdown.", nodeId); + renounce(leader.epoch()); + for (MetaLogListenerData listenerData : listeners) { + listenerData.listener.beginShutdown(); + } + shared.unregisterLogManager(this); + } + } catch (Exception e) { + log.error("Unexpected exception while sending beginShutdown callbacks", e); + } + shutdown = true; + }); + } + + @Override + public void close() throws InterruptedException { + log.debug("Node {}: closing.", nodeId); + beginShutdown(); + eventQueue.close(); + } + + @Override + public void initialize() throws Exception { + eventQueue.append(() -> { + log.debug("initialized local log manager for node " + nodeId); + initialized = true; + }); + } + + @Override + public void register(MetaLogListener listener) throws Exception { + CompletableFuture future = new CompletableFuture<>(); + eventQueue.append(() -> { + if (shutdown) { + log.info("Node {}: can't register because local log manager has " + + "already been shut down.", nodeId); + future.complete(null); + } else if (initialized) { + log.info("Node {}: registered MetaLogListener.", nodeId); + listeners.add(new MetaLogListenerData(listener)); + shared.electLeaderIfNeeded(); + scheduleLogCheck(); + future.complete(null); + } else { + log.info("Node {}: can't register because local log manager has not " + + "been initialized.", nodeId); + future.completeExceptionally(new RuntimeException( + "LocalLogManager was not initialized.")); + } + }); + future.get(); + } + + @Override + public long scheduleWrite(long epoch, List batch) { + return shared.tryAppend(nodeId, leader.epoch(), new LocalRecordBatch( + batch.stream().map(r -> r.message()).collect(Collectors.toList()))); + } + + @Override + public void renounce(long epoch) { + MetaLogLeader curLeader = leader; + MetaLogLeader nextLeader = new MetaLogLeader(-1, curLeader.epoch() + 1); + shared.tryAppend(nodeId, curLeader.epoch(), new LeaderChangeBatch(nextLeader)); + } + + @Override + public MetaLogLeader leader() { + return leader; + } + + @Override + public int nodeId() { + return nodeId; + } + + public List listeners() { + final CompletableFuture> future = new CompletableFuture<>(); + eventQueue.append(() -> { + future.complete(listeners.stream().map(l -> l.listener).collect(Collectors.toList())); + }); + try { + return future.get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + public void setMaxReadOffset(long maxReadOffset) { + CompletableFuture future = new CompletableFuture<>(); + eventQueue.append(() -> { + log.trace("Node {}: set maxReadOffset to {}.", nodeId, maxReadOffset); + this.maxReadOffset = maxReadOffset; + scheduleLogCheck(); + future.complete(null); + }); + try { + future.get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/metadata/src/main/resources/common/metadata/IsrChangeRecord.json b/metadata/src/main/resources/common/metadata/IsrChangeRecord.json index e45839090957c..fd8d834178744 100644 --- a/metadata/src/main/resources/common/metadata/IsrChangeRecord.json +++ b/metadata/src/main/resources/common/metadata/IsrChangeRecord.json @@ -28,6 +28,8 @@ { "name": "Leader", "type": "int32", "versions": "0+", "default": "-1", "about": "The lead replica, or -1 if there is no leader." }, { "name": "LeaderEpoch", "type": "int32", "versions": "0+", "default": "-1", - "about": "An epoch that gets incremented each time we change the ISR." } + "about": "An epoch that gets incremented each time we change the partition leader." }, + { "name": "PartitionEpoch", "type": "int32", "versions": "0+", "default": "-1", + "about": "An epoch that gets incremented each time we change anything in the partition." } ] } diff --git a/metadata/src/main/resources/common/metadata/PartitionRecord.json b/metadata/src/main/resources/common/metadata/PartitionRecord.json index 79c24c2f1ac1e..5cc7d1328c9dc 100644 --- a/metadata/src/main/resources/common/metadata/PartitionRecord.json +++ b/metadata/src/main/resources/common/metadata/PartitionRecord.json @@ -34,6 +34,8 @@ { "name": "Leader", "type": "int32", "versions": "0+", "default": "-1", "about": "The lead replica, or -1 if there is no leader." }, { "name": "LeaderEpoch", "type": "int32", "versions": "0+", "default": "-1", - "about": "An epoch that gets incremented each time we change the ISR." } + "about": "An epoch that gets incremented each time we change the partition leader." }, + { "name": "PartitionEpoch", "type": "int32", "versions": "0+", "default": "-1", + "about": "An epoch that gets incremented each time we change anything in the partition." } ] } diff --git a/metadata/src/test/java/org/apache/kafka/metadata/MetadataParserTest.java b/metadata/src/test/java/org/apache/kafka/metadata/MetadataParserTest.java index 6d673b3afddd4..41e968c4d5c8d 100644 --- a/metadata/src/test/java/org/apache/kafka/metadata/MetadataParserTest.java +++ b/metadata/src/test/java/org/apache/kafka/metadata/MetadataParserTest.java @@ -82,7 +82,7 @@ public void testMaxSerializedEventSizeCheck() { PartitionRecord partitionRecord = new PartitionRecord(). setReplicas(longReplicaList); ObjectSerializationCache cache = new ObjectSerializationCache(); - assertEquals("Event size would be 33554478, but the maximum serialized event " + + assertEquals("Event size would be 33554482, but the maximum serialized event " + "size is 33554432", assertThrows(RuntimeException.class, () -> { MetadataParser.size(partitionRecord, (short) 0, cache); }).getMessage()); diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTest.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTest.java index 9d4eb8b594e29..9dd6262ff66f9 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTest.java @@ -51,7 +51,7 @@ public void testCreateAndClose() throws Exception { } /** - * Test that the local log maanger will claim leadership. + * Test that the local log manager will claim leadership. */ @Test public void testClaimsLeadership() throws Exception { diff --git a/settings.gradle b/settings.gradle index 55f77f3bed379..fedfa9a650cc4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -29,6 +29,7 @@ include 'clients', 'log4j-appender', 'metadata', 'raft', + 'shell', 'streams', 'streams:examples', 'streams:streams-scala', diff --git a/shell/src/main/java/org/apache/kafka/shell/CatCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/CatCommandHandler.java new file mode 100644 index 0000000000000..3fc942795652d --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/CatCommandHandler.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.kafka.shell.MetadataNode.DirectoryNode; +import org.apache.kafka.shell.MetadataNode.FileNode; +import org.jline.reader.Candidate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Implements the cat command. + */ +public final class CatCommandHandler implements Commands.Handler { + private static final Logger log = LoggerFactory.getLogger(CatCommandHandler.class); + + public final static Commands.Type TYPE = new CatCommandType(); + + public static class CatCommandType implements Commands.Type { + private CatCommandType() { + } + + @Override + public String name() { + return "cat"; + } + + @Override + public String description() { + return "Show the contents of metadata nodes."; + } + + @Override + public boolean shellOnly() { + return false; + } + + @Override + public void addArguments(ArgumentParser parser) { + parser.addArgument("targets"). + nargs("+"). + help("The metadata nodes to display."); + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + return new CatCommandHandler(namespace.getList("targets")); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + CommandUtils.completePath(nodeManager, nextWords.get(nextWords.size() - 1), + candidates); + } + } + + private final List targets; + + public CatCommandHandler(List targets) { + this.targets = targets; + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) throws Exception { + log.trace("cat " + targets); + for (String target : targets) { + manager.visit(new GlobVisitor(target, entryOption -> { + if (entryOption.isPresent()) { + MetadataNode node = entryOption.get().node(); + if (node instanceof DirectoryNode) { + writer.println("cat: " + target + ": Is a directory"); + } else if (node instanceof FileNode) { + FileNode fileNode = (FileNode) node; + writer.println(fileNode.contents()); + } + } else { + writer.println("cat: " + target + ": No such file or directory."); + } + })); + } + } + + @Override + public int hashCode() { + return targets.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CatCommandHandler)) return false; + CatCommandHandler o = (CatCommandHandler) other; + if (!Objects.equals(o.targets, targets)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/CdCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/CdCommandHandler.java new file mode 100644 index 0000000000000..8d270e543285c --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/CdCommandHandler.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.kafka.shell.MetadataNode.DirectoryNode; +import org.jline.reader.Candidate; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; + +/** + * Implements the cd command. + */ +public final class CdCommandHandler implements Commands.Handler { + public final static Commands.Type TYPE = new CdCommandType(); + + public static class CdCommandType implements Commands.Type { + private CdCommandType() { + } + + @Override + public String name() { + return "cd"; + } + + @Override + public String description() { + return "Set the current working directory."; + } + + @Override + public boolean shellOnly() { + return true; + } + + @Override + public void addArguments(ArgumentParser parser) { + parser.addArgument("target"). + nargs("?"). + help("The directory to change to."); + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + return new CdCommandHandler(Optional.ofNullable(namespace.getString("target"))); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + if (nextWords.size() == 1) { + CommandUtils.completePath(nodeManager, nextWords.get(0), candidates); + } + } + } + + private final Optional target; + + public CdCommandHandler(Optional target) { + this.target = target; + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) throws Exception { + String effectiveTarget = target.orElse("/"); + manager.visit(new Consumer() { + @Override + public void accept(MetadataNodeManager.Data data) { + new GlobVisitor(effectiveTarget, entryOption -> { + if (entryOption.isPresent()) { + if (!(entryOption.get().node() instanceof DirectoryNode)) { + writer.println("cd: " + effectiveTarget + ": not a directory."); + } else { + data.setWorkingDirectory(entryOption.get().absolutePath()); + } + } else { + writer.println("cd: " + effectiveTarget + ": no such directory."); + } + }).accept(data); + } + }); + } + + @Override + public int hashCode() { + return target.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CdCommandHandler)) return false; + CdCommandHandler o = (CdCommandHandler) other; + if (!o.target.equals(target)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/CommandUtils.java b/shell/src/main/java/org/apache/kafka/shell/CommandUtils.java new file mode 100644 index 0000000000000..0639172e95d38 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/CommandUtils.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import org.apache.kafka.shell.MetadataNode.DirectoryNode; +import org.jline.reader.Candidate; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; + +/** + * Utility functions for command handlers. + */ +public final class CommandUtils { + /** + * Convert a list of paths into the effective list of paths which should be used. + * Empty strings will be removed. If no paths are given, the current working + * directory will be used. + * + * @param paths The input paths. Non-null. + * + * @return The output paths. + */ + public static List getEffectivePaths(List paths) { + List effectivePaths = new ArrayList<>(); + for (String path : paths) { + if (!path.isEmpty()) { + effectivePaths.add(path); + } + } + if (effectivePaths.isEmpty()) { + effectivePaths.add("."); + } + return effectivePaths; + } + + /** + * Generate a list of potential completions for a prefix of a command name. + * + * @param commandPrefix The command prefix. Non-null. + * @param candidates The list to add the output completions to. + */ + public static void completeCommand(String commandPrefix, List candidates) { + String command = Commands.TYPES.ceilingKey(commandPrefix); + while (command != null && command.startsWith(commandPrefix)) { + candidates.add(new Candidate(command)); + command = Commands.TYPES.higherKey(command); + } + } + + /** + * Convert a path to a list of path components. + * Multiple slashes in a row are treated the same as a single slash. + * Trailing slashes are ignored. + */ + public static List splitPath(String path) { + List results = new ArrayList<>(); + String[] components = path.split("/"); + for (int i = 0; i < components.length; i++) { + if (!components[i].isEmpty()) { + results.add(components[i]); + } + } + return results; + } + + public static List stripDotPathComponents(List input) { + List output = new ArrayList<>(); + for (String string : input) { + if (string.equals("..")) { + if (output.size() > 0) { + output.remove(output.size() - 1); + } + } else if (!string.equals(".")) { + output.add(string); + } + } + return output; + } + + /** + * Generate a list of potential completions for a path. + * + * @param nodeManager The NodeManager. + * @param pathPrefix The path prefix. Non-null. + * @param candidates The list to add the output completions to. + */ + public static void completePath(MetadataNodeManager nodeManager, + String pathPrefix, + List candidates) throws Exception { + nodeManager.visit(data -> { + String absolutePath = pathPrefix.startsWith("/") ? + pathPrefix : data.workingDirectory() + "/" + pathPrefix; + List pathComponents = stripDotPathComponents(splitPath(absolutePath)); + DirectoryNode directory = data.root(); + int numDirectories = pathPrefix.endsWith("/") ? + pathComponents.size() : pathComponents.size() - 1; + for (int i = 0; i < numDirectories; i++) { + MetadataNode node = directory.child(pathComponents.get(i)); + if (node == null || !(node instanceof DirectoryNode)) { + return; + } + directory = (DirectoryNode) node; + } + String lastComponent = ""; + if (numDirectories >= 0 && numDirectories < pathComponents.size()) { + lastComponent = pathComponents.get(numDirectories); + } + Entry candidate = + directory.children().ceilingEntry(lastComponent); + String effectivePrefix; + int lastSlash = pathPrefix.lastIndexOf('/'); + if (lastSlash < 0) { + effectivePrefix = ""; + } else { + effectivePrefix = pathPrefix.substring(0, lastSlash + 1); + } + while (candidate != null && candidate.getKey().startsWith(lastComponent)) { + StringBuilder candidateBuilder = new StringBuilder(); + candidateBuilder.append(effectivePrefix).append(candidate.getKey()); + boolean complete = true; + if (candidate.getValue() instanceof DirectoryNode) { + candidateBuilder.append("/"); + complete = false; + } + candidates.add(new Candidate(candidateBuilder.toString(), + candidateBuilder.toString(), null, null, null, null, complete)); + candidate = directory.children().higherEntry(candidate.getKey()); + } + }); + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/Commands.java b/shell/src/main/java/org/apache/kafka/shell/Commands.java new file mode 100644 index 0000000000000..db16411ebae3b --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/Commands.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.ArgumentParserException; +import net.sourceforge.argparse4j.inf.Namespace; +import net.sourceforge.argparse4j.inf.Subparser; +import net.sourceforge.argparse4j.inf.Subparsers; +import net.sourceforge.argparse4j.internal.HelpScreenException; +import org.jline.reader.Candidate; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.TreeMap; + +/** + * The commands for the Kafka metadata tool. + */ +public final class Commands { + /** + * A map from command names to command types. + */ + static final NavigableMap TYPES; + + static { + TreeMap typesMap = new TreeMap<>(); + for (Type type : Arrays.asList( + CatCommandHandler.TYPE, + CdCommandHandler.TYPE, + ExitCommandHandler.TYPE, + FindCommandHandler.TYPE, + HelpCommandHandler.TYPE, + HistoryCommandHandler.TYPE, + LsCommandHandler.TYPE, + ManCommandHandler.TYPE, + PwdCommandHandler.TYPE)) { + typesMap.put(type.name(), type); + } + TYPES = Collections.unmodifiableNavigableMap(typesMap); + } + + /** + * Command handler objects are instantiated with specific arguments to + * execute commands. + */ + public interface Handler { + void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) throws Exception; + } + + /** + * An object which describes a type of command handler. This includes + * information like its name, help text, and whether it should be accessible + * from non-interactive mode. + */ + public interface Type { + String name(); + String description(); + boolean shellOnly(); + void addArguments(ArgumentParser parser); + Handler createHandler(Namespace namespace); + void completeNext(MetadataNodeManager nodeManager, + List nextWords, + List candidates) throws Exception; + } + + private final ArgumentParser parser; + + /** + * Create the commands instance. + * + * @param addShellCommands True if we should include the shell-only commands. + */ + public Commands(boolean addShellCommands) { + this.parser = ArgumentParsers.newArgumentParser("", false); + Subparsers subparsers = this.parser.addSubparsers().dest("command"); + for (Type type : TYPES.values()) { + if (addShellCommands || !type.shellOnly()) { + Subparser subParser = subparsers.addParser(type.name()); + subParser.help(type.description()); + type.addArguments(subParser); + } + } + } + + ArgumentParser parser() { + return parser; + } + + /** + * Handle the given command. + * + * In general this function should not throw exceptions. Instead, it should + * return ErroneousCommandHandler if the input was invalid. + * + * @param arguments The command line arguments. + * @return The command handler. + */ + public Handler parseCommand(List arguments) { + List trimmedArguments = new ArrayList<>(arguments); + while (true) { + if (trimmedArguments.isEmpty()) { + return new NoOpCommandHandler(); + } + String last = trimmedArguments.get(trimmedArguments.size() - 1); + if (!last.isEmpty()) { + break; + } + trimmedArguments.remove(trimmedArguments.size() - 1); + } + Namespace namespace; + try { + namespace = parser.parseArgs(trimmedArguments.toArray(new String[0])); + } catch (HelpScreenException e) { + return new NoOpCommandHandler(); + } catch (ArgumentParserException e) { + return new ErroneousCommandHandler(e.getMessage()); + } + String command = namespace.get("command"); + if (!command.equals(trimmedArguments.get(0))) { + return new ErroneousCommandHandler("invalid choice: '" + + trimmedArguments.get(0) + "': did you mean '" + command + "'?"); + } + Type type = TYPES.get(command); + if (type == null) { + return new ErroneousCommandHandler("Unknown command specified: " + command); + } else { + return type.createHandler(namespace); + } + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/ErroneousCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/ErroneousCommandHandler.java new file mode 100644 index 0000000000000..d52c55f963087 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/ErroneousCommandHandler.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import java.io.PrintWriter; +import java.util.Objects; +import java.util.Optional; + +/** + * Handles erroneous commands. + */ +public final class ErroneousCommandHandler implements Commands.Handler { + private final String message; + + public ErroneousCommandHandler(String message) { + this.message = message; + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) { + writer.println(message); + } + + @Override + public int hashCode() { + return Objects.hashCode(message); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ErroneousCommandHandler)) return false; + ErroneousCommandHandler o = (ErroneousCommandHandler) other; + if (!Objects.equals(o.message, message)) return false; + return true; + } + + @Override + public String toString() { + return "ErroneousCommandHandler(" + message + ")"; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/ExitCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/ExitCommandHandler.java new file mode 100644 index 0000000000000..2b11b352a8f1e --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/ExitCommandHandler.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.kafka.common.utils.Exit; +import org.jline.reader.Candidate; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Optional; + +/** + * Implements the exit command. + */ +public final class ExitCommandHandler implements Commands.Handler { + public final static Commands.Type TYPE = new ExitCommandType(); + + public static class ExitCommandType implements Commands.Type { + private ExitCommandType() { + } + + @Override + public String name() { + return "exit"; + } + + @Override + public String description() { + return "Exit the metadata shell."; + } + + @Override + public boolean shellOnly() { + return true; + } + + @Override + public void addArguments(ArgumentParser parser) { + // nothing to do + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + return new ExitCommandHandler(); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + // nothing to do + } + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) { + Exit.exit(0); + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ExitCommandHandler)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/FindCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/FindCommandHandler.java new file mode 100644 index 0000000000000..6d9ae44654ba3 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/FindCommandHandler.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.kafka.shell.MetadataNode.DirectoryNode; +import org.jline.reader.Candidate; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; + +/** + * Implements the find command. + */ +public final class FindCommandHandler implements Commands.Handler { + public final static Commands.Type TYPE = new FindCommandType(); + + public static class FindCommandType implements Commands.Type { + private FindCommandType() { + } + + @Override + public String name() { + return "find"; + } + + @Override + public String description() { + return "Search for nodes in the directory hierarchy."; + } + + @Override + public boolean shellOnly() { + return false; + } + + @Override + public void addArguments(ArgumentParser parser) { + parser.addArgument("paths"). + nargs("*"). + help("The paths to start at."); + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + return new FindCommandHandler(namespace.getList("paths")); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + CommandUtils.completePath(nodeManager, nextWords.get(nextWords.size() - 1), + candidates); + } + } + + private final List paths; + + public FindCommandHandler(List paths) { + this.paths = paths; + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) throws Exception { + for (String path : CommandUtils.getEffectivePaths(paths)) { + manager.visit(new GlobVisitor(path, entryOption -> { + if (entryOption.isPresent()) { + find(writer, path, entryOption.get().node()); + } else { + writer.println("find: " + path + ": no such file or directory."); + } + })); + } + } + + private void find(PrintWriter writer, String path, MetadataNode node) { + writer.println(path); + if (node instanceof DirectoryNode) { + DirectoryNode directory = (DirectoryNode) node; + for (Entry entry : directory.children().entrySet()) { + String nextPath = path.equals("/") ? + path + entry.getKey() : path + "/" + entry.getKey(); + find(writer, nextPath, entry.getValue()); + } + } + } + + @Override + public int hashCode() { + return Objects.hashCode(paths); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof FindCommandHandler)) return false; + FindCommandHandler o = (FindCommandHandler) other; + if (!Objects.equals(o.paths, paths)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/GlobComponent.java b/shell/src/main/java/org/apache/kafka/shell/GlobComponent.java new file mode 100644 index 0000000000000..b93382b258e22 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/GlobComponent.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.regex.Pattern; + +/** + * Implements a per-path-component glob. + */ +public final class GlobComponent { + private static final Logger log = LoggerFactory.getLogger(GlobComponent.class); + + /** + * Returns true if the character is a special character for regular expressions. + */ + private static boolean isRegularExpressionSpecialCharacter(char ch) { + switch (ch) { + case '$': + case '(': + case ')': + case '+': + case '.': + case '[': + case ']': + case '^': + case '{': + case '|': + return true; + default: + break; + } + return false; + } + + /** + * Returns true if the character is a special character for globs. + */ + private static boolean isGlobSpecialCharacter(char ch) { + switch (ch) { + case '*': + case '?': + case '\\': + case '{': + case '}': + return true; + default: + break; + } + return false; + } + + /** + * Converts a glob string to a regular expression string. + * Returns null if the glob should be handled as a literal (can only match one string). + * Throws an exception if the glob is malformed. + */ + static String toRegularExpression(String glob) { + StringBuilder output = new StringBuilder("^"); + boolean literal = true; + boolean processingGroup = false; + + for (int i = 0; i < glob.length(); ) { + char c = glob.charAt(i++); + switch (c) { + case '?': + literal = false; + output.append("."); + break; + case '*': + literal = false; + output.append(".*"); + break; + case '\\': + if (i == glob.length()) { + output.append(c); + } else { + char next = glob.charAt(i); + i++; + if (isGlobSpecialCharacter(next) || + isRegularExpressionSpecialCharacter(next)) { + output.append('\\'); + } + output.append(next); + } + break; + case '{': + if (processingGroup) { + throw new RuntimeException("Can't nest glob groups."); + } + literal = false; + output.append("(?:(?:"); + processingGroup = true; + break; + case ',': + if (processingGroup) { + literal = false; + output.append(")|(?:"); + } else { + output.append(c); + } + break; + case '}': + if (processingGroup) { + literal = false; + output.append("))"); + processingGroup = false; + } else { + output.append(c); + } + break; + // TODO: handle character ranges + default: + if (isRegularExpressionSpecialCharacter(c)) { + output.append('\\'); + } + output.append(c); + } + } + if (processingGroup) { + throw new RuntimeException("Unterminated glob group."); + } + if (literal) { + return null; + } + output.append('$'); + return output.toString(); + } + + private final String component; + private final Pattern pattern; + + public GlobComponent(String component) { + this.component = component; + Pattern newPattern = null; + try { + String regularExpression = toRegularExpression(component); + if (regularExpression != null) { + newPattern = Pattern.compile(regularExpression); + } + } catch (RuntimeException e) { + log.debug("Invalid glob pattern: " + e.getMessage()); + } + this.pattern = newPattern; + } + + public String component() { + return component; + } + + public boolean literal() { + return pattern == null; + } + + public boolean matches(String nodeName) { + if (pattern == null) { + return component.equals(nodeName); + } else { + return pattern.matcher(nodeName).matches(); + } + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/GlobVisitor.java b/shell/src/main/java/org/apache/kafka/shell/GlobVisitor.java new file mode 100644 index 0000000000000..8081b7e4450ad --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/GlobVisitor.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import java.util.Arrays; +import java.util.List; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; + +/** + * Visits metadata paths based on a glob string. + */ +public final class GlobVisitor implements Consumer { + private final String glob; + private final Consumer> handler; + + public GlobVisitor(String glob, + Consumer> handler) { + this.glob = glob; + this.handler = handler; + } + + public static class MetadataNodeInfo { + private final String[] path; + private final MetadataNode node; + + MetadataNodeInfo(String[] path, MetadataNode node) { + this.path = path; + this.node = node; + } + + public String[] path() { + return path; + } + + public MetadataNode node() { + return node; + } + + public String lastPathComponent() { + if (path.length == 0) { + return "/"; + } else { + return path[path.length - 1]; + } + } + + public String absolutePath() { + return "/" + String.join("/", path); + } + + @Override + public int hashCode() { + return Objects.hash(path, node); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof MetadataNodeInfo)) return false; + MetadataNodeInfo other = (MetadataNodeInfo) o; + if (!Arrays.equals(path, other.path)) return false; + if (!node.equals(other.node)) return false; + return true; + } + + @Override + public String toString() { + StringBuilder bld = new StringBuilder("MetadataNodeInfo(path="); + for (int i = 0; i < path.length; i++) { + bld.append("/"); + bld.append(path[i]); + } + bld.append(", node=").append(node).append(")"); + return bld.toString(); + } + } + + @Override + public void accept(MetadataNodeManager.Data data) { + String fullGlob = glob.startsWith("/") ? glob : + data.workingDirectory() + "/" + glob; + List globComponents = + CommandUtils.stripDotPathComponents(CommandUtils.splitPath(fullGlob)); + if (!accept(globComponents, 0, data.root(), new String[0])) { + handler.accept(Optional.empty()); + } + } + + private boolean accept(List globComponents, + int componentIndex, + MetadataNode node, + String[] path) { + if (componentIndex >= globComponents.size()) { + handler.accept(Optional.of(new MetadataNodeInfo(path, node))); + return true; + } + String globComponentString = globComponents.get(componentIndex); + GlobComponent globComponent = new GlobComponent(globComponentString); + if (globComponent.literal()) { + if (!(node instanceof MetadataNode.DirectoryNode)) { + return false; + } + MetadataNode.DirectoryNode directory = (MetadataNode.DirectoryNode) node; + MetadataNode child = directory.child(globComponent.component()); + if (child == null) { + return false; + } + String[] newPath = new String[path.length + 1]; + System.arraycopy(path, 0, newPath, 0, path.length); + newPath[path.length] = globComponent.component(); + return accept(globComponents, componentIndex + 1, child, newPath); + } + if (!(node instanceof MetadataNode.DirectoryNode)) { + return false; + } + MetadataNode.DirectoryNode directory = (MetadataNode.DirectoryNode) node; + boolean matchedAny = false; + for (Entry entry : directory.children().entrySet()) { + String nodeName = entry.getKey(); + if (globComponent.matches(nodeName)) { + String[] newPath = new String[path.length + 1]; + System.arraycopy(path, 0, newPath, 0, path.length); + newPath[path.length] = nodeName; + if (accept(globComponents, componentIndex + 1, entry.getValue(), newPath)) { + matchedAny = true; + } + } + } + return matchedAny; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/HelpCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/HelpCommandHandler.java new file mode 100644 index 0000000000000..829274eefccf4 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/HelpCommandHandler.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.jline.reader.Candidate; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Optional; + +/** + * Implements the help command. + */ +public final class HelpCommandHandler implements Commands.Handler { + public final static Commands.Type TYPE = new HelpCommandType(); + + public static class HelpCommandType implements Commands.Type { + private HelpCommandType() { + } + + @Override + public String name() { + return "help"; + } + + @Override + public String description() { + return "Display this help message."; + } + + @Override + public boolean shellOnly() { + return true; + } + + @Override + public void addArguments(ArgumentParser parser) { + // nothing to do + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + return new HelpCommandHandler(); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + // nothing to do + } + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) { + writer.printf("Welcome to the Apache Kafka metadata shell.%n%n"); + new Commands(true).parser().printHelp(writer); + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof HelpCommandHandler)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/HistoryCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/HistoryCommandHandler.java new file mode 100644 index 0000000000000..edf9def4c878e --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/HistoryCommandHandler.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.jline.reader.Candidate; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Implements the history command. + */ +public final class HistoryCommandHandler implements Commands.Handler { + public final static Commands.Type TYPE = new HistoryCommandType(); + + public static class HistoryCommandType implements Commands.Type { + private HistoryCommandType() { + } + + @Override + public String name() { + return "history"; + } + + @Override + public String description() { + return "Print command history."; + } + + @Override + public boolean shellOnly() { + return true; + } + + @Override + public void addArguments(ArgumentParser parser) { + parser.addArgument("numEntriesToShow"). + nargs("?"). + type(Integer.class). + help("The number of entries to show."); + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + Integer numEntriesToShow = namespace.getInt("numEntriesToShow"); + return new HistoryCommandHandler(numEntriesToShow == null ? + Integer.MAX_VALUE : numEntriesToShow); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + // nothing to do + } + } + + private final int numEntriesToShow; + + public HistoryCommandHandler(int numEntriesToShow) { + this.numEntriesToShow = numEntriesToShow; + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) throws Exception { + if (!shell.isPresent()) { + throw new RuntimeException("The history command requires a shell."); + } + Iterator> iter = shell.get().history(numEntriesToShow); + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + writer.printf("% 5d %s%n", entry.getKey(), entry.getValue()); + } + } + + @Override + public int hashCode() { + return numEntriesToShow; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof HistoryCommandHandler)) return false; + HistoryCommandHandler o = (HistoryCommandHandler) other; + return o.numEntriesToShow == numEntriesToShow; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/InteractiveShell.java b/shell/src/main/java/org/apache/kafka/shell/InteractiveShell.java new file mode 100644 index 0000000000000..aa4d4ea56cf77 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/InteractiveShell.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import org.jline.reader.Candidate; +import org.jline.reader.Completer; +import org.jline.reader.EndOfFileException; +import org.jline.reader.History; +import org.jline.reader.LineReader; +import org.jline.reader.LineReaderBuilder; +import org.jline.reader.ParsedLine; +import org.jline.reader.Parser; +import org.jline.reader.UserInterruptException; +import org.jline.reader.impl.DefaultParser; +import org.jline.reader.impl.history.DefaultHistory; +import org.jline.terminal.Terminal; +import org.jline.terminal.TerminalBuilder; + +import java.io.IOException; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import java.util.Optional; + +/** + * The Kafka metadata shell. + */ +public final class InteractiveShell implements AutoCloseable { + static class MetadataShellCompleter implements Completer { + private final MetadataNodeManager nodeManager; + + MetadataShellCompleter(MetadataNodeManager nodeManager) { + this.nodeManager = nodeManager; + } + + @Override + public void complete(LineReader reader, ParsedLine line, List candidates) { + if (line.words().size() == 0) { + CommandUtils.completeCommand("", candidates); + } else if (line.words().size() == 1) { + CommandUtils.completeCommand(line.words().get(0), candidates); + } else { + Iterator iter = line.words().iterator(); + String command = iter.next(); + List nextWords = new ArrayList<>(); + while (iter.hasNext()) { + nextWords.add(iter.next()); + } + Commands.Type type = Commands.TYPES.get(command); + if (type == null) { + return; + } + try { + type.completeNext(nodeManager, nextWords, candidates); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + } + + private final MetadataNodeManager nodeManager; + private final Terminal terminal; + private final Parser parser; + private final History history; + private final MetadataShellCompleter completer; + private final LineReader reader; + + public InteractiveShell(MetadataNodeManager nodeManager) throws IOException { + this.nodeManager = nodeManager; + TerminalBuilder builder = TerminalBuilder.builder(). + system(true). + nativeSignals(true); + this.terminal = builder.build(); + this.parser = new DefaultParser(); + this.history = new DefaultHistory(); + this.completer = new MetadataShellCompleter(nodeManager); + this.reader = LineReaderBuilder.builder(). + terminal(terminal). + parser(parser). + history(history). + completer(completer). + option(LineReader.Option.AUTO_FRESH_LINE, false). + build(); + } + + public void runMainLoop() throws Exception { + terminal.writer().println("[ Kafka Metadata Shell ]"); + terminal.flush(); + Commands commands = new Commands(true); + while (true) { + try { + reader.readLine(">> "); + ParsedLine parsedLine = reader.getParsedLine(); + Commands.Handler handler = commands.parseCommand(parsedLine.words()); + handler.run(Optional.of(this), terminal.writer(), nodeManager); + terminal.writer().flush(); + } catch (UserInterruptException eof) { + // Handle the user pressing control-C. + terminal.writer().println("^C"); + } catch (EndOfFileException eof) { + return; + } + } + } + + public int screenWidth() { + return terminal.getWidth(); + } + + public Iterator> history(int numEntriesToShow) { + if (numEntriesToShow < 0) { + numEntriesToShow = 0; + } + int last = history.last(); + if (numEntriesToShow > last + 1) { + numEntriesToShow = last + 1; + } + int first = last - numEntriesToShow + 1; + if (first < history.first()) { + first = history.first(); + } + return new HistoryIterator(first, last); + } + + public class HistoryIterator implements Iterator> { + private int index; + private int last; + + HistoryIterator(int index, int last) { + this.index = index; + this.last = last; + } + + @Override + public boolean hasNext() { + return index <= last; + } + + @Override + public Entry next() { + if (index > last) { + throw new NoSuchElementException(); + } + int p = index++; + return new AbstractMap.SimpleImmutableEntry<>(p, history.get(p)); + } + } + + @Override + public void close() throws IOException { + terminal.close(); + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/LsCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/LsCommandHandler.java new file mode 100644 index 0000000000000..6260d122bfea3 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/LsCommandHandler.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.kafka.shell.GlobVisitor.MetadataNodeInfo; +import org.apache.kafka.shell.MetadataNode.DirectoryNode; +import org.apache.kafka.shell.MetadataNode.FileNode; +import org.jline.reader.Candidate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; + +/** + * Implements the ls command. + */ +public final class LsCommandHandler implements Commands.Handler { + private static final Logger log = LoggerFactory.getLogger(LsCommandHandler.class); + + public final static Commands.Type TYPE = new LsCommandType(); + + public static class LsCommandType implements Commands.Type { + private LsCommandType() { + } + + @Override + public String name() { + return "ls"; + } + + @Override + public String description() { + return "List metadata nodes."; + } + + @Override + public boolean shellOnly() { + return false; + } + + @Override + public void addArguments(ArgumentParser parser) { + parser.addArgument("targets"). + nargs("*"). + help("The metadata node paths to list."); + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + return new LsCommandHandler(namespace.getList("targets")); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + CommandUtils.completePath(nodeManager, nextWords.get(nextWords.size() - 1), + candidates); + } + } + + private final List targets; + + public LsCommandHandler(List targets) { + this.targets = targets; + } + + static class TargetDirectory { + private final String name; + private final List children; + + TargetDirectory(String name, List children) { + this.name = name; + this.children = children; + } + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) throws Exception { + List targetFiles = new ArrayList<>(); + List targetDirectories = new ArrayList<>(); + for (String target : CommandUtils.getEffectivePaths(targets)) { + manager.visit(new GlobVisitor(target, entryOption -> { + if (entryOption.isPresent()) { + MetadataNodeInfo info = entryOption.get(); + MetadataNode node = info.node(); + if (node instanceof DirectoryNode) { + DirectoryNode directory = (DirectoryNode) node; + List children = new ArrayList<>(); + children.addAll(directory.children().keySet()); + targetDirectories.add( + new TargetDirectory(info.lastPathComponent(), children)); + } else if (node instanceof FileNode) { + targetFiles.add(info.lastPathComponent()); + } + } else { + writer.println("ls: " + target + ": no such file or directory."); + } + })); + } + OptionalInt screenWidth = shell.isPresent() ? + OptionalInt.of(shell.get().screenWidth()) : OptionalInt.empty(); + log.trace("LS : targetFiles = {}, targetDirectories = {}, screenWidth = {}", + targetFiles, targetDirectories, screenWidth); + printTargets(writer, screenWidth, targetFiles, targetDirectories); + } + + static void printTargets(PrintWriter writer, + OptionalInt screenWidth, + List targetFiles, + List targetDirectories) { + printEntries(writer, "", screenWidth, targetFiles); + boolean needIntro = targetFiles.size() > 0 || targetDirectories.size() > 1; + boolean firstIntro = targetFiles.isEmpty(); + for (TargetDirectory targetDirectory : targetDirectories) { + String intro = ""; + if (needIntro) { + if (!firstIntro) { + intro = intro + String.format("%n"); + } + intro = intro + targetDirectory.name + ":"; + firstIntro = false; + } + log.trace("LS : targetDirectory name = {}, children = {}", + targetDirectory.name, targetDirectory.children); + printEntries(writer, intro, screenWidth, targetDirectory.children); + } + } + + static void printEntries(PrintWriter writer, + String intro, + OptionalInt screenWidth, + List entries) { + if (entries.isEmpty()) { + return; + } + if (!intro.isEmpty()) { + writer.println(intro); + } + ColumnSchema columnSchema = calculateColumnSchema(screenWidth, entries); + int numColumns = columnSchema.numColumns(); + int numLines = (entries.size() + numColumns - 1) / numColumns; + for (int line = 0; line < numLines; line++) { + StringBuilder output = new StringBuilder(); + for (int column = 0; column < numColumns; column++) { + int entryIndex = line + (column * columnSchema.entriesPerColumn()); + if (entryIndex < entries.size()) { + String entry = entries.get(entryIndex); + output.append(entry); + if (column < numColumns - 1) { + int width = columnSchema.columnWidth(column); + for (int i = 0; i < width - entry.length(); i++) { + output.append(" "); + } + } + } + } + writer.println(output.toString()); + } + } + + static ColumnSchema calculateColumnSchema(OptionalInt screenWidth, + List entries) { + if (!screenWidth.isPresent()) { + return new ColumnSchema(1, entries.size()); + } + int maxColumns = screenWidth.getAsInt() / 4; + if (maxColumns <= 1) { + return new ColumnSchema(1, entries.size()); + } + ColumnSchema[] schemas = new ColumnSchema[maxColumns]; + for (int numColumns = 1; numColumns <= maxColumns; numColumns++) { + schemas[numColumns - 1] = new ColumnSchema(numColumns, + (entries.size() + numColumns - 1) / numColumns); + } + for (int i = 0; i < entries.size(); i++) { + String entry = entries.get(i); + for (int s = 0; s < schemas.length; s++) { + ColumnSchema schema = schemas[s]; + schema.process(i, entry); + } + } + for (int s = schemas.length - 1; s > 0; s--) { + ColumnSchema schema = schemas[s]; + if (schema.columnWidths[schema.columnWidths.length - 1] != 0 && + schema.totalWidth() <= screenWidth.getAsInt()) { + return schema; + } + } + return schemas[0]; + } + + static class ColumnSchema { + private final int[] columnWidths; + private final int entriesPerColumn; + + ColumnSchema(int numColumns, int entriesPerColumn) { + this.columnWidths = new int[numColumns]; + this.entriesPerColumn = entriesPerColumn; + } + + ColumnSchema setColumnWidths(Integer... widths) { + for (int i = 0; i < widths.length; i++) { + columnWidths[i] = widths[i]; + } + return this; + } + + void process(int entryIndex, String output) { + int columnIndex = entryIndex / entriesPerColumn; + columnWidths[columnIndex] = Math.max( + columnWidths[columnIndex], output.length() + 2); + } + + int totalWidth() { + int total = 0; + for (int i = 0; i < columnWidths.length; i++) { + total += columnWidths[i]; + } + return total; + } + + int numColumns() { + return columnWidths.length; + } + + int columnWidth(int columnIndex) { + return columnWidths[columnIndex]; + } + + int entriesPerColumn() { + return entriesPerColumn; + } + + @Override + public int hashCode() { + return Objects.hash(columnWidths, entriesPerColumn); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ColumnSchema)) return false; + ColumnSchema other = (ColumnSchema) o; + if (entriesPerColumn != other.entriesPerColumn) return false; + if (!Arrays.equals(columnWidths, other.columnWidths)) return false; + return true; + } + + @Override + public String toString() { + StringBuilder bld = new StringBuilder("ColumnSchema(columnWidths=["); + String prefix = ""; + for (int i = 0; i < columnWidths.length; i++) { + bld.append(prefix); + bld.append(columnWidths[i]); + prefix = ", "; + } + bld.append("], entriesPerColumn=").append(entriesPerColumn).append(")"); + return bld.toString(); + } + } + + @Override + public int hashCode() { + return Objects.hashCode(targets); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof LsCommandHandler)) return false; + LsCommandHandler o = (LsCommandHandler) other; + if (!Objects.equals(o.targets, targets)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/ManCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/ManCommandHandler.java new file mode 100644 index 0000000000000..dcd0b8cd71629 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/ManCommandHandler.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.jline.reader.Candidate; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Optional; + +/** + * Implements the manual command. + */ +public final class ManCommandHandler implements Commands.Handler { + private final String cmd; + + public final static Commands.Type TYPE = new ManCommandType(); + + public static class ManCommandType implements Commands.Type { + private ManCommandType() { + } + + @Override + public String name() { + return "man"; + } + + @Override + public String description() { + return "Show the help text for a specific command."; + } + + @Override + public boolean shellOnly() { + return true; + } + + @Override + public void addArguments(ArgumentParser parser) { + parser.addArgument("cmd"). + nargs(1). + help("The command to get help text for."); + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + return new ManCommandHandler(namespace.getList("cmd").get(0)); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + if (nextWords.size() == 1) { + CommandUtils.completeCommand(nextWords.get(0), candidates); + } + } + } + + public ManCommandHandler(String cmd) { + this.cmd = cmd; + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) { + Commands.Type type = Commands.TYPES.get(cmd); + if (type == null) { + writer.println("man: unknown command " + cmd + + ". Type help to get a list of commands."); + } else { + ArgumentParser parser = ArgumentParsers.newArgumentParser(type.name(), false); + type.addArguments(parser); + writer.printf("%s: %s%n%n", cmd, type.description()); + parser.printHelp(writer); + } + } + + @Override + public int hashCode() { + return cmd.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ManCommandHandler)) return false; + ManCommandHandler o = (ManCommandHandler) other; + if (!o.cmd.equals(cmd)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/MetadataNode.java b/shell/src/main/java/org/apache/kafka/shell/MetadataNode.java new file mode 100644 index 0000000000000..3764a17b8b098 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/MetadataNode.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import java.util.NavigableMap; +import java.util.TreeMap; + +/** + * A node in the metadata tool. + */ +public interface MetadataNode { + class DirectoryNode implements MetadataNode { + private final TreeMap children = new TreeMap<>(); + + public DirectoryNode mkdirs(String... names) { + if (names.length == 0) { + throw new RuntimeException("Invalid zero-length path"); + } + DirectoryNode node = this; + for (int i = 0; i < names.length; i++) { + MetadataNode nextNode = node.children.get(names[i]); + if (nextNode == null) { + nextNode = new DirectoryNode(); + node.children.put(names[i], nextNode); + } else { + if (!(nextNode instanceof DirectoryNode)) { + throw new NotDirectoryException(); + } + } + node = (DirectoryNode) nextNode; + } + return node; + } + + public void rmrf(String... names) { + if (names.length == 0) { + throw new RuntimeException("Invalid zero-length path"); + } + DirectoryNode node = this; + for (int i = 0; i < names.length - 1; i++) { + MetadataNode nextNode = node.children.get(names[i]); + if (nextNode == null || !(nextNode instanceof DirectoryNode)) { + throw new RuntimeException("Unable to locate directory /" + + String.join("/", names)); + } + node = (DirectoryNode) nextNode; + } + node.children.remove(names[names.length - 1]); + } + + public FileNode create(String name) { + MetadataNode node = children.get(name); + if (node == null) { + node = new FileNode(); + children.put(name, node); + } else { + if (!(node instanceof FileNode)) { + throw new NotFileException(); + } + } + return (FileNode) node; + } + + public MetadataNode child(String component) { + return children.get(component); + } + + public NavigableMap children() { + return children; + } + + public void addChild(String name, DirectoryNode child) { + children.put(name, child); + } + + public DirectoryNode directory(String... names) { + if (names.length == 0) { + throw new RuntimeException("Invalid zero-length path"); + } + DirectoryNode node = this; + for (int i = 0; i < names.length; i++) { + MetadataNode nextNode = node.children.get(names[i]); + if (nextNode == null || !(nextNode instanceof DirectoryNode)) { + throw new RuntimeException("Unable to locate directory /" + + String.join("/", names)); + } + node = (DirectoryNode) nextNode; + } + return node; + } + + public FileNode file(String... names) { + if (names.length == 0) { + throw new RuntimeException("Invalid zero-length path"); + } + DirectoryNode node = this; + for (int i = 0; i < names.length - 1; i++) { + MetadataNode nextNode = node.children.get(names[i]); + if (nextNode == null || !(nextNode instanceof DirectoryNode)) { + throw new RuntimeException("Unable to locate file /" + + String.join("/", names)); + } + node = (DirectoryNode) nextNode; + } + MetadataNode nextNode = node.child(names[names.length - 1]); + if (nextNode == null || !(nextNode instanceof FileNode)) { + throw new RuntimeException("Unable to locate file /" + + String.join("/", names)); + } + return (FileNode) nextNode; + } + } + + class FileNode implements MetadataNode { + private String contents; + + void setContents(String contents) { + this.contents = contents; + } + + String contents() { + return contents; + } + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java b/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java new file mode 100644 index 0000000000000..7910285d48412 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.metadata.ConfigRecord; +import org.apache.kafka.common.metadata.FenceBrokerRecord; +import org.apache.kafka.common.metadata.IsrChangeRecord; +import org.apache.kafka.common.metadata.MetadataRecordType; +import org.apache.kafka.common.metadata.PartitionRecord; +import org.apache.kafka.common.metadata.PartitionRecordJsonConverter; +import org.apache.kafka.common.metadata.RegisterBrokerRecord; +import org.apache.kafka.common.metadata.RemoveTopicRecord; +import org.apache.kafka.common.metadata.TopicRecord; +import org.apache.kafka.common.metadata.UnfenceBrokerRecord; +import org.apache.kafka.common.metadata.UnregisterBrokerRecord; +import org.apache.kafka.common.protocol.ApiMessage; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.metalog.MetaLogLeader; +import org.apache.kafka.metalog.MetaLogListener; +import org.apache.kafka.queue.EventQueue; +import org.apache.kafka.queue.KafkaEventQueue; +import org.apache.kafka.raft.BatchReader; +import org.apache.kafka.raft.RaftClient; +import org.apache.kafka.shell.MetadataNode.DirectoryNode; +import org.apache.kafka.shell.MetadataNode.FileNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +/** + * Maintains the in-memory metadata for the metadata tool. + */ +public final class MetadataNodeManager implements AutoCloseable { + private static final Logger log = LoggerFactory.getLogger(MetadataNodeManager.class); + + public static class Data { + private final DirectoryNode root = new DirectoryNode(); + private String workingDirectory = "/"; + + public DirectoryNode root() { + return root; + } + + public String workingDirectory() { + return workingDirectory; + } + + public void setWorkingDirectory(String workingDirectory) { + this.workingDirectory = workingDirectory; + } + } + + class LogListener implements MetaLogListener, RaftClient.Listener { + @Override + public void handleCommit(BatchReader reader) { + try { + // TODO: handle lastOffset + while (reader.hasNext()) { + BatchReader.Batch batch = reader.next(); + for (ApiMessageAndVersion messageAndVersion : batch.records()) { + handleMessage(messageAndVersion.message()); + } + } + } finally { + reader.close(); + } + } + + @Override + public void handleCommits(long lastOffset, List messages) { + appendEvent("handleCommits", () -> { + log.error("handleCommits " + messages + " at offset " + lastOffset); + DirectoryNode dir = data.root.mkdirs("metadataQuorum"); + dir.create("offset").setContents(String.valueOf(lastOffset)); + for (ApiMessage message : messages) { + handleMessage(message); + } + }, null); + } + + @Override + public void handleNewLeader(MetaLogLeader leader) { + appendEvent("handleNewLeader", () -> { + log.error("handleNewLeader " + leader); + DirectoryNode dir = data.root.mkdirs("metadataQuorum"); + dir.create("leader").setContents(leader.toString()); + }, null); + } + + @Override + public void handleClaim(int epoch) { + // This shouldn't happen because we should never be the leader. + log.debug("RaftClient.Listener sent handleClaim(epoch=" + epoch + ")"); + } + + @Override + public void handleRenounce(long epoch) { + // This shouldn't happen because we should never be the leader. + log.debug("MetaLogListener sent handleRenounce(epoch=" + epoch + ")"); + } + + @Override + public void beginShutdown() { + log.debug("MetaLogListener sent beginShutdown"); + } + } + + private final Data data = new Data(); + private final LogListener logListener = new LogListener(); + private final ObjectMapper objectMapper; + private final KafkaEventQueue queue; + + public MetadataNodeManager() { + this.objectMapper = new ObjectMapper(); + this.objectMapper.registerModule(new Jdk8Module()); + this.queue = new KafkaEventQueue(Time.SYSTEM, + new LogContext("[node-manager-event-queue] "), ""); + } + + public void setup() throws Exception { + CompletableFuture future = new CompletableFuture<>(); + appendEvent("createShellNodes", () -> { + DirectoryNode directory = data.root().mkdirs("local"); + directory.create("version").setContents(AppInfoParser.getVersion()); + directory.create("commitId").setContents(AppInfoParser.getCommitId()); + future.complete(null); + }, future); + future.get(); + } + + public LogListener logListener() { + return logListener; + } + + @Override + public void close() throws Exception { + queue.close(); + } + + public void visit(Consumer consumer) throws Exception { + CompletableFuture future = new CompletableFuture<>(); + appendEvent("visit", () -> { + consumer.accept(data); + future.complete(null); + }, future); + future.get(); + } + + private void appendEvent(String name, Runnable runnable, CompletableFuture future) { + queue.append(new EventQueue.Event() { + @Override + public void run() throws Exception { + runnable.run(); + } + + @Override + public void handleException(Throwable e) { + log.error("Unexpected error while handling event " + name, e); + if (future != null) { + future.completeExceptionally(e); + } + } + }); + } + + private void handleMessage(ApiMessage message) { + try { + MetadataRecordType type = MetadataRecordType.fromId(message.apiKey()); + handleCommitImpl(type, message); + } catch (Exception e) { + log.error("Error processing record of type " + message.apiKey(), e); + } + } + + private void handleCommitImpl(MetadataRecordType type, ApiMessage message) + throws Exception { + switch (type) { + case REGISTER_BROKER_RECORD: { + DirectoryNode brokersNode = data.root.mkdirs("brokers"); + RegisterBrokerRecord record = (RegisterBrokerRecord) message; + DirectoryNode brokerNode = brokersNode. + mkdirs(Integer.toString(record.brokerId())); + FileNode registrationNode = brokerNode.create("registration"); + registrationNode.setContents(record.toString()); + brokerNode.create("isFenced").setContents("true"); + break; + } + case UNREGISTER_BROKER_RECORD: { + UnregisterBrokerRecord record = (UnregisterBrokerRecord) message; + data.root.rmrf("brokers", Integer.toString(record.brokerId())); + break; + } + case TOPIC_RECORD: { + TopicRecord record = (TopicRecord) message; + DirectoryNode topicsDirectory = data.root.mkdirs("topics"); + DirectoryNode topicDirectory = topicsDirectory.mkdirs(record.name()); + topicDirectory.create("id").setContents(record.topicId().toString()); + topicDirectory.create("name").setContents(record.name().toString()); + DirectoryNode topicIdsDirectory = data.root.mkdirs("topicIds"); + topicIdsDirectory.addChild(record.topicId().toString(), topicDirectory); + break; + } + case PARTITION_RECORD: { + PartitionRecord record = (PartitionRecord) message; + DirectoryNode topicDirectory = + data.root.mkdirs("topicIds").mkdirs(record.topicId().toString()); + DirectoryNode partitionDirectory = + topicDirectory.mkdirs(Integer.toString(record.partitionId())); + JsonNode node = PartitionRecordJsonConverter. + write(record, PartitionRecord.HIGHEST_SUPPORTED_VERSION); + partitionDirectory.create("data").setContents(node.toPrettyString()); + break; + } + case CONFIG_RECORD: { + ConfigRecord record = (ConfigRecord) message; + String typeString = ""; + switch (ConfigResource.Type.forId(record.resourceType())) { + case BROKER: + typeString = "broker"; + break; + case TOPIC: + typeString = "topic"; + break; + default: + throw new RuntimeException("Error processing CONFIG_RECORD: " + + "Can't handle ConfigResource.Type " + record.resourceType()); + } + DirectoryNode configDirectory = data.root.mkdirs("configs"). + mkdirs(typeString).mkdirs(record.resourceName()); + if (record.value() == null) { + configDirectory.rmrf(record.name()); + } else { + configDirectory.create(record.name()).setContents(record.value()); + } + break; + } + case ISR_CHANGE_RECORD: { + IsrChangeRecord record = (IsrChangeRecord) message; + FileNode file = data.root.file("topicIds", record.topicId().toString(), + Integer.toString(record.partitionId()), "data"); + JsonNode node = objectMapper.readTree(file.contents()); + PartitionRecord partition = PartitionRecordJsonConverter. + read(node, PartitionRecord.HIGHEST_SUPPORTED_VERSION); + partition.setIsr(record.isr()); + partition.setLeader(record.leader()); + partition.setLeaderEpoch(record.leaderEpoch()); + partition.setPartitionEpoch(record.partitionEpoch()); + file.setContents(PartitionRecordJsonConverter.write(partition, + PartitionRecord.HIGHEST_SUPPORTED_VERSION).toPrettyString()); + break; + } + case FENCE_BROKER_RECORD: { + FenceBrokerRecord record = (FenceBrokerRecord) message; + data.root.mkdirs("brokers", Integer.toString(record.id())). + create("isFenced").setContents("true"); + break; + } + case UNFENCE_BROKER_RECORD: { + UnfenceBrokerRecord record = (UnfenceBrokerRecord) message; + data.root.mkdirs("brokers", Integer.toString(record.id())). + create("isFenced").setContents("false"); + break; + } + case REMOVE_TOPIC_RECORD: { + RemoveTopicRecord record = (RemoveTopicRecord) message; + DirectoryNode topicsDirectory = + data.root.directory("topicIds", record.topicId().toString()); + String name = topicsDirectory.file("name").contents(); + data.root.rmrf("topics", name); + data.root.rmrf("topicIds", record.topicId().toString()); + break; + } + default: + throw new RuntimeException("Unhandled metadata record type"); + } + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/MetadataShell.java b/shell/src/main/java/org/apache/kafka/shell/MetadataShell.java new file mode 100644 index 0000000000000..b701310efb774 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/MetadataShell.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import kafka.raft.KafkaRaftManager; +import kafka.tools.TerseFailure; +import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.kafka.common.utils.Exit; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + + +/** + * The Kafka metadata shell. + */ +public final class MetadataShell { + private static final Logger log = LoggerFactory.getLogger(MetadataShell.class); + + public static class Builder { + private String snapshotPath; + + public Builder setSnapshotPath(String snapshotPath) { + this.snapshotPath = snapshotPath; + return this; + } + + public MetadataShell build() throws Exception { + MetadataNodeManager nodeManager = null; + SnapshotFileReader reader = null; + try { + nodeManager = new MetadataNodeManager(); + reader = new SnapshotFileReader(snapshotPath, nodeManager.logListener()); + return new MetadataShell(null, reader, nodeManager); + } catch (Throwable e) { + log.error("Initialization error", e); + if (reader != null) { + reader.close(); + } + if (nodeManager != null) { + nodeManager.close(); + } + throw e; + } + } + } + + private final KafkaRaftManager raftManager; + + private final SnapshotFileReader snapshotFileReader; + + private final MetadataNodeManager nodeManager; + + public MetadataShell(KafkaRaftManager raftManager, + SnapshotFileReader snapshotFileReader, + MetadataNodeManager nodeManager) { + this.raftManager = raftManager; + this.snapshotFileReader = snapshotFileReader; + this.nodeManager = nodeManager; + } + + public void run(List args) throws Exception { + nodeManager.setup(); + if (raftManager != null) { + raftManager.startup(); + raftManager.register(nodeManager.logListener()); + } else if (snapshotFileReader != null) { + snapshotFileReader.startup(); + } else { + throw new RuntimeException("Expected either a raft manager or snapshot reader"); + } + if (args == null || args.isEmpty()) { + // Interactive mode. + try (InteractiveShell shell = new InteractiveShell(nodeManager)) { + shell.runMainLoop(); + } + } else { + // Non-interactive mode. + Commands commands = new Commands(false); + try (PrintWriter writer = new PrintWriter(new BufferedWriter( + new OutputStreamWriter(System.out, StandardCharsets.UTF_8)))) { + Commands.Handler handler = commands.parseCommand(args); + handler.run(Optional.empty(), writer, nodeManager); + writer.flush(); + } + } + } + + public void close() throws Exception { + if (raftManager != null) { + raftManager.shutdown(); + } + if (snapshotFileReader != null) { + snapshotFileReader.close(); + } + nodeManager.close(); + } + + public static void main(String[] args) throws Exception { + ArgumentParser parser = ArgumentParsers + .newArgumentParser("metadata-tool") + .defaultHelp(true) + .description("The Apache Kafka metadata tool"); + parser.addArgument("--snapshot", "-s") + .type(String.class) + .help("The snapshot file to read."); + parser.addArgument("command") + .nargs("*") + .help("The command to run."); + Namespace res = parser.parseArgsOrFail(args); + try { + Builder builder = new Builder(); + builder.setSnapshotPath(res.getString("snapshot")); + Path tempDir = Files.createTempDirectory("MetadataShell"); + Exit.addShutdownHook("agent-shutdown-hook", () -> { + log.debug("Removing temporary directory " + tempDir.toAbsolutePath().toString()); + try { + Utils.delete(tempDir.toFile()); + } catch (Exception e) { + log.error("Got exception while removing temporary directory " + + tempDir.toAbsolutePath().toString()); + } + }); + MetadataShell shell = builder.build(); + shell.waitUntilCaughtUp(); + try { + shell.run(res.getList("command")); + } finally { + shell.close(); + } + Exit.exit(0); + } catch (TerseFailure e) { + System.err.println("Error: " + e.getMessage()); + Exit.exit(1); + } catch (Throwable e) { + System.err.println("Unexpected error: " + + (e.getMessage() == null ? "" : e.getMessage())); + e.printStackTrace(System.err); + Exit.exit(1); + } + } + + void waitUntilCaughtUp() throws ExecutionException, InterruptedException { + snapshotFileReader.caughtUpFuture().get(); + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/NoOpCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/NoOpCommandHandler.java new file mode 100644 index 0000000000000..1756ba76aa8de --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/NoOpCommandHandler.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import java.io.PrintWriter; +import java.util.Optional; + +/** + * Does nothing. + */ +public final class NoOpCommandHandler implements Commands.Handler { + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) { + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof NoOpCommandHandler)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/NotDirectoryException.java b/shell/src/main/java/org/apache/kafka/shell/NotDirectoryException.java new file mode 100644 index 0000000000000..692534758e27a --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/NotDirectoryException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +/** + * An exception that is thrown when a non-directory node is treated like a + * directory. + */ +public class NotDirectoryException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public NotDirectoryException() { + super(); + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/NotFileException.java b/shell/src/main/java/org/apache/kafka/shell/NotFileException.java new file mode 100644 index 0000000000000..cbc2a832d679c --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/NotFileException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +/** + * An exception that is thrown when a non-file node is treated like a + * file. + */ +public class NotFileException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public NotFileException() { + super(); + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/PwdCommandHandler.java b/shell/src/main/java/org/apache/kafka/shell/PwdCommandHandler.java new file mode 100644 index 0000000000000..1e5b5da39ef2b --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/PwdCommandHandler.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.Namespace; +import org.jline.reader.Candidate; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Optional; + +/** + * Implements the pwd command. + */ +public final class PwdCommandHandler implements Commands.Handler { + public final static Commands.Type TYPE = new PwdCommandType(); + + public static class PwdCommandType implements Commands.Type { + private PwdCommandType() { + } + + @Override + public String name() { + return "pwd"; + } + + @Override + public String description() { + return "Print the current working directory."; + } + + @Override + public boolean shellOnly() { + return true; + } + + @Override + public void addArguments(ArgumentParser parser) { + // nothing to do + } + + @Override + public Commands.Handler createHandler(Namespace namespace) { + return new PwdCommandHandler(); + } + + @Override + public void completeNext(MetadataNodeManager nodeManager, List nextWords, + List candidates) throws Exception { + // nothing to do + } + } + + @Override + public void run(Optional shell, + PrintWriter writer, + MetadataNodeManager manager) throws Exception { + manager.visit(data -> { + writer.println(data.workingDirectory()); + }); + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof PwdCommandHandler)) return false; + return true; + } +} diff --git a/shell/src/main/java/org/apache/kafka/shell/SnapshotFileReader.java b/shell/src/main/java/org/apache/kafka/shell/SnapshotFileReader.java new file mode 100644 index 0000000000000..e566be67d7d95 --- /dev/null +++ b/shell/src/main/java/org/apache/kafka/shell/SnapshotFileReader.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import org.apache.kafka.common.message.LeaderChangeMessage; +import org.apache.kafka.common.metadata.MetadataRecordType; +import org.apache.kafka.common.protocol.ApiMessage; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.record.ControlRecordType; +import org.apache.kafka.common.record.FileLogInputStream.FileChannelRecordBatch; +import org.apache.kafka.common.record.FileRecords; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.metalog.MetaLogLeader; +import org.apache.kafka.metalog.MetaLogListener; +import org.apache.kafka.queue.EventQueue; +import org.apache.kafka.queue.KafkaEventQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CompletableFuture; + + +/** + * Reads Kafka metadata snapshots. + */ +public final class SnapshotFileReader implements AutoCloseable { + private static final Logger log = LoggerFactory.getLogger(SnapshotFileReader.class); + + private final String snapshotPath; + private final MetaLogListener listener; + private final KafkaEventQueue queue; + private final CompletableFuture caughtUpFuture; + private FileRecords fileRecords; + private Iterator batchIterator; + + public SnapshotFileReader(String snapshotPath, MetaLogListener listener) { + this.snapshotPath = snapshotPath; + this.listener = listener; + this.queue = new KafkaEventQueue(Time.SYSTEM, + new LogContext("[snapshotReaderQueue] "), "snapshotReaderQueue_"); + this.caughtUpFuture = new CompletableFuture<>(); + } + + public void startup() throws Exception { + CompletableFuture future = new CompletableFuture<>(); + queue.append(new EventQueue.Event() { + @Override + public void run() throws Exception { + fileRecords = FileRecords.open(new File(snapshotPath), false); + batchIterator = fileRecords.batches().iterator(); + scheduleHandleNextBatch(); + future.complete(null); + } + + @Override + public void handleException(Throwable e) { + future.completeExceptionally(e); + beginShutdown("startup error"); + } + }); + future.get(); + } + + private void handleNextBatch() { + if (!batchIterator.hasNext()) { + beginShutdown("done"); + return; + } + FileChannelRecordBatch batch = batchIterator.next(); + if (batch.isControlBatch()) { + handleControlBatch(batch); + } else { + handleMetadataBatch(batch); + } + scheduleHandleNextBatch(); + } + + private void scheduleHandleNextBatch() { + queue.append(new EventQueue.Event() { + @Override + public void run() throws Exception { + handleNextBatch(); + } + + @Override + public void handleException(Throwable e) { + log.error("Unexpected error while handling a batch of events", e); + beginShutdown("handleBatch error"); + } + }); + } + + private void handleControlBatch(FileChannelRecordBatch batch) { + for (Iterator iter = batch.iterator(); iter.hasNext(); ) { + Record record = iter.next(); + try { + short typeId = ControlRecordType.parseTypeId(record.key()); + ControlRecordType type = ControlRecordType.fromTypeId(typeId); + switch (type) { + case LEADER_CHANGE: + LeaderChangeMessage message = new LeaderChangeMessage(); + message.read(new ByteBufferAccessor(record.value()), (short) 0); + listener.handleNewLeader(new MetaLogLeader(message.leaderId(), + batch.partitionLeaderEpoch())); + break; + default: + log.error("Ignoring control record with type {} at offset {}", + type, record.offset()); + } + } catch (Throwable e) { + log.error("unable to read control record at offset {}", record.offset(), e); + } + } + } + + private void handleMetadataBatch(FileChannelRecordBatch batch) { + List messages = new ArrayList<>(); + for (Iterator iter = batch.iterator(); iter.hasNext(); ) { + Record record = iter.next(); + ByteBufferAccessor accessor = new ByteBufferAccessor(record.value()); + try { + int apiKey = accessor.readUnsignedVarint(); + if (apiKey > Short.MAX_VALUE || apiKey < 0) { + throw new RuntimeException("Invalid apiKey value " + apiKey); + } + int apiVersion = accessor.readUnsignedVarint(); + if (apiVersion > Short.MAX_VALUE || apiVersion < 0) { + throw new RuntimeException("Invalid apiVersion value " + apiVersion); + } + ApiMessage message = MetadataRecordType.fromId((short) apiKey).newMetadataRecord(); + message.read(accessor, (short) apiVersion); + messages.add(message); + } catch (Throwable e) { + log.error("unable to read metadata record at offset {}", record.offset(), e); + } + } + listener.handleCommits(batch.lastOffset(), messages); + } + + public void beginShutdown(String reason) { + if (reason.equals("done")) { + caughtUpFuture.complete(null); + } else { + caughtUpFuture.completeExceptionally(new RuntimeException(reason)); + } + queue.beginShutdown(reason, new EventQueue.Event() { + @Override + public void run() throws Exception { + listener.beginShutdown(); + if (fileRecords != null) { + fileRecords.close(); + fileRecords = null; + } + batchIterator = null; + } + + @Override + public void handleException(Throwable e) { + log.error("shutdown error", e); + } + }); + } + + @Override + public void close() throws Exception { + beginShutdown("closing"); + queue.close(); + } + + public CompletableFuture caughtUpFuture() { + return caughtUpFuture; + } +} diff --git a/shell/src/test/java/org/apache/kafka/shell/CommandTest.java b/shell/src/test/java/org/apache/kafka/shell/CommandTest.java new file mode 100644 index 0000000000000..c896a06caa036 --- /dev/null +++ b/shell/src/test/java/org/apache/kafka/shell/CommandTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; + +@Timeout(value = 120000, unit = MILLISECONDS) +public class CommandTest { + @Test + public void testParseCommands() { + assertEquals(new CatCommandHandler(Arrays.asList("foo")), + new Commands(true).parseCommand(Arrays.asList("cat", "foo"))); + assertEquals(new CdCommandHandler(Optional.empty()), + new Commands(true).parseCommand(Arrays.asList("cd"))); + assertEquals(new CdCommandHandler(Optional.of("foo")), + new Commands(true).parseCommand(Arrays.asList("cd", "foo"))); + assertEquals(new ExitCommandHandler(), + new Commands(true).parseCommand(Arrays.asList("exit"))); + assertEquals(new HelpCommandHandler(), + new Commands(true).parseCommand(Arrays.asList("help"))); + assertEquals(new HistoryCommandHandler(3), + new Commands(true).parseCommand(Arrays.asList("history", "3"))); + assertEquals(new HistoryCommandHandler(Integer.MAX_VALUE), + new Commands(true).parseCommand(Arrays.asList("history"))); + assertEquals(new LsCommandHandler(Collections.emptyList()), + new Commands(true).parseCommand(Arrays.asList("ls"))); + assertEquals(new LsCommandHandler(Arrays.asList("abc", "123")), + new Commands(true).parseCommand(Arrays.asList("ls", "abc", "123"))); + assertEquals(new PwdCommandHandler(), + new Commands(true).parseCommand(Arrays.asList("pwd"))); + } + + @Test + public void testParseInvalidCommand() { + assertEquals(new ErroneousCommandHandler("invalid choice: 'blah' (choose " + + "from 'cat', 'cd', 'exit', 'find', 'help', 'history', 'ls', 'man', 'pwd')"), + new Commands(true).parseCommand(Arrays.asList("blah"))); + } + + @Test + public void testEmptyCommandLine() { + assertEquals(new NoOpCommandHandler(), + new Commands(true).parseCommand(Arrays.asList(""))); + assertEquals(new NoOpCommandHandler(), + new Commands(true).parseCommand(Collections.emptyList())); + } +} diff --git a/shell/src/test/java/org/apache/kafka/shell/CommandUtilsTest.java b/shell/src/test/java/org/apache/kafka/shell/CommandUtilsTest.java new file mode 100644 index 0000000000000..90c3b5c148950 --- /dev/null +++ b/shell/src/test/java/org/apache/kafka/shell/CommandUtilsTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.Arrays; + +@Timeout(value = 120000, unit = MILLISECONDS) +public class CommandUtilsTest { + @Test + public void testSplitPath() { + assertEquals(Arrays.asList("alpha", "beta"), + CommandUtils.splitPath("/alpha/beta")); + assertEquals(Arrays.asList("alpha", "beta"), + CommandUtils.splitPath("//alpha/beta/")); + } +} diff --git a/shell/src/test/java/org/apache/kafka/shell/GlobComponentTest.java b/shell/src/test/java/org/apache/kafka/shell/GlobComponentTest.java new file mode 100644 index 0000000000000..da3a7ec108127 --- /dev/null +++ b/shell/src/test/java/org/apache/kafka/shell/GlobComponentTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(value = 120000, unit = MILLISECONDS) +public class GlobComponentTest { + private void verifyIsLiteral(GlobComponent globComponent, String component) { + assertTrue(globComponent.literal()); + assertEquals(component, globComponent.component()); + assertTrue(globComponent.matches(component)); + assertFalse(globComponent.matches(component + "foo")); + } + + @Test + public void testLiteralComponent() { + verifyIsLiteral(new GlobComponent("abc"), "abc"); + verifyIsLiteral(new GlobComponent(""), ""); + verifyIsLiteral(new GlobComponent("foobar_123"), "foobar_123"); + verifyIsLiteral(new GlobComponent("$blah+"), "$blah+"); + } + + @Test + public void testToRegularExpression() { + assertEquals(null, GlobComponent.toRegularExpression("blah")); + assertEquals(null, GlobComponent.toRegularExpression("")); + assertEquals(null, GlobComponent.toRegularExpression("does not need a regex, actually")); + assertEquals("^\\$blah.*$", GlobComponent.toRegularExpression("$blah*")); + assertEquals("^.*$", GlobComponent.toRegularExpression("*")); + assertEquals("^foo(?:(?:bar)|(?:baz))$", GlobComponent.toRegularExpression("foo{bar,baz}")); + } + + @Test + public void testGlobMatch() { + GlobComponent star = new GlobComponent("*"); + assertFalse(star.literal()); + assertTrue(star.matches("")); + assertTrue(star.matches("anything")); + GlobComponent question = new GlobComponent("b?b"); + assertFalse(question.literal()); + assertFalse(question.matches("")); + assertTrue(question.matches("bob")); + assertTrue(question.matches("bib")); + assertFalse(question.matches("bic")); + GlobComponent foobarOrFoobaz = new GlobComponent("foo{bar,baz}"); + assertFalse(foobarOrFoobaz.literal()); + assertTrue(foobarOrFoobaz.matches("foobar")); + assertTrue(foobarOrFoobaz.matches("foobaz")); + assertFalse(foobarOrFoobaz.matches("foobah")); + assertFalse(foobarOrFoobaz.matches("foo")); + assertFalse(foobarOrFoobaz.matches("baz")); + } +} diff --git a/shell/src/test/java/org/apache/kafka/shell/GlobVisitorTest.java b/shell/src/test/java/org/apache/kafka/shell/GlobVisitorTest.java new file mode 100644 index 0000000000000..59eeb5db79e61 --- /dev/null +++ b/shell/src/test/java/org/apache/kafka/shell/GlobVisitorTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.kafka.shell.GlobVisitor.MetadataNodeInfo; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; + +@Timeout(value = 120000, unit = MILLISECONDS) +public class GlobVisitorTest { + static private final MetadataNodeManager.Data DATA; + + static { + DATA = new MetadataNodeManager.Data(); + DATA.root().mkdirs("alpha", "beta", "gamma"); + DATA.root().mkdirs("alpha", "theta"); + DATA.root().mkdirs("foo", "a"); + DATA.root().mkdirs("foo", "beta"); + DATA.root().mkdirs("zeta").create("c"); + DATA.root().mkdirs("zeta"); + DATA.root().create("zzz"); + DATA.setWorkingDirectory("foo"); + } + + static class InfoConsumer implements Consumer> { + private Optional> infos = null; + + @Override + public void accept(Optional info) { + if (infos == null) { + if (info.isPresent()) { + infos = Optional.of(new ArrayList<>()); + infos.get().add(info.get()); + } else { + infos = Optional.empty(); + } + } else { + if (info.isPresent()) { + infos.get().add(info.get()); + } else { + throw new RuntimeException("Saw non-empty info after seeing empty info"); + } + } + } + } + + @Test + public void testStarGlob() { + InfoConsumer consumer = new InfoConsumer(); + GlobVisitor visitor = new GlobVisitor("*", consumer); + visitor.accept(DATA); + assertEquals(Optional.of(Arrays.asList( + new MetadataNodeInfo(new String[] {"foo", "a"}, + DATA.root().directory("foo").child("a")), + new MetadataNodeInfo(new String[] {"foo", "beta"}, + DATA.root().directory("foo").child("beta")))), consumer.infos); + } + + @Test + public void testDotDot() { + InfoConsumer consumer = new InfoConsumer(); + GlobVisitor visitor = new GlobVisitor("..", consumer); + visitor.accept(DATA); + assertEquals(Optional.of(Arrays.asList( + new MetadataNodeInfo(new String[0], DATA.root()))), consumer.infos); + } + + @Test + public void testDoubleDotDot() { + InfoConsumer consumer = new InfoConsumer(); + GlobVisitor visitor = new GlobVisitor("../..", consumer); + visitor.accept(DATA); + assertEquals(Optional.of(Arrays.asList( + new MetadataNodeInfo(new String[0], DATA.root()))), consumer.infos); + } + + @Test + public void testZGlob() { + InfoConsumer consumer = new InfoConsumer(); + GlobVisitor visitor = new GlobVisitor("../z*", consumer); + visitor.accept(DATA); + assertEquals(Optional.of(Arrays.asList( + new MetadataNodeInfo(new String[] {"zeta"}, + DATA.root().child("zeta")), + new MetadataNodeInfo(new String[] {"zzz"}, + DATA.root().child("zzz")))), consumer.infos); + } + + @Test + public void testBetaOrThetaGlob() { + InfoConsumer consumer = new InfoConsumer(); + GlobVisitor visitor = new GlobVisitor("../*/{beta,theta}", consumer); + visitor.accept(DATA); + assertEquals(Optional.of(Arrays.asList( + new MetadataNodeInfo(new String[] {"alpha", "beta"}, + DATA.root().directory("alpha").child("beta")), + new MetadataNodeInfo(new String[] {"alpha", "theta"}, + DATA.root().directory("alpha").child("theta")), + new MetadataNodeInfo(new String[] {"foo", "beta"}, + DATA.root().directory("foo").child("beta")))), consumer.infos); + } + + @Test + public void testNotFoundGlob() { + InfoConsumer consumer = new InfoConsumer(); + GlobVisitor visitor = new GlobVisitor("epsilon", consumer); + visitor.accept(DATA); + assertEquals(Optional.empty(), consumer.infos); + } + + @Test + public void testAbsoluteGlob() { + InfoConsumer consumer = new InfoConsumer(); + GlobVisitor visitor = new GlobVisitor("/a?pha", consumer); + visitor.accept(DATA); + assertEquals(Optional.of(Arrays.asList( + new MetadataNodeInfo(new String[] {"alpha"}, + DATA.root().directory("alpha")))), consumer.infos); + } +} diff --git a/shell/src/test/java/org/apache/kafka/shell/LsCommandHandlerTest.java b/shell/src/test/java/org/apache/kafka/shell/LsCommandHandlerTest.java new file mode 100644 index 0000000000000..c845706f1b641 --- /dev/null +++ b/shell/src/test/java/org/apache/kafka/shell/LsCommandHandlerTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.kafka.shell.LsCommandHandler.ColumnSchema; +import org.apache.kafka.shell.LsCommandHandler.TargetDirectory; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.OptionalInt; + +@Timeout(value = 120000, unit = MILLISECONDS) +public class LsCommandHandlerTest { + @Test + public void testCalculateColumnSchema() { + assertEquals(new ColumnSchema(1, 3), + LsCommandHandler.calculateColumnSchema(OptionalInt.empty(), + Arrays.asList("abc", "def", "ghi"))); + assertEquals(new ColumnSchema(1, 2), + LsCommandHandler.calculateColumnSchema(OptionalInt.of(0), + Arrays.asList("abc", "def"))); + assertEquals(new ColumnSchema(3, 1).setColumnWidths(3, 8, 6), + LsCommandHandler.calculateColumnSchema(OptionalInt.of(80), + Arrays.asList("a", "abcdef", "beta"))); + assertEquals(new ColumnSchema(2, 3).setColumnWidths(10, 7), + LsCommandHandler.calculateColumnSchema(OptionalInt.of(18), + Arrays.asList("alphabet", "beta", "gamma", "theta", "zeta"))); + } + + @Test + public void testPrintEntries() throws Exception { + try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) { + try (PrintWriter writer = new PrintWriter(new OutputStreamWriter( + stream, StandardCharsets.UTF_8))) { + LsCommandHandler.printEntries(writer, "", OptionalInt.of(18), + Arrays.asList("alphabet", "beta", "gamma", "theta", "zeta")); + } + assertEquals(String.join(String.format("%n"), Arrays.asList( + "alphabet theta", + "beta zeta", + "gamma")), stream.toString().trim()); + } + } + + @Test + public void testPrintTargets() throws Exception { + try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) { + try (PrintWriter writer = new PrintWriter(new OutputStreamWriter( + stream, StandardCharsets.UTF_8))) { + LsCommandHandler.printTargets(writer, OptionalInt.of(18), + Arrays.asList("foo", "foobarbaz", "quux"), Arrays.asList( + new TargetDirectory("/some/dir", + Collections.singletonList("supercalifragalistic")), + new TargetDirectory("/some/other/dir", + Arrays.asList("capability", "delegation", "elephant", + "fungible", "green")))); + } + assertEquals(String.join(String.format("%n"), Arrays.asList( + "foo quux", + "foobarbaz ", + "", + "/some/dir:", + "supercalifragalistic", + "", + "/some/other/dir:", + "capability", + "delegation", + "elephant", + "fungible", + "green")), stream.toString().trim()); + } + } +} + diff --git a/shell/src/test/java/org/apache/kafka/shell/MetadataNodeTest.java b/shell/src/test/java/org/apache/kafka/shell/MetadataNodeTest.java new file mode 100644 index 0000000000000..42223c78c80e8 --- /dev/null +++ b/shell/src/test/java/org/apache/kafka/shell/MetadataNodeTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.shell; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.kafka.shell.MetadataNode.DirectoryNode; +import org.apache.kafka.shell.MetadataNode.FileNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; + +@Timeout(value = 120000, unit = MILLISECONDS) +public class MetadataNodeTest { + @Test + public void testMkdirs() { + DirectoryNode root = new DirectoryNode(); + DirectoryNode defNode = root.mkdirs("abc", "def"); + DirectoryNode defNode2 = root.mkdirs("abc", "def"); + assertTrue(defNode == defNode2); + DirectoryNode defNode3 = root.directory("abc", "def"); + assertTrue(defNode == defNode3); + root.mkdirs("ghi"); + assertEquals(new HashSet<>(Arrays.asList("abc", "ghi")), root.children().keySet()); + assertEquals(Collections.singleton("def"), root.mkdirs("abc").children().keySet()); + assertEquals(Collections.emptySet(), defNode.children().keySet()); + } + + @Test + public void testRmrf() { + DirectoryNode root = new DirectoryNode(); + DirectoryNode foo = root.mkdirs("foo"); + foo.mkdirs("a"); + foo.mkdirs("b"); + root.mkdirs("baz"); + assertEquals(new HashSet<>(Arrays.asList("foo", "baz")), root.children().keySet()); + root.rmrf("foo", "a"); + assertEquals(new HashSet<>(Arrays.asList("b")), foo.children().keySet()); + root.rmrf("foo"); + assertEquals(new HashSet<>(Collections.singleton("baz")), root.children().keySet()); + } + + @Test + public void testCreateFiles() { + DirectoryNode root = new DirectoryNode(); + DirectoryNode abcdNode = root.mkdirs("abcd"); + FileNode quuxNodde = abcdNode.create("quux"); + quuxNodde.setContents("quux contents"); + assertEquals("quux contents", quuxNodde.contents()); + assertThrows(NotDirectoryException.class, () -> root.mkdirs("abcd", "quux")); + } +} From d3612ebc775ea401e6170b3248d146b228d85d1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?= Date: Fri, 19 Feb 2021 17:07:01 -0800 Subject: [PATCH 031/243] KAFKA-9672: Leader with ISR as a superset of replicas (#9631) It is possible for the the controller to send LeaderAndIsr requests with an ISR that contains ids not in the replica set. This is used during reassignment so that the partition leader doesn't add replicas back to the ISR. This is needed because the controller updates ZK and the replicas through two rounds: 1. The first round of ZK updates and LeaderAndIsr requests shrinks the ISR. 2. The second round of ZK updates and LeaderAndIsr requests shrinks the replica set. This could be avoided by doing 1. and 2. in one round. Unfortunately the current controller implementation makes that non-trivial. This commit changes the leader to allow the state where the ISR contains ids that are not in the replica set and to remove such ids from the ISR if required. Reviewers: Jun Rao --- .../main/scala/kafka/cluster/Partition.scala | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala index cfd029b38f55f..99ae4ab85f8ef 100755 --- a/core/src/main/scala/kafka/cluster/Partition.scala +++ b/core/src/main/scala/kafka/cluster/Partition.scala @@ -352,10 +352,6 @@ class Partition(val topicPartition: TopicPartition, def getReplica(replicaId: Int): Option[Replica] = Option(remoteReplicasMap.get(replicaId)) - private def getReplicaOrException(replicaId: Int): Replica = getReplica(replicaId).getOrElse{ - throw new NotLeaderOrFollowerException(s"Replica with id $replicaId is not available on broker $localBrokerId") - } - private def checkCurrentLeaderEpoch(remoteLeaderEpochOpt: Optional[Integer]): Errors = { if (!remoteLeaderEpochOpt.isPresent) { Errors.NONE @@ -826,7 +822,7 @@ class Partition(val topicPartition: TopicPartition, case (brokerId, logEndOffset) => s"broker $brokerId: $logEndOffset" } - val curInSyncReplicaObjects = (curMaximalIsr - localBrokerId).map(getReplicaOrException) + val curInSyncReplicaObjects = (curMaximalIsr - localBrokerId).flatMap(getReplica) val replicaInfo = curInSyncReplicaObjects.map(replica => (replica.brokerId, replica.logEndOffset)) val localLogInfo = (localBrokerId, localLogOrException.logEndOffset) val (ackedReplicas, awaitingReplicas) = (replicaInfo + localLogInfo).partition { _._2 >= requiredOffset} @@ -948,11 +944,15 @@ class Partition(val topicPartition: TopicPartition, val outOfSyncReplicaIds = getOutOfSyncReplicas(replicaLagTimeMaxMs) if (outOfSyncReplicaIds.nonEmpty) { val outOfSyncReplicaLog = outOfSyncReplicaIds.map { replicaId => - s"(brokerId: $replicaId, endOffset: ${getReplicaOrException(replicaId).logEndOffset})" + val logEndOffsetMessage = getReplica(replicaId) + .map(_.logEndOffset.toString) + .getOrElse("unknown") + s"(brokerId: $replicaId, endOffset: $logEndOffsetMessage)" }.mkString(" ") val newIsrLog = (isrState.isr -- outOfSyncReplicaIds).mkString(",") info(s"Shrinking ISR from ${isrState.isr.mkString(",")} to $newIsrLog. " + - s"Leader: (highWatermark: ${leaderLog.highWatermark}, endOffset: ${leaderLog.logEndOffset}). " + + s"Leader: (highWatermark: ${leaderLog.highWatermark}, " + + s"endOffset: ${leaderLog.logEndOffset}). " + s"Out of sync replicas: $outOfSyncReplicaLog.") shrinkIsr(outOfSyncReplicaIds) @@ -978,9 +978,10 @@ class Partition(val topicPartition: TopicPartition, leaderEndOffset: Long, currentTimeMs: Long, maxLagMs: Long): Boolean = { - val followerReplica = getReplicaOrException(replicaId) - followerReplica.logEndOffset != leaderEndOffset && - (currentTimeMs - followerReplica.lastCaughtUpTimeMs) > maxLagMs + getReplica(replicaId).fold(true) { followerReplica => + followerReplica.logEndOffset != leaderEndOffset && + (currentTimeMs - followerReplica.lastCaughtUpTimeMs) > maxLagMs + } } /** From 5eac5a822fe4eb8691d2411fb25fcba1d4203b55 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Fri, 19 Feb 2021 18:03:23 -0800 Subject: [PATCH 032/243] KAFKA-12276: Add the quorum controller code (#10070) The quorum controller stores metadata in the KIP-500 metadata log, not in Apache ZooKeeper. Each controller node is a voter in the metadata quorum. The leader of the quorum is the active controller, which processes write requests. The followers are standby controllers, which replay the operations written to the log. If the active controller goes away, a standby controller can take its place. Like the ZooKeeper-based controller, the quorum controller is based on an event queue backed by a single-threaded executor. However, unlike the ZK-based controller, the quorum controller can have multiple operations in flight-- it does not need to wait for one operation to be finished before starting another. Therefore, calls into the QuorumController return CompleteableFuture objects which are completed with either a result or an error when the operation is done. The QuorumController will also time out operations that have been sitting on the queue too long without being processed. In this case, the future is completed with a TimeoutException. The controller uses timeline data structures to store multiple "versions" of its in-memory state simultaneously. "Read operations" read only committed state, which is slightly older than the most up-to-date in-memory state. "Write operations" read and write the latest in-memory state. However, we can not return a successful result for a write operation until its state has been committed to the log. Therefore, if a client receives an RPC response, it knows that the requested operation has been performed, and can not be undone by a controller failover. Reviewers: Jun Rao , Ron Dagostino --- build.gradle | 1 + checkstyle/import-control.xml | 2 + checkstyle/suppressions.xml | 6 + .../BrokerIdNotRegisteredException.java | 29 + .../apache/kafka/common/protocol/Errors.java | 6 +- .../common/requests/RequestResponseTest.java | 1 - .../scala/kafka/server/BrokerServer.scala | 3 + .../scala/kafka/server/ControllerApis.scala | 2 +- .../scala/kafka/server/ControllerServer.scala | 25 +- .../metadata/BrokerMetadataListener.scala | 10 +- .../server/metadata/MetadataPartitions.scala | 26 +- .../kafka/controller/BrokerControlState.java | 46 + .../kafka/controller/BrokerControlStates.java | 56 ++ .../controller/BrokerHeartbeatManager.java | 597 +++++++++++ .../kafka/controller/BrokersToIsrs.java | 314 ++++++ .../controller/ClientQuotaControlManager.java | 275 +++++ .../controller/ClusterControlManager.java | 346 +++++++ .../ConfigurationControlManager.java | 367 +++++++ .../apache/kafka/controller/Controller.java | 17 +- .../kafka/controller/ControllerMetrics.java | 29 + .../kafka/controller/ControllerPurgatory.java | 108 ++ .../kafka/controller/ControllerResult.java | 75 ++ .../controller/ControllerResultAndOffset.java | 69 ++ .../kafka/controller/DeferredEvent.java | 31 + .../controller/FeatureControlManager.java | 136 +++ .../kafka/controller/QuorumController.java | 941 ++++++++++++++++++ .../controller/QuorumControllerMetrics.java | 70 ++ .../controller/ReplicaPlacementPolicy.java | 47 + .../org/apache/kafka/controller/Replicas.java | 180 ++++ .../controller/ReplicationControlManager.java | 908 +++++++++++++++++ .../kafka/controller/ResultOrError.java | 4 +- .../SimpleReplicaPlacementPolicy.java | 77 ++ .../kafka/metadata/BrokerHeartbeatReply.java | 15 +- .../apache/kafka/metadata/UsableBroker.java | 61 ++ .../common/metadata/AccessControlRecord.json | 1 + ...Record.json => PartitionChangeRecord.json} | 17 +- .../common/metadata/PartitionRecord.json | 2 +- .../BrokerHeartbeatManagerTest.java | 296 ++++++ .../kafka/controller/BrokersToIsrsTest.java | 109 ++ .../ClientQuotaControlManagerTest.java | 238 +++++ .../controller/ClusterControlManagerTest.java | 150 +++ .../ConfigurationControlManagerTest.java | 203 ++++ .../controller/ControllerPurgatoryTest.java | 102 ++ .../kafka/controller/ControllerTestUtils.java | 51 + .../controller/FeatureControlManagerTest.java | 132 +++ .../controller/MockControllerMetrics.java | 47 + .../apache/kafka/controller/MockRandom.java | 34 + .../controller/QuorumControllerTest.java | 180 ++++ .../controller/QuorumControllerTestEnv.java | 88 ++ .../apache/kafka/controller/ReplicasTest.java | 96 ++ .../ReplicationControlManagerTest.java | 204 ++++ .../kafka/controller/ResultOrErrorTest.java | 65 ++ .../apache/kafka/metalog/LocalLogManager.java | 43 + .../kafka/metalog/LocalLogManagerTest.java | 1 + .../kafka/shell/MetadataNodeManager.java | 20 +- 55 files changed, 6914 insertions(+), 45 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/BrokerIdNotRegisteredException.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/BrokerControlState.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/BrokerControlStates.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/BrokerHeartbeatManager.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/BrokersToIsrs.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ControllerPurgatory.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/DeferredEvent.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/QuorumController.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/QuorumControllerMetrics.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ReplicaPlacementPolicy.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/Replicas.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java create mode 100644 metadata/src/main/java/org/apache/kafka/controller/SimpleReplicaPlacementPolicy.java create mode 100644 metadata/src/main/java/org/apache/kafka/metadata/UsableBroker.java rename metadata/src/main/resources/common/metadata/{IsrChangeRecord.json => PartitionChangeRecord.json} (63%) create mode 100644 metadata/src/test/java/org/apache/kafka/controller/BrokerHeartbeatManagerTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/BrokersToIsrsTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ControllerPurgatoryTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ControllerTestUtils.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/MockControllerMetrics.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/MockRandom.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTestEnv.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ReplicasTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ResultOrErrorTest.java diff --git a/build.gradle b/build.gradle index d03748848ce02..bf92f9645eb7c 100644 --- a/build.gradle +++ b/build.gradle @@ -1029,6 +1029,7 @@ project(':metadata') { compile project(':clients') compile libs.jacksonDatabind compile libs.jacksonJDK8Datatypes + compile libs.metrics compileOnly libs.log4j testCompile libs.junitJupiter testCompile libs.hamcrest diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 63ed7ab238b13..d9d504db178c5 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -203,8 +203,10 @@ + + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 1cfc630c0ea95..46fb97b3e197e 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -267,6 +267,12 @@ files="RequestResponseTest.java"/> + + + 0) { + if (controller.isActive()) { metadataResponseData.setControllerId(config.nodeId) } else { metadataResponseData.setControllerId(MetadataResponse.NO_CONTROLLER_ID) diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index 625ce5257b8e3..50fc6e2c4f774 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,24 +17,26 @@ package kafka.server -import java.util.concurrent.CompletableFuture +import java.util.concurrent.{CompletableFuture, TimeUnit} import java.util import java.util.concurrent.locks.ReentrantLock import kafka.cluster.Broker.ServerInfo +import kafka.log.LogConfig import kafka.metrics.{KafkaMetricsGroup, KafkaYammerMetrics, LinuxIoMetricsCollector} import kafka.network.SocketServer import kafka.raft.RaftManager import kafka.security.CredentialProvider import kafka.server.QuotaFactory.QuotaManagers import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache import org.apache.kafka.common.utils.{LogContext, Time} import org.apache.kafka.common.{ClusterResource, Endpoint} -import org.apache.kafka.controller.Controller +import org.apache.kafka.controller.{Controller, QuorumController, QuorumControllerMetrics} import org.apache.kafka.metadata.{ApiMessageAndVersion, VersionRange} import org.apache.kafka.metalog.MetaLogManager import org.apache.kafka.raft.RaftConfig @@ -138,7 +140,22 @@ class ControllerServer( socketServerFirstBoundPortFuture.complete(socketServer.boundPort( config.controllerListeners.head.listenerName)) - controller = null + val configDefs = Map(ConfigResource.Type.BROKER -> KafkaConfig.configDef, + ConfigResource.Type.TOPIC -> LogConfig.configDefCopy).asJava + val threadNamePrefixAsString = threadNamePrefix.getOrElse("") + controller = new QuorumController.Builder(config.nodeId). + setTime(time). + setThreadNamePrefix(threadNamePrefixAsString). + setConfigDefs(configDefs). + setLogManager(metaLogManager). + setDefaultReplicationFactor(config.defaultReplicationFactor.toShort). + setDefaultNumPartitions(config.numPartitions.intValue()). + setSessionTimeoutNs(TimeUnit.NANOSECONDS.convert(config.brokerSessionTimeoutMs.longValue(), + TimeUnit.MILLISECONDS)). + setMetrics(new QuorumControllerMetrics(KafkaYammerMetrics.defaultRegistry())). + build() + + quotaManagers = QuotaFactory.instantiate(config, metrics, time, threadNamePrefix.getOrElse("")) val controllerNodes = RaftConfig.quorumVoterStringsToNodes(controllerQuorumVotersFuture.get()).asScala diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala index 6185a35d3f35d..9c2bccadddcc8 100644 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala +++ b/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala @@ -152,8 +152,8 @@ class BrokerMetadataListener(brokerId: Int, case PARTITION_RECORD => handlePartitionRecord(imageBuilder, record.asInstanceOf[PartitionRecord]) case CONFIG_RECORD => handleConfigRecord(record.asInstanceOf[ConfigRecord]) - case ISR_CHANGE_RECORD => handleIsrChangeRecord(imageBuilder, - record.asInstanceOf[IsrChangeRecord]) + case PARTITION_CHANGE_RECORD => handlePartitionChangeRecord(imageBuilder, + record.asInstanceOf[PartitionChangeRecord]) case FENCE_BROKER_RECORD => handleFenceBrokerRecord(imageBuilder, record.asInstanceOf[FenceBrokerRecord]) case UNFENCE_BROKER_RECORD => handleUnfenceBrokerRecord(imageBuilder, @@ -203,9 +203,9 @@ class BrokerMetadataListener(brokerId: Int, configRepository.setConfig(resource, record.name(), record.value()) } - def handleIsrChangeRecord(imageBuilder: MetadataImageBuilder, - record: IsrChangeRecord): Unit = { - imageBuilder.partitionsBuilder().handleIsrChange(record) + def handlePartitionChangeRecord(imageBuilder: MetadataImageBuilder, + record: PartitionChangeRecord): Unit = { + imageBuilder.partitionsBuilder().handleChange(record) } def handleFenceBrokerRecord(imageBuilder: MetadataImageBuilder, diff --git a/core/src/main/scala/kafka/server/metadata/MetadataPartitions.scala b/core/src/main/scala/kafka/server/metadata/MetadataPartitions.scala index c3efac5cbea77..bd84e7a4d348b 100644 --- a/core/src/main/scala/kafka/server/metadata/MetadataPartitions.scala +++ b/core/src/main/scala/kafka/server/metadata/MetadataPartitions.scala @@ -23,13 +23,15 @@ import java.util.Collections import org.apache.kafka.common.message.LeaderAndIsrRequestData import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState -import org.apache.kafka.common.metadata.{IsrChangeRecord, PartitionRecord} +import org.apache.kafka.common.metadata.{PartitionChangeRecord, PartitionRecord} import org.apache.kafka.common.{TopicPartition, Uuid} import scala.jdk.CollectionConverters._ object MetadataPartition { + val NO_LEADER_CHANGE = -2 + def apply(name: String, record: PartitionRecord): MetadataPartition = { MetadataPartition(name, record.partitionId(), @@ -83,13 +85,23 @@ case class MetadataPartition(topicName: String, def isReplicaFor(brokerId: Int): Boolean = replicas.contains(Integer.valueOf(brokerId)) - def copyWithIsrChanges(record: IsrChangeRecord): MetadataPartition = { + def copyWithChanges(record: PartitionChangeRecord): MetadataPartition = { + val (newLeader, newLeaderEpoch) = if (record.leader() == MetadataPartition.NO_LEADER_CHANGE) { + (leaderId, leaderEpoch) + } else { + (record.leader(), leaderEpoch + 1) + } + val newIsr = if (record.isr() == null) { + isr + } else { + record.isr() + } MetadataPartition(topicName, partitionIndex, - record.leader(), - record.leaderEpoch(), + newLeader, + newLeaderEpoch, replicas, - record.isr(), + newIsr, offlineReplicas, addingReplicas, removingReplicas) @@ -113,14 +125,14 @@ class MetadataPartitionsBuilder(val brokerId: Int, } } - def handleIsrChange(record: IsrChangeRecord): Unit = { + def handleChange(record: PartitionChangeRecord): Unit = { Option(newIdMap.get(record.topicId())) match { case None => throw new RuntimeException(s"Unable to locate topic with ID ${record.topicId()}") case Some(name) => Option(newNameMap.get(name)) match { case None => throw new RuntimeException(s"Unable to locate topic with name $name") case Some(partitionMap) => Option(partitionMap.get(record.partitionId())) match { case None => throw new RuntimeException(s"Unable to locate $name-${record.partitionId}") - case Some(partition) => set(partition.copyWithIsrChanges(record)) + case Some(partition) => set(partition.copyWithChanges(record)) } } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/BrokerControlState.java b/metadata/src/main/java/org/apache/kafka/controller/BrokerControlState.java new file mode 100644 index 0000000000000..dfcf8ceb0da2d --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/BrokerControlState.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + + +public enum BrokerControlState { + FENCED(true, false), + UNFENCED(false, false), + CONTROLLED_SHUTDOWN(false, false), + SHUTDOWN_NOW(true, true); + + private final boolean fenced; + private final boolean shouldShutDown; + + BrokerControlState(boolean fenced, boolean shouldShutDown) { + this.fenced = fenced; + this.shouldShutDown = shouldShutDown; + } + + public boolean fenced() { + return fenced; + } + + public boolean shouldShutDown() { + return shouldShutDown; + } + + public boolean inControlledShutdown() { + return this == CONTROLLED_SHUTDOWN; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/BrokerControlStates.java b/metadata/src/main/java/org/apache/kafka/controller/BrokerControlStates.java new file mode 100644 index 0000000000000..660585223e770 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/BrokerControlStates.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.Objects; + + +class BrokerControlStates { + private final BrokerControlState current; + private final BrokerControlState next; + + BrokerControlStates(BrokerControlState current, BrokerControlState next) { + this.current = current; + this.next = next; + } + + BrokerControlState current() { + return current; + } + + BrokerControlState next() { + return next; + } + + @Override + public int hashCode() { + return Objects.hash(current, next); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof BrokerControlStates)) return false; + BrokerControlStates other = (BrokerControlStates) o; + return other.current == current && other.next == next; + } + + @Override + public String toString() { + return "BrokerControlStates(current=" + current + ", next=" + next + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/BrokerHeartbeatManager.java b/metadata/src/main/java/org/apache/kafka/controller/BrokerHeartbeatManager.java new file mode 100644 index 0000000000000..4a41fb8c7303d --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/BrokerHeartbeatManager.java @@ -0,0 +1,597 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.errors.InvalidReplicationFactorException; +import org.apache.kafka.common.message.BrokerHeartbeatRequestData; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.metadata.UsableBroker; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.apache.kafka.controller.BrokerControlState.FENCED; +import static org.apache.kafka.controller.BrokerControlState.CONTROLLED_SHUTDOWN; +import static org.apache.kafka.controller.BrokerControlState.SHUTDOWN_NOW; +import static org.apache.kafka.controller.BrokerControlState.UNFENCED; + + +/** + * The BrokerHeartbeatManager manages all the soft state associated with broker heartbeats. + * Soft state is state which does not appear in the metadata log. This state includes + * things like the last time each broker sent us a heartbeat, and whether the broker is + * trying to perform a controlled shutdown. + * + * Only the active controller has a BrokerHeartbeatManager, since only the active + * controller handles broker heartbeats. Standby controllers will create a heartbeat + * manager as part of the process of activating. This design minimizes the size of the + * metadata partition by excluding heartbeats from it. However, it does mean that after + * a controller failover, we may take some extra time to fence brokers, since the new + * active controller does not know when the last heartbeats were received from each. + */ +public class BrokerHeartbeatManager { + static class BrokerHeartbeatState { + /** + * The broker ID. + */ + private final int id; + + /** + * The last time we received a heartbeat from this broker, in monotonic nanoseconds. + * When this field is updated, we also may have to update the broker's position in + * the unfenced list. + */ + long lastContactNs; + + /** + * The last metadata offset which this broker reported. When this field is updated, + * we may also have to update the broker's position in the active set. + */ + long metadataOffset; + + /** + * The offset at which the broker should complete its controlled shutdown, or -1 + * if the broker is not performing a controlled shutdown. When this field is + * updated, we also have to update the broker's position in the shuttingDown set. + */ + private long controlledShutDownOffset; + + /** + * The previous entry in the unfenced list, or null if the broker is not in that list. + */ + private BrokerHeartbeatState prev; + + /** + * The next entry in the unfenced list, or null if the broker is not in that list. + */ + private BrokerHeartbeatState next; + + BrokerHeartbeatState(int id) { + this.id = id; + this.lastContactNs = 0; + this.prev = null; + this.next = null; + this.metadataOffset = -1; + this.controlledShutDownOffset = -1; + } + + /** + * Returns the broker ID. + */ + int id() { + return id; + } + + /** + * Returns true only if the broker is fenced. + */ + boolean fenced() { + return prev == null; + } + + /** + * Returns true only if the broker is in controlled shutdown state. + */ + boolean shuttingDown() { + return controlledShutDownOffset >= 0; + } + } + + static class MetadataOffsetComparator implements Comparator { + static final MetadataOffsetComparator INSTANCE = new MetadataOffsetComparator(); + + @Override + public int compare(BrokerHeartbeatState a, BrokerHeartbeatState b) { + if (a.metadataOffset < b.metadataOffset) { + return -1; + } else if (a.metadataOffset > b.metadataOffset) { + return 1; + } else if (a.id < b.id) { + return -1; + } else if (a.id > b.id) { + return 1; + } else { + return 0; + } + } + } + + static class BrokerHeartbeatStateList { + /** + * The head of the list of unfenced brokers. The list is sorted in ascending order + * of last contact time. + */ + private final BrokerHeartbeatState head; + + BrokerHeartbeatStateList() { + this.head = new BrokerHeartbeatState(-1); + head.prev = head; + head.next = head; + } + + /** + * Return the head of the list, or null if the list is empty. + */ + BrokerHeartbeatState first() { + BrokerHeartbeatState result = head.next; + return result == head ? null : result; + } + + /** + * Add the broker to the list. We start looking for a place to put it at the end + * of the list. + */ + void add(BrokerHeartbeatState broker) { + BrokerHeartbeatState cur = head.prev; + while (true) { + if (cur == head || cur.lastContactNs <= broker.lastContactNs) { + broker.next = cur.next; + cur.next.prev = broker; + broker.prev = cur; + cur.next = broker; + break; + } + cur = cur.prev; + } + } + + /** + * Remove a broker from the list. + */ + void remove(BrokerHeartbeatState broker) { + if (broker.next == null) { + throw new RuntimeException(broker + " is not in the list."); + } + broker.prev.next = broker.next; + broker.next.prev = broker.prev; + broker.prev = null; + broker.next = null; + } + + BrokerHeartbeatStateIterator iterator() { + return new BrokerHeartbeatStateIterator(head); + } + } + + static class BrokerHeartbeatStateIterator implements Iterator { + private final BrokerHeartbeatState head; + private BrokerHeartbeatState cur; + + BrokerHeartbeatStateIterator(BrokerHeartbeatState head) { + this.head = head; + this.cur = head; + } + + @Override + public boolean hasNext() { + return cur.next != head; + } + + @Override + public BrokerHeartbeatState next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + BrokerHeartbeatState result = cur.next; + cur = cur.next; + return result; + } + } + + private final Logger log; + + /** + * The Kafka clock object to use. + */ + private final Time time; + + /** + * The broker session timeout in nanoseconds. + */ + private final long sessionTimeoutNs; + + /** + * Maps broker IDs to heartbeat states. + */ + private final HashMap brokers; + + /** + * The list of unfenced brokers, sorted by last contact time. + */ + private final BrokerHeartbeatStateList unfenced; + + /** + * The set of active brokers. A broker is active if it is unfenced, and not shutting + * down. + */ + private final TreeSet active; + + BrokerHeartbeatManager(LogContext logContext, + Time time, + long sessionTimeoutNs) { + this.log = logContext.logger(BrokerHeartbeatManager.class); + this.time = time; + this.sessionTimeoutNs = sessionTimeoutNs; + this.brokers = new HashMap<>(); + this.unfenced = new BrokerHeartbeatStateList(); + this.active = new TreeSet<>(MetadataOffsetComparator.INSTANCE); + } + + // VisibleForTesting + Time time() { + return time; + } + + // VisibleForTesting + BrokerHeartbeatStateList unfenced() { + return unfenced; + } + + /** + * Mark a broker as fenced. + * + * @param brokerId The ID of the broker to mark as fenced. + */ + void fence(int brokerId) { + BrokerHeartbeatState broker = brokers.get(brokerId); + if (broker != null) { + untrack(broker); + } + } + + /** + * Remove a broker. + * + * @param brokerId The ID of the broker to remove. + */ + void remove(int brokerId) { + BrokerHeartbeatState broker = brokers.remove(brokerId); + if (broker != null) { + untrack(broker); + } + } + + /** + * Stop tracking the broker in the unfenced list and active set, if it was tracked + * in either of these. + * + * @param broker The broker state to stop tracking. + */ + private void untrack(BrokerHeartbeatState broker) { + if (!broker.fenced()) { + unfenced.remove(broker); + if (!broker.shuttingDown()) { + active.remove(broker); + } + } + } + + /** + * Check if the given broker has a valid session. + * + * @param brokerId The broker ID to check. + * + * @return True if the given broker has a valid session. + */ + boolean hasValidSession(int brokerId) { + BrokerHeartbeatState broker = brokers.get(brokerId); + if (broker == null) return false; + return hasValidSession(broker); + } + + /** + * Check if the given broker has a valid session. + * + * @param broker The broker to check. + * + * @return True if the given broker has a valid session. + */ + private boolean hasValidSession(BrokerHeartbeatState broker) { + if (broker.fenced()) { + return false; + } else { + return broker.lastContactNs + sessionTimeoutNs >= time.nanoseconds(); + } + } + + /** + * Update broker state, including lastContactNs. + * + * @param brokerId The broker ID. + * @param fenced True only if the broker is currently fenced. + * @param metadataOffset The latest metadata offset of the broker. + */ + void touch(int brokerId, boolean fenced, long metadataOffset) { + BrokerHeartbeatState broker = brokers.get(brokerId); + if (broker == null) { + broker = new BrokerHeartbeatState(brokerId); + brokers.put(brokerId, broker); + } else { + // Remove the broker from the unfenced list and/or the active set. Its + // position in either of those data structures depends on values we are + // changing here. We will re-add it if necessary at the end of this function. + untrack(broker); + } + broker.lastContactNs = time.nanoseconds(); + broker.metadataOffset = metadataOffset; + if (fenced) { + // If a broker is fenced, it leaves controlled shutdown. On its next heartbeat, + // it will shut down immediately. + broker.controlledShutDownOffset = -1; + } else { + unfenced.add(broker); + if (!broker.shuttingDown()) { + active.add(broker); + } + } + } + + long lowestActiveOffset() { + Iterator iterator = active.iterator(); + if (!iterator.hasNext()) { + return Long.MAX_VALUE; + } + BrokerHeartbeatState first = iterator.next(); + return first.metadataOffset; + } + + /** + * Mark a broker as being in the controlled shutdown state. + * + * @param brokerId The broker id. + * @param controlledShutDownOffset The offset at which controlled shutdown will be complete. + */ + void updateControlledShutdownOffset(int brokerId, long controlledShutDownOffset) { + BrokerHeartbeatState broker = brokers.get(brokerId); + if (broker == null) { + throw new RuntimeException("Unable to locate broker " + brokerId); + } + if (broker.fenced()) { + throw new RuntimeException("Fenced brokers cannot enter controlled shutdown."); + } + active.remove(broker); + broker.controlledShutDownOffset = controlledShutDownOffset; + log.debug("Updated the controlled shutdown offset for broker {} to {}.", + brokerId, controlledShutDownOffset); + } + + /** + * Return the time in monotonic nanoseconds at which we should check if a broker + * session needs to be expired. + */ + long nextCheckTimeNs() { + BrokerHeartbeatState broker = unfenced.first(); + if (broker == null) { + return Long.MAX_VALUE; + } else { + return broker.lastContactNs + sessionTimeoutNs; + } + } + + /** + * Find the stale brokers which haven't heartbeated in a long time, and which need to + * be fenced. + * + * @return A list of node IDs. + */ + List findStaleBrokers() { + List nodes = new ArrayList<>(); + BrokerHeartbeatStateIterator iterator = unfenced.iterator(); + while (iterator.hasNext()) { + BrokerHeartbeatState broker = iterator.next(); + if (hasValidSession(broker)) { + break; + } + nodes.add(broker.id); + } + return nodes; + } + + /** + * Place replicas on unfenced brokers. + * + * @param numPartitions The number of partitions to place. + * @param numReplicas The number of replicas for each partition. + * @param idToRack A function mapping broker id to broker rack. + * @param policy The replica placement policy to use. + * + * @return A list of replica lists. + * + * @throws InvalidReplicationFactorException If too many replicas were requested. + */ + List> placeReplicas(int numPartitions, short numReplicas, + Function> idToRack, + ReplicaPlacementPolicy policy) { + // TODO: support using fenced brokers here if necessary to get to the desired + // number of replicas. We probably need to add a fenced boolean in UsableBroker. + Iterator iterator = new UsableBrokerIterator( + unfenced.iterator(), idToRack); + return policy.createPlacement(numPartitions, numReplicas, iterator); + } + + static class UsableBrokerIterator implements Iterator { + private final Iterator iterator; + private final Function> idToRack; + private UsableBroker next; + + UsableBrokerIterator(Iterator iterator, + Function> idToRack) { + this.iterator = iterator; + this.idToRack = idToRack; + this.next = null; + } + + @Override + public boolean hasNext() { + if (next != null) { + return true; + } + BrokerHeartbeatState result; + do { + if (!iterator.hasNext()) { + return false; + } + result = iterator.next(); + } while (result.shuttingDown()); + Optional rack = idToRack.apply(result.id()); + next = new UsableBroker(result.id(), rack); + return true; + } + + @Override + public UsableBroker next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + UsableBroker result = next; + next = null; + return result; + } + } + + BrokerControlState currentBrokerState(BrokerHeartbeatState broker) { + if (broker.shuttingDown()) { + return CONTROLLED_SHUTDOWN; + } else if (broker.fenced()) { + return FENCED; + } else { + return UNFENCED; + } + } + + /** + * Calculate the next broker state for a broker that just sent a heartbeat request. + * + * @param brokerId The broker id. + * @param request The incoming heartbeat request. + * @param lastCommittedOffset The last committed offset of the quorum controller. + * @param hasLeaderships A callback which evaluates to true if the broker leads + * at least one partition. + * + * @return The current and next broker states. + */ + BrokerControlStates calculateNextBrokerState(int brokerId, + BrokerHeartbeatRequestData request, + long lastCommittedOffset, + Supplier hasLeaderships) { + BrokerHeartbeatState broker = brokers.getOrDefault(brokerId, + new BrokerHeartbeatState(brokerId)); + BrokerControlState currentState = currentBrokerState(broker); + switch (currentState) { + case FENCED: + if (request.wantShutDown()) { + log.info("Fenced broker {} has requested and been granted an immediate " + + "shutdown.", brokerId); + return new BrokerControlStates(currentState, SHUTDOWN_NOW); + } else if (!request.wantFence()) { + if (request.currentMetadataOffset() >= lastCommittedOffset) { + log.info("The request from broker {} to unfence has been granted " + + "because it has caught up with the last committed metadata " + + "offset {}.", brokerId, lastCommittedOffset); + return new BrokerControlStates(currentState, UNFENCED); + } else { + if (log.isDebugEnabled()) { + log.debug("The request from broker {} to unfence cannot yet " + + "be granted because it has not caught up with the last " + + "committed metadata offset {}. It is still at offset {}.", + brokerId, lastCommittedOffset, request.currentMetadataOffset()); + } + return new BrokerControlStates(currentState, FENCED); + } + } + return new BrokerControlStates(currentState, FENCED); + + case UNFENCED: + if (request.wantFence()) { + if (request.wantShutDown()) { + log.info("Unfenced broker {} has requested and been granted an " + + "immediate shutdown.", brokerId); + return new BrokerControlStates(currentState, SHUTDOWN_NOW); + } else { + log.info("Unfenced broker {} has requested and been granted " + + "fencing", brokerId); + return new BrokerControlStates(currentState, FENCED); + } + } else if (request.wantShutDown()) { + if (hasLeaderships.get()) { + log.info("Unfenced broker {} has requested and been granted a " + + "controlled shutdown.", brokerId); + return new BrokerControlStates(currentState, CONTROLLED_SHUTDOWN); + } else { + log.info("Unfenced broker {} has requested and been granted an " + + "immediate shutdown.", brokerId); + return new BrokerControlStates(currentState, SHUTDOWN_NOW); + } + } + return new BrokerControlStates(currentState, UNFENCED); + + case CONTROLLED_SHUTDOWN: + if (hasLeaderships.get()) { + log.debug("Broker {} is in controlled shutdown state, but can not " + + "shut down because more leaders still need to be moved.", brokerId); + return new BrokerControlStates(currentState, CONTROLLED_SHUTDOWN); + } + long lowestActiveOffset = lowestActiveOffset(); + if (broker.controlledShutDownOffset <= lowestActiveOffset) { + log.info("The request from broker {} to shut down has been granted " + + "since the lowest active offset {} is now greater than the " + + "broker's controlled shutdown offset {}.", brokerId, + lowestActiveOffset, broker.controlledShutDownOffset); + return new BrokerControlStates(currentState, SHUTDOWN_NOW); + } + log.debug("The request from broker {} to shut down can not yet be granted " + + "because the lowest active offset {} is not greater than the broker's " + + "shutdown offset {}.", brokerId, lowestActiveOffset, + broker.controlledShutDownOffset); + return new BrokerControlStates(currentState, CONTROLLED_SHUTDOWN); + + default: + return new BrokerControlStates(currentState, SHUTDOWN_NOW); + } + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/BrokersToIsrs.java b/metadata/src/main/java/org/apache/kafka/controller/BrokersToIsrs.java new file mode 100644 index 0000000000000..6b219eb943ad3 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/BrokersToIsrs.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.apache.kafka.timeline.TimelineHashMap; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import java.util.Objects; + +import static org.apache.kafka.controller.ReplicationControlManager.NO_LEADER; + + +/** + * Associates brokers with their in-sync partitions. + * + * This is useful when we need to remove a broker from all the ISRs, or move all leaders + * away from a broker. + * + * We also track all the partitions that currently have no leader. + * + * The core data structure is a map from broker IDs to topic maps. Each topic map relates + * topic UUIDs to arrays of partition IDs. + * + * Each entry in the array has a high bit which indicates that the broker is the leader + * for the given partition, as well as 31 low bits which contain the partition id. This + * works because partition IDs cannot be negative. + */ +public class BrokersToIsrs { + private final static int[] EMPTY = new int[0]; + + private final static int LEADER_FLAG = 0x8000_0000; + + private final static int REPLICA_MASK = 0x7fff_ffff; + + static class TopicPartition { + private final Uuid topicId; + private final int partitionId; + + TopicPartition(Uuid topicId, int partitionId) { + this.topicId = topicId; + this.partitionId = partitionId; + } + + public Uuid topicId() { + return topicId; + } + + public int partitionId() { + return partitionId; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof TopicPartition)) return false; + TopicPartition other = (TopicPartition) o; + return other.topicId.equals(topicId) && other.partitionId == partitionId; + } + + @Override + public int hashCode() { + return Objects.hash(topicId, partitionId); + } + + @Override + public String toString() { + return topicId + ":" + partitionId; + } + } + + static class PartitionsOnReplicaIterator implements Iterator { + private final Iterator> iterator; + private final boolean leaderOnly; + private int offset = 0; + Uuid uuid = Uuid.ZERO_UUID; + int[] replicas = EMPTY; + private TopicPartition next = null; + + PartitionsOnReplicaIterator(Map topicMap, boolean leaderOnly) { + this.iterator = topicMap.entrySet().iterator(); + this.leaderOnly = leaderOnly; + } + + @Override + public boolean hasNext() { + if (next != null) return true; + while (true) { + if (offset >= replicas.length) { + if (!iterator.hasNext()) return false; + offset = 0; + Entry entry = iterator.next(); + uuid = entry.getKey(); + replicas = entry.getValue(); + } + int replica = replicas[offset++]; + if ((!leaderOnly) || (replica & LEADER_FLAG) != 0) { + next = new TopicPartition(uuid, replica & REPLICA_MASK); + return true; + } + } + } + + @Override + public TopicPartition next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + TopicPartition result = next; + next = null; + return result; + } + } + + private final SnapshotRegistry snapshotRegistry; + + /** + * A map of broker IDs to the partitions that the broker is in the ISR for. + * Partitions with no isr members appear in this map under id NO_LEADER. + */ + private final TimelineHashMap> isrMembers; + + BrokersToIsrs(SnapshotRegistry snapshotRegistry) { + this.snapshotRegistry = snapshotRegistry; + this.isrMembers = new TimelineHashMap<>(snapshotRegistry, 0); + } + + /** + * Update our records of a partition's ISR. + * + * @param topicId The topic ID of the partition. + * @param partitionId The partition ID of the partition. + * @param prevIsr The previous ISR, or null if the partition is new. + * @param nextIsr The new ISR, or null if the partition is being removed. + * @param prevLeader The previous leader, or NO_LEADER if the partition had no leader. + * @param nextLeader The new leader, or NO_LEADER if the partition now has no leader. + */ + void update(Uuid topicId, int partitionId, int[] prevIsr, int[] nextIsr, + int prevLeader, int nextLeader) { + int[] prev; + if (prevIsr == null) { + prev = EMPTY; + } else { + if (prevLeader == NO_LEADER) { + prev = Replicas.copyWith(prevIsr, NO_LEADER); + } else { + prev = Replicas.clone(prevIsr); + } + Arrays.sort(prev); + } + int[] next; + if (nextIsr == null) { + next = EMPTY; + } else { + if (nextLeader == NO_LEADER) { + next = Replicas.copyWith(nextIsr, NO_LEADER); + } else { + next = Replicas.clone(nextIsr); + } + Arrays.sort(next); + } + int i = 0, j = 0; + while (true) { + if (i == prev.length) { + if (j == next.length) { + break; + } + int newReplica = next[j]; + add(newReplica, topicId, partitionId, newReplica == nextLeader); + j++; + } else if (j == next.length) { + int prevReplica = prev[i]; + remove(prevReplica, topicId, partitionId, prevReplica == prevLeader); + i++; + } else { + int prevReplica = prev[i]; + int newReplica = next[j]; + if (prevReplica < newReplica) { + remove(prevReplica, topicId, partitionId, prevReplica == prevLeader); + i++; + } else if (prevReplica > newReplica) { + add(newReplica, topicId, partitionId, newReplica == nextLeader); + j++; + } else { + boolean wasLeader = prevReplica == prevLeader; + boolean isLeader = prevReplica == nextLeader; + if (wasLeader != isLeader) { + change(prevReplica, topicId, partitionId, wasLeader, isLeader); + } + i++; + j++; + } + } + } + } + + private void add(int brokerId, Uuid topicId, int newPartition, boolean leader) { + if (leader) { + newPartition = newPartition | LEADER_FLAG; + } + TimelineHashMap topicMap = isrMembers.get(brokerId); + if (topicMap == null) { + topicMap = new TimelineHashMap<>(snapshotRegistry, 0); + isrMembers.put(brokerId, topicMap); + } + int[] partitions = topicMap.get(topicId); + int[] newPartitions; + if (partitions == null) { + newPartitions = new int[1]; + } else { + newPartitions = new int[partitions.length + 1]; + System.arraycopy(partitions, 0, newPartitions, 0, partitions.length); + } + newPartitions[newPartitions.length - 1] = newPartition; + topicMap.put(topicId, newPartitions); + } + + private void change(int brokerId, Uuid topicId, int partition, + boolean wasLeader, boolean isLeader) { + TimelineHashMap topicMap = isrMembers.get(brokerId); + if (topicMap == null) { + throw new RuntimeException("Broker " + brokerId + " has no isrMembers " + + "entry, so we can't change " + topicId + ":" + partition); + } + int[] partitions = topicMap.get(topicId); + if (partitions == null) { + throw new RuntimeException("Broker " + brokerId + " has no " + + "entry in isrMembers for topic " + topicId); + } + int[] newPartitions = new int[partitions.length]; + int target = wasLeader ? partition | LEADER_FLAG : partition; + for (int i = 0; i < partitions.length; i++) { + int cur = partitions[i]; + if (cur == target) { + newPartitions[i] = isLeader ? partition | LEADER_FLAG : partition; + } else { + newPartitions[i] = cur; + } + } + topicMap.put(topicId, newPartitions); + } + + private void remove(int brokerId, Uuid topicId, int removedPartition, boolean leader) { + if (leader) { + removedPartition = removedPartition | LEADER_FLAG; + } + TimelineHashMap topicMap = isrMembers.get(brokerId); + if (topicMap == null) { + throw new RuntimeException("Broker " + brokerId + " has no isrMembers " + + "entry, so we can't remove " + topicId + ":" + removedPartition); + } + int[] partitions = topicMap.get(topicId); + if (partitions == null) { + throw new RuntimeException("Broker " + brokerId + " has no " + + "entry in isrMembers for topic " + topicId); + } + if (partitions.length == 1) { + if (partitions[0] != removedPartition) { + throw new RuntimeException("Broker " + brokerId + " has no " + + "entry in isrMembers for " + topicId + ":" + removedPartition); + } + topicMap.remove(topicId); + if (topicMap.isEmpty()) { + isrMembers.remove(brokerId); + } + } else { + int[] newPartitions = new int[partitions.length - 1]; + int j = 0; + for (int i = 0; i < partitions.length; i++) { + int partition = partitions[i]; + if (partition != removedPartition) { + newPartitions[j++] = partition; + } + } + topicMap.put(topicId, newPartitions); + } + } + + PartitionsOnReplicaIterator iterator(int brokerId, boolean leadersOnly) { + Map topicMap = isrMembers.get(brokerId); + if (topicMap == null) { + topicMap = Collections.emptyMap(); + } + return new PartitionsOnReplicaIterator(topicMap, leadersOnly); + } + + PartitionsOnReplicaIterator noLeaderIterator() { + return iterator(NO_LEADER, true); + } + + boolean hasLeaderships(int brokerId) { + return iterator(brokerId, true).hasNext(); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java new file mode 100644 index 0000000000000..4aac9e4882f46 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.internals.QuotaConfigs; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.metadata.QuotaRecord; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.quota.ClientQuotaAlteration; +import org.apache.kafka.common.quota.ClientQuotaEntity; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.apache.kafka.timeline.TimelineHashMap; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; + + +public class ClientQuotaControlManager { + + private final SnapshotRegistry snapshotRegistry; + + final TimelineHashMap> clientQuotaData; + + ClientQuotaControlManager(SnapshotRegistry snapshotRegistry) { + this.snapshotRegistry = snapshotRegistry; + this.clientQuotaData = new TimelineHashMap<>(snapshotRegistry, 0); + } + + /** + * Determine the result of applying a batch of client quota alteration. Note + * that this method does not change the contents of memory. It just generates a + * result, that you can replay later if you wish using replay(). + * + * @param quotaAlterations List of client quota alterations to evaluate + * @return The result. + */ + ControllerResult> alterClientQuotas( + Collection quotaAlterations) { + List outputRecords = new ArrayList<>(); + Map outputResults = new HashMap<>(); + + quotaAlterations.forEach(quotaAlteration -> { + // Note that the values in this map may be null + Map alterations = new HashMap<>(quotaAlteration.ops().size()); + quotaAlteration.ops().forEach(op -> { + if (alterations.containsKey(op.key())) { + outputResults.put(quotaAlteration.entity(), ApiError.fromThrowable( + new InvalidRequestException("Duplicate quota key " + op.key() + + " not updating quota for this entity " + quotaAlteration.entity()))); + } else { + alterations.put(op.key(), op.value()); + } + }); + if (outputResults.containsKey(quotaAlteration.entity())) { + outputResults.put(quotaAlteration.entity(), ApiError.fromThrowable( + new InvalidRequestException("Ignoring duplicate entity " + quotaAlteration.entity()))); + } else { + alterClientQuotaEntity(quotaAlteration.entity(), alterations, outputRecords, outputResults); + } + }); + + return new ControllerResult<>(outputRecords, outputResults); + } + + /** + * Apply a quota record to the in-memory state. + * + * @param record A QuotaRecord instance. + */ + public void replay(QuotaRecord record) { + Map entityMap = new HashMap<>(2); + record.entity().forEach(entityData -> entityMap.put(entityData.entityType(), entityData.entityName())); + ClientQuotaEntity entity = new ClientQuotaEntity(entityMap); + Map quotas = clientQuotaData.get(entity); + if (quotas == null) { + quotas = new TimelineHashMap<>(snapshotRegistry, 0); + clientQuotaData.put(entity, quotas); + } + if (record.remove()) { + quotas.remove(record.key()); + if (quotas.size() == 0) { + clientQuotaData.remove(entity); + } + } else { + quotas.put(record.key(), record.value()); + } + } + + private void alterClientQuotaEntity( + ClientQuotaEntity entity, + Map newQuotaConfigs, + List outputRecords, + Map outputResults) { + + // Check entity types and sanitize the names + Map validatedEntityMap = new HashMap<>(3); + ApiError error = validateEntity(entity, validatedEntityMap); + if (error.isFailure()) { + outputResults.put(entity, error); + return; + } + + // Check the combination of entity types and get the config keys + Map configKeys = new HashMap<>(4); + error = configKeysForEntityType(validatedEntityMap, configKeys); + if (error.isFailure()) { + outputResults.put(entity, error); + return; + } + + // Don't share objects between different records + Supplier> recordEntitySupplier = () -> + validatedEntityMap.entrySet().stream().map(mapEntry -> new QuotaRecord.EntityData() + .setEntityType(mapEntry.getKey()) + .setEntityName(mapEntry.getValue())) + .collect(Collectors.toList()); + + List newRecords = new ArrayList<>(newQuotaConfigs.size()); + Map currentQuotas = clientQuotaData.getOrDefault(entity, Collections.emptyMap()); + newQuotaConfigs.forEach((key, newValue) -> { + if (newValue == null) { + if (currentQuotas.containsKey(key)) { + // Null value indicates removal + newRecords.add(new ApiMessageAndVersion(new QuotaRecord() + .setEntity(recordEntitySupplier.get()) + .setKey(key) + .setRemove(true), (short) 0)); + } + } else { + ApiError validationError = validateQuotaKeyValue(configKeys, key, newValue); + if (validationError.isFailure()) { + outputResults.put(entity, validationError); + } else { + final Double currentValue = currentQuotas.get(key); + if (!Objects.equals(currentValue, newValue)) { + // Only record the new value if it has changed + newRecords.add(new ApiMessageAndVersion(new QuotaRecord() + .setEntity(recordEntitySupplier.get()) + .setKey(key) + .setValue(newValue), (short) 0)); + } + } + } + }); + + outputRecords.addAll(newRecords); + outputResults.put(entity, ApiError.NONE); + } + + private ApiError configKeysForEntityType(Map entity, Map output) { + // We only allow certain combinations of quota entity types. Which type is in use determines which config + // keys are valid + boolean hasUser = entity.containsKey(ClientQuotaEntity.USER); + boolean hasClientId = entity.containsKey(ClientQuotaEntity.CLIENT_ID); + boolean hasIp = entity.containsKey(ClientQuotaEntity.IP); + + final Map configKeys; + if (hasUser && hasClientId && !hasIp) { + configKeys = QuotaConfigs.userConfigs().configKeys(); + } else if (hasUser && !hasClientId && !hasIp) { + configKeys = QuotaConfigs.userConfigs().configKeys(); + } else if (!hasUser && hasClientId && !hasIp) { + configKeys = QuotaConfigs.clientConfigs().configKeys(); + } else if (!hasUser && !hasClientId && hasIp) { + if (isValidIpEntity(entity.get(ClientQuotaEntity.IP))) { + configKeys = QuotaConfigs.ipConfigs().configKeys(); + } else { + return new ApiError(Errors.INVALID_REQUEST, entity.get(ClientQuotaEntity.IP) + " is not a valid IP or resolvable host."); + } + } else { + return new ApiError(Errors.INVALID_REQUEST, "Invalid empty client quota entity"); + } + + output.putAll(configKeys); + return ApiError.NONE; + } + + private ApiError validateQuotaKeyValue(Map validKeys, String key, Double value) { + // TODO can this validation be shared with alter configs? + // Ensure we have an allowed quota key + ConfigDef.ConfigKey configKey = validKeys.get(key); + if (configKey == null) { + return new ApiError(Errors.INVALID_REQUEST, "Invalid configuration key " + key); + } + + // Ensure the quota value is valid + switch (configKey.type()) { + case DOUBLE: + break; + case LONG: + Double epsilon = 1e-6; + Long longValue = Double.valueOf(value + epsilon).longValue(); + if (Math.abs(longValue.doubleValue() - value) > epsilon) { + return new ApiError(Errors.INVALID_REQUEST, + "Configuration " + key + " must be a Long value"); + } + break; + default: + return new ApiError(Errors.UNKNOWN_SERVER_ERROR, + "Unexpected config type " + configKey.type() + " should be Long or Double"); + } + return ApiError.NONE; + } + + // TODO move this somewhere common? + private boolean isValidIpEntity(String ip) { + if (Objects.nonNull(ip)) { + try { + InetAddress.getByName(ip); + return true; + } catch (UnknownHostException e) { + return false; + } + } else { + return true; + } + } + + private ApiError validateEntity(ClientQuotaEntity entity, Map validatedEntityMap) { + // Given a quota entity (which is a mapping of entity type to entity name), validate it's types + if (entity.entries().isEmpty()) { + return new ApiError(Errors.INVALID_REQUEST, "Invalid empty client quota entity"); + } + + for (Map.Entry entityEntry : entity.entries().entrySet()) { + String entityType = entityEntry.getKey(); + String entityName = entityEntry.getValue(); + if (validatedEntityMap.containsKey(entityType)) { + return new ApiError(Errors.INVALID_REQUEST, "Invalid empty client quota entity, duplicate entity entry " + entityType); + } + if (Objects.equals(entityType, ClientQuotaEntity.USER)) { + validatedEntityMap.put(ClientQuotaEntity.USER, entityName); + } else if (Objects.equals(entityType, ClientQuotaEntity.CLIENT_ID)) { + validatedEntityMap.put(ClientQuotaEntity.CLIENT_ID, entityName); + } else if (Objects.equals(entityType, ClientQuotaEntity.IP)) { + validatedEntityMap.put(ClientQuotaEntity.IP, entityName); + } else { + return new ApiError(Errors.INVALID_REQUEST, "Unhandled client quota entity type: " + entityType); + } + + if (entityName != null && entityName.isEmpty()) { + return new ApiError(Errors.INVALID_REQUEST, "Empty " + entityType + " not supported"); + } + } + + return ApiError.NONE; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java new file mode 100644 index 0000000000000..6e329c72a0e3f --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.Endpoint; +import org.apache.kafka.common.errors.DuplicateBrokerRegistrationException; +import org.apache.kafka.common.errors.StaleBrokerEpochException; +import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.message.BrokerRegistrationRequestData; +import org.apache.kafka.common.metadata.FenceBrokerRecord; +import org.apache.kafka.common.metadata.RegisterBrokerRecord; +import org.apache.kafka.common.metadata.UnfenceBrokerRecord; +import org.apache.kafka.common.metadata.UnregisterBrokerRecord; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.metadata.BrokerRegistration; +import org.apache.kafka.metadata.BrokerRegistrationReply; +import org.apache.kafka.metadata.FeatureMapAndEpoch; +import org.apache.kafka.metadata.VersionRange; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.apache.kafka.timeline.TimelineHashMap; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + + +/** + * The ClusterControlManager manages all the hard state associated with the Kafka cluster. + * Hard state is state which appears in the metadata log, such as broker registrations, + * brokers being fenced or unfenced, and broker feature versions. + */ +public class ClusterControlManager { + class ReadyBrokersFuture { + private final CompletableFuture future; + private final int minBrokers; + + ReadyBrokersFuture(CompletableFuture future, int minBrokers) { + this.future = future; + this.minBrokers = minBrokers; + } + + boolean check() { + int numUnfenced = 0; + for (BrokerRegistration registration : brokerRegistrations.values()) { + if (!registration.fenced()) { + numUnfenced++; + } + if (numUnfenced >= minBrokers) { + return true; + } + } + return false; + } + } + + /** + * The SLF4J log context. + */ + private final LogContext logContext; + + /** + * The SLF4J log object. + */ + private final Logger log; + + /** + * The Kafka clock object to use. + */ + private final Time time; + + /** + * How long sessions should last, in nanoseconds. + */ + private final long sessionTimeoutNs; + + /** + * The replica placement policy to use. + */ + private final ReplicaPlacementPolicy placementPolicy; + + /** + * Maps broker IDs to broker registrations. + */ + private final TimelineHashMap brokerRegistrations; + + /** + * The broker heartbeat manager, or null if this controller is on standby. + */ + private BrokerHeartbeatManager heartbeatManager; + + /** + * A future which is completed as soon as we have the given number of brokers + * ready. + */ + private Optional readyBrokersFuture; + + ClusterControlManager(LogContext logContext, + Time time, + SnapshotRegistry snapshotRegistry, + long sessionTimeoutNs, + ReplicaPlacementPolicy placementPolicy) { + this.logContext = logContext; + this.log = logContext.logger(ClusterControlManager.class); + this.time = time; + this.sessionTimeoutNs = sessionTimeoutNs; + this.placementPolicy = placementPolicy; + this.brokerRegistrations = new TimelineHashMap<>(snapshotRegistry, 0); + this.heartbeatManager = null; + this.readyBrokersFuture = Optional.empty(); + } + + /** + * Transition this ClusterControlManager to active. + */ + public void activate() { + heartbeatManager = new BrokerHeartbeatManager(logContext, time, sessionTimeoutNs); + for (BrokerRegistration registration : brokerRegistrations.values()) { + heartbeatManager.touch(registration.id(), registration.fenced(), -1); + } + } + + /** + * Transition this ClusterControlManager to standby. + */ + public void deactivate() { + heartbeatManager = null; + } + + Map brokerRegistrations() { + return brokerRegistrations; + } + + /** + * Process an incoming broker registration request. + */ + public ControllerResult registerBroker( + BrokerRegistrationRequestData request, + long brokerEpoch, + FeatureMapAndEpoch finalizedFeatures) { + if (heartbeatManager == null) { + throw new RuntimeException("ClusterControlManager is not active."); + } + int brokerId = request.brokerId(); + BrokerRegistration existing = brokerRegistrations.get(brokerId); + if (existing != null) { + if (heartbeatManager.hasValidSession(brokerId)) { + if (!existing.incarnationId().equals(request.incarnationId())) { + throw new DuplicateBrokerRegistrationException("Another broker is " + + "registered with that broker id."); + } + } else { + if (!existing.incarnationId().equals(request.incarnationId())) { + // Remove any existing session for the old broker incarnation. + heartbeatManager.remove(brokerId); + existing = null; + } + } + } + + RegisterBrokerRecord record = new RegisterBrokerRecord().setBrokerId(brokerId). + setIncarnationId(request.incarnationId()). + setBrokerEpoch(brokerEpoch). + setRack(request.rack()); + for (BrokerRegistrationRequestData.Listener listener : request.listeners()) { + record.endPoints().add(new RegisterBrokerRecord.BrokerEndpoint(). + setHost(listener.host()). + setName(listener.name()). + setPort(listener.port()). + setSecurityProtocol(listener.securityProtocol())); + } + for (BrokerRegistrationRequestData.Feature feature : request.features()) { + Optional finalized = finalizedFeatures.map().get(feature.name()); + if (finalized.isPresent()) { + if (!finalized.get().contains(new VersionRange(feature.minSupportedVersion(), + feature.maxSupportedVersion()))) { + throw new UnsupportedVersionException("Unable to register because " + + "the broker has an unsupported version of " + feature.name()); + } + } + record.features().add(new RegisterBrokerRecord.BrokerFeature(). + setName(feature.name()). + setMinSupportedVersion(feature.minSupportedVersion()). + setMaxSupportedVersion(feature.maxSupportedVersion())); + } + + if (existing == null) { + heartbeatManager.touch(brokerId, true, -1); + } else { + heartbeatManager.touch(brokerId, existing.fenced(), -1); + } + + List records = new ArrayList<>(); + records.add(new ApiMessageAndVersion(record, (short) 0)); + return new ControllerResult<>(records, new BrokerRegistrationReply(brokerEpoch)); + } + + public void replay(RegisterBrokerRecord record) { + int brokerId = record.brokerId(); + List listeners = new ArrayList<>(); + for (RegisterBrokerRecord.BrokerEndpoint endpoint : record.endPoints()) { + listeners.add(new Endpoint(endpoint.name(), + SecurityProtocol.forId(endpoint.securityProtocol()), + endpoint.host(), endpoint.port())); + } + Map features = new HashMap<>(); + for (RegisterBrokerRecord.BrokerFeature feature : record.features()) { + features.put(feature.name(), new VersionRange( + feature.minSupportedVersion(), feature.maxSupportedVersion())); + } + // Normally, all newly registered brokers start off in the fenced state. + // If this registration record is for a broker incarnation that was already + // registered, though, we preserve the existing fencing state. + boolean fenced = true; + BrokerRegistration prevRegistration = brokerRegistrations.get(brokerId); + if (prevRegistration != null && + prevRegistration.incarnationId().equals(record.incarnationId())) { + fenced = prevRegistration.fenced(); + } + // Update broker registrations. + brokerRegistrations.put(brokerId, new BrokerRegistration(brokerId, + record.brokerEpoch(), record.incarnationId(), listeners, features, + Optional.ofNullable(record.rack()), fenced)); + + if (prevRegistration == null) { + log.info("Registered new broker: {}", record); + } else if (prevRegistration.incarnationId().equals(record.incarnationId())) { + log.info("Re-registered broker incarnation: {}", record); + } else { + log.info("Re-registered broker id {}: {}", brokerId, record); + } + } + + public void replay(UnregisterBrokerRecord record) { + int brokerId = record.brokerId(); + BrokerRegistration registration = brokerRegistrations.get(brokerId); + if (registration == null) { + throw new RuntimeException(String.format("Unable to replay %s: no broker " + + "registration found for that id", record.toString())); + } else if (registration.epoch() != record.brokerEpoch()) { + throw new RuntimeException(String.format("Unable to replay %s: no broker " + + "registration with that epoch found", record.toString())); + } else { + brokerRegistrations.remove(brokerId); + log.info("Unregistered broker: {}", record); + } + } + + public void replay(FenceBrokerRecord record) { + int brokerId = record.id(); + BrokerRegistration registration = brokerRegistrations.get(brokerId); + if (registration == null) { + throw new RuntimeException(String.format("Unable to replay %s: no broker " + + "registration found for that id", record.toString())); + } else if (registration.epoch() != record.epoch()) { + throw new RuntimeException(String.format("Unable to replay %s: no broker " + + "registration with that epoch found", record.toString())); + } else { + brokerRegistrations.put(brokerId, registration.cloneWithFencing(true)); + log.info("Fenced broker: {}", record); + } + } + + public void replay(UnfenceBrokerRecord record) { + int brokerId = record.id(); + BrokerRegistration registration = brokerRegistrations.get(brokerId); + if (registration == null) { + throw new RuntimeException(String.format("Unable to replay %s: no broker " + + "registration found for that id", record.toString())); + } else if (registration.epoch() != record.epoch()) { + throw new RuntimeException(String.format("Unable to replay %s: no broker " + + "registration with that epoch found", record.toString())); + } else { + brokerRegistrations.put(brokerId, registration.cloneWithFencing(false)); + log.info("Unfenced broker: {}", record); + } + if (readyBrokersFuture.isPresent()) { + if (readyBrokersFuture.get().check()) { + readyBrokersFuture.get().future.complete(null); + readyBrokersFuture = Optional.empty(); + } + } + } + + public List> placeReplicas(int numPartitions, short numReplicas) { + if (heartbeatManager == null) { + throw new RuntimeException("ClusterControlManager is not active."); + } + return heartbeatManager.placeReplicas(numPartitions, numReplicas, + id -> brokerRegistrations.get(id).rack(), placementPolicy); + } + + public boolean unfenced(int brokerId) { + BrokerRegistration registration = brokerRegistrations.get(brokerId); + if (registration == null) return false; + return !registration.fenced(); + } + + BrokerHeartbeatManager heartbeatManager() { + if (heartbeatManager == null) { + throw new RuntimeException("ClusterControlManager is not active."); + } + return heartbeatManager; + } + + public void checkBrokerEpoch(int brokerId, long brokerEpoch) { + BrokerRegistration registration = brokerRegistrations.get(brokerId); + if (registration == null) { + throw new StaleBrokerEpochException("No broker registration found for " + + "broker id " + brokerId); + } + if (registration.epoch() != brokerEpoch) { + throw new StaleBrokerEpochException("Expected broker epoch " + + registration.epoch() + ", but got broker epoch " + brokerEpoch); + } + } + + public void addReadyBrokersFuture(CompletableFuture future, int minBrokers) { + readyBrokersFuture = Optional.of(new ReadyBrokersFuture(future, minBrokers)); + if (readyBrokersFuture.get().check()) { + readyBrokersFuture.get().future.complete(null); + readyBrokersFuture = Optional.empty(); + } + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java new file mode 100644 index 0000000000000..4402b3a117d83 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.clients.admin.AlterConfigOp.OpType; +import org.apache.kafka.common.config.ConfigDef.ConfigKey; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigResource.Type; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.metadata.ConfigRecord; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.apache.kafka.timeline.TimelineHashMap; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; + +import static org.apache.kafka.clients.admin.AlterConfigOp.OpType.APPEND; + +public class ConfigurationControlManager { + private final Logger log; + private final SnapshotRegistry snapshotRegistry; + private final Map configDefs; + private final TimelineHashMap> configData; + + ConfigurationControlManager(LogContext logContext, + SnapshotRegistry snapshotRegistry, + Map configDefs) { + this.log = logContext.logger(ConfigurationControlManager.class); + this.snapshotRegistry = snapshotRegistry; + this.configDefs = configDefs; + this.configData = new TimelineHashMap<>(snapshotRegistry, 0); + } + + /** + * Determine the result of applying a batch of incremental configuration changes. Note + * that this method does not change the contents of memory. It just generates a + * result, that you can replay later if you wish using replay(). + * + * Note that there can only be one result per ConfigResource. So if you try to modify + * several keys and one modification fails, the whole ConfigKey fails and nothing gets + * changed. + * + * @param configChanges Maps each resource to a map from config keys to + * operation data. + * @return The result. + */ + ControllerResult> incrementalAlterConfigs( + Map>> configChanges) { + List outputRecords = new ArrayList<>(); + Map outputResults = new HashMap<>(); + for (Entry>> resourceEntry : + configChanges.entrySet()) { + incrementalAlterConfigResource(resourceEntry.getKey(), + resourceEntry.getValue(), + outputRecords, + outputResults); + } + return new ControllerResult<>(outputRecords, outputResults); + } + + private void incrementalAlterConfigResource(ConfigResource configResource, + Map> keysToOps, + List outputRecords, + Map outputResults) { + ApiError error = checkConfigResource(configResource); + if (error.isFailure()) { + outputResults.put(configResource, error); + return; + } + List newRecords = new ArrayList<>(); + for (Entry> keysToOpsEntry : keysToOps.entrySet()) { + String key = keysToOpsEntry.getKey(); + String currentValue = null; + TimelineHashMap currentConfigs = configData.get(configResource); + if (currentConfigs != null) { + currentValue = currentConfigs.get(key); + } + String newValue = currentValue; + Entry opTypeAndNewValue = keysToOpsEntry.getValue(); + OpType opType = opTypeAndNewValue.getKey(); + String opValue = opTypeAndNewValue.getValue(); + switch (opType) { + case SET: + newValue = opValue; + break; + case DELETE: + if (opValue != null) { + outputResults.put(configResource, new ApiError( + Errors.INVALID_REQUEST, "A DELETE op was given with a " + + "non-null value.")); + return; + } + newValue = null; + break; + case APPEND: + case SUBTRACT: + if (!isSplittable(configResource.type(), key)) { + outputResults.put(configResource, new ApiError( + Errors.INVALID_CONFIG, "Can't " + opType + " to " + + "key " + key + " because its type is not LIST.")); + return; + } + List newValueParts = getParts(newValue, key, configResource); + if (opType == APPEND) { + if (!newValueParts.contains(opValue)) { + newValueParts.add(opValue); + } + newValue = String.join(",", newValueParts); + } else if (newValueParts.remove(opValue)) { + newValue = String.join(",", newValueParts); + } + break; + } + if (!Objects.equals(currentValue, newValue)) { + newRecords.add(new ApiMessageAndVersion(new ConfigRecord(). + setResourceType(configResource.type().id()). + setResourceName(configResource.name()). + setName(key). + setValue(newValue), (short) 0)); + } + } + outputRecords.addAll(newRecords); + outputResults.put(configResource, ApiError.NONE); + } + + /** + * Determine the result of applying a batch of legacy configuration changes. Note + * that this method does not change the contents of memory. It just generates a + * result, that you can replay later if you wish using replay(). + * + * @param newConfigs The new configurations to install for each resource. + * All existing configurations will be overwritten. + * @return The result. + */ + ControllerResult> legacyAlterConfigs( + Map> newConfigs) { + List outputRecords = new ArrayList<>(); + Map outputResults = new HashMap<>(); + for (Entry> resourceEntry : + newConfigs.entrySet()) { + legacyAlterConfigResource(resourceEntry.getKey(), + resourceEntry.getValue(), + outputRecords, + outputResults); + } + return new ControllerResult<>(outputRecords, outputResults); + } + + private void legacyAlterConfigResource(ConfigResource configResource, + Map newConfigs, + List outputRecords, + Map outputResults) { + ApiError error = checkConfigResource(configResource); + if (error.isFailure()) { + outputResults.put(configResource, error); + return; + } + List newRecords = new ArrayList<>(); + Map currentConfigs = configData.get(configResource); + if (currentConfigs == null) { + currentConfigs = Collections.emptyMap(); + } + for (Entry entry : newConfigs.entrySet()) { + String key = entry.getKey(); + String newValue = entry.getValue(); + String currentValue = currentConfigs.get(key); + if (!Objects.equals(newValue, currentValue)) { + newRecords.add(new ApiMessageAndVersion(new ConfigRecord(). + setResourceType(configResource.type().id()). + setResourceName(configResource.name()). + setName(key). + setValue(newValue), (short) 0)); + } + } + for (String key : currentConfigs.keySet()) { + if (!newConfigs.containsKey(key)) { + newRecords.add(new ApiMessageAndVersion(new ConfigRecord(). + setResourceType(configResource.type().id()). + setResourceName(configResource.name()). + setName(key). + setValue(null), (short) 0)); + } + } + outputRecords.addAll(newRecords); + outputResults.put(configResource, ApiError.NONE); + } + + private List getParts(String value, String key, ConfigResource configResource) { + if (value == null) { + value = getConfigValueDefault(configResource.type(), key); + } + List parts = new ArrayList<>(); + if (value == null) { + return parts; + } + String[] splitValues = value.split(","); + for (String splitValue : splitValues) { + if (!splitValue.isEmpty()) { + parts.add(splitValue); + } + } + return parts; + } + + static ApiError checkConfigResource(ConfigResource configResource) { + switch (configResource.type()) { + case BROKER_LOGGER: + // We do not handle resources of type BROKER_LOGGER in + // ConfigurationControlManager, since they are not persisted to the + // metadata log. + // + // When using incrementalAlterConfigs, we handle changes to BROKER_LOGGER + // in ControllerApis.scala. When using the legacy alterConfigs, + // BROKER_LOGGER is not supported at all. + return new ApiError(Errors.INVALID_REQUEST, "Unsupported " + + "configuration resource type BROKER_LOGGER "); + case BROKER: + // Note: A Resource with type BROKER and an empty name represents a + // cluster configuration that applies to all brokers. + if (!configResource.name().isEmpty()) { + try { + int brokerId = Integer.parseInt(configResource.name()); + if (brokerId < 0) { + return new ApiError(Errors.INVALID_REQUEST, "Illegal " + + "negative broker ID in BROKER resource."); + } + } catch (NumberFormatException e) { + return new ApiError(Errors.INVALID_REQUEST, "Illegal " + + "non-integral BROKER resource type name."); + } + } + return ApiError.NONE; + case TOPIC: + try { + Topic.validate(configResource.name()); + } catch (Exception e) { + return new ApiError(Errors.INVALID_REQUEST, "Illegal topic name."); + } + return ApiError.NONE; + case UNKNOWN: + return new ApiError(Errors.INVALID_REQUEST, "Unsupported configuration " + + "resource type UNKNOWN."); + default: + return new ApiError(Errors.INVALID_REQUEST, "Unsupported unexpected " + + "resource type"); + } + } + + boolean isSplittable(ConfigResource.Type type, String key) { + ConfigDef configDef = configDefs.get(type); + if (configDef == null) { + return false; + } + ConfigKey configKey = configDef.configKeys().get(key); + if (configKey == null) { + return false; + } + return configKey.type == ConfigDef.Type.LIST; + } + + String getConfigValueDefault(ConfigResource.Type type, String key) { + ConfigDef configDef = configDefs.get(type); + if (configDef == null) { + return null; + } + ConfigKey configKey = configDef.configKeys().get(key); + if (configKey == null || !configKey.hasDefault()) { + return null; + } + return ConfigDef.convertToString(configKey.defaultValue, configKey.type); + } + + /** + * Apply a configuration record to the in-memory state. + * + * @param record The ConfigRecord. + */ + void replay(ConfigRecord record) { + Type type = Type.forId(record.resourceType()); + ConfigResource configResource = new ConfigResource(type, record.resourceName()); + TimelineHashMap configs = configData.get(configResource); + if (configs == null) { + configs = new TimelineHashMap<>(snapshotRegistry, 0); + configData.put(configResource, configs); + } + if (record.value() == null) { + configs.remove(record.name()); + } else { + configs.put(record.name(), record.value()); + } + log.info("{}: set configuration {} to {}", configResource, record.name(), record.value()); + } + + // VisibleForTesting + Map getConfigs(ConfigResource configResource) { + Map map = configData.get(configResource); + if (map == null) { + return Collections.emptyMap(); + } else { + return Collections.unmodifiableMap(new HashMap<>(map)); + } + } + + public Map>> describeConfigs( + long lastCommittedOffset, Map> resources) { + Map>> results = new HashMap<>(); + for (Entry> resourceEntry : resources.entrySet()) { + ConfigResource resource = resourceEntry.getKey(); + ApiError error = checkConfigResource(resource); + if (error.isFailure()) { + results.put(resource, new ResultOrError<>(error)); + continue; + } + Map foundConfigs = new HashMap<>(); + TimelineHashMap configs = + configData.get(resource, lastCommittedOffset); + if (configs != null) { + Collection targetConfigs = resourceEntry.getValue(); + if (targetConfigs.isEmpty()) { + Iterator> iter = + configs.entrySet(lastCommittedOffset).iterator(); + while (iter.hasNext()) { + Entry entry = iter.next(); + foundConfigs.put(entry.getKey(), entry.getValue()); + } + } else { + for (String key : targetConfigs) { + String value = configs.get(key, lastCommittedOffset); + if (value != null) { + foundConfigs.put(key, value); + } + } + } + } + results.put(resource, new ResultOrError<>(foundConfigs)); + } + return results; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/Controller.java b/metadata/src/main/java/org/apache/kafka/controller/Controller.java index 0f6a54b1ad399..1ce63e0a46e61 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/Controller.java +++ b/metadata/src/main/java/org/apache/kafka/controller/Controller.java @@ -41,7 +41,7 @@ public interface Controller extends AutoCloseable { /** - * Change the in-sync replica sets for some partitions. + * Change partition ISRs. * * @param request The AlterIsrRequest data. * @@ -103,7 +103,7 @@ public interface Controller extends AutoCloseable { * @param configChanges The changes. * @param validateOnly True if we should validate the changes but not apply them. * - * @return A future yielding a map from partitions to error results. + * @return A future yielding a map from config resources to error results. */ CompletableFuture> incrementalAlterConfigs( Map>> configChanges, @@ -115,7 +115,7 @@ CompletableFuture> incrementalAlterConfigs( * @param newConfigs The new configuration maps to apply. * @param validateOnly True if we should validate the changes but not apply them. * - * @return A future yielding a map from partitions to error results. + * @return A future yielding a map from config resources to error results. */ CompletableFuture> legacyAlterConfigs( Map> newConfigs, boolean validateOnly); @@ -125,7 +125,7 @@ CompletableFuture> legacyAlterConfigs( * * @param request The broker heartbeat request. * - * @return A future yielding a heartbeat reply. + * @return A future yielding the broker heartbeat reply. */ CompletableFuture processBrokerHeartbeat( BrokerHeartbeatRequestData request); @@ -135,7 +135,7 @@ CompletableFuture processBrokerHeartbeat( * * @param request The registration request. * - * @return A future yielding a registration reply. + * @return A future yielding the broker registration reply. */ CompletableFuture registerBroker( BrokerRegistrationRequestData request); @@ -173,6 +173,13 @@ CompletableFuture> alterClientQuotas( */ long curClaimEpoch(); + /** + * Returns true if this controller is currently active. + */ + default boolean isActive() { + return curClaimEpoch() != -1; + } + /** * Blocks until we have shut down and freed all resources. */ diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java new file mode 100644 index 0000000000000..fd4f3befb805f --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + + +public interface ControllerMetrics { + void setActive(boolean active); + + boolean active(); + + void updateEventQueueTime(long durationMs); + + void updateEventQueueProcessingTime(long durationMs); +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerPurgatory.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerPurgatory.java new file mode 100644 index 0000000000000..ee6c1d1ebf7e5 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerPurgatory.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.TreeMap; + +/** + * The purgatory which holds events that have been started, but not yet completed. + * We wait for the high water mark of the metadata log to advance before completing + * them. + */ +class ControllerPurgatory { + /** + * A map from log offsets to events. Each event will be completed once the log + * advances past its offset. + */ + private final TreeMap> pending = new TreeMap<>(); + + /** + * Complete some purgatory entries. + * + * @param offset The offset which the high water mark has advanced to. + */ + void completeUpTo(long offset) { + Iterator>> iter = pending.entrySet().iterator(); + while (iter.hasNext()) { + Entry> entry = iter.next(); + if (entry.getKey() > offset) { + break; + } + for (DeferredEvent event : entry.getValue()) { + event.complete(null); + } + iter.remove(); + } + } + + /** + * Fail all the pending purgatory entries. + * + * @param exception The exception to fail the entries with. + */ + void failAll(Exception exception) { + Iterator>> iter = pending.entrySet().iterator(); + while (iter.hasNext()) { + Entry> entry = iter.next(); + for (DeferredEvent event : entry.getValue()) { + event.complete(exception); + } + iter.remove(); + } + } + + /** + * Add a new purgatory event. + * + * @param offset The offset to add the new event at. + * @param event The new event. + */ + void add(long offset, DeferredEvent event) { + if (!pending.isEmpty()) { + long lastKey = pending.lastKey(); + if (offset < lastKey) { + throw new RuntimeException("There is already a purgatory event with " + + "offset " + lastKey + ". We should not add one with an offset of " + + offset + " which " + "is lower than that."); + } + } + List events = pending.get(offset); + if (events == null) { + events = new ArrayList<>(); + pending.put(offset, events); + } + events.add(event); + } + + /** + * Get the offset of the highest pending event, or empty if there are no pending + * events. + */ + Optional highestPendingOffset() { + if (pending.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(pending.lastKey()); + } + } +} \ No newline at end of file diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java new file mode 100644 index 0000000000000..4906c8b0972a8 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.metadata.ApiMessageAndVersion; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + + +class ControllerResult { + private final List records; + private final T response; + + public ControllerResult(T response) { + this(new ArrayList<>(), response); + } + + public ControllerResult(List records, T response) { + Objects.requireNonNull(records); + this.records = records; + this.response = response; + } + + public List records() { + return records; + } + + public T response() { + return response; + } + + @Override + public boolean equals(Object o) { + if (o == null || (!o.getClass().equals(getClass()))) { + return false; + } + ControllerResult other = (ControllerResult) o; + return records.equals(other.records) && + Objects.equals(response, other.response); + } + + @Override + public int hashCode() { + return Objects.hash(records, response); + } + + @Override + public String toString() { + return "ControllerResult(records=" + String.join(",", + records.stream().map(r -> r.toString()).collect(Collectors.toList())) + + ", response=" + response + ")"; + } + + public ControllerResult withoutRecords() { + return new ControllerResult<>(new ArrayList<>(), response); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java new file mode 100644 index 0000000000000..5e483f773d5e2 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.metadata.ApiMessageAndVersion; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + + +class ControllerResultAndOffset extends ControllerResult { + private final long offset; + + public ControllerResultAndOffset(T response) { + super(new ArrayList<>(), response); + this.offset = -1; + } + + public ControllerResultAndOffset(long offset, + List records, + T response) { + super(records, response); + this.offset = offset; + } + + public long offset() { + return offset; + } + + @Override + public boolean equals(Object o) { + if (o == null || (!o.getClass().equals(getClass()))) { + return false; + } + ControllerResultAndOffset other = (ControllerResultAndOffset) o; + return records().equals(other.records()) && + response().equals(other.response()) && + offset == other.offset; + } + + @Override + public int hashCode() { + return Objects.hash(records(), response(), offset); + } + + @Override + public String toString() { + return "ControllerResultAndOffset(records=" + String.join(",", + records().stream().map(r -> r.toString()).collect(Collectors.toList())) + + ", response=" + response() + ", offset=" + offset + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/DeferredEvent.java b/metadata/src/main/java/org/apache/kafka/controller/DeferredEvent.java new file mode 100644 index 0000000000000..e1606f353207a --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/DeferredEvent.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +/** + * Represents a deferred event in the controller purgatory. + */ +interface DeferredEvent { + /** + * Complete the event. + * + * @param exception null if the event should be completed successfully; the + * error otherwise. + */ + void complete(Throwable exception); +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java new file mode 100644 index 0000000000000..25ff3fdcdd80e --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import org.apache.kafka.common.metadata.FeatureLevelRecord; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.metadata.FeatureMap; +import org.apache.kafka.metadata.FeatureMapAndEpoch; +import org.apache.kafka.metadata.VersionRange; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.apache.kafka.timeline.TimelineHashMap; +import org.apache.kafka.timeline.TimelineHashSet; + + +public class FeatureControlManager { + /** + * The features supported by this controller's software. + */ + private final Map supportedFeatures; + + /** + * Maps feature names to finalized version ranges. + */ + private final TimelineHashMap finalizedVersions; + + /** + * The latest feature epoch. + */ + private final TimelineHashSet epoch; + + FeatureControlManager(Map supportedFeatures, + SnapshotRegistry snapshotRegistry) { + this.supportedFeatures = supportedFeatures; + this.finalizedVersions = new TimelineHashMap<>(snapshotRegistry, 0); + this.epoch = new TimelineHashSet<>(snapshotRegistry, 0); + } + + ControllerResult> updateFeatures( + Map updates, Set downgradeables, + Map> brokerFeatures) { + TreeMap results = new TreeMap<>(); + List records = new ArrayList<>(); + for (Entry entry : updates.entrySet()) { + results.put(entry.getKey(), updateFeature(entry.getKey(), entry.getValue(), + downgradeables.contains(entry.getKey()), brokerFeatures, records)); + } + return new ControllerResult<>(records, results); + } + + private ApiError updateFeature(String featureName, + VersionRange newRange, + boolean downgradeable, + Map> brokerFeatures, + List records) { + if (newRange.min() <= 0) { + return new ApiError(Errors.INVALID_UPDATE_VERSION, + "The lower value for the new range cannot be less than 1."); + } + if (newRange.max() <= 0) { + return new ApiError(Errors.INVALID_UPDATE_VERSION, + "The upper value for the new range cannot be less than 1."); + } + VersionRange localRange = supportedFeatures.get(featureName); + if (localRange == null || !localRange.contains(newRange)) { + return new ApiError(Errors.INVALID_UPDATE_VERSION, + "The controller does not support the given feature range."); + } + for (Entry> brokerEntry : + brokerFeatures.entrySet()) { + VersionRange brokerRange = brokerEntry.getValue().get(featureName); + if (brokerRange == null || !brokerRange.contains(newRange)) { + return new ApiError(Errors.INVALID_UPDATE_VERSION, + "Broker " + brokerEntry.getKey() + " does not support the given " + + "feature range."); + } + } + VersionRange currentRange = finalizedVersions.get(featureName); + if (currentRange != null && currentRange.max() > newRange.max()) { + if (!downgradeable) { + return new ApiError(Errors.INVALID_UPDATE_VERSION, + "Can't downgrade the maximum version of this feature without " + + "setting downgradable to true."); + } + } + records.add(new ApiMessageAndVersion( + new FeatureLevelRecord().setName(featureName). + setMinFeatureLevel(newRange.min()).setMaxFeatureLevel(newRange.max()), + (short) 0)); + return ApiError.NONE; + } + + FeatureMapAndEpoch finalizedFeatures(long lastCommittedOffset) { + Map features = new HashMap<>(); + for (Entry entry : finalizedVersions.entrySet(lastCommittedOffset)) { + features.put(entry.getKey(), entry.getValue()); + } + long currentEpoch = -1; + Iterator iterator = epoch.iterator(lastCommittedOffset); + if (iterator.hasNext()) { + currentEpoch = iterator.next(); + } + return new FeatureMapAndEpoch(new FeatureMap(features), currentEpoch); + } + + void replay(FeatureLevelRecord record, long offset) { + finalizedVersions.put(record.name(), + new VersionRange(record.minFeatureLevel(), record.maxFeatureLevel())); + epoch.clear(); + epoch.add(offset); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java new file mode 100644 index 0000000000000..198097538985a --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -0,0 +1,941 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map.Entry; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import org.apache.kafka.clients.admin.AlterConfigOp.OpType; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.NotControllerException; +import org.apache.kafka.common.errors.UnknownServerException; +import org.apache.kafka.common.message.AlterIsrRequestData; +import org.apache.kafka.common.message.AlterIsrResponseData; +import org.apache.kafka.common.message.BrokerHeartbeatRequestData; +import org.apache.kafka.common.message.BrokerRegistrationRequestData; +import org.apache.kafka.common.message.CreateTopicsRequestData; +import org.apache.kafka.common.message.CreateTopicsResponseData; +import org.apache.kafka.common.message.ElectLeadersRequestData; +import org.apache.kafka.common.message.ElectLeadersResponseData; +import org.apache.kafka.common.metadata.ConfigRecord; +import org.apache.kafka.common.metadata.FenceBrokerRecord; +import org.apache.kafka.common.metadata.MetadataRecordType; +import org.apache.kafka.common.metadata.PartitionChangeRecord; +import org.apache.kafka.common.metadata.PartitionRecord; +import org.apache.kafka.common.metadata.QuotaRecord; +import org.apache.kafka.common.metadata.RegisterBrokerRecord; +import org.apache.kafka.common.metadata.TopicRecord; +import org.apache.kafka.common.metadata.UnfenceBrokerRecord; +import org.apache.kafka.common.metadata.UnregisterBrokerRecord; +import org.apache.kafka.common.protocol.ApiMessage; +import org.apache.kafka.common.quota.ClientQuotaAlteration; +import org.apache.kafka.common.quota.ClientQuotaEntity; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.metadata.BrokerHeartbeatReply; +import org.apache.kafka.metadata.BrokerRegistrationReply; +import org.apache.kafka.metadata.FeatureMapAndEpoch; +import org.apache.kafka.metadata.VersionRange; +import org.apache.kafka.metalog.MetaLogLeader; +import org.apache.kafka.metalog.MetaLogListener; +import org.apache.kafka.metalog.MetaLogManager; +import org.apache.kafka.queue.EventQueue.EarliestDeadlineFunction; +import org.apache.kafka.queue.EventQueue; +import org.apache.kafka.queue.KafkaEventQueue; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.slf4j.Logger; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; + + +/** + * QuorumController implements the main logic of the KIP-500 controller. + * + * The node which is the leader of the metadata log becomes the active controller. All + * other nodes remain in standby mode. Standby controllers cannot create new metadata log + * entries. They just replay the metadata log entries that the current active controller + * has created. + * + * The QuorumController is single-threaded. A single event handler thread performs most + * operations. This avoids the need for complex locking. + * + * The controller exposes an asynchronous, futures-based API to the world. This reflects + * the fact that the controller may have several operations in progress at any given + * point. The future associated with each operation will not be completed until the + * results of the operation have been made durable to the metadata log. + */ +public final class QuorumController implements Controller { + /** + * A builder class which creates the QuorumController. + */ + static public class Builder { + private final int nodeId; + private Time time = Time.SYSTEM; + private String threadNamePrefix = null; + private LogContext logContext = null; + private Map configDefs = Collections.emptyMap(); + private MetaLogManager logManager = null; + private Map supportedFeatures = Collections.emptyMap(); + private short defaultReplicationFactor = 3; + private int defaultNumPartitions = 1; + private ReplicaPlacementPolicy replicaPlacementPolicy = + new SimpleReplicaPlacementPolicy(new Random()); + private long sessionTimeoutNs = NANOSECONDS.convert(18, TimeUnit.SECONDS); + private ControllerMetrics controllerMetrics = null; + + public Builder(int nodeId) { + this.nodeId = nodeId; + } + + public Builder setTime(Time time) { + this.time = time; + return this; + } + + public Builder setThreadNamePrefix(String threadNamePrefix) { + this.threadNamePrefix = threadNamePrefix; + return this; + } + + public Builder setLogContext(LogContext logContext) { + this.logContext = logContext; + return this; + } + + public Builder setConfigDefs(Map configDefs) { + this.configDefs = configDefs; + return this; + } + + public Builder setLogManager(MetaLogManager logManager) { + this.logManager = logManager; + return this; + } + + public Builder setSupportedFeatures(Map supportedFeatures) { + this.supportedFeatures = supportedFeatures; + return this; + } + + public Builder setDefaultReplicationFactor(short defaultReplicationFactor) { + this.defaultReplicationFactor = defaultReplicationFactor; + return this; + } + + public Builder setDefaultNumPartitions(int defaultNumPartitions) { + this.defaultNumPartitions = defaultNumPartitions; + return this; + } + + public Builder setReplicaPlacementPolicy(ReplicaPlacementPolicy replicaPlacementPolicy) { + this.replicaPlacementPolicy = replicaPlacementPolicy; + return this; + } + + public Builder setSessionTimeoutNs(long sessionTimeoutNs) { + this.sessionTimeoutNs = sessionTimeoutNs; + return this; + } + + public Builder setMetrics(ControllerMetrics controllerMetrics) { + this.controllerMetrics = controllerMetrics; + return this; + } + + public QuorumController build() throws Exception { + if (logManager == null) { + throw new RuntimeException("You must set a metadata log manager."); + } + if (threadNamePrefix == null) { + threadNamePrefix = String.format("Node%d_", nodeId); + } + if (logContext == null) { + logContext = new LogContext(String.format("[Controller %d] ", nodeId)); + } + if (controllerMetrics == null) { + controllerMetrics = (ControllerMetrics) Class.forName( + "org.apache.kafka.controller.MockControllerMetrics").getConstructor().newInstance(); + } + KafkaEventQueue queue = null; + try { + queue = new KafkaEventQueue(time, logContext, threadNamePrefix); + return new QuorumController(logContext, nodeId, queue, time, configDefs, + logManager, supportedFeatures, defaultReplicationFactor, + defaultNumPartitions, replicaPlacementPolicy, sessionTimeoutNs, + controllerMetrics); + } catch (Exception e) { + Utils.closeQuietly(queue, "event queue"); + throw e; + } + } + } + + private static final String ACTIVE_CONTROLLER_EXCEPTION_TEXT_PREFIX = + "The active controller appears to be node "; + + private NotControllerException newNotControllerException() { + int latestController = logManager.leader().nodeId(); + if (latestController < 0) { + return new NotControllerException("No controller appears to be active."); + } else { + return new NotControllerException(ACTIVE_CONTROLLER_EXCEPTION_TEXT_PREFIX + + latestController); + } + } + + public static int exceptionToApparentController(NotControllerException e) { + if (e.getMessage().startsWith(ACTIVE_CONTROLLER_EXCEPTION_TEXT_PREFIX)) { + return Integer.parseInt(e.getMessage().substring( + ACTIVE_CONTROLLER_EXCEPTION_TEXT_PREFIX.length())); + } else { + return -1; + } + } + + private void handleEventEnd(String name, long startProcessingTimeNs) { + long endProcessingTime = time.nanoseconds(); + long deltaNs = endProcessingTime - startProcessingTimeNs; + log.debug("Processed {} in {} us", name, + MICROSECONDS.convert(deltaNs, NANOSECONDS)); + controllerMetrics.updateEventQueueProcessingTime(NANOSECONDS.toMillis(deltaNs)); + } + + private Throwable handleEventException(String name, + Optional startProcessingTimeNs, + Throwable exception) { + if (!startProcessingTimeNs.isPresent()) { + log.info("unable to start processing {} because of {}.", name, + exception.getClass().getSimpleName()); + if (exception instanceof ApiException) { + return exception; + } else { + return new UnknownServerException(exception); + } + } + long endProcessingTime = time.nanoseconds(); + long deltaNs = endProcessingTime - startProcessingTimeNs.get(); + long deltaUs = MICROSECONDS.convert(deltaNs, NANOSECONDS); + if (exception instanceof ApiException) { + log.info("{}: failed with {} in {} us", name, + exception.getClass().getSimpleName(), deltaUs); + return exception; + } + log.warn("{}: failed with unknown server exception {} at epoch {} in {} us. " + + "Reverting to last committed offset {}.", + this, exception.getClass().getSimpleName(), curClaimEpoch, deltaUs, + lastCommittedOffset, exception); + renounce(); + return new UnknownServerException(exception); + } + + /** + * A controller event for handling internal state changes, such as Raft inputs. + */ + class ControlEvent implements EventQueue.Event { + private final String name; + private final Runnable handler; + private long eventCreatedTimeNs = time.nanoseconds(); + private Optional startProcessingTimeNs = Optional.empty(); + + ControlEvent(String name, Runnable handler) { + this.name = name; + this.handler = handler; + } + + @Override + public void run() throws Exception { + long now = time.nanoseconds(); + controllerMetrics.updateEventQueueTime(NANOSECONDS.toMillis(now - eventCreatedTimeNs)); + startProcessingTimeNs = Optional.of(now); + log.debug("Executing {}.", this); + handler.run(); + handleEventEnd(this.toString(), startProcessingTimeNs.get()); + } + + @Override + public void handleException(Throwable exception) { + handleEventException(name, startProcessingTimeNs, exception); + } + + @Override + public String toString() { + return name; + } + } + + private void appendControlEvent(String name, Runnable handler) { + ControlEvent event = new ControlEvent(name, handler); + queue.append(event); + } + + /** + * A controller event that reads the committed internal state in order to expose it + * to an API. + */ + class ControllerReadEvent implements EventQueue.Event { + private final String name; + private final CompletableFuture future; + private final Supplier handler; + private long eventCreatedTimeNs = time.nanoseconds(); + private Optional startProcessingTimeNs = Optional.empty(); + + ControllerReadEvent(String name, Supplier handler) { + this.name = name; + this.future = new CompletableFuture(); + this.handler = handler; + } + + CompletableFuture future() { + return future; + } + + @Override + public void run() throws Exception { + long now = time.nanoseconds(); + controllerMetrics.updateEventQueueTime(NANOSECONDS.toMillis(now - eventCreatedTimeNs)); + startProcessingTimeNs = Optional.of(now); + T value = handler.get(); + handleEventEnd(this.toString(), startProcessingTimeNs.get()); + future.complete(value); + } + + @Override + public void handleException(Throwable exception) { + future.completeExceptionally( + handleEventException(name, startProcessingTimeNs, exception)); + } + + @Override + public String toString() { + return name + "(" + System.identityHashCode(this) + ")"; + } + } + + // VisibleForTesting + ReplicationControlManager replicationControl() { + return replicationControl; + } + + // VisibleForTesting + CompletableFuture appendReadEvent(String name, Supplier handler) { + ControllerReadEvent event = new ControllerReadEvent(name, handler); + queue.append(event); + return event.future(); + } + + interface ControllerWriteOperation { + /** + * Generate the metadata records needed to implement this controller write + * operation. In general, this operation should not modify the "hard state" of + * the controller. That modification will happen later on, when we replay the + * records generated by this function. + * + * There are cases where this function modifies the "soft state" of the + * controller. Mainly, this happens when we process cluster heartbeats. + * + * This function also generates an RPC result. In general, if the RPC resulted in + * an error, the RPC result will be an error, and the generated record list will + * be empty. This would happen if we tried to create a topic with incorrect + * parameters, for example. Of course, partial errors are possible for batch + * operations. + * + * @return A result containing a list of records, and the RPC result. + */ + ControllerResult generateRecordsAndResult() throws Exception; + + /** + * Once we've passed the records to the Raft layer, we will invoke this function + * with the end offset at which those records were placed. If there were no + * records to write, we'll just pass the last write offset. + */ + default void processBatchEndOffset(long offset) {} + } + + /** + * A controller event that modifies the controller state. + */ + class ControllerWriteEvent implements EventQueue.Event, DeferredEvent { + private final String name; + private final CompletableFuture future; + private final ControllerWriteOperation op; + private long eventCreatedTimeNs = time.nanoseconds(); + private Optional startProcessingTimeNs = Optional.empty(); + private ControllerResultAndOffset resultAndOffset; + + ControllerWriteEvent(String name, ControllerWriteOperation op) { + this.name = name; + this.future = new CompletableFuture(); + this.op = op; + this.resultAndOffset = null; + } + + CompletableFuture future() { + return future; + } + + @Override + public void run() throws Exception { + long now = time.nanoseconds(); + controllerMetrics.updateEventQueueTime(NANOSECONDS.toMillis(now - eventCreatedTimeNs)); + long controllerEpoch = curClaimEpoch; + if (controllerEpoch == -1) { + throw newNotControllerException(); + } + startProcessingTimeNs = Optional.of(now); + ControllerResult result = op.generateRecordsAndResult(); + if (result.records().isEmpty()) { + op.processBatchEndOffset(writeOffset); + // If the operation did not return any records, then it was actually just + // a read after all, and not a read + write. However, this read was done + // from the latest in-memory state, which might contain uncommitted data. + Optional maybeOffset = purgatory.highestPendingOffset(); + if (!maybeOffset.isPresent()) { + // If the purgatory is empty, there are no pending operations and no + // uncommitted state. We can return immediately. + resultAndOffset = new ControllerResultAndOffset<>(-1, + new ArrayList<>(), result.response()); + log.debug("Completing read-only operation {} immediately because " + + "the purgatory is empty.", this); + complete(null); + return; + } + // If there are operations in the purgatory, we want to wait for the latest + // one to complete before returning our result to the user. + resultAndOffset = new ControllerResultAndOffset<>(maybeOffset.get(), + result.records(), result.response()); + log.debug("Read-only operation {} will be completed when the log " + + "reaches offset {}", this, resultAndOffset.offset()); + } else { + // If the operation returned a batch of records, those records need to be + // written before we can return our result to the user. Here, we hand off + // the batch of records to the metadata log manager. They will be written + // out asynchronously. + long offset = logManager.scheduleWrite(controllerEpoch, result.records()); + op.processBatchEndOffset(offset); + writeOffset = offset; + resultAndOffset = new ControllerResultAndOffset<>(offset, + result.records(), result.response()); + for (ApiMessageAndVersion message : result.records()) { + replay(message.message()); + } + snapshotRegistry.createSnapshot(offset); + log.debug("Read-write operation {} will be completed when the log " + + "reaches offset {}.", this, resultAndOffset.offset()); + } + purgatory.add(resultAndOffset.offset(), this); + } + + @Override + public void handleException(Throwable exception) { + complete(exception); + } + + @Override + public void complete(Throwable exception) { + if (exception == null) { + handleEventEnd(this.toString(), startProcessingTimeNs.get()); + future.complete(resultAndOffset.response()); + } else { + future.completeExceptionally( + handleEventException(name, startProcessingTimeNs, exception)); + } + } + + @Override + public String toString() { + return name + "(" + System.identityHashCode(this) + ")"; + } + } + + private CompletableFuture appendWriteEvent(String name, + long timeoutMs, + ControllerWriteOperation op) { + ControllerWriteEvent event = new ControllerWriteEvent<>(name, op); + queue.appendWithDeadline(time.nanoseconds() + + NANOSECONDS.convert(timeoutMs, TimeUnit.MILLISECONDS), event); + return event.future(); + } + + private CompletableFuture appendWriteEvent(String name, + ControllerWriteOperation op) { + ControllerWriteEvent event = new ControllerWriteEvent<>(name, op); + queue.append(event); + return event.future(); + } + + class QuorumMetaLogListener implements MetaLogListener { + @Override + public void handleCommits(long offset, List messages) { + appendControlEvent("handleCommits[" + offset + "]", () -> { + if (curClaimEpoch == -1) { + // If the controller is a standby, replay the records that were + // created by the active controller. + if (log.isDebugEnabled()) { + if (log.isTraceEnabled()) { + log.trace("Replaying commits from the active node up to " + + "offset {}: {}.", offset, messages.stream(). + map(m -> m.toString()).collect(Collectors.joining(", "))); + } else { + log.debug("Replaying commits from the active node up to " + + "offset {}.", offset); + } + } + for (ApiMessage message : messages) { + replay(message); + } + } else { + // If the controller is active, the records were already replayed, + // so we don't need to do it here. + log.debug("Completing purgatory items up to offset {}.", offset); + + // Complete any events in the purgatory that were waiting for this offset. + purgatory.completeUpTo(offset); + + // Delete all snapshots older than the offset. + // TODO: add an exception here for when we're writing out a log snapshot + snapshotRegistry.deleteSnapshotsUpTo(offset); + } + lastCommittedOffset = offset; + }); + } + + @Override + public void handleNewLeader(MetaLogLeader newLeader) { + if (newLeader.nodeId() == nodeId) { + final long newEpoch = newLeader.epoch(); + appendControlEvent("handleClaim[" + newEpoch + "]", () -> { + long curEpoch = curClaimEpoch; + if (curEpoch != -1) { + throw new RuntimeException("Tried to claim controller epoch " + + newEpoch + ", but we never renounced controller epoch " + + curEpoch); + } + log.info("Becoming active at controller epoch {}.", newEpoch); + curClaimEpoch = newEpoch; + controllerMetrics.setActive(true); + writeOffset = lastCommittedOffset; + clusterControl.activate(); + }); + } + } + + @Override + public void handleRenounce(long oldEpoch) { + appendControlEvent("handleRenounce[" + oldEpoch + "]", () -> { + if (curClaimEpoch == oldEpoch) { + log.info("Renouncing the leadership at oldEpoch {} due to a metadata " + + "log event. Reverting to last committed offset {}.", curClaimEpoch, + lastCommittedOffset); + renounce(); + } + }); + } + + @Override + public void beginShutdown() { + queue.beginShutdown("MetaLogManager.Listener"); + } + } + + private void renounce() { + curClaimEpoch = -1; + controllerMetrics.setActive(false); + purgatory.failAll(newNotControllerException()); + snapshotRegistry.revertToSnapshot(lastCommittedOffset); + snapshotRegistry.deleteSnapshotsUpTo(lastCommittedOffset); + writeOffset = -1; + clusterControl.deactivate(); + cancelMaybeFenceReplicas(); + } + + private void scheduleDeferredWriteEvent(String name, long deadlineNs, + ControllerWriteOperation op) { + ControllerWriteEvent event = new ControllerWriteEvent<>(name, op); + queue.scheduleDeferred(name, new EarliestDeadlineFunction(deadlineNs), event); + event.future.exceptionally(e -> { + if (e instanceof UnknownServerException && e.getCause() != null && + e.getCause() instanceof RejectedExecutionException) { + log.error("Cancelling deferred write event {} because the event queue " + + "is now closed.", name); + return null; + } else if (e instanceof NotControllerException) { + log.debug("Cancelling deferred write event {} because this controller " + + "is no longer active.", name); + return null; + } + log.error("Unexpected exception while executing deferred write event {}. " + + "Rescheduling for a minute from now.", name, e); + scheduleDeferredWriteEvent(name, + deadlineNs + NANOSECONDS.convert(1, TimeUnit.MINUTES), op); + return null; + }); + } + + static final String MAYBE_FENCE_REPLICAS = "maybeFenceReplicas"; + + private void rescheduleMaybeFenceStaleBrokers() { + long nextCheckTimeNs = clusterControl.heartbeatManager().nextCheckTimeNs(); + if (nextCheckTimeNs == Long.MAX_VALUE) { + cancelMaybeFenceReplicas(); + return; + } + scheduleDeferredWriteEvent(MAYBE_FENCE_REPLICAS, nextCheckTimeNs, () -> { + ControllerResult result = replicationControl.maybeFenceStaleBrokers(); + rescheduleMaybeFenceStaleBrokers(); + return result; + }); + } + + private void cancelMaybeFenceReplicas() { + queue.cancelDeferred(MAYBE_FENCE_REPLICAS); + } + + @SuppressWarnings("unchecked") + private void replay(ApiMessage message) { + try { + MetadataRecordType type = MetadataRecordType.fromId(message.apiKey()); + switch (type) { + case REGISTER_BROKER_RECORD: + clusterControl.replay((RegisterBrokerRecord) message); + break; + case UNREGISTER_BROKER_RECORD: + clusterControl.replay((UnregisterBrokerRecord) message); + break; + case FENCE_BROKER_RECORD: + clusterControl.replay((FenceBrokerRecord) message); + break; + case UNFENCE_BROKER_RECORD: + clusterControl.replay((UnfenceBrokerRecord) message); + break; + case TOPIC_RECORD: + replicationControl.replay((TopicRecord) message); + break; + case PARTITION_RECORD: + replicationControl.replay((PartitionRecord) message); + break; + case CONFIG_RECORD: + configurationControl.replay((ConfigRecord) message); + break; + case QUOTA_RECORD: + clientQuotaControlManager.replay((QuotaRecord) message); + break; + case PARTITION_CHANGE_RECORD: + replicationControl.replay((PartitionChangeRecord) message); + break; + default: + throw new RuntimeException("Unhandled record type " + type); + } + } catch (Exception e) { + log.error("Error replaying record {}", message.toString(), e); + } + } + + private final Logger log; + + /** + * The ID of this controller node. + */ + private final int nodeId; + + /** + * The single-threaded queue that processes all of our events. + * It also processes timeouts. + */ + private final KafkaEventQueue queue; + + /** + * The Kafka clock object to use. + */ + private final Time time; + + /** + * The controller metrics. + */ + private final ControllerMetrics controllerMetrics; + + /** + * A registry for snapshot data. This must be accessed only by the event queue thread. + */ + private final SnapshotRegistry snapshotRegistry; + + /** + * The purgatory which holds deferred operations which are waiting for the metadata + * log's high water mark to advance. This must be accessed only by the event queue thread. + */ + private final ControllerPurgatory purgatory; + + /** + * An object which stores the controller's dynamic configuration. + * This must be accessed only by the event queue thread. + */ + private final ConfigurationControlManager configurationControl; + + /** + * An object which stores the controller's dynamic client quotas. + * This must be accessed only by the event queue thread. + */ + private final ClientQuotaControlManager clientQuotaControlManager; + + /** + * An object which stores the controller's view of the cluster. + * This must be accessed only by the event queue thread. + */ + private final ClusterControlManager clusterControl; + + /** + * An object which stores the controller's view of the cluster features. + * This must be accessed only by the event queue thread. + */ + private final FeatureControlManager featureControl; + + /** + * An object which stores the controller's view of topics and partitions. + * This must be accessed only by the event queue thread. + */ + private final ReplicationControlManager replicationControl; + + /** + * The interface that we use to mutate the Raft log. + */ + private final MetaLogManager logManager; + + /** + * The interface that receives callbacks from the Raft log. These callbacks are + * invoked from the Raft thread(s), not from the controller thread. + */ + private final QuorumMetaLogListener metaLogListener; + + /** + * If this controller is active, this is the non-negative controller epoch. + * Otherwise, this is -1. This variable must be modified only from the controller + * thread, but it can be read from other threads. + */ + private volatile long curClaimEpoch; + + /** + * The last offset we have committed, or -1 if we have not committed any offsets. + */ + private long lastCommittedOffset; + + /** + * If we have called scheduleWrite, this is the last offset we got back from it. + */ + private long writeOffset; + + private QuorumController(LogContext logContext, + int nodeId, + KafkaEventQueue queue, + Time time, + Map configDefs, + MetaLogManager logManager, + Map supportedFeatures, + short defaultReplicationFactor, + int defaultNumPartitions, + ReplicaPlacementPolicy replicaPlacementPolicy, + long sessionTimeoutNs, + ControllerMetrics controllerMetrics) throws Exception { + this.log = logContext.logger(QuorumController.class); + this.nodeId = nodeId; + this.queue = queue; + this.time = time; + this.controllerMetrics = controllerMetrics; + this.snapshotRegistry = new SnapshotRegistry(logContext); + snapshotRegistry.createSnapshot(-1); + this.purgatory = new ControllerPurgatory(); + this.configurationControl = new ConfigurationControlManager(logContext, + snapshotRegistry, configDefs); + this.clientQuotaControlManager = new ClientQuotaControlManager(snapshotRegistry); + this.clusterControl = new ClusterControlManager(logContext, time, + snapshotRegistry, sessionTimeoutNs, replicaPlacementPolicy); + this.featureControl = new FeatureControlManager(supportedFeatures, snapshotRegistry); + this.replicationControl = new ReplicationControlManager(snapshotRegistry, + logContext, new Random(), defaultReplicationFactor, defaultNumPartitions, + configurationControl, clusterControl); + this.logManager = logManager; + this.metaLogListener = new QuorumMetaLogListener(); + this.curClaimEpoch = -1L; + this.lastCommittedOffset = -1L; + this.writeOffset = -1L; + this.logManager.register(metaLogListener); + } + + @Override + public CompletableFuture alterIsr(AlterIsrRequestData request) { + return appendWriteEvent("alterIsr", () -> + replicationControl.alterIsr(request)); + } + + @Override + public CompletableFuture + createTopics(CreateTopicsRequestData request) { + return appendWriteEvent("createTopics", () -> + replicationControl.createTopics(request)); + } + + @Override + public CompletableFuture unregisterBroker(int brokerId) { + return appendWriteEvent("unregisterBroker", + () -> replicationControl.unregisterBroker(brokerId)); + } + + @Override + public CompletableFuture>>> + describeConfigs(Map> resources) { + return appendReadEvent("describeConfigs", () -> + configurationControl.describeConfigs(lastCommittedOffset, resources)); + } + + @Override + public CompletableFuture + electLeaders(ElectLeadersRequestData request) { + return appendWriteEvent("electLeaders", request.timeoutMs(), + () -> replicationControl.electLeaders(request)); + } + + @Override + public CompletableFuture finalizedFeatures() { + return appendReadEvent("getFinalizedFeatures", + () -> featureControl.finalizedFeatures(lastCommittedOffset)); + } + + @Override + public CompletableFuture> incrementalAlterConfigs( + Map>> configChanges, + boolean validateOnly) { + return appendWriteEvent("incrementalAlterConfigs", () -> { + ControllerResult> result = + configurationControl.incrementalAlterConfigs(configChanges); + if (validateOnly) { + return result.withoutRecords(); + } else { + return result; + } + }); + } + + @Override + public CompletableFuture> legacyAlterConfigs( + Map> newConfigs, boolean validateOnly) { + return appendWriteEvent("legacyAlterConfigs", () -> { + ControllerResult> result = + configurationControl.legacyAlterConfigs(newConfigs); + if (validateOnly) { + return result.withoutRecords(); + } else { + return result; + } + }); + } + + @Override + public CompletableFuture + processBrokerHeartbeat(BrokerHeartbeatRequestData request) { + return appendWriteEvent("processBrokerHeartbeat", + new ControllerWriteOperation() { + private final int brokerId = request.brokerId(); + private boolean inControlledShutdown = false; + + @Override + public ControllerResult generateRecordsAndResult() { + ControllerResult result = replicationControl. + processBrokerHeartbeat(request, lastCommittedOffset); + inControlledShutdown = result.response().inControlledShutdown(); + rescheduleMaybeFenceStaleBrokers(); + return result; + } + + @Override + public void processBatchEndOffset(long offset) { + if (inControlledShutdown) { + clusterControl.heartbeatManager(). + updateControlledShutdownOffset(brokerId, offset); + } + } + }); + } + + @Override + public CompletableFuture + registerBroker(BrokerRegistrationRequestData request) { + return appendWriteEvent("registerBroker", () -> { + ControllerResult result = clusterControl. + registerBroker(request, writeOffset + 1, featureControl. + finalizedFeatures(Long.MAX_VALUE)); + rescheduleMaybeFenceStaleBrokers(); + return result; + }); + } + + @Override + public CompletableFuture> alterClientQuotas( + Collection quotaAlterations, boolean validateOnly) { + return appendWriteEvent("alterClientQuotas", () -> { + ControllerResult> result = + clientQuotaControlManager.alterClientQuotas(quotaAlterations); + if (validateOnly) { + return result.withoutRecords(); + } else { + return result; + } + }); + } + + @Override + public CompletableFuture waitForReadyBrokers(int minBrokers) { + final CompletableFuture future = new CompletableFuture<>(); + appendControlEvent("waitForReadyBrokers", () -> { + clusterControl.addReadyBrokersFuture(future, minBrokers); + }); + return future; + } + + @Override + public void beginShutdown() { + queue.beginShutdown("QuorumController#beginShutdown"); + } + + public int nodeId() { + return nodeId; + } + + @Override + public long curClaimEpoch() { + return curClaimEpoch; + } + + @Override + public void close() throws InterruptedException { + queue.close(); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumControllerMetrics.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumControllerMetrics.java new file mode 100644 index 0000000000000..ad56faf3da99c --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumControllerMetrics.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import com.yammer.metrics.core.Gauge; +import com.yammer.metrics.core.Histogram; +import com.yammer.metrics.core.MetricName; +import com.yammer.metrics.core.MetricsRegistry; + + +public final class QuorumControllerMetrics implements ControllerMetrics { + private final static MetricName ACTIVE_CONTROLLER_COUNT = new MetricName( + "kafka.controller", "KafkaController", "ActiveControllerCount", null); + private final static MetricName EVENT_QUEUE_TIME_MS = new MetricName( + "kafka.controller", "ControllerEventManager", "EventQueueTimeMs", null); + private final static MetricName EVENT_QUEUE_PROCESSING_TIME_MS = new MetricName( + "kafka.controller", "ControllerEventManager", "EventQueueProcessingTimeMs", null); + + private volatile boolean active; + private final Gauge activeControllerCount; + private final Histogram eventQueueTime; + private final Histogram eventQueueProcessingTime; + + public QuorumControllerMetrics(MetricsRegistry registry) { + this.active = false; + this.activeControllerCount = registry.newGauge(ACTIVE_CONTROLLER_COUNT, new Gauge() { + @Override + public Integer value() { + return active ? 1 : 0; + } + }); + this.eventQueueTime = registry.newHistogram(EVENT_QUEUE_TIME_MS, true); + this.eventQueueProcessingTime = registry.newHistogram(EVENT_QUEUE_PROCESSING_TIME_MS, true); + } + + @Override + public void setActive(boolean active) { + this.active = active; + } + + @Override + public boolean active() { + return this.active; + } + + @Override + public void updateEventQueueTime(long durationMs) { + eventQueueTime.update(durationMs); + } + + @Override + public void updateEventQueueProcessingTime(long durationMs) { + eventQueueTime.update(durationMs); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ReplicaPlacementPolicy.java b/metadata/src/main/java/org/apache/kafka/controller/ReplicaPlacementPolicy.java new file mode 100644 index 0000000000000..44de85db34ef2 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ReplicaPlacementPolicy.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.Iterator; +import java.util.List; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.errors.InvalidReplicationFactorException; +import org.apache.kafka.metadata.UsableBroker; + + +/** + * The interface which a Kafka replica placement policy must implement. + */ +@InterfaceStability.Unstable +interface ReplicaPlacementPolicy { + /** + * Create a new replica placement. + * + * @param numPartitions The number of partitions to create placements for. + * @param numReplicas The number of replicas to create for each partitions. + * Must be positive. + * @param iterator An iterator that yields all the usable brokers. + * + * @return A list of replica lists. + * + * @throws InvalidReplicationFactorException If too many replicas were requested. + */ + List> createPlacement(int numPartitions, short numReplicas, + Iterator iterator) + throws InvalidReplicationFactorException; +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/Replicas.java b/metadata/src/main/java/org/apache/kafka/controller/Replicas.java new file mode 100644 index 0000000000000..104fffea58533 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/Replicas.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + + +public class Replicas { + /** + * An empty replica array. + */ + public final static int[] NONE = new int[0]; + + /** + * Convert an array of integers to a list of ints. + * + * @param array The input array. + * @return The output list. + */ + public static List toList(int[] array) { + if (array == null) return null; + ArrayList list = new ArrayList<>(array.length); + for (int i = 0; i < array.length; i++) { + list.add(array[i]); + } + return list; + } + + /** + * Convert a list of integers to an array of ints. + * + * @param list The input list. + * @return The output array. + */ + public static int[] toArray(List list) { + if (list == null) return null; + int[] array = new int[list.size()]; + for (int i = 0; i < list.size(); i++) { + array[i] = list.get(i); + } + return array; + } + + /** + * Copy an array of ints. + * + * @param array The input array. + * @return A copy of the array. + */ + public static int[] clone(int[] array) { + int[] clone = new int[array.length]; + System.arraycopy(array, 0, clone, 0, array.length); + return clone; + } + + /** + * Check that a replica set is valid. + * + * @param replicas The replica set. + * @return True if none of the replicas are negative, and there are no + * duplicates. + */ + public static boolean validate(int[] replicas) { + if (replicas.length == 0) return true; + int[] sortedReplicas = clone(replicas); + Arrays.sort(sortedReplicas); + int prev = sortedReplicas[0]; + if (prev < 0) return false; + for (int i = 1; i < sortedReplicas.length; i++) { + int replica = sortedReplicas[i]; + if (prev == replica) return false; + prev = replica; + } + return true; + } + + /** + * Check that an isr set is valid. + * + * @param replicas The replica set. + * @param isr The in-sync replica set. + * @return True if none of the in-sync replicas are negative, there are + * no duplicates, and all in-sync replicas are also replicas. + */ + public static boolean validateIsr(int[] replicas, int[] isr) { + if (isr.length == 0) return true; + if (replicas.length == 0) return false; + int[] sortedReplicas = clone(replicas); + Arrays.sort(sortedReplicas); + int[] sortedIsr = clone(isr); + Arrays.sort(sortedIsr); + int j = 0; + if (sortedIsr[0] < 0) return false; + int prevIsr = -1; + for (int i = 0; i < sortedIsr.length; i++) { + int curIsr = sortedIsr[i]; + if (prevIsr == curIsr) return false; + prevIsr = curIsr; + while (true) { + if (j == sortedReplicas.length) return false; + int curReplica = sortedReplicas[j++]; + if (curReplica == curIsr) break; + } + } + return true; + } + + /** + * Returns true if an array of replicas contains a specific value. + * + * @param replicas The replica array. + * @param value The value to look for. + * + * @return True only if the value is found in the array. + */ + public static boolean contains(int[] replicas, int value) { + for (int i = 0; i < replicas.length; i++) { + if (replicas[i] == value) return true; + } + return false; + } + + /** + * Copy a replica array without any occurrences of the given value. + * + * @param replicas The replica array. + * @param value The value to filter out. + * + * @return A new array without the given value. + */ + public static int[] copyWithout(int[] replicas, int value) { + int size = 0; + for (int i = 0; i < replicas.length; i++) { + if (replicas[i] != value) { + size++; + } + } + int[] result = new int[size]; + int j = 0; + for (int i = 0; i < replicas.length; i++) { + int replica = replicas[i]; + if (replica != value) { + result[j++] = replica; + } + } + return result; + } + + /** + * Copy a replica array with the given value. + * + * @param replicas The replica array. + * @param value The value to add. + * + * @return A new array with the given value. + */ + public static int[] copyWith(int[] replicas, int value) { + int[] newReplicas = new int[replicas.length + 1]; + System.arraycopy(replicas, 0, newReplicas, 0, replicas.length); + newReplicas[newReplicas.length - 1] = value; + return newReplicas; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java new file mode 100644 index 0000000000000..aa60f5127de4d --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @@ -0,0 +1,908 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.clients.admin.AlterConfigOp.OpType; +import org.apache.kafka.common.ElectionType; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.BrokerIdNotRegisteredException; +import org.apache.kafka.common.errors.InvalidReplicationFactorException; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.AlterIsrRequestData; +import org.apache.kafka.common.message.AlterIsrResponseData; +import org.apache.kafka.common.message.BrokerHeartbeatRequestData; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection; +import org.apache.kafka.common.message.CreateTopicsRequestData; +import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; +import org.apache.kafka.common.message.CreateTopicsResponseData; +import org.apache.kafka.common.message.ElectLeadersRequestData.TopicPartitions; +import org.apache.kafka.common.message.ElectLeadersRequestData; +import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult; +import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; +import org.apache.kafka.common.message.ElectLeadersResponseData; +import org.apache.kafka.common.metadata.FenceBrokerRecord; +import org.apache.kafka.common.metadata.PartitionChangeRecord; +import org.apache.kafka.common.metadata.PartitionRecord; +import org.apache.kafka.common.metadata.TopicRecord; +import org.apache.kafka.common.metadata.UnfenceBrokerRecord; +import org.apache.kafka.common.metadata.UnregisterBrokerRecord; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.controller.BrokersToIsrs.TopicPartition; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.metadata.BrokerHeartbeatReply; +import org.apache.kafka.metadata.BrokerRegistration; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.apache.kafka.timeline.TimelineHashMap; +import org.slf4j.Logger; + +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Random; + +import static org.apache.kafka.clients.admin.AlterConfigOp.OpType.SET; +import static org.apache.kafka.common.config.ConfigResource.Type.TOPIC; + + +/** + * The ReplicationControlManager is the part of the controller which deals with topics + * and partitions. It is responsible for managing the in-sync replica set and leader + * of each partition, as well as administrative tasks like creating or deleting topics. + */ +public class ReplicationControlManager { + /** + * A special value used to represent the leader for a partition with no leader. + */ + public static final int NO_LEADER = -1; + + /** + * A special value used to represent a PartitionChangeRecord that does not change the + * partition leader. + */ + public static final int NO_LEADER_CHANGE = -2; + + static class TopicControlInfo { + private final Uuid id; + private final TimelineHashMap parts; + + TopicControlInfo(SnapshotRegistry snapshotRegistry, Uuid id) { + this.id = id; + this.parts = new TimelineHashMap<>(snapshotRegistry, 0); + } + } + + static class PartitionControlInfo { + private final int[] replicas; + private final int[] isr; + private final int[] removingReplicas; + private final int[] addingReplicas; + private final int leader; + private final int leaderEpoch; + private final int partitionEpoch; + + PartitionControlInfo(PartitionRecord record) { + this(Replicas.toArray(record.replicas()), + Replicas.toArray(record.isr()), + Replicas.toArray(record.removingReplicas()), + Replicas.toArray(record.addingReplicas()), + record.leader(), + record.leaderEpoch(), + record.partitionEpoch()); + } + + PartitionControlInfo(int[] replicas, int[] isr, int[] removingReplicas, + int[] addingReplicas, int leader, int leaderEpoch, + int partitionEpoch) { + this.replicas = replicas; + this.isr = isr; + this.removingReplicas = removingReplicas; + this.addingReplicas = addingReplicas; + this.leader = leader; + this.leaderEpoch = leaderEpoch; + this.partitionEpoch = partitionEpoch; + } + + PartitionControlInfo merge(PartitionChangeRecord record) { + int[] newIsr = (record.isr() == null) ? isr : Replicas.toArray(record.isr()); + int newLeader; + int newLeaderEpoch; + if (record.leader() == NO_LEADER_CHANGE) { + newLeader = leader; + newLeaderEpoch = leaderEpoch; + } else { + newLeader = record.leader(); + newLeaderEpoch = leaderEpoch + 1; + } + return new PartitionControlInfo(replicas, + newIsr, + removingReplicas, + addingReplicas, + newLeader, + newLeaderEpoch, + partitionEpoch + 1); + } + + String diff(PartitionControlInfo prev) { + StringBuilder builder = new StringBuilder(); + String prefix = ""; + if (!Arrays.equals(replicas, prev.replicas)) { + builder.append(prefix).append("oldReplicas=").append(Arrays.toString(prev.replicas)); + prefix = ", "; + builder.append(prefix).append("newReplicas=").append(Arrays.toString(replicas)); + } + if (!Arrays.equals(isr, prev.isr)) { + builder.append(prefix).append("oldIsr=").append(Arrays.toString(prev.isr)); + prefix = ", "; + builder.append(prefix).append("newIsr=").append(Arrays.toString(isr)); + } + if (!Arrays.equals(removingReplicas, prev.removingReplicas)) { + builder.append(prefix).append("oldRemovingReplicas="). + append(Arrays.toString(prev.removingReplicas)); + prefix = ", "; + builder.append(prefix).append("newRemovingReplicas="). + append(Arrays.toString(removingReplicas)); + } + if (!Arrays.equals(addingReplicas, prev.addingReplicas)) { + builder.append(prefix).append("oldAddingReplicas="). + append(Arrays.toString(prev.addingReplicas)); + prefix = ", "; + builder.append(prefix).append("newAddingReplicas="). + append(Arrays.toString(addingReplicas)); + } + if (leader != prev.leader) { + builder.append(prefix).append("oldLeader=").append(prev.leader); + prefix = ", "; + builder.append(prefix).append("newLeader=").append(leader); + } + if (leaderEpoch != prev.leaderEpoch) { + builder.append(prefix).append("oldLeaderEpoch=").append(prev.leaderEpoch); + prefix = ", "; + builder.append(prefix).append("newLeaderEpoch=").append(leaderEpoch); + } + if (partitionEpoch != prev.partitionEpoch) { + builder.append(prefix).append("oldPartitionEpoch=").append(prev.partitionEpoch); + prefix = ", "; + builder.append(prefix).append("newPartitionEpoch=").append(partitionEpoch); + } + return builder.toString(); + } + + boolean hasLeader() { + return leader != NO_LEADER; + } + + int preferredReplica() { + return replicas.length == 0 ? NO_LEADER : replicas[0]; + } + + @Override + public int hashCode() { + return Objects.hash(replicas, isr, removingReplicas, addingReplicas, leader, + leaderEpoch, partitionEpoch); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof PartitionControlInfo)) return false; + PartitionControlInfo other = (PartitionControlInfo) o; + return diff(other).isEmpty(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("PartitionControlInfo("); + builder.append("replicas=").append(Arrays.toString(replicas)); + builder.append(", isr=").append(Arrays.toString(isr)); + builder.append(", removingReplicas=").append(Arrays.toString(removingReplicas)); + builder.append(", addingReplicas=").append(Arrays.toString(addingReplicas)); + builder.append(", leader=").append(leader); + builder.append(", leaderEpoch=").append(leaderEpoch); + builder.append(", partitionEpoch=").append(partitionEpoch); + builder.append(")"); + return builder.toString(); + } + } + + private final SnapshotRegistry snapshotRegistry; + private final Logger log; + + /** + * The random number generator used by this object. + */ + private final Random random; + + /** + * The KIP-464 default replication factor that is used if a CreateTopics request does + * not specify one. + */ + private final short defaultReplicationFactor; + + /** + * The KIP-464 default number of partitions that is used if a CreateTopics request does + * not specify a number of partitions. + */ + private final int defaultNumPartitions; + + /** + * A reference to the controller's configuration control manager. + */ + private final ConfigurationControlManager configurationControl; + + /** + * A reference to the controller's cluster control manager. + */ + final ClusterControlManager clusterControl; + + /** + * Maps topic names to topic UUIDs. + */ + private final TimelineHashMap topicsByName; + + /** + * Maps topic UUIDs to structures containing topic information, including partitions. + */ + private final TimelineHashMap topics; + + /** + * A map of broker IDs to the partitions that the broker is in the ISR for. + */ + private final BrokersToIsrs brokersToIsrs; + + ReplicationControlManager(SnapshotRegistry snapshotRegistry, + LogContext logContext, + Random random, + short defaultReplicationFactor, + int defaultNumPartitions, + ConfigurationControlManager configurationControl, + ClusterControlManager clusterControl) { + this.snapshotRegistry = snapshotRegistry; + this.log = logContext.logger(ReplicationControlManager.class); + this.random = random; + this.defaultReplicationFactor = defaultReplicationFactor; + this.defaultNumPartitions = defaultNumPartitions; + this.configurationControl = configurationControl; + this.clusterControl = clusterControl; + this.topicsByName = new TimelineHashMap<>(snapshotRegistry, 0); + this.topics = new TimelineHashMap<>(snapshotRegistry, 0); + this.brokersToIsrs = new BrokersToIsrs(snapshotRegistry); + } + + public void replay(TopicRecord record) { + topicsByName.put(record.name(), record.topicId()); + topics.put(record.topicId(), new TopicControlInfo(snapshotRegistry, record.topicId())); + log.info("Created topic {} with ID {}.", record.name(), record.topicId()); + } + + public void replay(PartitionRecord record) { + TopicControlInfo topicInfo = topics.get(record.topicId()); + if (topicInfo == null) { + throw new RuntimeException("Tried to create partition " + record.topicId() + + ":" + record.partitionId() + ", but no topic with that ID was found."); + } + PartitionControlInfo newPartInfo = new PartitionControlInfo(record); + PartitionControlInfo prevPartInfo = topicInfo.parts.get(record.partitionId()); + if (prevPartInfo == null) { + log.info("Created partition {}:{} with {}.", record.topicId(), + record.partitionId(), newPartInfo.toString()); + topicInfo.parts.put(record.partitionId(), newPartInfo); + brokersToIsrs.update(record.topicId(), record.partitionId(), null, + newPartInfo.isr, NO_LEADER, newPartInfo.leader); + } else { + String diff = newPartInfo.diff(prevPartInfo); + if (!diff.isEmpty()) { + log.info("Modified partition {}:{}: {}.", record.topicId(), + record.partitionId(), diff); + topicInfo.parts.put(record.partitionId(), newPartInfo); + brokersToIsrs.update(record.topicId(), record.partitionId(), + prevPartInfo.isr, newPartInfo.isr, prevPartInfo.leader, + newPartInfo.leader); + } + } + } + + public void replay(PartitionChangeRecord record) { + TopicControlInfo topicInfo = topics.get(record.topicId()); + if (topicInfo == null) { + throw new RuntimeException("Tried to create partition " + record.topicId() + + ":" + record.partitionId() + ", but no topic with that ID was found."); + } + PartitionControlInfo prevPartitionInfo = topicInfo.parts.get(record.partitionId()); + if (prevPartitionInfo == null) { + throw new RuntimeException("Tried to create partition " + record.topicId() + + ":" + record.partitionId() + ", but no partition with that id was found."); + } + PartitionControlInfo newPartitionInfo = prevPartitionInfo.merge(record); + topicInfo.parts.put(record.partitionId(), newPartitionInfo); + brokersToIsrs.update(record.topicId(), record.partitionId(), + prevPartitionInfo.isr, newPartitionInfo.isr, prevPartitionInfo.leader, + newPartitionInfo.leader); + log.debug("Applied ISR change record: {}", record.toString()); + } + + ControllerResult + createTopics(CreateTopicsRequestData request) { + Map topicErrors = new HashMap<>(); + List records = new ArrayList<>(); + + // Check the topic names. + validateNewTopicNames(topicErrors, request.topics()); + + // Identify topics that already exist and mark them with the appropriate error + request.topics().stream().filter(creatableTopic -> topicsByName.containsKey(creatableTopic.name())) + .forEach(t -> topicErrors.put(t.name(), new ApiError(Errors.TOPIC_ALREADY_EXISTS))); + + // Verify that the configurations for the new topics are OK, and figure out what + // ConfigRecords should be created. + Map>> configChanges = + computeConfigChanges(topicErrors, request.topics()); + ControllerResult> configResult = + configurationControl.incrementalAlterConfigs(configChanges); + for (Entry entry : configResult.response().entrySet()) { + if (entry.getValue().isFailure()) { + topicErrors.put(entry.getKey().name(), entry.getValue()); + } + } + records.addAll(configResult.records()); + + // Try to create whatever topics are needed. + Map successes = new HashMap<>(); + for (CreatableTopic topic : request.topics()) { + if (topicErrors.containsKey(topic.name())) continue; + ApiError error = createTopic(topic, records, successes); + if (error.isFailure()) { + topicErrors.put(topic.name(), error); + } + } + + // Create responses for all topics. + CreateTopicsResponseData data = new CreateTopicsResponseData(); + StringBuilder resultsBuilder = new StringBuilder(); + String resultsPrefix = ""; + for (CreatableTopic topic : request.topics()) { + ApiError error = topicErrors.get(topic.name()); + if (error != null) { + data.topics().add(new CreatableTopicResult(). + setName(topic.name()). + setErrorCode(error.error().code()). + setErrorMessage(error.message())); + resultsBuilder.append(resultsPrefix).append(topic).append(": "). + append(error.error()).append(" (").append(error.message()).append(")"); + resultsPrefix = ", "; + continue; + } + CreatableTopicResult result = successes.get(topic.name()); + data.topics().add(result); + resultsBuilder.append(resultsPrefix).append(topic).append(": "). + append("SUCCESS"); + resultsPrefix = ", "; + } + log.info("createTopics result(s): {}", resultsBuilder.toString()); + return new ControllerResult<>(records, data); + } + + private ApiError createTopic(CreatableTopic topic, + List records, + Map successes) { + Map newParts = new HashMap<>(); + if (!topic.assignments().isEmpty()) { + if (topic.replicationFactor() != -1) { + return new ApiError(Errors.INVALID_REQUEST, + "A manual partition assignment was specified, but replication " + + "factor was not set to -1."); + } + if (topic.numPartitions() != -1) { + return new ApiError(Errors.INVALID_REQUEST, + "A manual partition assignment was specified, but numPartitions " + + "was not set to -1."); + } + for (CreatableReplicaAssignment assignment : topic.assignments()) { + if (newParts.containsKey(assignment.partitionIndex())) { + return new ApiError(Errors.INVALID_REPLICA_ASSIGNMENT, + "Found multiple manual partition assignments for partition " + + assignment.partitionIndex()); + } + HashSet brokerIds = new HashSet<>(); + for (int brokerId : assignment.brokerIds()) { + if (!brokerIds.add(brokerId)) { + return new ApiError(Errors.INVALID_REPLICA_ASSIGNMENT, + "The manual partition assignment specifies the same node " + + "id more than once."); + } else if (!clusterControl.unfenced(brokerId)) { + return new ApiError(Errors.INVALID_REPLICA_ASSIGNMENT, + "The manual partition assignment contains node " + brokerId + + ", but that node is not usable."); + } + } + int[] replicas = new int[assignment.brokerIds().size()]; + for (int i = 0; i < replicas.length; i++) { + replicas[i] = assignment.brokerIds().get(i); + } + int[] isr = new int[assignment.brokerIds().size()]; + for (int i = 0; i < replicas.length; i++) { + isr[i] = assignment.brokerIds().get(i); + } + newParts.put(assignment.partitionIndex(), + new PartitionControlInfo(replicas, isr, null, null, isr[0], 0, 0)); + } + } else if (topic.replicationFactor() < -1 || topic.replicationFactor() == 0) { + return new ApiError(Errors.INVALID_REPLICATION_FACTOR, + "Replication factor was set to an invalid non-positive value."); + } else if (!topic.assignments().isEmpty()) { + return new ApiError(Errors.INVALID_REQUEST, + "Replication factor was not set to -1 but a manual partition " + + "assignment was specified."); + } else if (topic.numPartitions() < -1 || topic.numPartitions() == 0) { + return new ApiError(Errors.INVALID_PARTITIONS, + "Number of partitions was set to an invalid non-positive value."); + } else { + int numPartitions = topic.numPartitions() == -1 ? + defaultNumPartitions : topic.numPartitions(); + short replicationFactor = topic.replicationFactor() == -1 ? + defaultReplicationFactor : topic.replicationFactor(); + try { + List> replicas = clusterControl. + placeReplicas(numPartitions, replicationFactor); + for (int partitionId = 0; partitionId < replicas.size(); partitionId++) { + int[] r = Replicas.toArray(replicas.get(partitionId)); + newParts.put(partitionId, + new PartitionControlInfo(r, r, null, null, r[0], 0, 0)); + } + } catch (InvalidReplicationFactorException e) { + return new ApiError(Errors.INVALID_REPLICATION_FACTOR, + "Unable to replicate the partition " + replicationFactor + + " times: " + e.getMessage()); + } + } + Uuid topicId = new Uuid(random.nextLong(), random.nextLong()); + successes.put(topic.name(), new CreatableTopicResult(). + setName(topic.name()). + setTopicId(topicId). + setErrorCode((short) 0). + setErrorMessage(null). + setNumPartitions(newParts.size()). + setReplicationFactor((short) newParts.get(0).replicas.length)); + records.add(new ApiMessageAndVersion(new TopicRecord(). + setName(topic.name()). + setTopicId(topicId), (short) 0)); + for (Entry partEntry : newParts.entrySet()) { + int partitionIndex = partEntry.getKey(); + PartitionControlInfo info = partEntry.getValue(); + records.add(new ApiMessageAndVersion(new PartitionRecord(). + setPartitionId(partitionIndex). + setTopicId(topicId). + setReplicas(Replicas.toList(info.replicas)). + setIsr(Replicas.toList(info.isr)). + setRemovingReplicas(null). + setAddingReplicas(null). + setLeader(info.leader). + setLeaderEpoch(info.leaderEpoch). + setPartitionEpoch(0), (short) 0)); + } + return ApiError.NONE; + } + + static void validateNewTopicNames(Map topicErrors, + CreatableTopicCollection topics) { + for (CreatableTopic topic : topics) { + if (topicErrors.containsKey(topic.name())) continue; + try { + Topic.validate(topic.name()); + } catch (InvalidTopicException e) { + topicErrors.put(topic.name(), + new ApiError(Errors.INVALID_TOPIC_EXCEPTION, e.getMessage())); + } + } + } + + static Map>> + computeConfigChanges(Map topicErrors, + CreatableTopicCollection topics) { + Map>> configChanges = new HashMap<>(); + for (CreatableTopic topic : topics) { + if (topicErrors.containsKey(topic.name())) continue; + Map> topicConfigs = new HashMap<>(); + for (CreateTopicsRequestData.CreateableTopicConfig config : topic.configs()) { + topicConfigs.put(config.name(), new SimpleImmutableEntry<>(SET, config.value())); + } + if (!topicConfigs.isEmpty()) { + configChanges.put(new ConfigResource(TOPIC, topic.name()), topicConfigs); + } + } + return configChanges; + } + + // VisibleForTesting + PartitionControlInfo getPartition(Uuid topicId, int partitionId) { + TopicControlInfo topic = topics.get(topicId); + if (topic == null) { + return null; + } + return topic.parts.get(partitionId); + } + + // VisibleForTesting + BrokersToIsrs brokersToIsrs() { + return brokersToIsrs; + } + + ControllerResult alterIsr(AlterIsrRequestData request) { + clusterControl.checkBrokerEpoch(request.brokerId(), request.brokerEpoch()); + AlterIsrResponseData response = new AlterIsrResponseData(); + List records = new ArrayList<>(); + for (AlterIsrRequestData.TopicData topicData : request.topics()) { + AlterIsrResponseData.TopicData responseTopicData = + new AlterIsrResponseData.TopicData().setName(topicData.name()); + response.topics().add(responseTopicData); + Uuid topicId = topicsByName.get(topicData.name()); + if (topicId == null || !topics.containsKey(topicId)) { + for (AlterIsrRequestData.PartitionData partitionData : topicData.partitions()) { + responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). + setPartitionIndex(partitionData.partitionIndex()). + setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code())); + } + continue; + } + TopicControlInfo topic = topics.get(topicId); + for (AlterIsrRequestData.PartitionData partitionData : topicData.partitions()) { + PartitionControlInfo partition = topic.parts.get(partitionData.partitionIndex()); + if (partition == null) { + responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). + setPartitionIndex(partitionData.partitionIndex()). + setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code())); + continue; + } + if (partitionData.leaderEpoch() != partition.leaderEpoch) { + responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). + setPartitionIndex(partitionData.partitionIndex()). + setErrorCode(Errors.FENCED_LEADER_EPOCH.code())); + continue; + } + if (partitionData.currentIsrVersion() != partition.partitionEpoch) { + responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). + setPartitionIndex(partitionData.partitionIndex()). + setErrorCode(Errors.INVALID_UPDATE_VERSION.code())); + continue; + } + int[] newIsr = Replicas.toArray(partitionData.newIsr()); + if (!Replicas.validateIsr(partition.replicas, newIsr)) { + responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). + setPartitionIndex(partitionData.partitionIndex()). + setErrorCode(Errors.INVALID_REQUEST.code())); + continue; + } + if (!Replicas.contains(newIsr, partition.leader)) { + // An alterIsr request can't remove the current leader. + responseTopicData.partitions().add(new AlterIsrResponseData.PartitionData(). + setPartitionIndex(partitionData.partitionIndex()). + setErrorCode(Errors.INVALID_REQUEST.code())); + continue; + } + records.add(new ApiMessageAndVersion(new PartitionChangeRecord(). + setPartitionId(partitionData.partitionIndex()). + setTopicId(topic.id). + setIsr(partitionData.newIsr()), (short) 0)); + } + } + return new ControllerResult<>(records, response); + } + + /** + * Generate the appropriate records to handle a broker being fenced. + * + * First, we remove this broker from any non-singleton ISR. Then we generate a + * FenceBrokerRecord. + * + * @param brokerId The broker id. + * @param records The record list to append to. + */ + + void handleBrokerFenced(int brokerId, List records) { + BrokerRegistration brokerRegistration = clusterControl.brokerRegistrations().get(brokerId); + if (brokerRegistration == null) { + throw new RuntimeException("Can't find broker registration for broker " + brokerId); + } + handleNodeDeactivated(brokerId, records); + records.add(new ApiMessageAndVersion(new FenceBrokerRecord(). + setId(brokerId).setEpoch(brokerRegistration.epoch()), (short) 0)); + } + + /** + * Generate the appropriate records to handle a broker being unregistered. + * + * First, we remove this broker from any non-singleton ISR. Then we generate an + * UnregisterBrokerRecord. + * + * @param brokerId The broker id. + * @param brokerEpoch The broker epoch. + * @param records The record list to append to. + */ + void handleBrokerUnregistered(int brokerId, long brokerEpoch, + List records) { + handleNodeDeactivated(brokerId, records); + records.add(new ApiMessageAndVersion(new UnregisterBrokerRecord(). + setBrokerId(brokerId).setBrokerEpoch(brokerEpoch), (short) 0)); + } + + /** + * Handle a broker being deactivated. This means we remove it from any ISR that has + * more than one element. We do not remove the broker from ISRs where it is the only + * member since this would preclude clean leader election in the future. + * It is removed as the leader for all partitions it leads. + * + * @param brokerId The broker id. + * @param records The record list to append to. + */ + void handleNodeDeactivated(int brokerId, List records) { + Iterator iterator = brokersToIsrs.iterator(brokerId, false); + while (iterator.hasNext()) { + TopicPartition topicPartition = iterator.next(); + TopicControlInfo topic = topics.get(topicPartition.topicId()); + if (topic == null) { + throw new RuntimeException("Topic ID " + topicPartition.topicId() + " existed in " + + "isrMembers, but not in the topics map."); + } + PartitionControlInfo partition = topic.parts.get(topicPartition.partitionId()); + if (partition == null) { + throw new RuntimeException("Partition " + topicPartition + + " existed in isrMembers, but not in the partitions map."); + } + PartitionChangeRecord record = new PartitionChangeRecord(). + setPartitionId(topicPartition.partitionId()). + setTopicId(topic.id); + int[] newIsr = Replicas.copyWithout(partition.isr, brokerId); + if (newIsr.length == 0) { + // We don't want to shrink the ISR to size 0. So, leave the node in the ISR. + if (record.leader() != NO_LEADER) { + // The partition is now leaderless, so set its leader to -1. + record.setLeader(-1); + records.add(new ApiMessageAndVersion(record, (short) 0)); + } + } else { + record.setIsr(Replicas.toList(newIsr)); + if (partition.leader == brokerId) { + // The fenced node will no longer be the leader. + int newLeader = bestLeader(partition.replicas, newIsr, false); + record.setLeader(newLeader); + } else { + // Bump the partition leader epoch. + record.setLeader(partition.leader); + } + records.add(new ApiMessageAndVersion(record, (short) 0)); + } + } + } + + /** + * Generate the appropriate records to handle a broker becoming unfenced. + * + * First, we create an UnfenceBrokerRecord. Then, we check if if there are any + * partitions that don't currently have a leader that should be led by the newly + * unfenced broker. + * + * @param brokerId The broker id. + * @param brokerEpoch The broker epoch. + * @param records The record list to append to. + */ + void handleBrokerUnfenced(int brokerId, long brokerEpoch, List records) { + records.add(new ApiMessageAndVersion(new UnfenceBrokerRecord(). + setId(brokerId).setEpoch(brokerEpoch), (short) 0)); + handleNodeActivated(brokerId, records); + } + + /** + * Handle a broker being activated. This means we check if it can become the leader + * for any partition that currently has no leader (aka offline partition). + * + * @param brokerId The broker id. + * @param records The record list to append to. + */ + void handleNodeActivated(int brokerId, List records) { + Iterator iterator = brokersToIsrs.noLeaderIterator(); + while (iterator.hasNext()) { + TopicPartition topicPartition = iterator.next(); + TopicControlInfo topic = topics.get(topicPartition.topicId()); + if (topic == null) { + throw new RuntimeException("Topic ID " + topicPartition.topicId() + " existed in " + + "isrMembers, but not in the topics map."); + } + PartitionControlInfo partition = topic.parts.get(topicPartition.partitionId()); + if (partition == null) { + throw new RuntimeException("Partition " + topicPartition + + " existed in isrMembers, but not in the partitions map."); + } + // TODO: if this partition is configured for unclean leader election, + // check the replica set rather than the ISR. + if (Replicas.contains(partition.isr, brokerId)) { + records.add(new ApiMessageAndVersion(new PartitionChangeRecord(). + setPartitionId(topicPartition.partitionId()). + setTopicId(topic.id). + setLeader(brokerId), (short) 0)); + } + } + } + + ControllerResult electLeaders(ElectLeadersRequestData request) { + boolean unclean = electionIsUnclean(request.electionType()); + List records = new ArrayList<>(); + ElectLeadersResponseData response = new ElectLeadersResponseData(); + for (TopicPartitions topic : request.topicPartitions()) { + ReplicaElectionResult topicResults = + new ReplicaElectionResult().setTopic(topic.topic()); + response.replicaElectionResults().add(topicResults); + for (int partitionId : topic.partitions()) { + ApiError error = electLeader(topic.topic(), partitionId, unclean, records); + topicResults.partitionResult().add(new PartitionResult(). + setPartitionId(partitionId). + setErrorCode(error.error().code()). + setErrorMessage(error.message())); + } + } + return new ControllerResult<>(records, response); + } + + static boolean electionIsUnclean(byte electionType) { + ElectionType type; + try { + type = ElectionType.valueOf(electionType); + } catch (IllegalArgumentException e) { + throw new InvalidRequestException("Unknown election type " + (int) electionType); + } + return type == ElectionType.UNCLEAN; + } + + ApiError electLeader(String topic, int partitionId, boolean unclean, + List records) { + Uuid topicId = topicsByName.get(topic); + if (topicId == null) { + return new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, + "No such topic as " + topic); + } + TopicControlInfo topicInfo = topics.get(topicId); + if (topicInfo == null) { + return new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, + "No such topic id as " + topicId); + } + PartitionControlInfo partitionInfo = topicInfo.parts.get(partitionId); + if (partitionInfo == null) { + return new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, + "No such partition as " + topic + "-" + partitionId); + } + int newLeader = bestLeader(partitionInfo.replicas, partitionInfo.isr, unclean); + if (newLeader == NO_LEADER) { + // If we can't find any leader for the partition, return an error. + return new ApiError(Errors.LEADER_NOT_AVAILABLE, + "Unable to find any leader for the partition."); + } + if (newLeader == partitionInfo.leader) { + // If the new leader we picked is the same as the current leader, there is + // nothing to do. + return ApiError.NONE; + } + if (partitionInfo.hasLeader() && newLeader != partitionInfo.preferredReplica()) { + // It is not worth moving away from a valid leader to a new leader unless the + // new leader is the preferred replica. + return ApiError.NONE; + } + PartitionChangeRecord record = new PartitionChangeRecord(). + setPartitionId(partitionId). + setTopicId(topicId); + if (unclean && !Replicas.contains(partitionInfo.isr, newLeader)) { + // If the election was unclean, we may have to forcibly add the replica to + // the ISR. This can result in data loss! + record.setIsr(Collections.singletonList(newLeader)); + } + record.setLeader(newLeader); + records.add(new ApiMessageAndVersion(record, (short) 0)); + return ApiError.NONE; + } + + ControllerResult processBrokerHeartbeat( + BrokerHeartbeatRequestData request, long lastCommittedOffset) { + int brokerId = request.brokerId(); + long brokerEpoch = request.brokerEpoch(); + clusterControl.checkBrokerEpoch(brokerId, brokerEpoch); + BrokerHeartbeatManager heartbeatManager = clusterControl.heartbeatManager(); + BrokerControlStates states = heartbeatManager.calculateNextBrokerState(brokerId, + request, lastCommittedOffset, () -> brokersToIsrs.hasLeaderships(brokerId)); + List records = new ArrayList<>(); + if (states.current() != states.next()) { + switch (states.next()) { + case FENCED: + handleBrokerFenced(brokerId, records); + break; + case UNFENCED: + handleBrokerUnfenced(brokerId, brokerEpoch, records); + break; + case CONTROLLED_SHUTDOWN: + // Note: we always bump the leader epoch of each partition that the + // shutting down broker is in here. This prevents the broker from + // getting re-added to the ISR later. + handleNodeDeactivated(brokerId, records); + break; + case SHUTDOWN_NOW: + handleBrokerFenced(brokerId, records); + break; + } + } + heartbeatManager.touch(brokerId, + states.next().fenced(), + request.currentMetadataOffset()); + boolean isCaughtUp = request.currentMetadataOffset() >= lastCommittedOffset; + BrokerHeartbeatReply reply = new BrokerHeartbeatReply(isCaughtUp, + states.next().fenced(), + states.next().inControlledShutdown(), + states.next().shouldShutDown()); + return new ControllerResult<>(records, reply); + } + + int bestLeader(int[] replicas, int[] isr, boolean unclean) { + for (int i = 0; i < replicas.length; i++) { + int replica = replicas[i]; + if (Replicas.contains(isr, replica)) { + return replica; + } + } + if (unclean) { + for (int i = 0; i < replicas.length; i++) { + int replica = replicas[i]; + if (clusterControl.unfenced(replica)) { + return replica; + } + } + } + return NO_LEADER; + } + + public ControllerResult unregisterBroker(int brokerId) { + BrokerRegistration registration = clusterControl.brokerRegistrations().get(brokerId); + if (registration == null) { + throw new BrokerIdNotRegisteredException("Broker ID " + brokerId + + " is not currently registered"); + } + List records = new ArrayList<>(); + handleBrokerUnregistered(brokerId, registration.epoch(), records); + return new ControllerResult<>(records, null); + } + + ControllerResult maybeFenceStaleBrokers() { + List records = new ArrayList<>(); + BrokerHeartbeatManager heartbeatManager = clusterControl.heartbeatManager(); + List staleBrokers = heartbeatManager.findStaleBrokers(); + for (int brokerId : staleBrokers) { + log.info("Fencing broker {} because its session has timed out.", brokerId); + handleBrokerFenced(brokerId, records); + heartbeatManager.fence(brokerId); + } + return new ControllerResult<>(records, null); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ResultOrError.java b/metadata/src/main/java/org/apache/kafka/controller/ResultOrError.java index 82e2b49123f8d..6d910e4a86986 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ResultOrError.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ResultOrError.java @@ -64,7 +64,7 @@ public boolean equals(Object o) { return false; } ResultOrError other = (ResultOrError) o; - return error.equals(other.error) && + return Objects.equals(error, other.error) && Objects.equals(result, other.result); } @@ -75,7 +75,7 @@ public int hashCode() { @Override public String toString() { - if (error.isSuccess()) { + if (error == null) { return "ResultOrError(" + result + ")"; } else { return "ResultOrError(" + error + ")"; diff --git a/metadata/src/main/java/org/apache/kafka/controller/SimpleReplicaPlacementPolicy.java b/metadata/src/main/java/org/apache/kafka/controller/SimpleReplicaPlacementPolicy.java new file mode 100644 index 0000000000000..95e96cd62f4af --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/SimpleReplicaPlacementPolicy.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +import org.apache.kafka.common.errors.InvalidReplicationFactorException; +import org.apache.kafka.metadata.UsableBroker; + + +/** + * A simple uniformly random placement policy. + * + * TODO: implement the current "striped" placement policy, plus rack aware placement + * policies, etc. + */ +public class SimpleReplicaPlacementPolicy implements ReplicaPlacementPolicy { + private final Random random; + + public SimpleReplicaPlacementPolicy(Random random) { + this.random = random; + } + + @Override + public List> createPlacement(int numPartitions, + short numReplicas, + Iterator iterator) { + List usable = new ArrayList<>(); + while (iterator.hasNext()) { + usable.add(iterator.next()); + } + if (usable.size() < numReplicas) { + throw new InvalidReplicationFactorException("there are only " + usable.size() + + " usable brokers"); + } + List> results = new ArrayList<>(); + for (int p = 0; p < numPartitions; p++) { + List choices = new ArrayList<>(); + // TODO: rack-awareness + List indexes = new ArrayList<>(); + int initialIndex = random.nextInt(usable.size()); + for (int i = 0; i < numReplicas; i++) { + indexes.add((initialIndex + i) % usable.size()); + } + indexes.sort(Integer::compareTo); + Iterator iter = usable.iterator(); + for (int i = 0; choices.size() < indexes.size(); i++) { + int brokerId = iter.next().id(); + if (indexes.get(choices.size()) == i) { + choices.add(brokerId); + } + } + Collections.shuffle(choices, random); + results.add(choices); + } + return results; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/metadata/BrokerHeartbeatReply.java b/metadata/src/main/java/org/apache/kafka/metadata/BrokerHeartbeatReply.java index 5ab2a52dc0ef9..c936601e9d21e 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/BrokerHeartbeatReply.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/BrokerHeartbeatReply.java @@ -31,6 +31,11 @@ public class BrokerHeartbeatReply { */ private final boolean isFenced; + /** + * True if the broker is currently in a controlled shutdown state. + */ + private final boolean inControlledShutdown; + /** * True if the heartbeat reply should tell the broker that it should shut down. */ @@ -38,9 +43,11 @@ public class BrokerHeartbeatReply { public BrokerHeartbeatReply(boolean isCaughtUp, boolean isFenced, + boolean inControlledShutdown, boolean shouldShutDown) { this.isCaughtUp = isCaughtUp; this.isFenced = isFenced; + this.inControlledShutdown = inControlledShutdown; this.shouldShutDown = shouldShutDown; } @@ -52,13 +59,17 @@ public boolean isFenced() { return isFenced; } + public boolean inControlledShutdown() { + return inControlledShutdown; + } + public boolean shouldShutDown() { return shouldShutDown; } @Override public int hashCode() { - return Objects.hash(isCaughtUp, isFenced, shouldShutDown); + return Objects.hash(isCaughtUp, isFenced, inControlledShutdown, shouldShutDown); } @Override @@ -67,6 +78,7 @@ public boolean equals(Object o) { BrokerHeartbeatReply other = (BrokerHeartbeatReply) o; return other.isCaughtUp == isCaughtUp && other.isFenced == isFenced && + other.inControlledShutdown == inControlledShutdown && other.shouldShutDown == shouldShutDown; } @@ -74,6 +86,7 @@ public boolean equals(Object o) { public String toString() { return "BrokerHeartbeatReply(isCaughtUp=" + isCaughtUp + ", isFenced=" + isFenced + + ", inControlledShutdown=" + inControlledShutdown + ", shouldShutDown = " + shouldShutDown + ")"; } diff --git a/metadata/src/main/java/org/apache/kafka/metadata/UsableBroker.java b/metadata/src/main/java/org/apache/kafka/metadata/UsableBroker.java new file mode 100644 index 0000000000000..9a2db10fc0ab1 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/metadata/UsableBroker.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.metadata; + +import java.util.Objects; +import java.util.Optional; + + +/** + * A broker where a replica can be placed. + */ +public class UsableBroker { + private final int id; + + private final Optional rack; + + public UsableBroker(int id, Optional rack) { + this.id = id; + this.rack = rack; + } + + public int id() { + return id; + } + + public Optional rack() { + return rack; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof UsableBroker)) return false; + UsableBroker other = (UsableBroker) o; + return other.id == id && other.rack.equals(rack); + } + + @Override + public int hashCode() { + return Objects.hash(id, rack); + } + + @Override + public String toString() { + return "UsableBroker(id=" + id + ", rack=" + rack + ")"; + } +} diff --git a/metadata/src/main/resources/common/metadata/AccessControlRecord.json b/metadata/src/main/resources/common/metadata/AccessControlRecord.json index 3acf4698bbf45..deef33c2e71f3 100644 --- a/metadata/src/main/resources/common/metadata/AccessControlRecord.json +++ b/metadata/src/main/resources/common/metadata/AccessControlRecord.json @@ -18,6 +18,7 @@ "type": "metadata", "name": "AccessControlRecord", "validVersions": "0", + "flexibleVersions": "0+", "fields": [ { "name": "ResourceType", "type": "int8", "versions": "0+", "about": "The resource type" }, diff --git a/metadata/src/main/resources/common/metadata/IsrChangeRecord.json b/metadata/src/main/resources/common/metadata/PartitionChangeRecord.json similarity index 63% rename from metadata/src/main/resources/common/metadata/IsrChangeRecord.json rename to metadata/src/main/resources/common/metadata/PartitionChangeRecord.json index fd8d834178744..070e5c9529992 100644 --- a/metadata/src/main/resources/common/metadata/IsrChangeRecord.json +++ b/metadata/src/main/resources/common/metadata/PartitionChangeRecord.json @@ -16,20 +16,19 @@ { "apiKey": 5, "type": "metadata", - "name": "IsrChangeRecord", + "name": "PartitionChangeRecord", "validVersions": "0", + "flexibleVersions": "0+", "fields": [ { "name": "PartitionId", "type": "int32", "versions": "0+", "default": "-1", "about": "The partition id." }, { "name": "TopicId", "type": "uuid", "versions": "0+", "about": "The unique ID of this topic." }, - { "name": "Isr", "type": "[]int32", "versions": "0+", - "about": "The in-sync replicas of this partition" }, - { "name": "Leader", "type": "int32", "versions": "0+", "default": "-1", - "about": "The lead replica, or -1 if there is no leader." }, - { "name": "LeaderEpoch", "type": "int32", "versions": "0+", "default": "-1", - "about": "An epoch that gets incremented each time we change the partition leader." }, - { "name": "PartitionEpoch", "type": "int32", "versions": "0+", "default": "-1", - "about": "An epoch that gets incremented each time we change anything in the partition." } + { "name": "Isr", "type": "[]int32", "default": "null", + "versions": "0+", "nullableVersions": "0+", "taggedVersions": "0+", "tag": 0, + "about": "null if the ISR didn't change; the new in-sync replicas otherwise." }, + { "name": "Leader", "type": "int32", "default": "-2", + "versions": "0+", "taggedVersions": "0+", "tag": 1, + "about": "-1 if there is now no leader; -2 if the leader didn't change; the new leader otherwise." } ] } diff --git a/metadata/src/main/resources/common/metadata/PartitionRecord.json b/metadata/src/main/resources/common/metadata/PartitionRecord.json index 5cc7d1328c9dc..2a92c21f09870 100644 --- a/metadata/src/main/resources/common/metadata/PartitionRecord.json +++ b/metadata/src/main/resources/common/metadata/PartitionRecord.json @@ -34,7 +34,7 @@ { "name": "Leader", "type": "int32", "versions": "0+", "default": "-1", "about": "The lead replica, or -1 if there is no leader." }, { "name": "LeaderEpoch", "type": "int32", "versions": "0+", "default": "-1", - "about": "An epoch that gets incremented each time we change the partition leader." }, + "about": "The epoch of the partition leader." }, { "name": "PartitionEpoch", "type": "int32", "versions": "0+", "default": "-1", "about": "An epoch that gets incremented each time we change anything in the partition." } ] diff --git a/metadata/src/test/java/org/apache/kafka/controller/BrokerHeartbeatManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/BrokerHeartbeatManagerTest.java new file mode 100644 index 0000000000000..d70cc5c37d4ac --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/BrokerHeartbeatManagerTest.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import org.apache.kafka.common.message.BrokerHeartbeatRequestData; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.controller.BrokerHeartbeatManager.BrokerHeartbeatState; +import org.apache.kafka.controller.BrokerHeartbeatManager.BrokerHeartbeatStateIterator; +import org.apache.kafka.controller.BrokerHeartbeatManager.BrokerHeartbeatStateList; +import org.apache.kafka.controller.BrokerHeartbeatManager.UsableBrokerIterator; +import org.apache.kafka.metadata.UsableBroker; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.apache.kafka.controller.BrokerControlState.CONTROLLED_SHUTDOWN; +import static org.apache.kafka.controller.BrokerControlState.FENCED; +import static org.apache.kafka.controller.BrokerControlState.SHUTDOWN_NOW; +import static org.apache.kafka.controller.BrokerControlState.UNFENCED; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(40) +public class BrokerHeartbeatManagerTest { + private static BrokerHeartbeatManager newBrokerHeartbeatManager() { + LogContext logContext = new LogContext(); + MockTime time = new MockTime(0, 1_000_000, 0); + return new BrokerHeartbeatManager(logContext, time, 10_000_000); + } + + @Test + public void testHasValidSession() { + BrokerHeartbeatManager manager = newBrokerHeartbeatManager(); + MockTime time = (MockTime) manager.time(); + assertFalse(manager.hasValidSession(0)); + manager.touch(0, false, 0); + time.sleep(5); + manager.touch(1, false, 0); + manager.touch(2, false, 0); + assertTrue(manager.hasValidSession(0)); + assertTrue(manager.hasValidSession(1)); + assertTrue(manager.hasValidSession(2)); + assertFalse(manager.hasValidSession(3)); + time.sleep(6); + assertFalse(manager.hasValidSession(0)); + assertTrue(manager.hasValidSession(1)); + assertTrue(manager.hasValidSession(2)); + assertFalse(manager.hasValidSession(3)); + manager.remove(2); + assertFalse(manager.hasValidSession(2)); + manager.remove(1); + assertFalse(manager.hasValidSession(1)); + } + + @Test + public void testFindStaleBrokers() { + BrokerHeartbeatManager manager = newBrokerHeartbeatManager(); + MockTime time = (MockTime) manager.time(); + assertFalse(manager.hasValidSession(0)); + manager.touch(0, false, 0); + time.sleep(5); + manager.touch(1, false, 0); + time.sleep(1); + manager.touch(2, false, 0); + + Iterator iter = manager.unfenced().iterator(); + assertEquals(0, iter.next().id()); + assertEquals(1, iter.next().id()); + assertEquals(2, iter.next().id()); + assertFalse(iter.hasNext()); + assertEquals(Collections.emptyList(), manager.findStaleBrokers()); + + time.sleep(5); + assertEquals(Collections.singletonList(0), manager.findStaleBrokers()); + manager.fence(0); + assertEquals(Collections.emptyList(), manager.findStaleBrokers()); + iter = manager.unfenced().iterator(); + assertEquals(1, iter.next().id()); + assertEquals(2, iter.next().id()); + assertFalse(iter.hasNext()); + + time.sleep(20); + assertEquals(Arrays.asList(1, 2), manager.findStaleBrokers()); + manager.fence(1); + manager.fence(2); + assertEquals(Collections.emptyList(), manager.findStaleBrokers()); + iter = manager.unfenced().iterator(); + assertFalse(iter.hasNext()); + } + + @Test + public void testNextCheckTimeNs() { + BrokerHeartbeatManager manager = newBrokerHeartbeatManager(); + MockTime time = (MockTime) manager.time(); + assertEquals(Long.MAX_VALUE, manager.nextCheckTimeNs()); + manager.touch(0, false, 0); + time.sleep(2); + manager.touch(1, false, 0); + time.sleep(1); + manager.touch(2, false, 0); + time.sleep(1); + manager.touch(3, false, 0); + assertEquals(Collections.emptyList(), manager.findStaleBrokers()); + assertEquals(10_000_000, manager.nextCheckTimeNs()); + time.sleep(7); + assertEquals(10_000_000, manager.nextCheckTimeNs()); + assertEquals(Collections.singletonList(0), manager.findStaleBrokers()); + manager.fence(0); + assertEquals(12_000_000, manager.nextCheckTimeNs()); + time.sleep(3); + assertEquals(Arrays.asList(1, 2), manager.findStaleBrokers()); + manager.fence(1); + manager.fence(2); + assertEquals(14_000_000, manager.nextCheckTimeNs()); + } + + @Test + public void testMetadataOffsetComparator() { + TreeSet set = + new TreeSet<>(BrokerHeartbeatManager.MetadataOffsetComparator.INSTANCE); + BrokerHeartbeatState broker1 = new BrokerHeartbeatState(1); + BrokerHeartbeatState broker2 = new BrokerHeartbeatState(2); + BrokerHeartbeatState broker3 = new BrokerHeartbeatState(3); + set.add(broker1); + set.add(broker2); + set.add(broker3); + Iterator iterator = set.iterator(); + assertEquals(broker1, iterator.next()); + assertEquals(broker2, iterator.next()); + assertEquals(broker3, iterator.next()); + assertFalse(iterator.hasNext()); + assertTrue(set.remove(broker1)); + assertTrue(set.remove(broker2)); + assertTrue(set.remove(broker3)); + assertTrue(set.isEmpty()); + broker1.metadataOffset = 800; + broker2.metadataOffset = 400; + broker3.metadataOffset = 100; + set.add(broker1); + set.add(broker2); + set.add(broker3); + iterator = set.iterator(); + assertEquals(broker3, iterator.next()); + assertEquals(broker2, iterator.next()); + assertEquals(broker1, iterator.next()); + assertFalse(iterator.hasNext()); + } + + private static Set usableBrokersToSet(BrokerHeartbeatManager manager) { + Set brokers = new HashSet<>(); + for (Iterator iterator = new UsableBrokerIterator( + manager.unfenced().iterator(), + id -> id % 2 == 0 ? Optional.of("rack1") : Optional.of("rack2")); + iterator.hasNext(); ) { + brokers.add(iterator.next()); + } + return brokers; + } + + @Test + public void testUsableBrokerIterator() { + BrokerHeartbeatManager manager = newBrokerHeartbeatManager(); + assertEquals(Collections.emptySet(), usableBrokersToSet(manager)); + manager.touch(0, false, 100); + manager.touch(1, false, 100); + manager.touch(2, false, 98); + manager.touch(3, false, 100); + manager.touch(4, true, 100); + assertEquals(98L, manager.lowestActiveOffset()); + Set expected = new HashSet<>(); + expected.add(new UsableBroker(0, Optional.of("rack1"))); + expected.add(new UsableBroker(1, Optional.of("rack2"))); + expected.add(new UsableBroker(2, Optional.of("rack1"))); + expected.add(new UsableBroker(3, Optional.of("rack2"))); + assertEquals(expected, usableBrokersToSet(manager)); + manager.updateControlledShutdownOffset(2, 0); + assertEquals(100L, manager.lowestActiveOffset()); + assertThrows(RuntimeException.class, + () -> manager.updateControlledShutdownOffset(4, 0)); + manager.touch(4, false, 100); + manager.updateControlledShutdownOffset(4, 0); + expected.remove(new UsableBroker(2, Optional.of("rack1"))); + assertEquals(expected, usableBrokersToSet(manager)); + } + + @Test + public void testBrokerHeartbeatStateList() { + BrokerHeartbeatStateList list = new BrokerHeartbeatStateList(); + assertEquals(null, list.first()); + BrokerHeartbeatStateIterator iterator = list.iterator(); + assertFalse(iterator.hasNext()); + BrokerHeartbeatState broker0 = new BrokerHeartbeatState(0); + broker0.lastContactNs = 200; + BrokerHeartbeatState broker1 = new BrokerHeartbeatState(1); + broker1.lastContactNs = 100; + BrokerHeartbeatState broker2 = new BrokerHeartbeatState(2); + broker2.lastContactNs = 50; + BrokerHeartbeatState broker3 = new BrokerHeartbeatState(3); + broker3.lastContactNs = 150; + list.add(broker0); + list.add(broker1); + list.add(broker2); + list.add(broker3); + assertEquals(broker2, list.first()); + iterator = list.iterator(); + assertEquals(broker2, iterator.next()); + assertEquals(broker1, iterator.next()); + assertEquals(broker3, iterator.next()); + assertEquals(broker0, iterator.next()); + assertFalse(iterator.hasNext()); + list.remove(broker1); + iterator = list.iterator(); + assertEquals(broker2, iterator.next()); + assertEquals(broker3, iterator.next()); + assertEquals(broker0, iterator.next()); + assertFalse(iterator.hasNext()); + } + + @Test + public void testCalculateNextBrokerState() { + BrokerHeartbeatManager manager = newBrokerHeartbeatManager(); + manager.touch(0, true, 100); + manager.touch(1, false, 98); + manager.touch(2, false, 100); + manager.touch(3, false, 100); + manager.touch(4, true, 100); + manager.touch(5, false, 99); + manager.updateControlledShutdownOffset(5, 99); + + assertEquals(98L, manager.lowestActiveOffset()); + + assertEquals(new BrokerControlStates(FENCED, SHUTDOWN_NOW), + manager.calculateNextBrokerState(0, + new BrokerHeartbeatRequestData().setWantShutDown(true), 100, () -> false)); + assertEquals(new BrokerControlStates(FENCED, UNFENCED), + manager.calculateNextBrokerState(0, + new BrokerHeartbeatRequestData().setWantFence(false). + setCurrentMetadataOffset(100), 100, () -> false)); + assertEquals(new BrokerControlStates(FENCED, FENCED), + manager.calculateNextBrokerState(0, + new BrokerHeartbeatRequestData().setWantFence(false). + setCurrentMetadataOffset(50), 100, () -> false)); + assertEquals(new BrokerControlStates(FENCED, FENCED), + manager.calculateNextBrokerState(0, + new BrokerHeartbeatRequestData().setWantFence(true), 100, () -> false)); + + assertEquals(new BrokerControlStates(UNFENCED, CONTROLLED_SHUTDOWN), + manager.calculateNextBrokerState(1, + new BrokerHeartbeatRequestData().setWantShutDown(true), 100, () -> true)); + assertEquals(new BrokerControlStates(UNFENCED, SHUTDOWN_NOW), + manager.calculateNextBrokerState(1, + new BrokerHeartbeatRequestData().setWantShutDown(true), 100, () -> false)); + assertEquals(new BrokerControlStates(UNFENCED, UNFENCED), + manager.calculateNextBrokerState(1, + new BrokerHeartbeatRequestData().setWantFence(false), 100, () -> false)); + + assertEquals(new BrokerControlStates(CONTROLLED_SHUTDOWN, CONTROLLED_SHUTDOWN), + manager.calculateNextBrokerState(5, + new BrokerHeartbeatRequestData().setWantShutDown(true), 100, () -> true)); + assertEquals(new BrokerControlStates(CONTROLLED_SHUTDOWN, CONTROLLED_SHUTDOWN), + manager.calculateNextBrokerState(5, + new BrokerHeartbeatRequestData().setWantShutDown(true), 100, () -> false)); + manager.fence(1); + assertEquals(new BrokerControlStates(CONTROLLED_SHUTDOWN, SHUTDOWN_NOW), + manager.calculateNextBrokerState(5, + new BrokerHeartbeatRequestData().setWantShutDown(true), 100, () -> false)); + assertEquals(new BrokerControlStates(CONTROLLED_SHUTDOWN, CONTROLLED_SHUTDOWN), + manager.calculateNextBrokerState(5, + new BrokerHeartbeatRequestData().setWantShutDown(true), 100, () -> true)); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/BrokersToIsrsTest.java b/metadata/src/test/java/org/apache/kafka/controller/BrokersToIsrsTest.java new file mode 100644 index 0000000000000..6f124ad8ac4d8 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/BrokersToIsrsTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.controller.BrokersToIsrs.PartitionsOnReplicaIterator; +import org.apache.kafka.controller.BrokersToIsrs.TopicPartition; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + + +@Timeout(40) +public class BrokersToIsrsTest { + private static final Uuid[] UUIDS = new Uuid[] { + Uuid.fromString("z5XgH_fQSAK3-RYoF2ymgw"), + Uuid.fromString("U52uRe20RsGI0RvpcTx33Q") + }; + + private static Set toSet(TopicPartition... partitions) { + HashSet set = new HashSet<>(); + for (TopicPartition partition : partitions) { + set.add(partition); + } + return set; + } + + private static Set toSet(PartitionsOnReplicaIterator iterator) { + HashSet set = new HashSet<>(); + while (iterator.hasNext()) { + set.add(iterator.next()); + } + return set; + } + + @Test + public void testIterator() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + BrokersToIsrs brokersToIsrs = new BrokersToIsrs(snapshotRegistry); + assertEquals(toSet(), toSet(brokersToIsrs.iterator(1, false))); + brokersToIsrs.update(UUIDS[0], 0, null, new int[] {1, 2, 3}, -1, 1); + brokersToIsrs.update(UUIDS[1], 1, null, new int[] {2, 3, 4}, -1, 4); + assertEquals(toSet(new TopicPartition(UUIDS[0], 0)), + toSet(brokersToIsrs.iterator(1, false))); + assertEquals(toSet(new TopicPartition(UUIDS[0], 0), + new TopicPartition(UUIDS[1], 1)), + toSet(brokersToIsrs.iterator(2, false))); + assertEquals(toSet(new TopicPartition(UUIDS[1], 1)), + toSet(brokersToIsrs.iterator(4, false))); + assertEquals(toSet(), toSet(brokersToIsrs.iterator(5, false))); + brokersToIsrs.update(UUIDS[1], 2, null, new int[] {3, 2, 1}, -1, 3); + assertEquals(toSet(new TopicPartition(UUIDS[0], 0), + new TopicPartition(UUIDS[1], 1), + new TopicPartition(UUIDS[1], 2)), + toSet(brokersToIsrs.iterator(2, false))); + } + + @Test + public void testLeadersOnlyIterator() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + BrokersToIsrs brokersToIsrs = new BrokersToIsrs(snapshotRegistry); + brokersToIsrs.update(UUIDS[0], 0, null, new int[]{1, 2, 3}, -1, 1); + brokersToIsrs.update(UUIDS[1], 1, null, new int[]{2, 3, 4}, -1, 4); + assertEquals(toSet(new TopicPartition(UUIDS[0], 0)), + toSet(brokersToIsrs.iterator(1, true))); + assertEquals(toSet(), toSet(brokersToIsrs.iterator(2, true))); + assertEquals(toSet(new TopicPartition(UUIDS[1], 1)), + toSet(brokersToIsrs.iterator(4, true))); + brokersToIsrs.update(UUIDS[0], 0, new int[]{1, 2, 3}, new int[]{1, 2, 3}, 1, 2); + assertEquals(toSet(), toSet(brokersToIsrs.iterator(1, true))); + assertEquals(toSet(new TopicPartition(UUIDS[0], 0)), + toSet(brokersToIsrs.iterator(2, true))); + } + + @Test + public void testNoLeader() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + BrokersToIsrs brokersToIsrs = new BrokersToIsrs(snapshotRegistry); + brokersToIsrs.update(UUIDS[0], 2, null, new int[]{1, 2, 3}, -1, 3); + assertEquals(toSet(new TopicPartition(UUIDS[0], 2)), + toSet(brokersToIsrs.iterator(3, true))); + assertEquals(toSet(), toSet(brokersToIsrs.iterator(2, true))); + assertEquals(toSet(), toSet(brokersToIsrs.noLeaderIterator())); + brokersToIsrs.update(UUIDS[0], 2, new int[]{1, 2, 3}, new int[]{1, 2, 3}, 3, -1); + assertEquals(toSet(new TopicPartition(UUIDS[0], 2)), + toSet(brokersToIsrs.noLeaderIterator())); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java new file mode 100644 index 0000000000000..c4e8da87104f3 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.config.internals.QuotaConfigs; +import org.apache.kafka.common.metadata.QuotaRecord; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.quota.ClientQuotaAlteration; +import org.apache.kafka.common.quota.ClientQuotaEntity; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Timeout(value = 40) +public class ClientQuotaControlManagerTest { + + @Test + public void testInvalidEntityTypes() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ClientQuotaControlManager manager = new ClientQuotaControlManager(snapshotRegistry); + + // Unknown type "foo" + assertInvalidEntity(manager, entity("foo", "bar")); + + // Null type + assertInvalidEntity(manager, entity(null, "null")); + + // Valid + unknown combo + assertInvalidEntity(manager, entity(ClientQuotaEntity.USER, "user-1", "foo", "bar")); + assertInvalidEntity(manager, entity("foo", "bar", ClientQuotaEntity.IP, "1.2.3.4")); + + // Invalid combinations + assertInvalidEntity(manager, entity(ClientQuotaEntity.USER, "user-1", ClientQuotaEntity.IP, "1.2.3.4")); + assertInvalidEntity(manager, entity(ClientQuotaEntity.CLIENT_ID, "user-1", ClientQuotaEntity.IP, "1.2.3.4")); + + // Empty + assertInvalidEntity(manager, new ClientQuotaEntity(Collections.emptyMap())); + } + + private void assertInvalidEntity(ClientQuotaControlManager manager, ClientQuotaEntity entity) { + List alters = new ArrayList<>(); + entityQuotaToAlterations(entity, quotas(QuotaConfigs.PRODUCER_BYTE_RATE_OVERRIDE_CONFIG, 10000.0), alters::add); + ControllerResult> result = manager.alterClientQuotas(alters); + assertEquals(Errors.INVALID_REQUEST, result.response().get(entity).error()); + assertEquals(0, result.records().size()); + } + + @Test + public void testAlterAndRemove() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ClientQuotaControlManager manager = new ClientQuotaControlManager(snapshotRegistry); + + ClientQuotaEntity userEntity = userEntity("user-1"); + List alters = new ArrayList<>(); + + // Add one quota + entityQuotaToAlterations(userEntity, quotas(QuotaConfigs.PRODUCER_BYTE_RATE_OVERRIDE_CONFIG, 10000.0), alters::add); + alterQuotas(alters, manager); + assertEquals(1, manager.clientQuotaData.get(userEntity).size()); + assertEquals(10000.0, manager.clientQuotaData.get(userEntity).get(QuotaConfigs.PRODUCER_BYTE_RATE_OVERRIDE_CONFIG), 1e-6); + + // Replace it and add another + alters.clear(); + entityQuotaToAlterations(userEntity, quotas( + QuotaConfigs.PRODUCER_BYTE_RATE_OVERRIDE_CONFIG, 10001.0, + QuotaConfigs.CONSUMER_BYTE_RATE_OVERRIDE_CONFIG, 20000.0 + ), alters::add); + alterQuotas(alters, manager); + assertEquals(2, manager.clientQuotaData.get(userEntity).size()); + assertEquals(10001.0, manager.clientQuotaData.get(userEntity).get(QuotaConfigs.PRODUCER_BYTE_RATE_OVERRIDE_CONFIG), 1e-6); + assertEquals(20000.0, manager.clientQuotaData.get(userEntity).get(QuotaConfigs.CONSUMER_BYTE_RATE_OVERRIDE_CONFIG), 1e-6); + + // Remove one of the quotas, the other remains + alters.clear(); + entityQuotaToAlterations(userEntity, quotas( + QuotaConfigs.PRODUCER_BYTE_RATE_OVERRIDE_CONFIG, null + ), alters::add); + alterQuotas(alters, manager); + assertEquals(1, manager.clientQuotaData.get(userEntity).size()); + assertEquals(20000.0, manager.clientQuotaData.get(userEntity).get(QuotaConfigs.CONSUMER_BYTE_RATE_OVERRIDE_CONFIG), 1e-6); + + // Remove non-existent quota, no change + alters.clear(); + entityQuotaToAlterations(userEntity, quotas( + QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, null + ), alters::add); + alterQuotas(alters, manager); + assertEquals(1, manager.clientQuotaData.get(userEntity).size()); + assertEquals(20000.0, manager.clientQuotaData.get(userEntity).get(QuotaConfigs.CONSUMER_BYTE_RATE_OVERRIDE_CONFIG), 1e-6); + + // All quotas removed, we should cleanup the map + alters.clear(); + entityQuotaToAlterations(userEntity, quotas( + QuotaConfigs.CONSUMER_BYTE_RATE_OVERRIDE_CONFIG, null + ), alters::add); + alterQuotas(alters, manager); + assertFalse(manager.clientQuotaData.containsKey(userEntity)); + + // Remove non-existent quota, again no change + alters.clear(); + entityQuotaToAlterations(userEntity, quotas( + QuotaConfigs.CONSUMER_BYTE_RATE_OVERRIDE_CONFIG, null + ), alters::add); + alterQuotas(alters, manager); + assertFalse(manager.clientQuotaData.containsKey(userEntity)); + + // Mixed update + alters.clear(); + Map quotas = new HashMap<>(4); + quotas.put(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 99.0); + quotas.put(QuotaConfigs.CONTROLLER_MUTATION_RATE_OVERRIDE_CONFIG, null); + quotas.put(QuotaConfigs.PRODUCER_BYTE_RATE_OVERRIDE_CONFIG, 10002.0); + quotas.put(QuotaConfigs.CONSUMER_BYTE_RATE_OVERRIDE_CONFIG, 20001.0); + + entityQuotaToAlterations(userEntity, quotas, alters::add); + alterQuotas(alters, manager); + assertEquals(3, manager.clientQuotaData.get(userEntity).size()); + assertEquals(20001.0, manager.clientQuotaData.get(userEntity).get(QuotaConfigs.CONSUMER_BYTE_RATE_OVERRIDE_CONFIG), 1e-6); + assertEquals(10002.0, manager.clientQuotaData.get(userEntity).get(QuotaConfigs.PRODUCER_BYTE_RATE_OVERRIDE_CONFIG), 1e-6); + assertEquals(99.0, manager.clientQuotaData.get(userEntity).get(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG), 1e-6); + } + + @Test + public void testEntityTypes() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ClientQuotaControlManager manager = new ClientQuotaControlManager(snapshotRegistry); + + Map> quotasToTest = new HashMap<>(); + quotasToTest.put(userClientEntity("user-1", "client-id-1"), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 50.50)); + quotasToTest.put(userClientEntity("user-2", "client-id-1"), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 51.51)); + quotasToTest.put(userClientEntity("user-3", "client-id-2"), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 52.52)); + quotasToTest.put(userClientEntity(null, "client-id-1"), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 53.53)); + quotasToTest.put(userClientEntity("user-1", null), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 54.54)); + quotasToTest.put(userClientEntity("user-3", null), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 55.55)); + quotasToTest.put(userEntity("user-1"), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 56.56)); + quotasToTest.put(userEntity("user-2"), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 57.57)); + quotasToTest.put(userEntity("user-3"), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 58.58)); + quotasToTest.put(userEntity(null), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 59.59)); + quotasToTest.put(clientEntity("client-id-2"), + quotas(QuotaConfigs.REQUEST_PERCENTAGE_OVERRIDE_CONFIG, 60.60)); + + + List alters = new ArrayList<>(); + quotasToTest.forEach((entity, quota) -> entityQuotaToAlterations(entity, quota, alters::add)); + alterQuotas(alters, manager); + } + + static void entityQuotaToAlterations(ClientQuotaEntity entity, Map quota, + Consumer acceptor) { + Collection ops = quota.entrySet().stream() + .map(quotaEntry -> new ClientQuotaAlteration.Op(quotaEntry.getKey(), quotaEntry.getValue())) + .collect(Collectors.toList()); + acceptor.accept(new ClientQuotaAlteration(entity, ops)); + } + + static void alterQuotas(List alterations, ClientQuotaControlManager manager) { + ControllerResult> result = manager.alterClientQuotas(alterations); + assertTrue(result.response().values().stream().allMatch(ApiError::isSuccess)); + result.records().forEach(apiMessageAndVersion -> manager.replay((QuotaRecord) apiMessageAndVersion.message())); + } + + static Map quotas(String key, Double value) { + return Collections.singletonMap(key, value); + } + + static Map quotas(String key1, Double value1, String key2, Double value2) { + Map quotas = new HashMap<>(2); + quotas.put(key1, value1); + quotas.put(key2, value2); + return quotas; + } + + static ClientQuotaEntity entity(String type, String name) { + return new ClientQuotaEntity(Collections.singletonMap(type, name)); + } + + static ClientQuotaEntity entity(String type1, String name1, String type2, String name2) { + Map entries = new HashMap<>(2); + entries.put(type1, name1); + entries.put(type2, name2); + return new ClientQuotaEntity(entries); + } + + static ClientQuotaEntity userEntity(String user) { + return new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, user)); + } + + static ClientQuotaEntity clientEntity(String clientId) { + return new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.CLIENT_ID, clientId)); + } + + static ClientQuotaEntity userClientEntity(String user, String clientId) { + Map entries = new HashMap<>(2); + entries.put(ClientQuotaEntity.USER, user); + entries.put(ClientQuotaEntity.CLIENT_ID, clientId); + return new ClientQuotaEntity(entries); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java new file mode 100644 index 0000000000000..c410a686344d2 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import org.apache.kafka.common.Endpoint; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.StaleBrokerEpochException; +import org.apache.kafka.common.metadata.RegisterBrokerRecord; +import org.apache.kafka.common.metadata.UnfenceBrokerRecord; +import org.apache.kafka.common.metadata.UnregisterBrokerRecord; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.metadata.BrokerRegistration; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(value = 40) +public class ClusterControlManagerTest { + @Test + public void testReplay() { + MockTime time = new MockTime(0, 0, 0); + + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ClusterControlManager clusterControl = new ClusterControlManager( + new LogContext(), time, snapshotRegistry, 1000, + new SimpleReplicaPlacementPolicy(new Random())); + clusterControl.activate(); + assertFalse(clusterControl.unfenced(0)); + + RegisterBrokerRecord brokerRecord = new RegisterBrokerRecord().setBrokerEpoch(100).setBrokerId(1); + brokerRecord.endPoints().add(new RegisterBrokerRecord.BrokerEndpoint(). + setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). + setPort((short) 9092). + setName("PLAINTEXT"). + setHost("example.com")); + clusterControl.replay(brokerRecord); + clusterControl.checkBrokerEpoch(1, 100); + assertThrows(StaleBrokerEpochException.class, + () -> clusterControl.checkBrokerEpoch(1, 101)); + assertThrows(StaleBrokerEpochException.class, + () -> clusterControl.checkBrokerEpoch(2, 100)); + assertFalse(clusterControl.unfenced(0)); + assertFalse(clusterControl.unfenced(1)); + + UnfenceBrokerRecord unfenceBrokerRecord = + new UnfenceBrokerRecord().setId(1).setEpoch(100); + clusterControl.replay(unfenceBrokerRecord); + assertFalse(clusterControl.unfenced(0)); + assertTrue(clusterControl.unfenced(1)); + } + + @Test + public void testUnregister() throws Exception { + RegisterBrokerRecord brokerRecord = new RegisterBrokerRecord(). + setBrokerId(1). + setBrokerEpoch(100). + setIncarnationId(Uuid.fromString("fPZv1VBsRFmnlRvmGcOW9w")). + setRack("arack"); + brokerRecord.endPoints().add(new RegisterBrokerRecord.BrokerEndpoint(). + setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). + setPort((short) 9092). + setName("PLAINTEXT"). + setHost("example.com")); + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ClusterControlManager clusterControl = new ClusterControlManager( + new LogContext(), new MockTime(0, 0, 0), snapshotRegistry, 1000, + new SimpleReplicaPlacementPolicy(new Random())); + clusterControl.activate(); + clusterControl.replay(brokerRecord); + assertEquals(new BrokerRegistration(1, 100, + Uuid.fromString("fPZv1VBsRFmnlRvmGcOW9w"), Collections.singletonMap("PLAINTEXT", + new Endpoint("PLAINTEXT", SecurityProtocol.PLAINTEXT, "example.com", 9092)), + Collections.emptyMap(), Optional.of("arack"), true), + clusterControl.brokerRegistrations().get(1)); + UnregisterBrokerRecord unregisterRecord = new UnregisterBrokerRecord(). + setBrokerId(1). + setBrokerEpoch(100); + clusterControl.replay(unregisterRecord); + assertFalse(clusterControl.brokerRegistrations().containsKey(1)); + } + + @ParameterizedTest + @ValueSource(ints = {3, 10}) + public void testPlaceReplicas(int numUsableBrokers) throws Exception { + MockTime time = new MockTime(0, 0, 0); + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + MockRandom random = new MockRandom(); + ClusterControlManager clusterControl = new ClusterControlManager( + new LogContext(), time, snapshotRegistry, 1000, + new SimpleReplicaPlacementPolicy(random)); + clusterControl.activate(); + for (int i = 0; i < numUsableBrokers; i++) { + RegisterBrokerRecord brokerRecord = + new RegisterBrokerRecord().setBrokerEpoch(100).setBrokerId(i); + brokerRecord.endPoints().add(new RegisterBrokerRecord.BrokerEndpoint(). + setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). + setPort((short) 9092). + setName("PLAINTEXT"). + setHost("example.com")); + clusterControl.replay(brokerRecord); + UnfenceBrokerRecord unfenceRecord = + new UnfenceBrokerRecord().setId(i).setEpoch(100); + clusterControl.replay(unfenceRecord); + clusterControl.heartbeatManager().touch(i, false, 0); + } + for (int i = 0; i < numUsableBrokers; i++) { + assertTrue(clusterControl.unfenced(i), + String.format("broker %d was not unfenced.", i)); + } + for (int i = 0; i < 100; i++) { + List> results = clusterControl.placeReplicas(1, (short) 3); + HashSet seen = new HashSet<>(); + for (Integer result : results.get(0)) { + assertTrue(result >= 0); + assertTrue(result < numUsableBrokers); + assertTrue(seen.add(result)); + } + } + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java new file mode 100644 index 0000000000000..49a55338309b5 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.metadata.ConfigRecord; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.timeline.SnapshotRegistry; + +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.apache.kafka.clients.admin.AlterConfigOp.OpType.APPEND; +import static org.apache.kafka.clients.admin.AlterConfigOp.OpType.DELETE; +import static org.apache.kafka.clients.admin.AlterConfigOp.OpType.SET; +import static org.apache.kafka.common.config.ConfigResource.Type.BROKER; +import static org.apache.kafka.common.config.ConfigResource.Type.BROKER_LOGGER; +import static org.apache.kafka.common.config.ConfigResource.Type.TOPIC; +import static org.apache.kafka.common.config.ConfigResource.Type.UNKNOWN; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(value = 40) +public class ConfigurationControlManagerTest { + + static final Map CONFIGS = new HashMap<>(); + + static { + CONFIGS.put(BROKER, new ConfigDef(). + define("foo.bar", ConfigDef.Type.LIST, "1", ConfigDef.Importance.HIGH, "foo bar"). + define("baz", ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "baz"). + define("quux", ConfigDef.Type.INT, ConfigDef.Importance.HIGH, "quux")); + CONFIGS.put(TOPIC, new ConfigDef(). + define("abc", ConfigDef.Type.LIST, ConfigDef.Importance.HIGH, "abc"). + define("def", ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "def"). + define("ghi", ConfigDef.Type.BOOLEAN, true, ConfigDef.Importance.HIGH, "ghi")); + } + + static final ConfigResource BROKER0 = new ConfigResource(BROKER, "0"); + static final ConfigResource MYTOPIC = new ConfigResource(TOPIC, "mytopic"); + + @SuppressWarnings("unchecked") + private static Map toMap(Entry... entries) { + Map map = new HashMap<>(); + for (Entry entry : entries) { + map.put(entry.getKey(), entry.getValue()); + } + return map; + } + + static Entry entry(A a, B b) { + return new SimpleImmutableEntry<>(a, b); + } + + @Test + public void testReplay() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ConfigurationControlManager manager = + new ConfigurationControlManager(new LogContext(), snapshotRegistry, CONFIGS); + assertEquals(Collections.emptyMap(), manager.getConfigs(BROKER0)); + manager.replay(new ConfigRecord(). + setResourceType(BROKER.id()).setResourceName("0"). + setName("foo.bar").setValue("1,2")); + assertEquals(Collections.singletonMap("foo.bar", "1,2"), + manager.getConfigs(BROKER0)); + manager.replay(new ConfigRecord(). + setResourceType(BROKER.id()).setResourceName("0"). + setName("foo.bar").setValue(null)); + assertEquals(Collections.emptyMap(), manager.getConfigs(BROKER0)); + manager.replay(new ConfigRecord(). + setResourceType(TOPIC.id()).setResourceName("mytopic"). + setName("abc").setValue("x,y,z")); + manager.replay(new ConfigRecord(). + setResourceType(TOPIC.id()).setResourceName("mytopic"). + setName("def").setValue("blah")); + assertEquals(toMap(entry("abc", "x,y,z"), entry("def", "blah")), + manager.getConfigs(MYTOPIC)); + } + + @Test + public void testCheckConfigResource() { + assertEquals(new ApiError(Errors.INVALID_REQUEST, "Unsupported " + + "configuration resource type BROKER_LOGGER ").toString(), + ConfigurationControlManager.checkConfigResource( + new ConfigResource(BROKER_LOGGER, "kafka.server.FetchContext")).toString()); + assertEquals(new ApiError(Errors.INVALID_REQUEST, "Illegal topic name.").toString(), + ConfigurationControlManager.checkConfigResource( + new ConfigResource(TOPIC, "* @ invalid$")).toString()); + assertEquals(new ApiError(Errors.INVALID_REQUEST, "Illegal topic name.").toString(), + ConfigurationControlManager.checkConfigResource( + new ConfigResource(TOPIC, "")).toString()); + assertEquals(new ApiError(Errors.INVALID_REQUEST, "Illegal non-integral " + + "BROKER resource type name.").toString(), + ConfigurationControlManager.checkConfigResource( + new ConfigResource(BROKER, "bob")).toString()); + assertEquals(new ApiError(Errors.NONE, null).toString(), + ConfigurationControlManager.checkConfigResource( + new ConfigResource(BROKER, "")).toString()); + assertEquals(new ApiError(Errors.INVALID_REQUEST, "Unsupported configuration " + + "resource type UNKNOWN.").toString(), + ConfigurationControlManager.checkConfigResource( + new ConfigResource(UNKNOWN, "bob")).toString()); + } + + @Test + public void testIncrementalAlterConfigs() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ConfigurationControlManager manager = + new ConfigurationControlManager(new LogContext(), snapshotRegistry, CONFIGS); + assertEquals(new ControllerResult>(Collections.singletonList( + new ApiMessageAndVersion(new ConfigRecord(). + setResourceType(TOPIC.id()).setResourceName("mytopic"). + setName("abc").setValue("123"), (short) 0)), + toMap(entry(BROKER0, new ApiError( + Errors.INVALID_REQUEST, "A DELETE op was given with a non-null value.")), + entry(MYTOPIC, ApiError.NONE))), + manager.incrementalAlterConfigs(toMap(entry(BROKER0, toMap( + entry("foo.bar", entry(DELETE, "abc")), + entry("quux", entry(SET, "abc")))), + entry(MYTOPIC, toMap( + entry("abc", entry(APPEND, "123"))))))); + } + + @Test + public void testIsSplittable() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ConfigurationControlManager manager = + new ConfigurationControlManager(new LogContext(), snapshotRegistry, CONFIGS); + assertTrue(manager.isSplittable(BROKER, "foo.bar")); + assertFalse(manager.isSplittable(BROKER, "baz")); + assertFalse(manager.isSplittable(BROKER, "foo.baz.quux")); + assertFalse(manager.isSplittable(TOPIC, "baz")); + assertTrue(manager.isSplittable(TOPIC, "abc")); + } + + @Test + public void testGetConfigValueDefault() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ConfigurationControlManager manager = + new ConfigurationControlManager(new LogContext(), snapshotRegistry, CONFIGS); + assertEquals("1", manager.getConfigValueDefault(BROKER, "foo.bar")); + assertEquals(null, manager.getConfigValueDefault(BROKER, "foo.baz.quux")); + assertEquals(null, manager.getConfigValueDefault(TOPIC, "abc")); + assertEquals("true", manager.getConfigValueDefault(TOPIC, "ghi")); + } + + @Test + public void testLegacyAlterConfigs() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ConfigurationControlManager manager = + new ConfigurationControlManager(new LogContext(), snapshotRegistry, CONFIGS); + List expectedRecords1 = Arrays.asList( + new ApiMessageAndVersion(new ConfigRecord(). + setResourceType(TOPIC.id()).setResourceName("mytopic"). + setName("abc").setValue("456"), (short) 0), + new ApiMessageAndVersion(new ConfigRecord(). + setResourceType(TOPIC.id()).setResourceName("mytopic"). + setName("def").setValue("901"), (short) 0)); + assertEquals(new ControllerResult>( + expectedRecords1, + toMap(entry(MYTOPIC, ApiError.NONE))), + manager.legacyAlterConfigs(toMap(entry(MYTOPIC, toMap( + entry("abc", "456"), entry("def", "901")))))); + for (ApiMessageAndVersion message : expectedRecords1) { + manager.replay((ConfigRecord) message.message()); + } + assertEquals(new ControllerResult>(Arrays.asList( + new ApiMessageAndVersion(new ConfigRecord(). + setResourceType(TOPIC.id()).setResourceName("mytopic"). + setName("abc").setValue(null), (short) 0)), + toMap(entry(MYTOPIC, ApiError.NONE))), + manager.legacyAlterConfigs(toMap(entry(MYTOPIC, toMap( + entry("def", "901")))))); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/ControllerPurgatoryTest.java b/metadata/src/test/java/org/apache/kafka/controller/ControllerPurgatoryTest.java new file mode 100644 index 0000000000000..57953e1b38cf1 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ControllerPurgatoryTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Timeout(value = 40) +public class ControllerPurgatoryTest { + + static class SampleDeferredEvent implements DeferredEvent { + private final CompletableFuture future = new CompletableFuture<>(); + + @Override + public void complete(Throwable exception) { + if (exception != null) { + future.completeExceptionally(exception); + } else { + future.complete(null); + } + } + + CompletableFuture future() { + return future; + } + } + + @Test + public void testCompleteEvents() { + ControllerPurgatory purgatory = new ControllerPurgatory(); + SampleDeferredEvent event1 = new SampleDeferredEvent(); + SampleDeferredEvent event2 = new SampleDeferredEvent(); + SampleDeferredEvent event3 = new SampleDeferredEvent(); + purgatory.add(1, event1); + assertEquals(Optional.of(1L), purgatory.highestPendingOffset()); + purgatory.add(1, event2); + assertEquals(Optional.of(1L), purgatory.highestPendingOffset()); + purgatory.add(3, event3); + assertEquals(Optional.of(3L), purgatory.highestPendingOffset()); + purgatory.completeUpTo(2); + assertTrue(event1.future.isDone()); + assertTrue(event2.future.isDone()); + assertFalse(event3.future.isDone()); + purgatory.completeUpTo(4); + assertTrue(event3.future.isDone()); + assertEquals(Optional.empty(), purgatory.highestPendingOffset()); + } + + @Test + public void testFailOnIncorrectOrdering() { + ControllerPurgatory purgatory = new ControllerPurgatory(); + SampleDeferredEvent event1 = new SampleDeferredEvent(); + SampleDeferredEvent event2 = new SampleDeferredEvent(); + purgatory.add(2, event1); + assertThrows(RuntimeException.class, () -> purgatory.add(1, event2)); + } + + @Test + public void testFailEvents() { + ControllerPurgatory purgatory = new ControllerPurgatory(); + SampleDeferredEvent event1 = new SampleDeferredEvent(); + SampleDeferredEvent event2 = new SampleDeferredEvent(); + SampleDeferredEvent event3 = new SampleDeferredEvent(); + purgatory.add(1, event1); + purgatory.add(3, event2); + purgatory.add(3, event3); + purgatory.completeUpTo(2); + assertTrue(event1.future.isDone()); + assertFalse(event2.future.isDone()); + assertFalse(event3.future.isDone()); + purgatory.failAll(new RuntimeException("failed")); + assertTrue(event2.future.isDone()); + assertTrue(event3.future.isDone()); + assertEquals(RuntimeException.class, assertThrows(ExecutionException.class, + () -> event2.future.get()).getCause().getClass()); + assertEquals(RuntimeException.class, assertThrows(ExecutionException.class, + () -> event3.future.get()).getCause().getClass()); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/ControllerTestUtils.java b/metadata/src/test/java/org/apache/kafka/controller/ControllerTestUtils.java new file mode 100644 index 0000000000000..746c7efb55836 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ControllerTestUtils.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.protocol.ApiMessage; +import org.apache.kafka.metadata.ApiMessageAndVersion; + +import java.lang.reflect.Method; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + + +public class ControllerTestUtils { + public static void replayAll(Object target, + List recordsAndVersions) throws Exception { + for (ApiMessageAndVersion recordAndVersion : recordsAndVersions) { + ApiMessage record = recordAndVersion.message(); + try { + Method method = target.getClass().getMethod("replay", record.getClass()); + method.invoke(target, record); + } catch (NoSuchMethodException e) { + // ignore + } + } + } + + public static Set iteratorToSet(Iterator iterator) { + HashSet set = new HashSet<>(); + while (iterator.hasNext()) { + set.add(iterator.next()); + } + return set; + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java new file mode 100644 index 0000000000000..8687cc8f562d8 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.metadata.FeatureLevelRecord; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.metadata.FeatureMap; +import org.apache.kafka.metadata.FeatureMapAndEpoch; +import org.apache.kafka.metadata.VersionRange; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; + + +@Timeout(value = 40) +public class FeatureControlManagerTest { + @SuppressWarnings("unchecked") + private static Map rangeMap(Object... args) { + Map result = new HashMap<>(); + for (int i = 0; i < args.length; i += 3) { + String feature = (String) args[i]; + Integer low = (Integer) args[i + 1]; + Integer high = (Integer) args[i + 2]; + result.put(feature, new VersionRange(low.shortValue(), high.shortValue())); + } + return result; + } + + @Test + public void testUpdateFeatures() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + snapshotRegistry.createSnapshot(-1); + FeatureControlManager manager = new FeatureControlManager( + rangeMap("foo", 1, 2), snapshotRegistry); + assertEquals(new FeatureMapAndEpoch(new FeatureMap(Collections.emptyMap()), -1), + manager.finalizedFeatures(-1)); + assertEquals(new ControllerResult<>(Collections. + singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, + "The controller does not support the given feature range."))), + manager.updateFeatures(rangeMap("foo", 1, 3), + new HashSet<>(Arrays.asList("foo")), + Collections.emptyMap())); + ControllerResult> result = manager.updateFeatures( + rangeMap("foo", 1, 2, "bar", 1, 1), Collections.emptySet(), + Collections.emptyMap()); + Map expectedMap = new HashMap<>(); + expectedMap.put("foo", ApiError.NONE); + expectedMap.put("bar", new ApiError(Errors.INVALID_UPDATE_VERSION, + "The controller does not support the given feature range.")); + assertEquals(expectedMap, result.response()); + List expectedMessages = new ArrayList<>(); + expectedMessages.add(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName("foo").setMinFeatureLevel((short) 1).setMaxFeatureLevel((short) 2), + (short) 0)); + assertEquals(expectedMessages, result.records()); + } + + @Test + public void testReplay() { + FeatureLevelRecord record = new FeatureLevelRecord(). + setName("foo").setMinFeatureLevel((short) 1).setMaxFeatureLevel((short) 2); + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + snapshotRegistry.createSnapshot(-1); + FeatureControlManager manager = new FeatureControlManager( + rangeMap("foo", 1, 2), snapshotRegistry); + manager.replay(record, 123); + snapshotRegistry.createSnapshot(123); + assertEquals(new FeatureMapAndEpoch(new FeatureMap(rangeMap("foo", 1, 2)), 123), + manager.finalizedFeatures(123)); + } + + @Test + public void testUpdateFeaturesErrorCases() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + FeatureControlManager manager = new FeatureControlManager( + rangeMap("foo", 1, 5, "bar", 1, 2), snapshotRegistry); + assertEquals(new ControllerResult<>(Collections. + singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, + "Broker 5 does not support the given feature range."))), + manager.updateFeatures(rangeMap("foo", 1, 3), + new HashSet<>(Arrays.asList("foo")), + Collections.singletonMap(5, rangeMap()))); + + ControllerResult> result = manager.updateFeatures( + rangeMap("foo", 1, 3), Collections.emptySet(), Collections.emptyMap()); + assertEquals(Collections.singletonMap("foo", ApiError.NONE), result.response()); + manager.replay((FeatureLevelRecord) result.records().get(0).message(), 3); + snapshotRegistry.createSnapshot(3); + + assertEquals(new ControllerResult<>(Collections. + singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, + "Can't downgrade the maximum version of this feature without " + + "setting downgradable to true."))), + manager.updateFeatures(rangeMap("foo", 1, 2), + Collections.emptySet(), Collections.emptyMap())); + + assertEquals(new ControllerResult<>( + Collections.singletonList(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName("foo").setMinFeatureLevel((short) 1).setMaxFeatureLevel((short) 2), + (short) 0)), + Collections.singletonMap("foo", ApiError.NONE)), + manager.updateFeatures(rangeMap("foo", 1, 2), + new HashSet<>(Collections.singletonList("foo")), Collections.emptyMap())); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/MockControllerMetrics.java b/metadata/src/test/java/org/apache/kafka/controller/MockControllerMetrics.java new file mode 100644 index 0000000000000..4e6523e37f286 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/MockControllerMetrics.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + + +public final class MockControllerMetrics implements ControllerMetrics { + private volatile boolean active; + + public MockControllerMetrics() { + this.active = false; + } + + @Override + public void setActive(boolean active) { + this.active = active; + } + + @Override + public boolean active() { + return this.active; + } + + @Override + public void updateEventQueueTime(long durationMs) { + // nothing to do + } + + @Override + public void updateEventQueueProcessingTime(long durationMs) { + // nothing to do + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/MockRandom.java b/metadata/src/test/java/org/apache/kafka/controller/MockRandom.java new file mode 100644 index 0000000000000..c42a158b660b1 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/MockRandom.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.Random; + + +/** + * A subclass of Random with a fixed seed and generation algorithm. + */ +public class MockRandom extends Random { + private long state = 17; + + @Override + protected int next(int bits) { + state = (state * 2862933555777941757L) + 3037000493L; + return (int) (state >>> (64 - bits)); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java new file mode 100644 index 0000000000000..16b52d1578391 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.message.BrokerHeartbeatRequestData; +import org.apache.kafka.common.message.BrokerRegistrationRequestData.Listener; +import org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection; +import org.apache.kafka.common.message.BrokerRegistrationRequestData; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection; +import org.apache.kafka.common.message.CreateTopicsRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.controller.BrokersToIsrs.TopicPartition; +import org.apache.kafka.metadata.BrokerHeartbeatReply; +import org.apache.kafka.metadata.BrokerRegistrationReply; +import org.apache.kafka.metalog.LocalLogManagerTestEnv; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.kafka.clients.admin.AlterConfigOp.OpType.SET; +import static org.apache.kafka.controller.ConfigurationControlManagerTest.BROKER0; +import static org.apache.kafka.controller.ConfigurationControlManagerTest.CONFIGS; +import static org.apache.kafka.controller.ConfigurationControlManagerTest.entry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(value = 40) +public class QuorumControllerTest { + private static final Logger log = + LoggerFactory.getLogger(QuorumControllerTest.class); + + /** + * Test creating a new QuorumController and closing it. + */ + @Test + public void testCreateAndClose() throws Throwable { + try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1)) { + try (QuorumControllerTestEnv controlEnv = + new QuorumControllerTestEnv(logEnv, __ -> { })) { + } + } + } + + /** + * Test setting some configuration values and reading them back. + */ + @Test + public void testConfigurationOperations() throws Throwable { + try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1)) { + try (QuorumControllerTestEnv controlEnv = + new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) { + testConfigurationOperations(controlEnv.activeController()); + } + } + } + + private void testConfigurationOperations(QuorumController controller) throws Throwable { + assertEquals(Collections.singletonMap(BROKER0, ApiError.NONE), + controller.incrementalAlterConfigs(Collections.singletonMap( + BROKER0, Collections.singletonMap("baz", entry(SET, "123"))), true).get()); + assertEquals(Collections.singletonMap(BROKER0, + new ResultOrError<>(Collections.emptyMap())), + controller.describeConfigs(Collections.singletonMap( + BROKER0, Collections.emptyList())).get()); + assertEquals(Collections.singletonMap(BROKER0, ApiError.NONE), + controller.incrementalAlterConfigs(Collections.singletonMap( + BROKER0, Collections.singletonMap("baz", entry(SET, "123"))), false).get()); + assertEquals(Collections.singletonMap(BROKER0, new ResultOrError<>(Collections. + singletonMap("baz", "123"))), + controller.describeConfigs(Collections.singletonMap( + BROKER0, Collections.emptyList())).get()); + } + + /** + * Test that an incrementalAlterConfigs operation doesn't complete until the records + * can be written to the metadata log. + */ + @Test + public void testDelayedConfigurationOperations() throws Throwable { + try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1)) { + try (QuorumControllerTestEnv controlEnv = + new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) { + testDelayedConfigurationOperations(logEnv, controlEnv.activeController()); + } + } + } + + private void testDelayedConfigurationOperations(LocalLogManagerTestEnv logEnv, + QuorumController controller) + throws Throwable { + logEnv.logManagers().forEach(m -> m.setMaxReadOffset(0L)); + CompletableFuture> future1 = + controller.incrementalAlterConfigs(Collections.singletonMap( + BROKER0, Collections.singletonMap("baz", entry(SET, "123"))), false); + assertFalse(future1.isDone()); + assertEquals(Collections.singletonMap(BROKER0, + new ResultOrError<>(Collections.emptyMap())), + controller.describeConfigs(Collections.singletonMap( + BROKER0, Collections.emptyList())).get()); + logEnv.logManagers().forEach(m -> m.setMaxReadOffset(1L)); + assertEquals(Collections.singletonMap(BROKER0, ApiError.NONE), future1.get()); + } + + @Test + public void testUnregisterBroker() throws Throwable { + try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1)) { + try (QuorumControllerTestEnv controlEnv = + new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) { + ListenerCollection listeners = new ListenerCollection(); + listeners.add(new Listener().setName("PLAINTEXT"). + setHost("localhost").setPort(9092)); + QuorumController active = controlEnv.activeController(); + CompletableFuture reply = active.registerBroker( + new BrokerRegistrationRequestData(). + setBrokerId(0). + setClusterId(Uuid.fromString("06B-K3N1TBCNYFgruEVP0Q")). + setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwBA")). + setListeners(listeners)); + assertEquals(0L, reply.get().epoch()); + CreateTopicsRequestData createTopicsRequestData = + new CreateTopicsRequestData().setTopics( + new CreatableTopicCollection(Collections.singleton( + new CreatableTopic().setName("foo").setNumPartitions(1). + setReplicationFactor((short) 1)).iterator())); + // TODO: place on a fenced broker if we have no choice + assertEquals(Errors.INVALID_REPLICATION_FACTOR.code(), active.createTopics( + createTopicsRequestData).get().topics().find("foo").errorCode()); + assertEquals(new BrokerHeartbeatReply(true, false, false, false), + active.processBrokerHeartbeat(new BrokerHeartbeatRequestData(). + setWantFence(false).setBrokerEpoch(0L).setBrokerId(0). + setCurrentMetadataOffset(100000L)).get()); + assertEquals(Errors.NONE.code(), active.createTopics( + createTopicsRequestData).get().topics().find("foo").errorCode()); + CompletableFuture topicPartitionFuture = active.appendReadEvent( + "debugGetPartition", () -> { + Iterator iterator = active. + replicationControl().brokersToIsrs().iterator(0, true); + assertTrue(iterator.hasNext()); + return iterator.next(); + }); + assertEquals(0, topicPartitionFuture.get().partitionId()); + active.unregisterBroker(0).get(); + topicPartitionFuture = active.appendReadEvent( + "debugGetPartition", () -> { + Iterator iterator = active. + replicationControl().brokersToIsrs().noLeaderIterator(); + assertTrue(iterator.hasNext()); + return iterator.next(); + }); + assertEquals(0, topicPartitionFuture.get().partitionId()); + } + } + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTestEnv.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTestEnv.java new file mode 100644 index 0000000000000..99270422fcf2c --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTestEnv.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.metalog.LocalLogManagerTestEnv; +import org.apache.kafka.test.TestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +public class QuorumControllerTestEnv implements AutoCloseable { + private static final Logger log = + LoggerFactory.getLogger(QuorumControllerTestEnv.class); + + private final List controllers; + + public QuorumControllerTestEnv(LocalLogManagerTestEnv logEnv, + Consumer builderConsumer) + throws Exception { + int numControllers = logEnv.logManagers().size(); + this.controllers = new ArrayList<>(numControllers); + try { + for (int i = 0; i < numControllers; i++) { + QuorumController.Builder builder = new QuorumController.Builder(i); + builder.setLogManager(logEnv.logManagers().get(i)); + builderConsumer.accept(builder); + this.controllers.add(builder.build()); + } + } catch (Exception e) { + close(); + throw e; + } + } + + QuorumController activeController() throws InterruptedException { + AtomicReference value = new AtomicReference<>(null); + TestUtils.retryOnExceptionWithTimeout(3, 20000, () -> { + QuorumController activeController = null; + for (QuorumController controller : controllers) { + if (controller.isActive()) { + if (activeController != null) { + throw new RuntimeException("node " + activeController.nodeId() + + " thinks it's the leader, but so does " + controller.nodeId()); + } + activeController = controller; + } + } + if (activeController == null) { + throw new RuntimeException("No leader found."); + } + value.set(activeController); + }); + return value.get(); + } + + public List controllers() { + return controllers; + } + + @Override + public void close() throws InterruptedException { + for (QuorumController controller : controllers) { + controller.beginShutdown(); + } + for (QuorumController controller : controllers) { + controller.close(); + } + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/ReplicasTest.java b/metadata/src/test/java/org/apache/kafka/controller/ReplicasTest.java new file mode 100644 index 0000000000000..6947c70400042 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ReplicasTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(40) +public class ReplicasTest { + @Test + public void testToList() { + assertEquals(Arrays.asList(1, 2, 3, 4), Replicas.toList(new int[] {1, 2, 3, 4})); + assertEquals(Arrays.asList(), Replicas.toList(Replicas.NONE)); + assertEquals(Arrays.asList(2), Replicas.toList(new int[] {2})); + } + + @Test + public void testToArray() { + assertArrayEquals(new int[] {3, 2, 1}, Replicas.toArray(Arrays.asList(3, 2, 1))); + assertArrayEquals(new int[] {}, Replicas.toArray(Arrays.asList())); + assertArrayEquals(new int[] {2}, Replicas.toArray(Arrays.asList(2))); + } + + @Test + public void testClone() { + assertArrayEquals(new int[]{3, 2, 1}, Replicas.clone(new int[]{3, 2, 1})); + assertArrayEquals(new int[]{}, Replicas.clone(new int[]{})); + assertArrayEquals(new int[]{2}, Replicas.clone(new int[]{2})); + } + + @Test + public void testValidate() { + assertTrue(Replicas.validate(new int[] {})); + assertTrue(Replicas.validate(new int[] {3})); + assertTrue(Replicas.validate(new int[] {3, 1, 2, 6})); + assertFalse(Replicas.validate(new int[] {3, 3})); + assertFalse(Replicas.validate(new int[] {4, -1, 3})); + assertFalse(Replicas.validate(new int[] {-1})); + assertFalse(Replicas.validate(new int[] {3, 1, 2, 6, 1})); + assertTrue(Replicas.validate(new int[] {1, 100})); + } + + @Test + public void testValidateIsr() { + assertTrue(Replicas.validateIsr(new int[] {}, new int[] {})); + assertTrue(Replicas.validateIsr(new int[] {1, 2, 3}, new int[] {})); + assertTrue(Replicas.validateIsr(new int[] {1, 2, 3}, new int[] {1, 2, 3})); + assertTrue(Replicas.validateIsr(new int[] {3, 1, 2}, new int[] {2, 1})); + assertFalse(Replicas.validateIsr(new int[] {3, 1, 2}, new int[] {4, 1})); + assertFalse(Replicas.validateIsr(new int[] {1, 2, 4}, new int[] {4, 4})); + } + + @Test + public void testContains() { + assertTrue(Replicas.contains(new int[] {3, 0, 1}, 0)); + assertFalse(Replicas.contains(new int[] {}, 0)); + assertTrue(Replicas.contains(new int[] {1}, 1)); + } + + @Test + public void testCopyWithout() { + assertArrayEquals(new int[] {}, Replicas.copyWithout(new int[] {}, 0)); + assertArrayEquals(new int[] {}, Replicas.copyWithout(new int[] {1}, 1)); + assertArrayEquals(new int[] {1, 3}, Replicas.copyWithout(new int[] {1, 2, 3}, 2)); + assertArrayEquals(new int[] {4, 1}, Replicas.copyWithout(new int[] {4, 2, 2, 1}, 2)); + } + + @Test + public void testCopyWith() { + assertArrayEquals(new int[] {-1}, Replicas.copyWith(new int[] {}, -1)); + assertArrayEquals(new int[] {1, 2, 3, 4}, Replicas.copyWith(new int[] {1, 2, 3}, 4)); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java new file mode 100644 index 0000000000000..9cc4173bf7cf1 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.kafka.common.message.BrokerHeartbeatRequestData; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection; +import org.apache.kafka.common.message.CreateTopicsRequestData; +import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; +import org.apache.kafka.common.message.CreateTopicsResponseData; +import org.apache.kafka.common.metadata.RegisterBrokerRecord; +import org.apache.kafka.common.metadata.TopicRecord; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ApiError; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.metadata.ApiMessageAndVersion; +import org.apache.kafka.metadata.BrokerHeartbeatReply; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.apache.kafka.common.protocol.Errors.INVALID_TOPIC_EXCEPTION; +import static org.apache.kafka.controller.BrokersToIsrs.TopicPartition; +import static org.apache.kafka.controller.ReplicationControlManager.PartitionControlInfo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +@Timeout(40) +public class ReplicationControlManagerTest { + private static ReplicationControlManager newReplicationControlManager() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + LogContext logContext = new LogContext(); + MockTime time = new MockTime(); + MockRandom random = new MockRandom(); + ClusterControlManager clusterControl = new ClusterControlManager( + logContext, time, snapshotRegistry, 1000, + new SimpleReplicaPlacementPolicy(random)); + clusterControl.activate(); + ConfigurationControlManager configurationControl = new ConfigurationControlManager( + new LogContext(), snapshotRegistry, Collections.emptyMap()); + return new ReplicationControlManager(snapshotRegistry, + new LogContext(), + random, + (short) 3, + 1, + configurationControl, + clusterControl); + } + + private static void registerBroker(int brokerId, ClusterControlManager clusterControl) { + RegisterBrokerRecord brokerRecord = new RegisterBrokerRecord(). + setBrokerEpoch(100).setBrokerId(brokerId); + brokerRecord.endPoints().add(new RegisterBrokerRecord.BrokerEndpoint(). + setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). + setPort((short) 9092 + brokerId). + setName("PLAINTEXT"). + setHost("localhost")); + clusterControl.replay(brokerRecord); + } + + private static void unfenceBroker(int brokerId, + ReplicationControlManager replicationControl) throws Exception { + ControllerResult result = replicationControl. + processBrokerHeartbeat(new BrokerHeartbeatRequestData(). + setBrokerId(brokerId).setBrokerEpoch(100).setCurrentMetadataOffset(1). + setWantFence(false).setWantShutDown(false), 0); + assertEquals(new BrokerHeartbeatReply(true, false, false, false), result.response()); + ControllerTestUtils.replayAll(replicationControl.clusterControl, result.records()); + } + + @Test + public void testCreateTopics() throws Exception { + ReplicationControlManager replicationControl = newReplicationControlManager(); + CreateTopicsRequestData request = new CreateTopicsRequestData(); + request.topics().add(new CreatableTopic().setName("foo"). + setNumPartitions(-1).setReplicationFactor((short) -1)); + ControllerResult result = + replicationControl.createTopics(request); + CreateTopicsResponseData expectedResponse = new CreateTopicsResponseData(); + expectedResponse.topics().add(new CreatableTopicResult().setName("foo"). + setErrorCode(Errors.INVALID_REPLICATION_FACTOR.code()). + setErrorMessage("Unable to replicate the partition 3 times: there are only 0 usable brokers")); + assertEquals(expectedResponse, result.response()); + + registerBroker(0, replicationControl.clusterControl); + unfenceBroker(0, replicationControl); + registerBroker(1, replicationControl.clusterControl); + unfenceBroker(1, replicationControl); + registerBroker(2, replicationControl.clusterControl); + unfenceBroker(2, replicationControl); + ControllerResult result2 = + replicationControl.createTopics(request); + CreateTopicsResponseData expectedResponse2 = new CreateTopicsResponseData(); + expectedResponse2.topics().add(new CreatableTopicResult().setName("foo"). + setNumPartitions(1).setReplicationFactor((short) 3). + setErrorMessage(null).setErrorCode((short) 0). + setTopicId(result2.response().topics().find("foo").topicId())); + assertEquals(expectedResponse2, result2.response()); + ControllerTestUtils.replayAll(replicationControl, result2.records()); + assertEquals(new PartitionControlInfo(new int[] {2, 0, 1}, + new int[] {2, 0, 1}, null, null, 2, 0, 0), + replicationControl.getPartition( + ((TopicRecord) result2.records().get(0).message()).topicId(), 0)); + ControllerResult result3 = + replicationControl.createTopics(request); + CreateTopicsResponseData expectedResponse3 = new CreateTopicsResponseData(); + expectedResponse3.topics().add(new CreatableTopicResult().setName("foo"). + setErrorCode(Errors.TOPIC_ALREADY_EXISTS.code()). + setErrorMessage(Errors.TOPIC_ALREADY_EXISTS.exception().getMessage())); + assertEquals(expectedResponse3, result3.response()); + } + + @Test + public void testValidateNewTopicNames() { + Map topicErrors = new HashMap<>(); + CreatableTopicCollection topics = new CreatableTopicCollection(); + topics.add(new CreatableTopic().setName("")); + topics.add(new CreatableTopic().setName("woo")); + topics.add(new CreatableTopic().setName(".")); + ReplicationControlManager.validateNewTopicNames(topicErrors, topics); + Map expectedTopicErrors = new HashMap<>(); + expectedTopicErrors.put("", new ApiError(INVALID_TOPIC_EXCEPTION, + "Topic name is illegal, it can't be empty")); + expectedTopicErrors.put(".", new ApiError(INVALID_TOPIC_EXCEPTION, + "Topic name cannot be \".\" or \"..\"")); + assertEquals(expectedTopicErrors, topicErrors); + } + + private static CreatableTopicResult createTestTopic( + ReplicationControlManager replicationControl, String name, + int[][] replicas) throws Exception { + assertFalse(replicas.length == 0); + CreateTopicsRequestData request = new CreateTopicsRequestData(); + CreatableTopic topic = new CreatableTopic().setName(name); + topic.setNumPartitions(-1).setReplicationFactor((short) -1); + for (int i = 0; i < replicas.length; i++) { + topic.assignments().add(new CreatableReplicaAssignment(). + setPartitionIndex(i).setBrokerIds(Replicas.toList(replicas[i]))); + } + request.topics().add(topic); + ControllerResult result = + replicationControl.createTopics(request); + CreatableTopicResult topicResult = result.response().topics().find(name); + assertNotNull(topicResult); + assertEquals((short) 0, topicResult.errorCode()); + assertEquals(replicas.length, topicResult.numPartitions()); + assertEquals(replicas[0].length, topicResult.replicationFactor()); + ControllerTestUtils.replayAll(replicationControl, result.records()); + return topicResult; + } + + @Test + public void testRemoveLeaderships() throws Exception { + ReplicationControlManager replicationControl = newReplicationControlManager(); + for (int i = 0; i < 6; i++) { + registerBroker(i, replicationControl.clusterControl); + unfenceBroker(i, replicationControl); + } + CreatableTopicResult result = createTestTopic(replicationControl, "foo", + new int[][] { + new int[] {0, 1, 2}, + new int[] {1, 2, 3}, + new int[] {2, 3, 0}, + new int[] {0, 2, 1} + }); + Set expectedPartitions = new HashSet<>(); + expectedPartitions.add(new TopicPartition(result.topicId(), 0)); + expectedPartitions.add(new TopicPartition(result.topicId(), 3)); + assertEquals(expectedPartitions, ControllerTestUtils. + iteratorToSet(replicationControl.brokersToIsrs().iterator(0, true))); + List records = new ArrayList<>(); + replicationControl.handleNodeDeactivated(0, records); + ControllerTestUtils.replayAll(replicationControl, records); + assertEquals(Collections.emptySet(), ControllerTestUtils. + iteratorToSet(replicationControl.brokersToIsrs().iterator(0, true))); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/controller/ResultOrErrorTest.java b/metadata/src/test/java/org/apache/kafka/controller/ResultOrErrorTest.java new file mode 100644 index 0000000000000..7d42b2edafb29 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ResultOrErrorTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.ApiError; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(value = 40) +public class ResultOrErrorTest { + @Test + public void testError() { + ResultOrError resultOrError = + new ResultOrError<>(Errors.INVALID_REQUEST, "missing foobar"); + assertTrue(resultOrError.isError()); + assertFalse(resultOrError.isResult()); + assertEquals(null, resultOrError.result()); + assertEquals(new ApiError(Errors.INVALID_REQUEST, "missing foobar"), + resultOrError.error()); + } + + @Test + public void testResult() { + ResultOrError resultOrError = new ResultOrError<>(123); + assertFalse(resultOrError.isError()); + assertTrue(resultOrError.isResult()); + assertEquals(123, resultOrError.result()); + assertEquals(null, resultOrError.error()); + } + + @Test + public void testEquals() { + ResultOrError a = new ResultOrError<>(Errors.INVALID_REQUEST, "missing foobar"); + ResultOrError b = new ResultOrError<>("abcd"); + assertFalse(a.equals(b)); + assertFalse(b.equals(a)); + assertTrue(a.equals(a)); + assertTrue(b.equals(b)); + ResultOrError c = new ResultOrError<>(Errors.INVALID_REQUEST, "missing baz"); + assertFalse(a.equals(c)); + assertFalse(c.equals(a)); + assertTrue(c.equals(c)); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java index ef85314e0ef2f..7b6cf06212e8e 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java @@ -39,6 +39,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; + /** * The LocalLogManager is a test implementation that relies on the contents of memory. */ @@ -111,9 +112,26 @@ public String toString() { public static class SharedLogData { private final Logger log = LoggerFactory.getLogger(SharedLogData.class); + + /** + * Maps node IDs to the matching log managers. + */ private final HashMap logManagers = new HashMap<>(); + + /** + * Maps offsets to record batches. + */ private final TreeMap batches = new TreeMap<>(); + + /** + * The current leader. + */ private MetaLogLeader leader = new MetaLogLeader(-1, -1); + + /** + * The start offset of the last batch that was created, or -1 if no batches have + * been created. + */ private long prevOffset = -1; synchronized void registerLogManager(LocalLogManager logManager) { @@ -197,20 +215,45 @@ private static class MetaLogListenerData { private final Logger log; + /** + * The node ID of this local log manager. Each log manager must have a unique ID. + */ private final int nodeId; + /** + * A reference to the in-memory state that unites all the log managers in use. + */ private final SharedLogData shared; + /** + * The event queue used by this local log manager. + */ private final EventQueue eventQueue; + /** + * Whether this LocalLogManager has been initialized. + */ private boolean initialized = false; + /** + * Whether this LocalLogManager has been shut down. + */ private boolean shutdown = false; + /** + * An offset that the log manager will not read beyond. This exists only for testing + * purposes. + */ private long maxReadOffset = Long.MAX_VALUE; + /** + * The listener objects attached to this local log manager. + */ private final List listeners = new ArrayList<>(); + /** + * The current leader, as seen by this log manager. + */ private volatile MetaLogLeader leader = new MetaLogLeader(-1, -1); public LocalLogManager(LogContext logContext, diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTest.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTest.java index 9dd6262ff66f9..ac578fb635807 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTest.java @@ -34,6 +34,7 @@ import static org.apache.kafka.metalog.MockMetaLogManagerListener.SHUTDOWN; import static org.junit.jupiter.api.Assertions.assertEquals; + @Timeout(value = 40) public class LocalLogManagerTest { private static final Logger log = LoggerFactory.getLogger(LocalLogManagerTest.class); diff --git a/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java b/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java index 7910285d48412..fafccfae99dee 100644 --- a/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java +++ b/shell/src/main/java/org/apache/kafka/shell/MetadataNodeManager.java @@ -23,8 +23,8 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.metadata.ConfigRecord; import org.apache.kafka.common.metadata.FenceBrokerRecord; -import org.apache.kafka.common.metadata.IsrChangeRecord; import org.apache.kafka.common.metadata.MetadataRecordType; +import org.apache.kafka.common.metadata.PartitionChangeRecord; import org.apache.kafka.common.metadata.PartitionRecord; import org.apache.kafka.common.metadata.PartitionRecordJsonConverter; import org.apache.kafka.common.metadata.RegisterBrokerRecord; @@ -56,6 +56,8 @@ * Maintains the in-memory metadata for the metadata tool. */ public final class MetadataNodeManager implements AutoCloseable { + private static final int NO_LEADER_CHANGE = -2; + private static final Logger log = LoggerFactory.getLogger(MetadataNodeManager.class); public static class Data { @@ -259,17 +261,21 @@ private void handleCommitImpl(MetadataRecordType type, ApiMessage message) } break; } - case ISR_CHANGE_RECORD: { - IsrChangeRecord record = (IsrChangeRecord) message; + case PARTITION_CHANGE_RECORD: { + PartitionChangeRecord record = (PartitionChangeRecord) message; FileNode file = data.root.file("topicIds", record.topicId().toString(), Integer.toString(record.partitionId()), "data"); JsonNode node = objectMapper.readTree(file.contents()); PartitionRecord partition = PartitionRecordJsonConverter. read(node, PartitionRecord.HIGHEST_SUPPORTED_VERSION); - partition.setIsr(record.isr()); - partition.setLeader(record.leader()); - partition.setLeaderEpoch(record.leaderEpoch()); - partition.setPartitionEpoch(record.partitionEpoch()); + if (record.isr() != null) { + partition.setIsr(record.isr()); + } + if (record.leader() != NO_LEADER_CHANGE) { + partition.setLeader(record.leader()); + partition.setLeaderEpoch(partition.leaderEpoch() + 1); + } + partition.setPartitionEpoch(partition.partitionEpoch() + 1); file.setContents(PartitionRecordJsonConverter.write(partition, PartitionRecord.HIGHEST_SUPPORTED_VERSION).toPrettyString()); break; From 6bab96da043b5d097bf05ddf93044f776ff24568 Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Sat, 20 Feb 2021 11:27:46 +0800 Subject: [PATCH 033/243] KAFKA-12335 Upgrade junit from 5.7.0 to 5.7.1 (#10145) Reviewers: Ismael Juma --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 30193bc19f6c6..de4634a934258 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -80,7 +80,7 @@ versions += [ jaxrs: "2.1.1", jfreechart: "1.0.0", jopt: "5.0.4", - junit: "5.7.0", + junit: "5.7.1", kafka_0100: "0.10.0.1", kafka_0101: "0.10.1.1", kafka_0102: "0.10.2.2", From a2d46110321632430dd813383991afac41555fe1 Mon Sep 17 00:00:00 2001 From: nicolasguyomar Date: Sat, 20 Feb 2021 04:33:20 +0100 Subject: [PATCH 034/243] MINOR: Enhance the documentation with the metric unit which is milliseconds (#10148) Reviewers: Mickael Maison , Chia-Ping Tsai --- .../clients/consumer/internals/KafkaConsumerMetrics.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java index 0e750abc99b68..71332b8814cf1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/KafkaConsumerMetrics.java @@ -51,11 +51,11 @@ public KafkaConsumerMetrics(Metrics metrics, String metricGrpPrefix) { this.timeBetweenPollSensor = metrics.sensor("time-between-poll"); this.timeBetweenPollSensor.add(metrics.metricName("time-between-poll-avg", metricGroupName, - "The average delay between invocations of poll()."), + "The average delay between invocations of poll() in milliseconds."), new Avg()); this.timeBetweenPollSensor.add(metrics.metricName("time-between-poll-max", metricGroupName, - "The max delay between invocations of poll()."), + "The max delay between invocations of poll() in milliseconds."), new Max()); this.pollIdleSensor = metrics.sensor("poll-idle-ratio-avg"); From 236ddda3601b0337b49b248ca4064546ed853371 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Fri, 19 Feb 2021 22:38:43 -0500 Subject: [PATCH 035/243] MINOR: AbstractCoordinator should log with its subclass (#10149) Reviewers: Chia-Ping Tsai --- .../kafka/clients/consumer/internals/AbstractCoordinator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java index 3eb86dbaee75a..34f81db306d33 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java @@ -158,7 +158,7 @@ public AbstractCoordinator(GroupRebalanceConfig rebalanceConfig, Objects.requireNonNull(rebalanceConfig.groupId, "Expected a non-null group id for coordinator construction"); this.rebalanceConfig = rebalanceConfig; - this.log = logContext.logger(AbstractCoordinator.class); + this.log = logContext.logger(this.getClass()); this.client = client; this.time = time; this.heartbeat = new Heartbeat(rebalanceConfig, time); From 954c090ffc378a63ce3c3c9a72b87724fcd2cd6c Mon Sep 17 00:00:00 2001 From: CHUN-HAO TANG Date: Sat, 20 Feb 2021 11:44:29 +0800 Subject: [PATCH 036/243] MINOR: apply Utils.isBlank to code base (#10124) Reviewers: Chia-Ping Tsai --- .../org/apache/kafka/clients/admin/KafkaAdminClient.java | 2 +- .../org/apache/kafka/clients/consumer/KafkaConsumer.java | 4 ++-- .../internals/unsecured/OAuthBearerUnsecuredJws.java | 7 ++++--- .../OAuthBearerUnsecuredLoginCallbackHandler.java | 9 +++------ .../OAuthBearerUnsecuredValidatorCallbackHandler.java | 3 +-- .../internals/unsecured/OAuthBearerScopeUtilsTest.java | 3 ++- .../main/java/org/apache/kafka/connect/data/Values.java | 4 ++-- .../org/apache/kafka/connect/health/AbstractState.java | 6 ++++-- .../org/apache/kafka/connect/health/ConnectorHealth.java | 5 +++-- .../java/org/apache/kafka/connect/data/ValuesTest.java | 3 ++- .../basic/auth/extension/PropertyFileLoginModule.java | 3 ++- .../apache/kafka/connect/runtime/ConnectorConfig.java | 2 +- .../kafka/connect/runtime/SinkConnectorConfig.java | 7 ++++--- .../org/apache/kafka/connect/runtime/WorkerConfig.java | 5 +++-- .../connect/runtime/distributed/DistributedHerder.java | 2 +- .../apache/kafka/connect/runtime/isolation/Plugins.java | 3 +-- .../apache/kafka/connect/runtime/rest/RestServer.java | 7 ++++--- .../kafka/connect/transforms/TimestampConverter.java | 4 ++-- core/src/main/scala/kafka/network/SocketServer.scala | 4 ++-- core/src/main/scala/kafka/server/KafkaServer.scala | 4 ++-- core/src/main/scala/kafka/utils/Log4jController.scala | 5 +++-- .../java/org/apache/kafka/streams/kstream/Printed.java | 3 ++- .../java/org/apache/kafka/streams/state/HostInfo.java | 3 ++- 23 files changed, 53 insertions(+), 45 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 3296fb8be5f68..60c8d63b17a8f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -4540,7 +4540,7 @@ public UpdateFeaturesResult updateFeatures(final Map feat final Map> updateFutures = new HashMap<>(); for (final Map.Entry entry : featureUpdates.entrySet()) { final String feature = entry.getKey(); - if (feature.trim().isEmpty()) { + if (Utils.isBlank(feature)) { throw new IllegalArgumentException("Provided feature can not be empty."); } updateFutures.put(entry.getKey(), new KafkaFutureImpl<>()); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index e60eebe239c47..f7760bbe890a4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -957,7 +957,7 @@ public void subscribe(Collection topics, ConsumerRebalanceListener liste this.unsubscribe(); } else { for (String topic : topics) { - if (topic == null || topic.trim().isEmpty()) + if (Utils.isBlank(topic)) throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic"); } @@ -1108,7 +1108,7 @@ public void assign(Collection partitions) { } else { for (TopicPartition tp : partitions) { String topic = (tp != null) ? tp.topic() : null; - if (topic == null || topic.trim().isEmpty()) + if (Utils.isBlank(topic)) throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic"); } fetcher.clearBufferedDataForUnassignedPartitions(partitions); diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredJws.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredJws.java index b5b301650ef4f..fa175b3d34ce4 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredJws.java +++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredJws.java @@ -31,6 +31,7 @@ import java.util.Set; import org.apache.kafka.common.security.oauthbearer.OAuthBearerToken; +import org.apache.kafka.common.utils.Utils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -103,7 +104,7 @@ public OAuthBearerUnsecuredJws(String compactSerialization, String principalClai OAuthBearerValidationResult.newFailure("No expiration time in JWT")); lifetime = convertClaimTimeInSecondsToMs(expirationTimeSeconds); String principalName = claim(this.principalClaimName, String.class); - if (principalName == null || principalName.trim().isEmpty()) + if (Utils.isBlank(principalName)) throw new OAuthBearerIllegalTokenException(OAuthBearerValidationResult .newFailure("No principal name in JWT claim: " + this.principalClaimName)); this.principalName = principalName; @@ -345,7 +346,7 @@ private Set calculateScope() { String scopeClaimName = scopeClaimName(); if (isClaimType(scopeClaimName, String.class)) { String scopeClaimValue = claim(scopeClaimName, String.class); - if (scopeClaimValue.trim().isEmpty()) + if (Utils.isBlank(scopeClaimValue)) return Collections.emptySet(); else { Set retval = new HashSet<>(); @@ -360,7 +361,7 @@ private Set calculateScope() { List stringList = (List) scopeClaimValue; Set retval = new HashSet<>(); for (String scope : stringList) { - if (scope != null && !scope.trim().isEmpty()) { + if (!Utils.isBlank(scope)) { retval.add(scope.trim()); } } diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredLoginCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredLoginCallbackHandler.java index e7a4f2cc798d1..eb4c7db131964 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredLoginCallbackHandler.java +++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredLoginCallbackHandler.java @@ -45,6 +45,7 @@ import org.apache.kafka.common.security.oauthbearer.OAuthBearerTokenCallback; import org.apache.kafka.common.security.oauthbearer.internals.OAuthBearerClientInitialResponse; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -191,13 +192,9 @@ private void handleTokenCallback(OAuthBearerTokenCallback callback) { throw new OAuthBearerConfigException("Extensions provided in login context without a token"); } String principalClaimNameValue = optionValue(PRINCIPAL_CLAIM_NAME_OPTION); - String principalClaimName = principalClaimNameValue != null && !principalClaimNameValue.trim().isEmpty() - ? principalClaimNameValue.trim() - : DEFAULT_PRINCIPAL_CLAIM_NAME; + String principalClaimName = Utils.isBlank(principalClaimNameValue) ? DEFAULT_PRINCIPAL_CLAIM_NAME : principalClaimNameValue.trim(); String scopeClaimNameValue = optionValue(SCOPE_CLAIM_NAME_OPTION); - String scopeClaimName = scopeClaimNameValue != null && !scopeClaimNameValue.trim().isEmpty() - ? scopeClaimNameValue.trim() - : DEFAULT_SCOPE_CLAIM_NAME; + String scopeClaimName = Utils.isBlank(scopeClaimNameValue) ? DEFAULT_SCOPE_CLAIM_NAME : scopeClaimNameValue.trim(); String headerJson = "{" + claimOrHeaderJsonText("alg", "none") + "}"; String lifetimeSecondsValueToUse = optionValue(LIFETIME_SECONDS_OPTION, DEFAULT_LIFETIME_SECONDS_ONE_HOUR); String claimsJson; diff --git a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredValidatorCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredValidatorCallbackHandler.java index a9dc059128bce..7a81521518cd1 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredValidatorCallbackHandler.java +++ b/clients/src/main/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerUnsecuredValidatorCallbackHandler.java @@ -195,8 +195,7 @@ private int allowableClockSkewMs() { String allowableClockSkewMsValue = option(ALLOWABLE_CLOCK_SKEW_MILLIS_OPTION); int allowableClockSkewMs = 0; try { - allowableClockSkewMs = allowableClockSkewMsValue == null || allowableClockSkewMsValue.trim().isEmpty() ? 0 - : Integer.parseInt(allowableClockSkewMsValue.trim()); + allowableClockSkewMs = Utils.isBlank(allowableClockSkewMsValue) ? 0 : Integer.parseInt(allowableClockSkewMsValue.trim()); } catch (NumberFormatException e) { throw new OAuthBearerConfigException(e.getMessage(), e); } diff --git a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerScopeUtilsTest.java b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerScopeUtilsTest.java index 6c1496b7d8cc9..f65440e91da1d 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerScopeUtilsTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/unsecured/OAuthBearerScopeUtilsTest.java @@ -21,6 +21,7 @@ import java.util.List; +import org.apache.kafka.common.utils.Utils; import org.junit.jupiter.api.Test; public class OAuthBearerScopeUtilsTest { @@ -28,7 +29,7 @@ public class OAuthBearerScopeUtilsTest { public void validScope() { for (String validScope : new String[] {"", " ", "scope1", " scope1 ", "scope1 Scope2", "scope1 Scope2"}) { List parsedScope = OAuthBearerScopeUtils.parseScope(validScope); - if (validScope.trim().isEmpty()) { + if (Utils.isBlank(validScope)) { assertTrue(parsedScope.isEmpty()); } else if (validScope.contains("Scope2")) { assertTrue(parsedScope.size() == 2 && parsedScope.get(0).equals("scope1") diff --git a/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java b/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java index 067c91bbe3025..31f4183743bc1 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java @@ -891,7 +891,7 @@ protected static SchemaAndValue parse(Parser parser, boolean embedded) throws No } String token = parser.next(); - if (token.trim().isEmpty()) { + if (Utils.isBlank(token)) { return new SchemaAndValue(Schema.STRING_SCHEMA, token); } token = token.trim(); @@ -1253,7 +1253,7 @@ protected boolean isNext(String expected, boolean ignoreLeadingAndTrailingWhites nextToken = consumeNextToken(); } if (ignoreLeadingAndTrailingWhitespace) { - while (nextToken.trim().isEmpty() && canConsumeNextToken()) { + while (Utils.isBlank(nextToken) && canConsumeNextToken()) { nextToken = consumeNextToken(); } } diff --git a/connect/api/src/main/java/org/apache/kafka/connect/health/AbstractState.java b/connect/api/src/main/java/org/apache/kafka/connect/health/AbstractState.java index f707b3c3026ed..ff6571550af66 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/health/AbstractState.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/health/AbstractState.java @@ -19,6 +19,8 @@ import java.util.Objects; +import org.apache.kafka.common.utils.Utils; + /** * Provides the current status along with identifier for Connect worker and tasks. */ @@ -36,10 +38,10 @@ public abstract class AbstractState { * @param traceMessage any error trace message associated with the connector or the task; may be null or empty */ public AbstractState(String state, String workerId, String traceMessage) { - if (state == null || state.trim().isEmpty()) { + if (Utils.isBlank(state)) { throw new IllegalArgumentException("State must not be null or empty"); } - if (workerId == null || workerId.trim().isEmpty()) { + if (Utils.isBlank(workerId)) { throw new IllegalArgumentException("Worker ID must not be null or empty"); } this.state = state; diff --git a/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorHealth.java b/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorHealth.java index 12fa6b76aff1e..1f781574f52a9 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorHealth.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/health/ConnectorHealth.java @@ -16,10 +16,11 @@ */ package org.apache.kafka.connect.health; - import java.util.Map; import java.util.Objects; +import org.apache.kafka.common.utils.Utils; + /** * Provides basic health information about the connector and its tasks. */ @@ -35,7 +36,7 @@ public ConnectorHealth(String name, ConnectorState connectorState, Map tasks, ConnectorType type) { - if (name == null || name.trim().isEmpty()) { + if (Utils.isBlank(name)) { throw new IllegalArgumentException("Connector name is required"); } Objects.requireNonNull(connectorState, "connectorState can't be null"); diff --git a/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java b/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java index 01fbae76a52fb..3700a6ee4e6cc 100644 --- a/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java +++ b/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.connect.data; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.data.Schema.Type; import org.apache.kafka.connect.data.Values.Parser; import org.apache.kafka.connect.errors.DataException; @@ -911,7 +912,7 @@ protected void assertParsed(String input, String... expectedTokens) { protected void assertConsumable(Parser parser, String... expectedTokens) { for (String expectedToken : expectedTokens) { - if (!expectedToken.trim().isEmpty()) { + if (!Utils.isBlank(expectedToken)) { int position = parser.mark(); assertTrue(parser.canConsume(expectedToken.trim())); parser.rewindTo(position); diff --git a/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/PropertyFileLoginModule.java b/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/PropertyFileLoginModule.java index 8a26dc352438c..8b8e324d977eb 100644 --- a/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/PropertyFileLoginModule.java +++ b/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/PropertyFileLoginModule.java @@ -18,6 +18,7 @@ package org.apache.kafka.connect.rest.basic.auth.extension; import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,7 +60,7 @@ public class PropertyFileLoginModule implements LoginModule { public void initialize(Subject subject, CallbackHandler callbackHandler, Map sharedState, Map options) { this.callbackHandler = callbackHandler; fileName = (String) options.get(FILE_OPTIONS); - if (fileName == null || fileName.trim().isEmpty()) { + if (Utils.isBlank(fileName)) { throw new ConfigException("Property Credentials file must be specified"); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java index ca9d33c1f5e05..4ba1ddd6dad45 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java @@ -494,7 +494,7 @@ ConfigDef getConfigDefFromConfigProvidingClass(String key, Class cls) { .filter(c -> Modifier.isPublic(c.getModifiers())) .map(Class::getName) .collect(Collectors.joining(", ")); - String message = childClassNames.trim().isEmpty() ? + String message = Utils.isBlank(childClassNames) ? aliasKind + " is abstract and cannot be created." : aliasKind + " is abstract and cannot be created. Did you mean " + childClassNames + "?"; throw new ConfigException(key, String.valueOf(cls), message); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java index 415d46f6f8694..93c2cb458ab98 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SinkConnectorConfig.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigDef.Importance; import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.sink.SinkTask; @@ -125,17 +126,17 @@ public static void validate(Map props) { public static boolean hasTopicsConfig(Map props) { String topicsStr = props.get(TOPICS_CONFIG); - return topicsStr != null && !topicsStr.trim().isEmpty(); + return !Utils.isBlank(topicsStr); } public static boolean hasTopicsRegexConfig(Map props) { String topicsRegexStr = props.get(TOPICS_REGEX_CONFIG); - return topicsRegexStr != null && !topicsRegexStr.trim().isEmpty(); + return !Utils.isBlank(topicsRegexStr); } public static boolean hasDlqTopicConfig(Map props) { String dqlTopicStr = props.get(DLQ_TOPIC_NAME_CONFIG); - return dqlTopicStr != null && !dqlTopicStr.trim().isEmpty(); + return !Utils.isBlank(dqlTopicStr); } @SuppressWarnings("unchecked") diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java index 58a9ce3d5ac9a..e140e594f3784 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.kafka.connect.storage.Converter; @@ -516,7 +517,7 @@ public void ensureValid(String name, Object value) { if (!(item instanceof String)) { throw new ConfigException("Invalid type for admin listener (expected String)."); } - if (((String) item).trim().isEmpty()) { + if (Utils.isBlank((String) item)) { throw new ConfigException("Empty listener found when parsing list."); } } @@ -527,7 +528,7 @@ private static class ResponseHttpHeadersValidator implements ConfigDef.Validator @Override public void ensureValid(String name, Object value) { String strValue = (String) value; - if (strValue == null || strValue.trim().isEmpty()) { + if (Utils.isBlank(strValue)) { return; } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java index 16dfbf970212c..b4dfb4de6e8b7 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java @@ -1447,7 +1447,7 @@ private void reconfigureConnector(final String connName, final Callback cb forwardRequestExecutor.submit(() -> { try { String leaderUrl = leaderUrl(); - if (leaderUrl == null || leaderUrl.trim().isEmpty()) { + if (Utils.isBlank(leaderUrl)) { cb.onCompletion(new ConnectException("Request to leader to " + "reconfigure connector tasks failed " + "because the URL of the leader's REST interface is empty!"), null); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java index d507059eacc8c..6ab8a7660446d 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/Plugins.java @@ -445,8 +445,7 @@ public T newPlugin(String klassName, AbstractConfig config, Class pluginK plugin = newPlugin(klass); if (plugin instanceof Versioned) { Versioned versionedPlugin = (Versioned) plugin; - if (versionedPlugin.version() == null || versionedPlugin.version().trim() - .isEmpty()) { + if (Utils.isBlank(versionedPlugin.version())) { throw new ConnectException("Version not defined for '" + klassName + "'"); } } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java index 136c616a41084..8f371bbfaee40 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.health.ConnectClusterDetails; import org.apache.kafka.connect.rest.ConnectRestExtension; @@ -275,19 +276,19 @@ public void initializeResources(Herder herder) { } String allowedOrigins = config.getString(WorkerConfig.ACCESS_CONTROL_ALLOW_ORIGIN_CONFIG); - if (allowedOrigins != null && !allowedOrigins.trim().isEmpty()) { + if (!Utils.isBlank(allowedOrigins)) { FilterHolder filterHolder = new FilterHolder(new CrossOriginFilter()); filterHolder.setName("cross-origin"); filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, allowedOrigins); String allowedMethods = config.getString(WorkerConfig.ACCESS_CONTROL_ALLOW_METHODS_CONFIG); - if (allowedMethods != null && !allowedOrigins.trim().isEmpty()) { + if (!Utils.isBlank(allowedMethods)) { filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, allowedMethods); } context.addFilter(filterHolder, "/*", EnumSet.of(DispatcherType.REQUEST)); } String headerConfig = config.getString(WorkerConfig.RESPONSE_HTTP_HEADERS_CONFIG); - if (headerConfig != null && !headerConfig.trim().isEmpty()) { + if (!Utils.isBlank(headerConfig)) { configureHttpResponsHeaderFilter(context); } diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/TimestampConverter.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/TimestampConverter.java index b92dc848fc0ca..a8d5cecb64a3b 100644 --- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/TimestampConverter.java +++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/TimestampConverter.java @@ -255,11 +255,11 @@ public void configure(Map configs) { throw new ConfigException("Unknown timestamp type in TimestampConverter: " + type + ". Valid values are " + Utils.join(VALID_TYPES, ", ") + "."); } - if (type.equals(TYPE_STRING) && formatPattern.trim().isEmpty()) { + if (type.equals(TYPE_STRING) && Utils.isBlank(formatPattern)) { throw new ConfigException("TimestampConverter requires format option to be specified when using string timestamps"); } SimpleDateFormat format = null; - if (formatPattern != null && !formatPattern.trim().isEmpty()) { + if (!Utils.isBlank(formatPattern)) { try { format = new SimpleDateFormat(formatPattern); format.setTimeZone(UTC); diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 24df39f6ef2ef..a4a990c276f69 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -47,7 +47,7 @@ import org.apache.kafka.common.network.{ChannelBuilder, ChannelBuilders, ClientI import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.requests.{ApiVersionsRequest, RequestContext, RequestHeader} import org.apache.kafka.common.security.auth.SecurityProtocol -import org.apache.kafka.common.utils.{KafkaThread, LogContext, Time} +import org.apache.kafka.common.utils.{KafkaThread, LogContext, Time, Utils} import org.apache.kafka.common.{Endpoint, KafkaException, MetricName, Reconfigurable} import org.slf4j.event.Level @@ -650,7 +650,7 @@ private[kafka] class Acceptor(val endPoint: EndPoint, */ private def openServerSocket(host: String, port: Int): ServerSocketChannel = { val socketAddress = - if (host == null || host.trim.isEmpty) + if (Utils.isBlank(host)) new InetSocketAddress(port) else new InetSocketAddress(host, port) diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 4daee0866d8e9..90c57f2594362 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -46,7 +46,7 @@ import org.apache.kafka.common.requests.{ControlledShutdownRequest, ControlledSh import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache import org.apache.kafka.common.security.{JaasContext, JaasUtils} -import org.apache.kafka.common.utils.{AppInfoParser, LogContext, Time} +import org.apache.kafka.common.utils.{AppInfoParser, LogContext, Time, Utils} import org.apache.kafka.common.{Endpoint, Node} import org.apache.kafka.metadata.BrokerState import org.apache.kafka.server.authorizer.Authorizer @@ -477,7 +477,7 @@ class KafkaServer( } val updatedEndpoints = listeners.map(endpoint => - if (endpoint.host == null || endpoint.host.trim.isEmpty) + if (Utils.isBlank(endpoint.host)) endpoint.copy(host = InetAddress.getLocalHost.getCanonicalHostName) else endpoint diff --git a/core/src/main/scala/kafka/utils/Log4jController.scala b/core/src/main/scala/kafka/utils/Log4jController.scala index a02fdb69e14d1..0d54c74e07542 100755 --- a/core/src/main/scala/kafka/utils/Log4jController.scala +++ b/core/src/main/scala/kafka/utils/Log4jController.scala @@ -20,6 +20,7 @@ package kafka.utils import java.util import java.util.Locale +import org.apache.kafka.common.utils.Utils import org.apache.log4j.{Level, LogManager, Logger} import scala.collection.mutable @@ -71,7 +72,7 @@ object Log4jController { */ def logLevel(loggerName: String, logLevel: String): Boolean = { val log = existingLogger(loggerName) - if (!loggerName.trim.isEmpty && !logLevel.trim.isEmpty && log != null) { + if (!Utils.isBlank(loggerName) && !Utils.isBlank(logLevel) && log != null) { log.setLevel(Level.toLevel(logLevel.toUpperCase(Locale.ROOT))) true } @@ -80,7 +81,7 @@ object Log4jController { def unsetLogLevel(loggerName: String): Boolean = { val log = existingLogger(loggerName) - if (!loggerName.trim.isEmpty && log != null) { + if (!Utils.isBlank(loggerName) && log != null) { log.setLevel(null) true } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Printed.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Printed.java index fdcd9cb335f0d..6a3d1e53ee05e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Printed.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Printed.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.streams.kstream; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.errors.TopologyException; import java.io.IOException; @@ -63,7 +64,7 @@ protected Printed(final Printed printed) { */ public static Printed toFile(final String filePath) { Objects.requireNonNull(filePath, "filePath can't be null"); - if (filePath.trim().isEmpty()) { + if (Utils.isBlank(filePath)) { throw new TopologyException("filePath can't be an empty string"); } try { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/HostInfo.java b/streams/src/main/java/org/apache/kafka/streams/state/HostInfo.java index 6293cf5a78937..70bdcdad7dc86 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/HostInfo.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/HostInfo.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor; @@ -55,7 +56,7 @@ public HostInfo(final String host, * @return a new HostInfo or null if endPoint is null or has no characters */ public static HostInfo buildFromEndpoint(final String endPoint) { - if (endPoint == null || endPoint.trim().isEmpty()) { + if (Utils.isBlank(endPoint)) { return null; } From 1a09bac0301a15fe7967e9a0c5bf11d34120561b Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Sat, 20 Feb 2021 12:38:09 -0800 Subject: [PATCH 037/243] MINOR: Remove redundant log close in `KafkaRaftClient` (#10168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch fixes a small shutdown bug. Current logic closes the log twice: once in `KafkaRaftClient`, and once in `RaftManager`. This can lead to errors like the following: ``` [2021-02-18 18:35:12,643] WARN (kafka.utils.CoreUtils$) java.nio.channels.ClosedChannelException at java.base/sun.nio.ch.FileChannelImpl.ensureOpen(FileChannelImpl.java:150) at java.base/sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:452) at org.apache.kafka.common.record.FileRecords.flush(FileRecords.java:197) at org.apache.kafka.common.record.FileRecords.close(FileRecords.java:204) at kafka.log.LogSegment.$anonfun$close$4(LogSegment.scala:592) at kafka.utils.CoreUtils$.swallow(CoreUtils.scala:68) at kafka.log.LogSegment.close(LogSegment.scala:592) at kafka.log.Log.$anonfun$close$4(Log.scala:1038) at kafka.log.Log.$anonfun$close$4$adapted(Log.scala:1038) at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:563) at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:561) at scala.collection.AbstractIterable.foreach(Iterable.scala:919) at kafka.log.Log.$anonfun$close$3(Log.scala:1038) at kafka.log.Log.close(Log.scala:2433) at kafka.raft.KafkaMetadataLog.close(KafkaMetadataLog.scala:295) at kafka.raft.KafkaRaftManager.shutdown(RaftManager.scala:150) ``` I have tended to view `RaftManager` as owning the lifecycle of the log, so I removed the extra call to close in `KafkaRaftClient`. Reviewers: José Armando García Sancio , Ismael Juma --- raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java | 1 - 1 file changed, 1 deletion(-) diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index b964a877c803d..560d47098bead 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -2262,7 +2262,6 @@ public SnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) throws IOExce @Override public void close() { - log.close(); if (kafkaRaftMetrics != null) { kafkaRaftMetrics.close(); } From c71ec552d100973cda65da3228ba3f2ab8039221 Mon Sep 17 00:00:00 2001 From: Lee Dongjin Date: Mon, 22 Feb 2021 23:19:06 +0530 Subject: [PATCH 038/243] KAFKA-12324: Upgrade jetty to fix CVE-2020-27218 Here is the fix. The reason of [CVE-2020-27218](https://nvd.nist.gov/vuln/detail/CVE-2020-27218) was [Incorrect recycling of `HttpInput`](https://bugs.eclipse.org/bugs/show_bug.cgi?id=568892) and [patched in 9.4.35.v20201120](https://github.com/eclipse/jetty.project/security/advisories/GHSA-86wm-rrjm-8wh8). This PR updates Jetty dependency into the following version, 9.4.36.v20210114. Author: Lee Dongjin Reviewers: Manikumar Reddy Closes #10177 from dongjinleekr/feature/KAFKA-12324 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index de4634a934258..2606ea4c2b79f 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -69,7 +69,7 @@ versions += [ jackson: "2.10.5", jacksonDatabind: "2.10.5.1", jacoco: "0.8.5", - jetty: "9.4.33.v20201020", + jetty: "9.4.36.v20210114", jersey: "2.31", jline: "3.12.1", jmh: "1.27", From 95f51539c8d0b88bd7f285011d42e2d1117107de Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Mon, 22 Feb 2021 10:16:48 -0800 Subject: [PATCH 039/243] MINOR: Raft request thread should discover api versions (#10157) We do not plan to rely on the IBP in order to determine API versions for raft requests. Instead, we want to discover them through the ApiVersions API. This patch enables the flag to do so. In addition, this patch adds unsupported version as well as authentication version checking to all of the derivatives of `InterBrokerSendThread` which rely on dynamic api version discovery. Test cases for these checks have been added. Reviewers: Ismael Juma , Chia-Ping Tsai , Boyang Chen --- .../kafka/raft/KafkaNetworkChannel.scala | 14 ++- .../main/scala/kafka/raft/RaftManager.scala | 2 +- .../scala/kafka/server/AlterIsrManager.scala | 14 ++- .../BrokerToControllerChannelManager.scala | 18 +++- .../kafka/server/ForwardingManager.scala | 47 ++++++---- .../BrokerToControllerRequestThreadTest.scala | 94 ++++++++++++++++++- .../kafka/raft/KafkaNetworkChannelTest.scala | 15 ++- .../kafka/server/AlterIsrManagerTest.scala | 42 ++++++--- .../kafka/server/ForwardingManagerTest.scala | 42 +++++++++ ...MockBrokerToControllerChannelManager.scala | 4 +- 10 files changed, 247 insertions(+), 45 deletions(-) diff --git a/core/src/main/scala/kafka/raft/KafkaNetworkChannel.scala b/core/src/main/scala/kafka/raft/KafkaNetworkChannel.scala index 68f7b4a87fb72..d99039132d8b4 100644 --- a/core/src/main/scala/kafka/raft/KafkaNetworkChannel.scala +++ b/core/src/main/scala/kafka/raft/KafkaNetworkChannel.scala @@ -124,8 +124,18 @@ class KafkaNetworkChannel( } def onComplete(clientResponse: ClientResponse): Unit = { - val response = if (clientResponse.authenticationException != null) { - errorResponse(request.data, Errors.CLUSTER_AUTHORIZATION_FAILED) + val response = if (clientResponse.versionMismatch != null) { + error(s"Request $request failed due to unsupported version error", + clientResponse.versionMismatch) + errorResponse(request.data, Errors.UNSUPPORTED_VERSION) + } else if (clientResponse.authenticationException != null) { + // For now we treat authentication errors as retriable. We use the + // `NETWORK_EXCEPTION` error code for lack of a good alternative. + // Note that `BrokerToControllerChannelManager` will still log the + // authentication errors so that users have a chance to fix the problem. + error(s"Request $request failed due to authentication error", + clientResponse.authenticationException) + errorResponse(request.data, Errors.NETWORK_EXCEPTION) } else if (clientResponse.wasDisconnected()) { errorResponse(request.data, Errors.BROKER_NOT_AVAILABLE) } else { diff --git a/core/src/main/scala/kafka/raft/RaftManager.scala b/core/src/main/scala/kafka/raft/RaftManager.scala index 7bf34b091dde1..ecf89348ccaf9 100644 --- a/core/src/main/scala/kafka/raft/RaftManager.scala +++ b/core/src/main/scala/kafka/raft/RaftManager.scala @@ -271,7 +271,7 @@ class KafkaRaftManager[T]( val maxInflightRequestsPerConnection = 1 val reconnectBackoffMs = 50 val reconnectBackoffMsMs = 500 - val discoverBrokerVersions = false + val discoverBrokerVersions = true new NetworkClient( selector, diff --git a/core/src/main/scala/kafka/server/AlterIsrManager.scala b/core/src/main/scala/kafka/server/AlterIsrManager.scala index b58ca89da4045..9ad734f708c9b 100644 --- a/core/src/main/scala/kafka/server/AlterIsrManager.scala +++ b/core/src/main/scala/kafka/server/AlterIsrManager.scala @@ -165,9 +165,19 @@ class DefaultAlterIsrManager( new ControllerRequestCompletionHandler { override def onComplete(response: ClientResponse): Unit = { debug(s"Received AlterIsr response $response") - val body = response.responseBody().asInstanceOf[AlterIsrResponse] val error = try { - handleAlterIsrResponse(body, message.brokerEpoch, inflightAlterIsrItems) + if (response.authenticationException != null) { + // For now we treat authentication errors as retriable. We use the + // `NETWORK_EXCEPTION` error code for lack of a good alternative. + // Note that `BrokerToControllerChannelManager` will still log the + // authentication errors so that users have a chance to fix the problem. + Errors.NETWORK_EXCEPTION + } else if (response.versionMismatch != null) { + Errors.UNSUPPORTED_VERSION + } else { + val body = response.responseBody().asInstanceOf[AlterIsrResponse] + handleAlterIsrResponse(body, message.brokerEpoch, inflightAlterIsrItems) + } } finally { // clear the flag so future requests can proceed clearInFlightRequest() diff --git a/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala b/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala index 3b535220994dc..621c8671b9ec4 100644 --- a/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala +++ b/core/src/main/scala/kafka/server/BrokerToControllerChannelManager.scala @@ -327,10 +327,18 @@ class BrokerToControllerRequestThread( None } - private[server] def handleResponse(request: BrokerToControllerQueueItem)(response: ClientResponse): Unit = { - if (response.wasDisconnected()) { + private[server] def handleResponse(queueItem: BrokerToControllerQueueItem)(response: ClientResponse): Unit = { + if (response.authenticationException != null) { + error(s"Request ${queueItem.request} failed due to authentication error with controller", + response.authenticationException) + queueItem.callback.onComplete(response) + } else if (response.versionMismatch != null) { + error(s"Request ${queueItem.request} failed due to unsupported version error", + response.versionMismatch) + queueItem.callback.onComplete(response) + } else if (response.wasDisconnected()) { updateControllerAddress(null) - requestQueue.putFirst(request) + requestQueue.putFirst(queueItem) } else if (response.responseBody().errorCounts().containsKey(Errors.NOT_CONTROLLER)) { // just close the controller connection and wait for metadata cache update in doWork activeControllerAddress().foreach { controllerAddress => { @@ -338,9 +346,9 @@ class BrokerToControllerRequestThread( updateControllerAddress(null) }} - requestQueue.putFirst(request) + requestQueue.putFirst(queueItem) } else { - request.callback.onComplete(response) + queueItem.callback.onComplete(response) } } diff --git a/core/src/main/scala/kafka/server/ForwardingManager.scala b/core/src/main/scala/kafka/server/ForwardingManager.scala index b77bd3456359e..a6e22e15ec5a3 100644 --- a/core/src/main/scala/kafka/server/ForwardingManager.scala +++ b/core/src/main/scala/kafka/server/ForwardingManager.scala @@ -106,29 +106,40 @@ class ForwardingManagerImpl( class ForwardingResponseHandler extends ControllerRequestCompletionHandler { override def onComplete(clientResponse: ClientResponse): Unit = { - val envelopeResponse = clientResponse.responseBody.asInstanceOf[EnvelopeResponse] - val envelopeError = envelopeResponse.error() val requestBody = request.body[AbstractRequest] - // Unsupported version indicates an incompatibility between controller and client API versions. This - // could happen when the controller changed after the connection was established. The forwarding broker - // should close the connection with the client and let it reinitialize the connection and refresh - // the controller API versions. - if (envelopeError == Errors.UNSUPPORTED_VERSION) { - responseCallback(None) + if (clientResponse.versionMismatch != null) { + debug(s"Returning `UNKNOWN_SERVER_ERROR` in response to request $requestBody " + + s"due to unexpected version error", clientResponse.versionMismatch) + responseCallback(Some(requestBody.getErrorResponse(Errors.UNKNOWN_SERVER_ERROR.exception))) + } else if (clientResponse.authenticationException != null) { + debug(s"Returning `UNKNOWN_SERVER_ERROR` in response to request $requestBody " + + s"due to authentication error", clientResponse.authenticationException) + responseCallback(Some(requestBody.getErrorResponse(Errors.UNKNOWN_SERVER_ERROR.exception))) } else { - val response = if (envelopeError != Errors.NONE) { - // A general envelope error indicates broker misconfiguration (e.g. the principal serde - // might not be defined on the receiving broker). In this case, we do not return - // the error directly to the client since it would not be expected. Instead we - // return `UNKNOWN_SERVER_ERROR` so that the user knows that there is a problem - // on the broker. - debug(s"Forwarded request $request failed with an error in the envelope response $envelopeError") - requestBody.getErrorResponse(Errors.UNKNOWN_SERVER_ERROR.exception) + val envelopeResponse = clientResponse.responseBody.asInstanceOf[EnvelopeResponse] + val envelopeError = envelopeResponse.error() + + // Unsupported version indicates an incompatibility between controller and client API versions. This + // could happen when the controller changed after the connection was established. The forwarding broker + // should close the connection with the client and let it reinitialize the connection and refresh + // the controller API versions. + if (envelopeError == Errors.UNSUPPORTED_VERSION) { + responseCallback(None) } else { - parseResponse(envelopeResponse.responseData, requestBody, request.header) + val response = if (envelopeError != Errors.NONE) { + // A general envelope error indicates broker misconfiguration (e.g. the principal serde + // might not be defined on the receiving broker). In this case, we do not return + // the error directly to the client since it would not be expected. Instead we + // return `UNKNOWN_SERVER_ERROR` so that the user knows that there is a problem + // on the broker. + debug(s"Forwarded request $request failed with an error in the envelope response $envelopeError") + requestBody.getErrorResponse(Errors.UNKNOWN_SERVER_ERROR.exception) + } else { + parseResponse(envelopeResponse.responseData, requestBody, request.header) + } + responseCallback(Option(response)) } - responseCallback(Option(response)) } } diff --git a/core/src/test/scala/kafka/server/BrokerToControllerRequestThreadTest.scala b/core/src/test/scala/kafka/server/BrokerToControllerRequestThreadTest.scala index f02d4ac5db617..676eb349b6156 100644 --- a/core/src/test/scala/kafka/server/BrokerToControllerRequestThreadTest.scala +++ b/core/src/test/scala/kafka/server/BrokerToControllerRequestThreadTest.scala @@ -18,16 +18,16 @@ package kafka.server import java.util.Collections -import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} import kafka.utils.TestUtils import org.apache.kafka.clients.{ClientResponse, ManualMetadataUpdater, Metadata, MockClient} import org.apache.kafka.common.Node import org.apache.kafka.common.message.MetadataRequestData -import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.{AbstractRequest, MetadataRequest, MetadataResponse, RequestTestUtils} import org.apache.kafka.common.utils.MockTime -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test import org.mockito.Mockito._ @@ -253,6 +253,94 @@ class BrokerToControllerRequestThreadTest { assertTrue(completionHandler.timedOut.get()) } + @Test + def testUnsupportedVersionHandling(): Unit = { + val time = new MockTime() + val config = new KafkaConfig(TestUtils.createBrokerConfig(1, "localhost:2181")) + val controllerId = 2 + + val metadata = mock(classOf[Metadata]) + val mockClient = new MockClient(time, metadata) + + val controllerNodeProvider = mock(classOf[ControllerNodeProvider]) + val activeController = new Node(controllerId, "host", 1234) + + when(controllerNodeProvider.get()).thenReturn(Some(activeController)) + + val callbackResponse = new AtomicReference[ClientResponse]() + val completionHandler = new ControllerRequestCompletionHandler { + override def onTimeout(): Unit = fail("Unexpected timeout exception") + override def onComplete(response: ClientResponse): Unit = callbackResponse.set(response) + } + + val queueItem = BrokerToControllerQueueItem( + time.milliseconds(), + new MetadataRequest.Builder(new MetadataRequestData()), + completionHandler + ) + + mockClient.prepareUnsupportedVersionResponse(request => request.apiKey == ApiKeys.METADATA) + + val testRequestThread = new BrokerToControllerRequestThread(mockClient, new ManualMetadataUpdater(), controllerNodeProvider, + config, time, "", retryTimeoutMs = Long.MaxValue) + + testRequestThread.enqueue(queueItem) + pollUntil(testRequestThread, () => callbackResponse.get != null) + assertNotNull(callbackResponse.get.versionMismatch) + } + + @Test + def testAuthenticationExceptionHandling(): Unit = { + val time = new MockTime() + val config = new KafkaConfig(TestUtils.createBrokerConfig(1, "localhost:2181")) + val controllerId = 2 + + val metadata = mock(classOf[Metadata]) + val mockClient = new MockClient(time, metadata) + + val controllerNodeProvider = mock(classOf[ControllerNodeProvider]) + val activeController = new Node(controllerId, "host", 1234) + + when(controllerNodeProvider.get()).thenReturn(Some(activeController)) + + val callbackResponse = new AtomicReference[ClientResponse]() + val completionHandler = new ControllerRequestCompletionHandler { + override def onTimeout(): Unit = fail("Unexpected timeout exception") + override def onComplete(response: ClientResponse): Unit = callbackResponse.set(response) + } + + val queueItem = BrokerToControllerQueueItem( + time.milliseconds(), + new MetadataRequest.Builder(new MetadataRequestData()), + completionHandler + ) + + mockClient.createPendingAuthenticationError(activeController, 50) + + val testRequestThread = new BrokerToControllerRequestThread(mockClient, new ManualMetadataUpdater(), controllerNodeProvider, + config, time, "", retryTimeoutMs = Long.MaxValue) + + testRequestThread.enqueue(queueItem) + pollUntil(testRequestThread, () => callbackResponse.get != null) + assertNotNull(callbackResponse.get.authenticationException) + } + + private def pollUntil( + requestThread: BrokerToControllerRequestThread, + condition: () => Boolean, + maxRetries: Int = 10 + ): Unit = { + var tries = 0 + do { + requestThread.doWork() + tries += 1 + } while (!condition.apply() && tries < maxRetries) + + if (!condition.apply()) { + fail(s"Condition failed to be met after polling $tries times") + } + } + class TestRequestCompletionHandler( expectedResponse: Option[MetadataResponse] = None ) extends ControllerRequestCompletionHandler { diff --git a/core/src/test/scala/unit/kafka/raft/KafkaNetworkChannelTest.scala b/core/src/test/scala/unit/kafka/raft/KafkaNetworkChannelTest.scala index 0f755b1c2f629..41eac22e4eaec 100644 --- a/core/src/test/scala/unit/kafka/raft/KafkaNetworkChannelTest.scala +++ b/core/src/test/scala/unit/kafka/raft/KafkaNetworkChannelTest.scala @@ -117,7 +117,7 @@ class KafkaNetworkChannelTest { for (apiKey <- RaftApis) { client.createPendingAuthenticationError(destinationNode, 100) - sendAndAssertErrorResponse(apiKey, destinationId, Errors.CLUSTER_AUTHORIZATION_FAILED) + sendAndAssertErrorResponse(apiKey, destinationId, Errors.NETWORK_EXCEPTION) // reset to clear backoff time client.reset() @@ -145,6 +145,19 @@ class KafkaNetworkChannelTest { } } + @Test + def testUnsupportedVersionError(): Unit = { + val destinationId = 2 + val destinationNode = new Node(destinationId, "127.0.0.1", 9092) + channel.updateEndpoint(destinationId, new InetAddressSpec( + new InetSocketAddress(destinationNode.host, destinationNode.port))) + + for (apiKey <- RaftApis) { + client.prepareUnsupportedVersionResponse(request => request.apiKey == apiKey) + sendAndAssertErrorResponse(apiKey, destinationId, Errors.UNSUPPORTED_VERSION) + } + } + private def sendTestRequest( apiKey: ApiKeys, destinationId: Int, diff --git a/core/src/test/scala/unit/kafka/server/AlterIsrManagerTest.scala b/core/src/test/scala/unit/kafka/server/AlterIsrManagerTest.scala index f0ae4b53e4343..1074fd3157c2f 100644 --- a/core/src/test/scala/unit/kafka/server/AlterIsrManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/AlterIsrManagerTest.scala @@ -19,11 +19,13 @@ package kafka.server import java.util.Collections import java.util.concurrent.atomic.AtomicInteger + import kafka.api.LeaderAndIsr import kafka.utils.{MockScheduler, MockTime} import kafka.zk.KafkaZkClient import org.apache.kafka.clients.ClientResponse import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.errors.{AuthenticationException, UnsupportedVersionException} import org.apache.kafka.common.message.AlterIsrResponseData import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.Errors @@ -127,20 +129,39 @@ class AlterIsrManagerTest { @Test def testAuthorizationFailed(): Unit = { - checkTopLevelError(Errors.CLUSTER_AUTHORIZATION_FAILED) + testRetryOnTopLevelError(Errors.CLUSTER_AUTHORIZATION_FAILED) } @Test def testStaleBrokerEpoch(): Unit = { - checkTopLevelError(Errors.STALE_BROKER_EPOCH) + testRetryOnTopLevelError(Errors.STALE_BROKER_EPOCH) } @Test def testUnknownServer(): Unit = { - checkTopLevelError(Errors.UNKNOWN_SERVER_ERROR) + testRetryOnTopLevelError(Errors.UNKNOWN_SERVER_ERROR) + } + + @Test + def testRetryOnAuthenticationFailure(): Unit = { + testRetryOnErrorResponse(new ClientResponse(null, null, "", 0L, 0L, + false, null, new AuthenticationException("authentication failed"), null)) + } + + @Test + def testRetryOnUnsupportedVersionError(): Unit = { + testRetryOnErrorResponse(new ClientResponse(null, null, "", 0L, 0L, + false, new UnsupportedVersionException("unsupported version"), null, null)) } - private def checkTopLevelError(error: Errors): Unit = { + private def testRetryOnTopLevelError(error: Errors): Unit = { + val alterIsrResp = new AlterIsrResponse(new AlterIsrResponseData().setErrorCode(error.code)) + val response = new ClientResponse(null, null, "", 0L, 0L, + false, null, null, alterIsrResp) + testRetryOnErrorResponse(response) + } + + private def testRetryOnErrorResponse(response: ClientResponse): Unit = { val leaderAndIsr = new LeaderAndIsr(1, 1, List(1,2,3), 10) val isrs = Seq(AlterIsrItem(tp0, leaderAndIsr, _ => { }, 0)) val callbackCapture = EasyMock.newCapture[ControllerRequestCompletionHandler]() @@ -156,10 +177,7 @@ class AlterIsrManagerTest { EasyMock.verify(brokerToController) - var alterIsrResp = new AlterIsrResponse(new AlterIsrResponseData().setErrorCode(error.code)) - var resp = new ClientResponse(null, null, "", 0L, 0L, - false, null, null, alterIsrResp) - callbackCapture.getValue.onComplete(resp) + callbackCapture.getValue.onComplete(response) // Any top-level error, we want to retry, so we don't clear items from the pending map assertTrue(alterIsrManager.unsentIsrUpdates.containsKey(tp0)) @@ -173,10 +191,10 @@ class AlterIsrManagerTest { scheduler.tick() // After a successful response, we can submit another AlterIsrItem - alterIsrResp = partitionResponse(tp0, Errors.NONE) - resp = new ClientResponse(null, null, "", 0L, 0L, - false, null, null, alterIsrResp) - callbackCapture.getValue.onComplete(resp) + val retryAlterIsrResponse = partitionResponse(tp0, Errors.NONE) + val retryResponse = new ClientResponse(null, null, "", 0L, 0L, + false, null, null, retryAlterIsrResponse) + callbackCapture.getValue.onComplete(retryResponse) EasyMock.verify(brokerToController) diff --git a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala index 9240af6a10857..2fefdac46c05c 100644 --- a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala @@ -155,6 +155,48 @@ class ForwardingManagerTest { assertEquals(Map(Errors.REQUEST_TIMED_OUT -> 1).asJava, alterConfigResponse.errorCounts) } + @Test + def testUnsupportedVersionFromNetworkClient(): Unit = { + val requestCorrelationId = 27 + val clientPrincipal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "client") + val (requestHeader, requestBuffer) = buildRequest(testAlterConfigRequest, requestCorrelationId) + val request = buildRequest(requestHeader, requestBuffer, clientPrincipal) + + val controllerNode = new Node(0, "host", 1234) + Mockito.when(controllerNodeProvider.get()).thenReturn(Some(controllerNode)) + + client.prepareUnsupportedVersionResponse(req => req.apiKey == requestHeader.apiKey) + + val response = new AtomicReference[AbstractResponse]() + forwardingManager.forwardRequest(request, res => res.foreach(response.set)) + brokerToController.poll() + assertNotNull(response.get) + + val alterConfigResponse = response.get.asInstanceOf[AlterConfigsResponse] + assertEquals(Map(Errors.UNKNOWN_SERVER_ERROR -> 1).asJava, alterConfigResponse.errorCounts) + } + + @Test + def testFailedAuthentication(): Unit = { + val requestCorrelationId = 27 + val clientPrincipal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "client") + val (requestHeader, requestBuffer) = buildRequest(testAlterConfigRequest, requestCorrelationId) + val request = buildRequest(requestHeader, requestBuffer, clientPrincipal) + + val controllerNode = new Node(0, "host", 1234) + Mockito.when(controllerNodeProvider.get()).thenReturn(Some(controllerNode)) + + client.createPendingAuthenticationError(controllerNode, 50) + + val response = new AtomicReference[AbstractResponse]() + forwardingManager.forwardRequest(request, res => res.foreach(response.set)) + brokerToController.poll() + assertNotNull(response.get) + + val alterConfigResponse = response.get.asInstanceOf[AlterConfigsResponse] + assertEquals(Map(Errors.UNKNOWN_SERVER_ERROR -> 1).asJava, alterConfigResponse.errorCounts) + } + private def buildRequest( body: AbstractRequest, correlationId: Int diff --git a/core/src/test/scala/unit/kafka/server/MockBrokerToControllerChannelManager.scala b/core/src/test/scala/unit/kafka/server/MockBrokerToControllerChannelManager.scala index a795b91d5157e..febd06f354d6d 100644 --- a/core/src/test/scala/unit/kafka/server/MockBrokerToControllerChannelManager.scala +++ b/core/src/test/scala/unit/kafka/server/MockBrokerToControllerChannelManager.scala @@ -53,7 +53,9 @@ class MockBrokerToControllerChannelManager( } private[server] def handleResponse(request: BrokerToControllerQueueItem)(response: ClientResponse): Unit = { - if (response.wasDisconnected() || response.responseBody.errorCounts.containsKey(Errors.NOT_CONTROLLER)) { + if (response.authenticationException != null || response.versionMismatch != null) { + request.callback.onComplete(response) + } else if (response.wasDisconnected() || response.responseBody.errorCounts.containsKey(Errors.NOT_CONTROLLER)) { unsentQueue.addFirst(request) } else { request.callback.onComplete(response) From 0711d1558250c60807dfa815d46907eac8bb4b98 Mon Sep 17 00:00:00 2001 From: Ron Dagostino Date: Mon, 22 Feb 2021 16:57:17 -0500 Subject: [PATCH 040/243] MINOR: Test the new KIP-500 quorum mode in ducktape (#10105) Add the necessary test annotations to test the new KIP-500 quorum broker mode in many of our ducktape tests. This mode is tested in addition to the classic Apache ZooKeeper mode. This PR also adds a new sanity_checks/bounce_test.py system test that runs through a simple produce/bounce/produce series of events. Finally, this PR adds @cluster annotations to dozens of system tests that were missing them. The lack of this annotation was causing these tests to grab the entire cluster of nodes. Adding the @cluster annotation dramatically reduced the time needed to run these tests. Reviewers: Colin P. McCabe , Ismael Juma --- tests/kafkatest/sanity_checks/test_bounce.py | 72 +++++++++++++++++++ .../sanity_checks/test_console_consumer.py | 16 +++-- .../test_performance_services.py | 16 +++-- .../sanity_checks/test_verifiable_producer.py | 32 ++++++--- tests/kafkatest/services/console_consumer.py | 9 +-- .../performance/consumer_performance.py | 10 +-- .../performance/end_to_end_latency.py | 13 ++-- .../performance/producer_performance.py | 4 +- .../client_compatibility_features_test.py | 19 ++--- ...ient_compatibility_produce_consume_test.py | 15 ++-- .../tests/client/compression_test.py | 13 ++-- .../client/consumer_rolling_upgrade_test.py | 8 ++- tests/kafkatest/tests/client/consumer_test.py | 36 +++++----- .../client/message_format_change_test.py | 24 ++++--- .../kafkatest/tests/client/pluggable_test.py | 7 +- tests/kafkatest/tests/connect/connect_test.py | 16 +++-- .../compatibility_test_new_broker_test.py | 61 ++++++++-------- .../tests/core/consume_bench_test.py | 40 +++++++---- .../tests/core/consumer_group_command_test.py | 18 ++--- .../tests/core/delegation_token_test.py | 2 + tests/kafkatest/tests/core/downgrade_test.py | 2 +- .../tests/core/fetch_from_follower_test.py | 14 ++-- .../tests/core/get_offset_shell_test.py | 23 +++--- .../core/group_mode_transactions_test.py | 29 +++++--- .../tests/core/produce_bench_test.py | 21 ++++-- .../tests/core/replica_scale_test.py | 22 +++--- .../kafkatest/tests/core/replication_test.py | 50 +++++++++---- .../tests/core/round_trip_fault_test.py | 54 ++++++++++---- tests/kafkatest/tests/core/security_test.py | 14 ++-- .../kafkatest/tests/core/transactions_test.py | 31 +++++--- tests/kafkatest/tests/core/upgrade_test.py | 4 +- tests/kafkatest/tests/end_to_end.py | 6 +- tests/kafkatest/tests/kafka_test.py | 10 +-- .../streams_application_upgrade_test.py | 2 + .../streams_broker_compatibility_test.py | 5 ++ .../streams_broker_down_resilience_test.py | 5 ++ ...eams_cooperative_rebalance_upgrade_test.py | 2 + .../tests/streams/streams_optimized_test.py | 3 +- .../streams/streams_shutdown_deadlock_test.py | 2 + .../tests/streams/streams_smoke_test.py | 5 +- .../streams/streams_standby_replica_test.py | 2 + .../streams/streams_static_membership_test.py | 2 + .../tests/streams/streams_upgrade_test.py | 2 + .../tests/tools/log4j_appender_test.py | 16 +++-- .../tests/tools/log_compaction_test.py | 14 ++-- .../tests/tools/replica_verification_test.py | 16 +++-- tests/kafkatest/version.py | 12 ++++ 47 files changed, 538 insertions(+), 261 deletions(-) create mode 100644 tests/kafkatest/sanity_checks/test_bounce.py diff --git a/tests/kafkatest/sanity_checks/test_bounce.py b/tests/kafkatest/sanity_checks/test_bounce.py new file mode 100644 index 0000000000000..c01f23b0cbaa4 --- /dev/null +++ b/tests/kafkatest/sanity_checks/test_bounce.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ducktape.mark import parametrize +from ducktape.mark.resource import cluster +from ducktape.tests.test import Test +from ducktape.utils.util import wait_until + +from kafkatest.services.kafka import KafkaService, quorum +from kafkatest.services.verifiable_producer import VerifiableProducer +from kafkatest.services.zookeeper import ZookeeperService + + +class TestBounce(Test): + """Sanity checks on verifiable producer service class with cluster roll.""" + def __init__(self, test_context): + super(TestBounce, self).__init__(test_context) + + self.topic = "topic" + self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None + self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, + topics={self.topic: {"partitions": 1, "replication-factor": 1}}, + controller_num_nodes_override=3 if quorum.for_test(test_context) == quorum.remote_raft else 1) + self.num_messages = 1000 + + def create_producer(self): + # This will produce to source kafka cluster + self.producer = VerifiableProducer(self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, + max_messages=self.num_messages, throughput=self.num_messages // 10) + def setUp(self): + if self.zk: + self.zk.start() + + @cluster(num_nodes=6) + @parametrize(metadata_quorum=quorum.remote_raft) + @cluster(num_nodes=4) + @parametrize(metadata_quorum=quorum.colocated_raft) + @cluster(num_nodes=4) + @parametrize(metadata_quorum=quorum.zk) + def test_simple_run(self, metadata_quorum): + """ + Test that we can start VerifiableProducer on the current branch snapshot version, and + verify that we can produce a small number of messages both before and after a subsequent roll. + """ + self.kafka.start() + for first_time in [True, False]: + self.create_producer() + self.producer.start() + wait_until(lambda: self.producer.num_acked > 5, timeout_sec=15, + err_msg="Producer failed to start in a reasonable amount of time.") + + self.producer.wait() + num_produced = self.producer.num_acked + assert num_produced == self.num_messages, "num_produced: %d, num_messages: %d" % (num_produced, self.num_messages) + if first_time: + self.producer.stop() + if self.kafka.quorum_info.using_raft and self.kafka.remote_controller_quorum: + self.kafka.remote_controller_quorum.restart_cluster() + self.kafka.restart_cluster() diff --git a/tests/kafkatest/sanity_checks/test_console_consumer.py b/tests/kafkatest/sanity_checks/test_console_consumer.py index 686cd42e123e5..0847ce0cb41d6 100644 --- a/tests/kafkatest/sanity_checks/test_console_consumer.py +++ b/tests/kafkatest/sanity_checks/test_console_consumer.py @@ -21,7 +21,7 @@ from ducktape.utils.util import wait_until from kafkatest.services.console_consumer import ConsoleConsumer -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.zookeeper import ZookeeperService from kafkatest.utils.remote_account import line_count, file_exists @@ -34,20 +34,22 @@ def __init__(self, test_context): super(ConsoleConsumerTest, self).__init__(test_context) self.topic = "topic" - self.zk = ZookeeperService(test_context, num_nodes=1) + self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, zk_chroot="/kafka", topics={self.topic: {"partitions": 1, "replication-factor": 1}}) self.consumer = ConsoleConsumer(self.test_context, num_nodes=1, kafka=self.kafka, topic=self.topic) def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() @cluster(num_nodes=3) - @matrix(security_protocol=['PLAINTEXT', 'SSL']) + @matrix(security_protocol=['PLAINTEXT', 'SSL'], metadata_quorum=quorum.all_raft) @cluster(num_nodes=4) - @matrix(security_protocol=['SASL_SSL'], sasl_mechanism=['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512']) - @matrix(security_protocol=['SASL_PLAINTEXT', 'SASL_SSL']) - def test_lifecycle(self, security_protocol, sasl_mechanism='GSSAPI'): + @matrix(security_protocol=['SASL_SSL'], sasl_mechanism=['PLAIN'], metadata_quorum=quorum.all_raft) + @matrix(security_protocol=['SASL_SSL'], sasl_mechanism=['SCRAM-SHA-256', 'SCRAM-SHA-512']) # SCRAM not yet supported with Raft + @matrix(security_protocol=['SASL_PLAINTEXT', 'SASL_SSL'], metadata_quorum=quorum.all_raft) + def test_lifecycle(self, security_protocol, sasl_mechanism='GSSAPI', metadata_quorum=quorum.zk): """Check that console consumer starts/stops properly, and that we are capturing log output.""" self.kafka.security_protocol = security_protocol diff --git a/tests/kafkatest/sanity_checks/test_performance_services.py b/tests/kafkatest/sanity_checks/test_performance_services.py index 280152c0f8621..f0d1a48bf04fe 100644 --- a/tests/kafkatest/sanity_checks/test_performance_services.py +++ b/tests/kafkatest/sanity_checks/test_performance_services.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ducktape.mark import parametrize +from ducktape.mark import matrix, parametrize from ducktape.mark.resource import cluster from ducktape.tests.test import Test -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.performance import ProducerPerformanceService, ConsumerPerformanceService, EndToEndLatencyService from kafkatest.services.performance import latency, compute_aggregate_throughput from kafkatest.services.zookeeper import ZookeeperService @@ -31,10 +31,11 @@ def __init__(self, test_context): self.num_records = 10000 self.topic = "topic" - self.zk = ZookeeperService(test_context, 1) + self.zk = ZookeeperService(test_context, 1) if quorum.for_test(test_context) == quorum.zk else None def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() @cluster(num_nodes=5) # We are keeping 0.8.2 here so that we don't inadvertently break support for it. Since this is just a sanity check, @@ -43,8 +44,9 @@ def setUp(self): @parametrize(version=str(LATEST_0_9), new_consumer=False) @parametrize(version=str(LATEST_0_9)) @parametrize(version=str(LATEST_1_1), new_consumer=False) - @parametrize(version=str(DEV_BRANCH)) - def test_version(self, version=str(LATEST_0_9), new_consumer=True): + @cluster(num_nodes=5) + @matrix(version=[str(DEV_BRANCH)], metadata_quorum=quorum.all) + def test_version(self, version=str(LATEST_0_9), new_consumer=True, metadata_quorum=quorum.zk): """ Sanity check out producer performance service - verify that we can run the service with a small number of messages. The actual stats here are pretty meaningless since the number of messages is quite small. @@ -67,6 +69,7 @@ def test_version(self, version=str(LATEST_0_9), new_consumer=True): 'buffer.memory': 64*1024*1024}) self.producer_perf.run() producer_perf_data = compute_aggregate_throughput(self.producer_perf) + assert producer_perf_data['records_per_sec'] > 0 # check basic run of end to end latency self.end_to_end = EndToEndLatencyService( @@ -82,6 +85,7 @@ def test_version(self, version=str(LATEST_0_9), new_consumer=True): self.consumer_perf.group = "test-consumer-group" self.consumer_perf.run() consumer_perf_data = compute_aggregate_throughput(self.consumer_perf) + assert consumer_perf_data['records_per_sec'] > 0 return { "producer_performance": producer_perf_data, diff --git a/tests/kafkatest/sanity_checks/test_verifiable_producer.py b/tests/kafkatest/sanity_checks/test_verifiable_producer.py index 5a95e48898949..32961f1995dff 100644 --- a/tests/kafkatest/sanity_checks/test_verifiable_producer.py +++ b/tests/kafkatest/sanity_checks/test_verifiable_producer.py @@ -14,12 +14,12 @@ # limitations under the License. -from ducktape.mark import parametrize +from ducktape.mark import matrix, parametrize from ducktape.mark.resource import cluster from ducktape.tests.test import Test from ducktape.utils.util import wait_until -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.zookeeper import ZookeeperService from kafkatest.utils import is_version @@ -32,7 +32,7 @@ def __init__(self, test_context): super(TestVerifiableProducer, self).__init__(test_context) self.topic = "topic" - self.zk = ZookeeperService(test_context, num_nodes=1) + self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, topics={self.topic: {"partitions": 1, "replication-factor": 1}}) @@ -41,24 +41,40 @@ def __init__(self, test_context): self.producer = VerifiableProducer(test_context, num_nodes=1, kafka=self.kafka, topic=self.topic, max_messages=self.num_messages, throughput=self.num_messages // 10) def setUp(self): - self.zk.start() - self.kafka.start() + if self.zk: + self.zk.start() @cluster(num_nodes=3) @parametrize(producer_version=str(LATEST_0_8_2)) @parametrize(producer_version=str(LATEST_0_9)) @parametrize(producer_version=str(LATEST_0_10_0)) @parametrize(producer_version=str(LATEST_0_10_1)) - @parametrize(producer_version=str(DEV_BRANCH)) - def test_simple_run(self, producer_version=DEV_BRANCH): + @matrix(producer_version=[str(DEV_BRANCH)], security_protocol=['PLAINTEXT', 'SSL'], metadata_quorum=quorum.all) + @cluster(num_nodes=4) + @matrix(producer_version=[str(DEV_BRANCH)], security_protocol=['SASL_SSL'], sasl_mechanism=['PLAIN', 'GSSAPI'], + metadata_quorum=quorum.all) + def test_simple_run(self, producer_version, security_protocol = 'PLAINTEXT', sasl_mechanism='PLAIN', + metadata_quorum=quorum.zk): """ Test that we can start VerifiableProducer on the current branch snapshot version or against the 0.8.2 jar, and verify that we can produce a small number of messages. """ + self.kafka.security_protocol = security_protocol + self.kafka.client_sasl_mechanism = sasl_mechanism + self.kafka.interbroker_security_protocol = security_protocol + self.kafka.interbroker_sasl_mechanism = sasl_mechanism + if self.kafka.quorum_info.using_raft: + controller_quorum = self.kafka.controller_quorum + controller_quorum.controller_security_protocol = security_protocol + controller_quorum.controller_sasl_mechanism = sasl_mechanism + controller_quorum.intercontroller_security_protocol = security_protocol + controller_quorum.intercontroller_sasl_mechanism = sasl_mechanism + self.kafka.start() + node = self.producer.nodes[0] node.version = KafkaVersion(producer_version) self.producer.start() - wait_until(lambda: self.producer.num_acked > 5, timeout_sec=5, + wait_until(lambda: self.producer.num_acked > 5, timeout_sec=15, err_msg="Producer failed to start in a reasonable amount of time.") # using version.vstring (distutils.version.LooseVersion) is a tricky way of ensuring diff --git a/tests/kafkatest/services/console_consumer.py b/tests/kafkatest/services/console_consumer.py index 14b450c0871e2..32e714543cd0a 100644 --- a/tests/kafkatest/services/console_consumer.py +++ b/tests/kafkatest/services/console_consumer.py @@ -21,7 +21,7 @@ from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin from kafkatest.services.monitor.jmx import JmxMixin, JmxTool -from kafkatest.version import DEV_BRANCH, LATEST_0_8_2, LATEST_0_9, LATEST_0_10_0, V_0_9_0_0, V_0_10_0_0, V_0_11_0_0, V_2_0_0 +from kafkatest.version import DEV_BRANCH, LATEST_0_8_2, LATEST_0_9, LATEST_0_10_0, V_0_10_0_0, V_0_11_0_0, V_2_0_0 from kafkatest.services.kafka.util import fix_opts_for_new_jvm """ @@ -151,7 +151,9 @@ def prop_file(self, node): def start_cmd(self, node): """Return the start command appropriate for the given node.""" args = self.args.copy() - args['zk_connect'] = self.kafka.zk_connect_setting() + args['broker_list'] = self.kafka.bootstrap_servers(self.security_config.security_protocol) + if not self.new_consumer: + args['zk_connect'] = self.kafka.zk_connect_setting() args['stdout'] = ConsoleConsumer.STDOUT_CAPTURE args['stderr'] = ConsoleConsumer.STDERR_CAPTURE args['log_dir'] = ConsoleConsumer.LOG_DIR @@ -160,7 +162,6 @@ def start_cmd(self, node): args['stdout'] = ConsoleConsumer.STDOUT_CAPTURE args['jmx_port'] = self.jmx_port args['console_consumer'] = self.path.script("kafka-console-consumer.sh", node) - args['broker_list'] = self.kafka.bootstrap_servers(self.security_config.security_protocol) if self.kafka_opts_override: args['kafka_opts'] = "\"%s\"" % self.kafka_opts_override @@ -177,7 +178,7 @@ def start_cmd(self, node): "--consumer.config %(config_file)s " % args if self.new_consumer: - assert node.version >= V_0_9_0_0, \ + assert node.version.consumer_supports_bootstrap_server(), \ "new_consumer is only supported if version >= 0.9.0.0, version %s" % str(node.version) if node.version <= LATEST_0_10_0: cmd += " --new-consumer" diff --git a/tests/kafkatest/services/performance/consumer_performance.py b/tests/kafkatest/services/performance/consumer_performance.py index 930a68fb0e91d..6df8dfb6be86a 100644 --- a/tests/kafkatest/services/performance/consumer_performance.py +++ b/tests/kafkatest/services/performance/consumer_performance.py @@ -18,7 +18,7 @@ from kafkatest.services.performance import PerformanceService from kafkatest.services.security.security_config import SecurityConfig -from kafkatest.version import DEV_BRANCH, V_0_9_0_0, V_2_0_0, LATEST_0_10_0 +from kafkatest.version import DEV_BRANCH, V_2_0_0, LATEST_0_10_0 class ConsumerPerformanceService(PerformanceService): @@ -79,14 +79,14 @@ def __init__(self, context, num_nodes, kafka, topic, messages, version=DEV_BRANC self.new_consumer = new_consumer self.settings = settings - assert version >= V_0_9_0_0 or (not new_consumer), \ + assert version.consumer_supports_bootstrap_server() or (not new_consumer), \ "new_consumer is only supported if version >= 0.9.0.0, version %s" % str(version) assert version < V_2_0_0 or new_consumer, \ "new_consumer==false is only supported if version < 2.0.0, version %s" % str(version) security_protocol = self.security_config.security_protocol - assert version >= V_0_9_0_0 or security_protocol == SecurityConfig.PLAINTEXT, \ + assert version.consumer_supports_bootstrap_server() or security_protocol == SecurityConfig.PLAINTEXT, \ "Security protocol %s is only supported if version >= 0.9.0.0, version %s" % (self.security_config, str(version)) # These less-frequently used settings can be updated manually after instantiation @@ -142,7 +142,7 @@ def start_cmd(self, node): for key, value in self.args(node.version).items(): cmd += " --%s %s" % (key, value) - if node.version >= V_0_9_0_0: + if node.version.consumer_supports_bootstrap_server(): # This is only used for security settings cmd += " --consumer.config %s" % ConsumerPerformanceService.CONFIG_FILE @@ -155,7 +155,7 @@ def start_cmd(self, node): def parse_results(self, line, version): parts = line.split(',') - if version >= V_0_9_0_0: + if version.consumer_supports_bootstrap_server(): result = { 'total_mb': float(parts[2]), 'mbps': float(parts[3]), diff --git a/tests/kafkatest/services/performance/end_to_end_latency.py b/tests/kafkatest/services/performance/end_to_end_latency.py index 2c7f69a04d60d..3cde3ef1a5d40 100644 --- a/tests/kafkatest/services/performance/end_to_end_latency.py +++ b/tests/kafkatest/services/performance/end_to_end_latency.py @@ -17,7 +17,7 @@ from kafkatest.services.performance import PerformanceService from kafkatest.services.security.security_config import SecurityConfig -from kafkatest.version import DEV_BRANCH, V_0_9_0_0 +from kafkatest.version import DEV_BRANCH @@ -53,7 +53,7 @@ def __init__(self, context, num_nodes, kafka, topic, num_records, compression_ty security_protocol = self.security_config.security_protocol - if version < V_0_9_0_0: + if not version.consumer_supports_bootstrap_server(): assert security_protocol == SecurityConfig.PLAINTEXT, \ "Security protocol %s is only supported if version >= 0.9.0.0, version %s" % (self.security_config, str(version)) assert compression_type == "none", \ @@ -74,15 +74,18 @@ def __init__(self, context, num_nodes, kafka, topic, num_records, compression_ty def start_cmd(self, node): args = self.args.copy() args.update({ - 'zk_connect': self.kafka.zk_connect_setting(), 'bootstrap_servers': self.kafka.bootstrap_servers(self.security_config.security_protocol), 'config_file': EndToEndLatencyService.CONFIG_FILE, 'kafka_run_class': self.path.script("kafka-run-class.sh", node), 'java_class_name': self.java_class_name() }) + if not node.version.consumer_supports_bootstrap_server(): + args.update({ + 'zk_connect': self.kafka.zk_connect_setting(), + }) cmd = "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % EndToEndLatencyService.LOG4J_CONFIG - if node.version >= V_0_9_0_0: + if node.version.consumer_supports_bootstrap_server(): cmd += "KAFKA_OPTS=%(kafka_opts)s %(kafka_run_class)s %(java_class_name)s " % args cmd += "%(bootstrap_servers)s %(topic)s %(num_records)d %(acks)d %(message_bytes)d %(config_file)s" % args else: @@ -102,7 +105,7 @@ def _worker(self, idx, node): node.account.create_file(EndToEndLatencyService.LOG4J_CONFIG, log_config) client_config = str(self.security_config) - if node.version >= V_0_9_0_0: + if node.version.consumer_supports_bootstrap_server(): client_config += "compression_type=%(compression_type)s" % self.args node.account.create_file(EndToEndLatencyService.CONFIG_FILE, client_config) diff --git a/tests/kafkatest/services/performance/producer_performance.py b/tests/kafkatest/services/performance/producer_performance.py index 3c4369883040d..a990d4fe04527 100644 --- a/tests/kafkatest/services/performance/producer_performance.py +++ b/tests/kafkatest/services/performance/producer_performance.py @@ -22,7 +22,7 @@ from kafkatest.services.monitor.http import HttpMetricsCollector from kafkatest.services.performance import PerformanceService from kafkatest.services.security.security_config import SecurityConfig -from kafkatest.version import DEV_BRANCH, V_0_9_0_0 +from kafkatest.version import DEV_BRANCH class ProducerPerformanceService(HttpMetricsCollector, PerformanceService): @@ -55,7 +55,7 @@ def __init__(self, context, num_nodes, kafka, topic, num_records, record_size, t self.security_config = kafka.security_config.client_config() security_protocol = self.security_config.security_protocol - assert version >= V_0_9_0_0 or security_protocol == SecurityConfig.PLAINTEXT, \ + assert version.consumer_supports_bootstrap_server() or security_protocol == SecurityConfig.PLAINTEXT, \ "Security protocol %s is only supported if version >= 0.9.0.0, version %s" % (self.security_config, str(version)) self.args = { diff --git a/tests/kafkatest/tests/client/client_compatibility_features_test.py b/tests/kafkatest/tests/client/client_compatibility_features_test.py index 4e7aeed6a832a..d98dffa3b8549 100644 --- a/tests/kafkatest/tests/client/client_compatibility_features_test.py +++ b/tests/kafkatest/tests/client/client_compatibility_features_test.py @@ -19,11 +19,12 @@ import time from random import randint -from ducktape.mark import parametrize +from ducktape.mark import matrix, parametrize +from ducktape.mark.resource import cluster from ducktape.tests.test import TestContext from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from ducktape.tests.test import Test from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, LATEST_2_5, LATEST_2_6, LATEST_2_7, V_0_11_0_0, V_0_10_1_0, KafkaVersion @@ -69,7 +70,7 @@ def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ClientCompatibilityFeaturesTest, self).__init__(test_context=test_context) - self.zk = ZookeeperService(test_context, num_nodes=3) + self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test(test_context) == quorum.zk else None # Generate a unique topic name topic_name = "client_compat_features_topic_%d%d" % (int(time.time()), randint(0, 2147483647)) @@ -81,11 +82,11 @@ def __init__(self, test_context): def invoke_compatibility_program(self, features): # Run the compatibility test on the first Kafka node. - node = self.zk.nodes[0] + node = self.kafka.nodes[0] cmd = ("%s org.apache.kafka.tools.ClientCompatibilityTest " "--bootstrap-server %s " "--num-cluster-nodes %d " - "--topic %s " % (self.zk.path.script("kafka-run-class.sh", node), + "--topic %s " % (self.kafka.path.script("kafka-run-class.sh", node), self.kafka.bootstrap_servers(), len(self.kafka.nodes), list(self.topics.keys())[0])) @@ -107,7 +108,8 @@ def invoke_compatibility_program(self, features): self.logger.info("** Command failed. See %s for log messages." % ssh_log_file) raise - @parametrize(broker_version=str(DEV_BRANCH)) + @cluster(num_nodes=7) + @matrix(broker_version=[str(DEV_BRANCH)], metadata_quorum=quorum.all_non_upgrade) @parametrize(broker_version=str(LATEST_0_10_0)) @parametrize(broker_version=str(LATEST_0_10_1)) @parametrize(broker_version=str(LATEST_0_10_2)) @@ -122,8 +124,9 @@ def invoke_compatibility_program(self, features): @parametrize(broker_version=str(LATEST_2_5)) @parametrize(broker_version=str(LATEST_2_6)) @parametrize(broker_version=str(LATEST_2_7)) - def run_compatibility_test(self, broker_version): - self.zk.start() + def run_compatibility_test(self, broker_version, metadata_quorum=quorum.zk): + if self.zk: + self.zk.start() self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() features = get_broker_features(broker_version) diff --git a/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py b/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py index 52d41f5aa1a52..317d0dd40fd64 100644 --- a/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py +++ b/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ducktape.mark import parametrize +from ducktape.mark import matrix, parametrize +from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest @@ -34,7 +35,7 @@ def __init__(self, test_context): super(ClientCompatibilityProduceConsumeTest, self).__init__(test_context=test_context) self.topic = "test_topic" - self.zk = ZookeeperService(test_context, num_nodes=3) + self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, topics={self.topic:{ "partitions": 10, "replication-factor": 2}}) @@ -46,13 +47,15 @@ def __init__(self, test_context): self.num_consumers = 1 def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(ClientCompatibilityProduceConsumeTest, self).min_cluster_size() + self.num_producers + self.num_consumers - @parametrize(broker_version=str(DEV_BRANCH)) + @cluster(num_nodes=9) + @matrix(broker_version=[str(DEV_BRANCH)], metadata_quorum=quorum.all_non_upgrade) @parametrize(broker_version=str(LATEST_0_10_0)) @parametrize(broker_version=str(LATEST_0_10_1)) @parametrize(broker_version=str(LATEST_0_10_2)) @@ -67,7 +70,7 @@ def min_cluster_size(self): @parametrize(broker_version=str(LATEST_2_5)) @parametrize(broker_version=str(LATEST_2_6)) @parametrize(broker_version=str(LATEST_2_7)) - def test_produce_consume(self, broker_version): + def test_produce_consume(self, broker_version, metadata_quorum=quorum.zk): print("running producer_consumer_compat with broker_version = %s" % broker_version, flush=True) self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.security_protocol = "PLAINTEXT" diff --git a/tests/kafkatest/tests/client/compression_test.py b/tests/kafkatest/tests/client/compression_test.py index 23b30eac24c0e..37ce52d7efca0 100644 --- a/tests/kafkatest/tests/client/compression_test.py +++ b/tests/kafkatest/tests/client/compression_test.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ducktape.mark import parametrize +from ducktape.mark import matrix from ducktape.utils.util import wait_until from ducktape.mark.resource import cluster from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest @@ -36,7 +36,7 @@ def __init__(self, test_context): super(CompressionTest, self).__init__(test_context=test_context) self.topic = "test_topic" - self.zk = ZookeeperService(test_context, num_nodes=1) + self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, topics={self.topic: { "partitions": 10, "replication-factor": 1}}) @@ -48,15 +48,16 @@ def __init__(self, test_context): self.num_consumers = 1 def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def min_cluster_size(self): # Override this since we're adding services outside of the constructor return super(CompressionTest, self).min_cluster_size() + self.num_producers + self.num_consumers @cluster(num_nodes=8) - @parametrize(compression_types=COMPRESSION_TYPES) - def test_compressed_topic(self, compression_types): + @matrix(compression_types=[COMPRESSION_TYPES], metadata_quorum=quorum.all_non_upgrade) + def test_compressed_topic(self, compression_types, metadata_quorum=quorum.zk): """Test produce => consume => validate for compressed topics Setup: 1 zk, 1 kafka node, 1 topic with partitions=10, replication-factor=1 diff --git a/tests/kafkatest/tests/client/consumer_rolling_upgrade_test.py b/tests/kafkatest/tests/client/consumer_rolling_upgrade_test.py index 638a3fc068fbd..5beacf23c6376 100644 --- a/tests/kafkatest/tests/client/consumer_rolling_upgrade_test.py +++ b/tests/kafkatest/tests/client/consumer_rolling_upgrade_test.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ducktape.mark import matrix from ducktape.mark.resource import cluster from kafkatest.tests.verifiable_consumer_test import VerifiableConsumerTest -from kafkatest.services.kafka import TopicPartition +from kafkatest.services.kafka import TopicPartition, quorum class ConsumerRollingUpgradeTest(VerifiableConsumerTest): TOPIC = "test_topic" @@ -47,7 +48,8 @@ def _verify_roundrobin_assignment(self, consumer): "Mismatched assignment: %s" % assignment @cluster(num_nodes=4) - def rolling_update_test(self): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def rolling_update_test(self, metadata_quorum=quorum.zk): """ Verify rolling updates of partition assignment strategies works correctly. In this test, we use a rolling restart to change the group's assignment strategy from "range" @@ -70,7 +72,7 @@ def rolling_update_test(self): consumer.start_node(consumer.nodes[0]) self.await_all_members(consumer) self._verify_range_assignment(consumer) - + # now restart the other node and verify that we have switched to round-robin consumer.stop_node(consumer.nodes[1]) consumer.start_node(consumer.nodes[1]) diff --git a/tests/kafkatest/tests/client/consumer_test.py b/tests/kafkatest/tests/client/consumer_test.py index 4a9e89d8c9ae2..f41748078ca72 100644 --- a/tests/kafkatest/tests/client/consumer_test.py +++ b/tests/kafkatest/tests/client/consumer_test.py @@ -18,7 +18,7 @@ from ducktape.mark.resource import cluster from kafkatest.tests.verifiable_consumer_test import VerifiableConsumerTest -from kafkatest.services.kafka import TopicPartition +from kafkatest.services.kafka import TopicPartition, quorum import signal @@ -75,7 +75,8 @@ def setup_consumer(self, topic, **kwargs): return consumer @cluster(num_nodes=7) - def test_broker_rolling_bounce(self): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_broker_rolling_bounce(self, metadata_quorum=quorum.zk): """ Verify correct consumer behavior when the brokers are consecutively restarted. @@ -117,8 +118,8 @@ def test_broker_rolling_bounce(self): (consumer.total_consumed(), consumer.current_position(partition)) @cluster(num_nodes=7) - @matrix(clean_shutdown=[True], bounce_mode=["all", "rolling"]) - def test_consumer_bounce(self, clean_shutdown, bounce_mode): + @matrix(clean_shutdown=[True], bounce_mode=["all", "rolling"], metadata_quorum=quorum.all_non_upgrade) + def test_consumer_bounce(self, clean_shutdown, bounce_mode, metadata_quorum=quorum.zk): """ Verify correct consumer behavior when the consumers in the group are consecutively restarted. @@ -160,8 +161,8 @@ def test_consumer_bounce(self, clean_shutdown, bounce_mode): (consumer.current_position(partition), consumer.total_consumed()) @cluster(num_nodes=7) - @matrix(clean_shutdown=[True], static_membership=[True, False], bounce_mode=["all", "rolling"], num_bounces=[5]) - def test_static_consumer_bounce(self, clean_shutdown, static_membership, bounce_mode, num_bounces): + @matrix(clean_shutdown=[True], static_membership=[True, False], bounce_mode=["all", "rolling"], num_bounces=[5], metadata_quorum=quorum.all_non_upgrade) + def test_static_consumer_bounce(self, clean_shutdown, static_membership, bounce_mode, num_bounces, metadata_quorum=quorum.zk): """ Verify correct static consumer behavior when the consumers in the group are restarted. In order to make sure the behavior of static members are different from dynamic ones, we take both static and dynamic @@ -222,8 +223,8 @@ def test_static_consumer_bounce(self, clean_shutdown, static_membership, bounce_ (consumer.current_position(partition), consumer.total_consumed()) @cluster(num_nodes=7) - @matrix(bounce_mode=["all", "rolling"]) - def test_static_consumer_persisted_after_rejoin(self, bounce_mode): + @matrix(bounce_mode=["all", "rolling"], metadata_quorum=quorum.all_non_upgrade) + def test_static_consumer_persisted_after_rejoin(self, bounce_mode, metadata_quorum=quorum.zk): """ Verify that the updated member.id(updated_member_id) caused by static member rejoin would be persisted. If not, after the brokers rolling bounce, the migrated group coordinator would load the stale persisted member.id and @@ -253,8 +254,8 @@ def test_static_consumer_persisted_after_rejoin(self, bounce_mode): self.rolling_bounce_brokers(consumer, num_bounces=1) @cluster(num_nodes=10) - @matrix(num_conflict_consumers=[1, 2], fencing_stage=["stable", "all"]) - def test_fencing_static_consumer(self, num_conflict_consumers, fencing_stage): + @matrix(num_conflict_consumers=[1, 2], fencing_stage=["stable", "all"], metadata_quorum=quorum.all_non_upgrade) + def test_fencing_static_consumer(self, num_conflict_consumers, fencing_stage, metadata_quorum=quorum.zk): """ Verify correct static consumer behavior when there are conflicting consumers with same group.instance.id. @@ -306,8 +307,8 @@ def test_fencing_static_consumer(self, num_conflict_consumers, fencing_stage): ) @cluster(num_nodes=7) - @matrix(clean_shutdown=[True], enable_autocommit=[True, False]) - def test_consumer_failure(self, clean_shutdown, enable_autocommit): + @matrix(clean_shutdown=[True], enable_autocommit=[True, False], metadata_quorum=quorum.all_non_upgrade) + def test_consumer_failure(self, clean_shutdown, enable_autocommit, metadata_quorum=quorum.zk): partition = TopicPartition(self.TOPIC, 0) consumer = self.setup_consumer(self.TOPIC, enable_autocommit=enable_autocommit) @@ -353,8 +354,8 @@ def test_consumer_failure(self, clean_shutdown, enable_autocommit): (consumer.last_commit(partition), consumer.current_position(partition)) @cluster(num_nodes=7) - @matrix(clean_shutdown=[True, False], enable_autocommit=[True, False]) - def test_broker_failure(self, clean_shutdown, enable_autocommit): + @matrix(clean_shutdown=[True, False], enable_autocommit=[True, False], metadata_quorum=quorum.all_non_upgrade) + def test_broker_failure(self, clean_shutdown, enable_autocommit, metadata_quorum=quorum.zk): partition = TopicPartition(self.TOPIC, 0) consumer = self.setup_consumer(self.TOPIC, enable_autocommit=enable_autocommit) @@ -390,7 +391,8 @@ def test_broker_failure(self, clean_shutdown, enable_autocommit): (consumer.last_commit(partition), consumer.current_position(partition)) @cluster(num_nodes=7) - def test_group_consumption(self): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_group_consumption(self, metadata_quorum=quorum.zk): """ Verifies correct group rebalance behavior as consumers are started and stopped. In particular, this test verifies that the partition is readable after every @@ -442,8 +444,8 @@ def __init__(self, test_context): @cluster(num_nodes=6) @matrix(assignment_strategy=["org.apache.kafka.clients.consumer.RangeAssignor", "org.apache.kafka.clients.consumer.RoundRobinAssignor", - "org.apache.kafka.clients.consumer.StickyAssignor"]) - def test_valid_assignment(self, assignment_strategy): + "org.apache.kafka.clients.consumer.StickyAssignor"], metadata_quorum=quorum.all_non_upgrade) + def test_valid_assignment(self, assignment_strategy, metadata_quorum=quorum.zk): """ Verify assignment strategy correctness: each partition is assigned to exactly one consumer instance. diff --git a/tests/kafkatest/tests/client/message_format_change_test.py b/tests/kafkatest/tests/client/message_format_change_test.py index 1388330c6a00d..41e0f95fe8a66 100644 --- a/tests/kafkatest/tests/client/message_format_change_test.py +++ b/tests/kafkatest/tests/client/message_format_change_test.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ducktape.mark import parametrize +from ducktape.mark import matrix from ducktape.utils.util import wait_until from ducktape.mark.resource import cluster from kafkatest.services.console_consumer import ConsoleConsumer -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.zookeeper import ZookeeperService from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest @@ -32,9 +32,10 @@ def __init__(self, test_context): def setUp(self): self.topic = "test_topic" - self.zk = ZookeeperService(self.test_context, num_nodes=1) - - self.zk.start() + self.zk = ZookeeperService(self.test_context, num_nodes=1) if quorum.for_test(self.test_context) == quorum.zk else None + + if self.zk: + self.zk.start() # Producer and consumer self.producer_throughput = 10000 @@ -58,10 +59,10 @@ def produce_and_consume(self, producer_version, consumer_version, group): err_msg="Producer did not produce all messages in reasonable amount of time")) @cluster(num_nodes=12) - @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(DEV_BRANCH)) - @parametrize(producer_version=str(LATEST_0_10), consumer_version=str(LATEST_0_10)) - @parametrize(producer_version=str(LATEST_0_9), consumer_version=str(LATEST_0_9)) - def test_compatibility(self, producer_version, consumer_version): + @matrix(producer_version=[str(DEV_BRANCH)], consumer_version=[str(DEV_BRANCH)], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_10)], consumer_version=[str(LATEST_0_10)], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_9)], consumer_version=[str(LATEST_0_9)], metadata_quorum=quorum.all_non_upgrade) + def test_compatibility(self, producer_version, consumer_version, metadata_quorum=quorum.zk): """ This tests performs the following checks: The workload is a mix of 0.9.x, 0.10.x and 0.11.x producers and consumers that produce to and consume from a DEV_BRANCH cluster @@ -81,8 +82,9 @@ def test_compatibility(self, producer_version, consumer_version): self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=DEV_BRANCH, topics={self.topic: { "partitions": 3, "replication-factor": 3, - 'configs': {"min.insync.replicas": 2}}}) - + 'configs': {"min.insync.replicas": 2}}}, + controller_num_nodes_override=1) + self.kafka.start() self.logger.info("First format change to 0.9.0") self.kafka.alter_message_format(self.topic, str(LATEST_0_9)) diff --git a/tests/kafkatest/tests/client/pluggable_test.py b/tests/kafkatest/tests/client/pluggable_test.py index a2599d8b5572a..36b9172f18351 100644 --- a/tests/kafkatest/tests/client/pluggable_test.py +++ b/tests/kafkatest/tests/client/pluggable_test.py @@ -13,8 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ducktape.mark import matrix +from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until +from kafkatest.services.kafka import quorum from kafkatest.tests.verifiable_consumer_test import VerifiableConsumerTest class PluggableConsumerTest(VerifiableConsumerTest): @@ -29,7 +32,9 @@ def __init__(self, test_context): self.TOPIC : { 'partitions': self.NUM_PARTITIONS, 'replication-factor': 1 }, }) - def test_start_stop(self): + @cluster(num_nodes=4) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_start_stop(self, metadata_quorum=quorum.zk): """ Test that a pluggable VerifiableConsumer module load works """ diff --git a/tests/kafkatest/tests/connect/connect_test.py b/tests/kafkatest/tests/connect/connect_test.py index 580d9f376c627..1a7f6abfeb8b7 100644 --- a/tests/kafkatest/tests/connect/connect_test.py +++ b/tests/kafkatest/tests/connect/connect_test.py @@ -16,12 +16,12 @@ from ducktape.tests.test import Test from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until -from ducktape.mark import parametrize +from ducktape.mark import matrix, parametrize from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.errors import TimeoutError from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.connect import ConnectServiceBase, ConnectStandaloneService, ErrorTolerance from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.services.security.security_config import SecurityConfig @@ -63,7 +63,7 @@ def __init__(self, test_context): 'test' : { 'partitions': 1, 'replication-factor': 1 } } - self.zk = ZookeeperService(test_context, self.num_zk) + self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None @cluster(num_nodes=5) @parametrize(converter="org.apache.kafka.connect.json.JsonConverter", schemas=True) @@ -71,8 +71,9 @@ def __init__(self, test_context): @parametrize(converter="org.apache.kafka.connect.storage.StringConverter", schemas=None) @parametrize(security_protocol=SecurityConfig.PLAINTEXT) @cluster(num_nodes=6) - @parametrize(security_protocol=SecurityConfig.SASL_SSL) - def test_file_source_and_sink(self, converter="org.apache.kafka.connect.json.JsonConverter", schemas=True, security_protocol='PLAINTEXT'): + @matrix(security_protocol=[SecurityConfig.SASL_SSL], metadata_quorum=quorum.all_non_upgrade) + def test_file_source_and_sink(self, converter="org.apache.kafka.connect.json.JsonConverter", schemas=True, security_protocol='PLAINTEXT', + metadata_quorum=quorum.zk): """ Validates basic end-to-end functionality of Connect standalone using the file source and sink converters. Includes parameterizations to test different converters (which also test per-connector converter overrides), schema/schemaless @@ -88,14 +89,15 @@ def test_file_source_and_sink(self, converter="org.apache.kafka.connect.json.Jso self.kafka = KafkaService(self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=security_protocol, - topics=self.topics) + topics=self.topics, controller_num_nodes_override=self.num_zk) self.source = ConnectStandaloneService(self.test_context, self.kafka, [self.INPUT_FILE, self.OFFSETS_FILE]) self.sink = ConnectStandaloneService(self.test_context, self.kafka, [self.OUTPUT_FILE, self.OFFSETS_FILE]) self.consumer_validator = ConsoleConsumer(self.test_context, 1, self.kafka, self.TOPIC_TEST, consumer_timeout_ms=10000) - self.zk.start() + if self.zk: + self.zk.start() self.kafka.start() self.source.set_configs(lambda node: self.render("connect-standalone.properties", node=node), [self.render("connect-file-source.properties")]) diff --git a/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py b/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py index 5474112cae3da..db8aa1d0743ca 100644 --- a/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py +++ b/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ducktape.mark import parametrize +from ducktape.mark import matrix, parametrize from ducktape.utils.util import wait_until from ducktape.mark.resource import cluster from kafkatest.services.console_consumer import ConsoleConsumer -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.kafka import config_property from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.zookeeper import ZookeeperService @@ -33,9 +33,10 @@ def __init__(self, test_context): def setUp(self): self.topic = "test_topic" - self.zk = ZookeeperService(self.test_context, num_nodes=1) - - self.zk.start() + self.zk = ZookeeperService(self.test_context, num_nodes=1) if quorum.for_test(self.test_context) == quorum.zk else None + + if self.zk: + self.zk.start() # Producer and consumer self.producer_throughput = 10000 @@ -44,39 +45,41 @@ def setUp(self): self.messages_per_producer = 1000 @cluster(num_nodes=6) - @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(DEV_BRANCH), compression_types=["snappy"], timestamp_type=str("LogAppendTime")) - @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(DEV_BRANCH), compression_types=["none"], timestamp_type=str("LogAppendTime")) + @matrix(producer_version=[str(DEV_BRANCH)], consumer_version=[str(DEV_BRANCH)], compression_types=[["snappy"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(DEV_BRANCH)], consumer_version=[str(DEV_BRANCH)], compression_types=[["none"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(LATEST_0_9), compression_types=["none"], new_consumer=False, timestamp_type=None) - @parametrize(producer_version=str(DEV_BRANCH), consumer_version=str(LATEST_0_9), compression_types=["snappy"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_2), consumer_version=str(LATEST_2_2), compression_types=["none"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_3), consumer_version=str(LATEST_2_3), compression_types=["none"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_4), consumer_version=str(LATEST_2_4), compression_types=["none"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_5), consumer_version=str(LATEST_2_5), compression_types=["none"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_6), consumer_version=str(LATEST_2_6), compression_types=["none"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_7), consumer_version=str(LATEST_2_7), compression_types=["none"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_1), consumer_version=str(LATEST_2_1), compression_types=["zstd"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_0), consumer_version=str(LATEST_2_0), compression_types=["snappy"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_1_1), consumer_version=str(LATEST_1_1), compression_types=["lz4"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_1_0), consumer_version=str(LATEST_1_0), compression_types=["none"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_0_11_0), consumer_version=str(LATEST_0_11_0), compression_types=["gzip"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_0_10_2), consumer_version=str(LATEST_0_10_2), compression_types=["lz4"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_0_10_1), consumer_version=str(LATEST_0_10_1), compression_types=["snappy"], timestamp_type=str("LogAppendTime")) - @parametrize(producer_version=str(LATEST_0_10_0), consumer_version=str(LATEST_0_10_0), compression_types=["snappy"], timestamp_type=str("LogAppendTime")) - @parametrize(producer_version=str(LATEST_0_9), consumer_version=str(DEV_BRANCH), compression_types=["none"], timestamp_type=None) - @parametrize(producer_version=str(LATEST_0_9), consumer_version=str(DEV_BRANCH), compression_types=["snappy"], timestamp_type=None) - @parametrize(producer_version=str(LATEST_0_9), consumer_version=str(LATEST_0_9), compression_types=["snappy"], timestamp_type=str("LogAppendTime")) + @matrix(producer_version=[str(DEV_BRANCH)], consumer_version=[str(LATEST_0_9)], compression_types=[["snappy"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_2_2)], consumer_version=[str(LATEST_2_2)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_2_3)], consumer_version=[str(LATEST_2_3)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_2_4)], consumer_version=[str(LATEST_2_4)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_2_5)], consumer_version=[str(LATEST_2_5)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_2_6)], consumer_version=[str(LATEST_2_6)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_2_7)], consumer_version=[str(LATEST_2_7)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_2_1)], consumer_version=[str(LATEST_2_1)], compression_types=[["zstd"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_2_0)], consumer_version=[str(LATEST_2_0)], compression_types=[["snappy"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_1_1)], consumer_version=[str(LATEST_1_1)], compression_types=[["lz4"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_1_0)], consumer_version=[str(LATEST_1_0)], compression_types=[["none"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_11_0)], consumer_version=[str(LATEST_0_11_0)], compression_types=[["gzip"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_10_2)], consumer_version=[str(LATEST_0_10_2)], compression_types=[["lz4"]], timestamp_type=[str("CreateTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_10_1)], consumer_version=[str(LATEST_0_10_1)], compression_types=[["snappy"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_10_0)], consumer_version=[str(LATEST_0_10_0)], compression_types=[["snappy"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_9)], consumer_version=[str(DEV_BRANCH)], compression_types=[["none"]], timestamp_type=[None], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_9)], consumer_version=[str(DEV_BRANCH)], compression_types=[["snappy"]], timestamp_type=[None], metadata_quorum=quorum.all_non_upgrade) + @matrix(producer_version=[str(LATEST_0_9)], consumer_version=[str(LATEST_0_9)], compression_types=[["snappy"]], timestamp_type=[str("LogAppendTime")], metadata_quorum=quorum.all_non_upgrade) @parametrize(producer_version=str(LATEST_0_8_2), consumer_version=str(LATEST_0_8_2), compression_types=["none"], new_consumer=False, timestamp_type=None) - def test_compatibility(self, producer_version, consumer_version, compression_types, new_consumer=True, timestamp_type=None): - + def test_compatibility(self, producer_version, consumer_version, compression_types, new_consumer=True, timestamp_type=None, metadata_quorum=quorum.zk): + if not new_consumer and metadata_quorum != quorum.zk: + raise Exception("ZooKeeper-based consumers are not supported when using a Raft-based metadata quorum") self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk, version=DEV_BRANCH, topics={self.topic: { "partitions": 3, "replication-factor": 3, - 'configs': {"min.insync.replicas": 2}}}) + 'configs': {"min.insync.replicas": 2}}}, + controller_num_nodes_override=1) for node in self.kafka.nodes: if timestamp_type is not None: node.config[config_property.MESSAGE_TIMESTAMP_TYPE] = timestamp_type self.kafka.start() - + self.producer = VerifiableProducer(self.test_context, self.num_producers, self.kafka, self.topic, throughput=self.producer_throughput, message_validator=is_int, diff --git a/tests/kafkatest/tests/core/consume_bench_test.py b/tests/kafkatest/tests/core/consume_bench_test.py index e731270646d4a..ce08d80832ab8 100644 --- a/tests/kafkatest/tests/core/consume_bench_test.py +++ b/tests/kafkatest/tests/core/consume_bench_test.py @@ -14,9 +14,10 @@ # limitations under the License. import json -from ducktape.mark import parametrize +from ducktape.mark import matrix +from ducktape.mark.resource import cluster from ducktape.tests.test import Test -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.trogdor.produce_bench_workload import ProduceBenchWorkloadService, ProduceBenchWorkloadSpec from kafkatest.services.trogdor.consume_bench_workload import ConsumeBenchWorkloadService, ConsumeBenchWorkloadSpec from kafkatest.services.trogdor.task_spec import TaskSpec @@ -28,7 +29,7 @@ class ConsumeBenchTest(Test): def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ConsumeBenchTest, self).__init__(test_context) - self.zk = ZookeeperService(test_context, num_nodes=3) + self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk) self.producer_workload_service = ProduceBenchWorkloadService(test_context, self.kafka) self.consumer_workload_service = ConsumeBenchWorkloadService(test_context, self.kafka) @@ -41,13 +42,15 @@ def __init__(self, test_context): def setUp(self): self.trogdor.start() - self.zk.start() + if self.zk: + self.zk.start() self.kafka.start() def teardown(self): self.trogdor.stop() self.kafka.stop() - self.zk.stop() + if self.zk: + self.zk.stop() def produce_messages(self, topics, max_messages=10000): produce_spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, @@ -64,9 +67,10 @@ def produce_messages(self, topics, max_messages=10000): produce_workload.wait_for_done(timeout_sec=180) self.logger.debug("Produce workload finished") - @parametrize(topics=["consume_bench_topic[0-5]"]) # topic subscription - @parametrize(topics=["consume_bench_topic[0-5]:[0-4]"]) # manual topic assignment - def test_consume_bench(self, topics): + @cluster(num_nodes=10) + @matrix(topics=[["consume_bench_topic[0-5]"]], metadata_quorum=quorum.all_non_upgrade) # topic subscription + @matrix(topics=[["consume_bench_topic[0-5]:[0-4]"]], metadata_quorum=quorum.all_non_upgrade) # manual topic assignment + def test_consume_bench(self, topics, metadata_quorum=quorum.zk): """ Runs a ConsumeBench workload to consume messages """ @@ -86,7 +90,9 @@ def test_consume_bench(self, topics): tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2)) - def test_single_partition(self): + @cluster(num_nodes=10) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_single_partition(self, metadata_quorum=quorum.zk): """ Run a ConsumeBench against a single partition """ @@ -107,7 +113,9 @@ def test_single_partition(self): tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2)) - def test_multiple_consumers_random_group_topics(self): + @cluster(num_nodes=10) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_multiple_consumers_random_group_topics(self, metadata_quorum=quorum.zk): """ Runs multiple consumers group to read messages from topics. Since a consumerGroup isn't specified, each consumer should read from all topics independently @@ -129,7 +137,9 @@ def test_multiple_consumers_random_group_topics(self): tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2)) - def test_two_consumers_specified_group_topics(self): + @cluster(num_nodes=10) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_two_consumers_specified_group_topics(self, metadata_quorum=quorum.zk): """ Runs two consumers in the same consumer group to read messages from topics. Since a consumerGroup is specified, each consumer should dynamically get assigned a partition from group @@ -152,7 +162,9 @@ def test_two_consumers_specified_group_topics(self): tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2)) - def test_multiple_consumers_random_group_partitions(self): + @cluster(num_nodes=10) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_multiple_consumers_random_group_partitions(self, metadata_quorum=quorum.zk): """ Runs multiple consumers in to read messages from specific partitions. Since a consumerGroup isn't specified, each consumer will get assigned a random group @@ -175,7 +187,9 @@ def test_multiple_consumers_random_group_partitions(self): tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2)) - def test_multiple_consumers_specified_group_partitions_should_raise(self): + @cluster(num_nodes=10) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_multiple_consumers_specified_group_partitions_should_raise(self, metadata_quorum=quorum.zk): """ Runs multiple consumers in the same group to read messages from specific partitions. It is an invalid configuration to provide a consumer group and specific partitions. diff --git a/tests/kafkatest/tests/core/consumer_group_command_test.py b/tests/kafkatest/tests/core/consumer_group_command_test.py index 871e2761ade25..f81eec8de8cf8 100644 --- a/tests/kafkatest/tests/core/consumer_group_command_test.py +++ b/tests/kafkatest/tests/core/consumer_group_command_test.py @@ -20,7 +20,7 @@ from ducktape.mark.resource import cluster from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.services.security.security_config import SecurityConfig @@ -45,16 +45,18 @@ def __init__(self, test_context): self.topics = { TOPIC: {'partitions': 1, 'replication-factor': 1} } - self.zk = ZookeeperService(test_context, self.num_zk) + self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, - interbroker_security_protocol=interbroker_security_protocol, topics=self.topics) + interbroker_security_protocol=interbroker_security_protocol, topics=self.topics, + controller_num_nodes_override=self.num_zk) self.kafka.start() def start_consumer(self): @@ -88,8 +90,8 @@ def setup_and_verify(self, security_protocol, group=None): self.consumer.stop() @cluster(num_nodes=3) - @matrix(security_protocol=['PLAINTEXT', 'SSL']) - def test_list_consumer_groups(self, security_protocol='PLAINTEXT'): + @matrix(security_protocol=['PLAINTEXT', 'SSL'], metadata_quorum=quorum.all_non_upgrade) + def test_list_consumer_groups(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if ConsumerGroupCommand is listing correct consumer groups :return: None @@ -97,8 +99,8 @@ def test_list_consumer_groups(self, security_protocol='PLAINTEXT'): self.setup_and_verify(security_protocol) @cluster(num_nodes=3) - @matrix(security_protocol=['PLAINTEXT', 'SSL']) - def test_describe_consumer_group(self, security_protocol='PLAINTEXT'): + @matrix(security_protocol=['PLAINTEXT', 'SSL'], metadata_quorum=quorum.all_non_upgrade) + def test_describe_consumer_group(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if ConsumerGroupCommand is describing a consumer group correctly :return: None diff --git a/tests/kafkatest/tests/core/delegation_token_test.py b/tests/kafkatest/tests/core/delegation_token_test.py index feb593522e07c..5fe8d126210ba 100644 --- a/tests/kafkatest/tests/core/delegation_token_test.py +++ b/tests/kafkatest/tests/core/delegation_token_test.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ducktape.mark.resource import cluster from ducktape.tests.test import Test from ducktape.utils.util import wait_until from kafkatest.services.kafka import config_property, KafkaService @@ -109,6 +110,7 @@ def renew_delegation_token(self): self.delegation_tokens.renew_delegation_token(dt["hmac"], new_expirydate_ms) + @cluster(num_nodes=5) def test_delegation_token_lifecycle(self): self.kafka.start() self.delegation_tokens = DelegationTokens(self.kafka, self.test_context) diff --git a/tests/kafkatest/tests/core/downgrade_test.py b/tests/kafkatest/tests/core/downgrade_test.py index beb103a7d82db..489ae7c2b6efa 100644 --- a/tests/kafkatest/tests/core/downgrade_test.py +++ b/tests/kafkatest/tests/core/downgrade_test.py @@ -53,7 +53,7 @@ def downgrade_to(self, kafka_version): self.wait_until_rejoin() def setup_services(self, kafka_version, compression_types, security_protocol, static_membership): - self.create_zookeeper() + self.create_zookeeper_if_necessary() self.zk.start() self.create_kafka(num_nodes=3, diff --git a/tests/kafkatest/tests/core/fetch_from_follower_test.py b/tests/kafkatest/tests/core/fetch_from_follower_test.py index ef3772880c82a..fab5cfa6269ae 100644 --- a/tests/kafkatest/tests/core/fetch_from_follower_test.py +++ b/tests/kafkatest/tests/core/fetch_from_follower_test.py @@ -16,10 +16,11 @@ import time from collections import defaultdict +from ducktape.mark import matrix from ducktape.mark.resource import cluster from kafkatest.services.console_consumer import ConsoleConsumer -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.monitor.jmx import JmxTool from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.zookeeper import ZookeeperService @@ -36,7 +37,7 @@ def __init__(self, test_context): super(FetchFromFollowerTest, self).__init__(test_context=test_context) self.jmx_tool = JmxTool(test_context, jmx_poll_ms=100) self.topic = "test_topic" - self.zk = ZookeeperService(test_context, num_nodes=1) + self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk, @@ -53,7 +54,8 @@ def __init__(self, test_context): 1: [("broker.rack", "rack-a")], 2: [("broker.rack", "rack-b")], 3: [("broker.rack", "rack-c")] - }) + }, + controller_num_nodes_override=1) self.producer_throughput = 1000 self.num_producers = 1 @@ -63,11 +65,13 @@ def min_cluster_size(self): return super(FetchFromFollowerTest, self).min_cluster_size() + self.num_producers * 2 + self.num_consumers * 2 def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() self.kafka.start() @cluster(num_nodes=9) - def test_consumer_preferred_read_replica(self): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_consumer_preferred_read_replica(self, metadata_quorum=quorum.zk): """ This test starts up brokers with "broker.rack" and "replica.selector.class" configurations set. The replica selector is set to the rack-aware implementation. One of the brokers has a different rack than the other two. diff --git a/tests/kafkatest/tests/core/get_offset_shell_test.py b/tests/kafkatest/tests/core/get_offset_shell_test.py index 3f226c1d0bcc5..b24c5ac42005c 100644 --- a/tests/kafkatest/tests/core/get_offset_shell_test.py +++ b/tests/kafkatest/tests/core/get_offset_shell_test.py @@ -16,11 +16,12 @@ from ducktape.utils.util import wait_until from ducktape.tests.test import Test +from ducktape.mark import matrix from ducktape.mark.resource import cluster from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.console_consumer import ConsoleConsumer MAX_MESSAGES = 100 @@ -63,10 +64,11 @@ def __init__(self, test_context): TOPIC_TEST_TOPIC_PARTITIONS2: {'partitions': 2, 'replication-factor': REPLICATION_FACTOR} } - self.zk = ZookeeperService(test_context, self.num_zk) + self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( @@ -103,7 +105,8 @@ def extract_message_count_sum(self, **kwargs): return sum @cluster(num_nodes=3) - def test_get_offset_shell_topic_name(self, security_protocol='PLAINTEXT'): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_get_offset_shell_topic_name(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --topic argument with a simple name correctly :return: None @@ -116,7 +119,8 @@ def test_get_offset_shell_topic_name(self, security_protocol='PLAINTEXT'): timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") @cluster(num_nodes=4) - def test_get_offset_shell_topic_pattern(self, security_protocol='PLAINTEXT'): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_get_offset_shell_topic_pattern(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --topic argument with a pattern correctly :return: None @@ -130,7 +134,8 @@ def test_get_offset_shell_topic_pattern(self, security_protocol='PLAINTEXT'): timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") @cluster(num_nodes=3) - def test_get_offset_shell_partitions(self, security_protocol='PLAINTEXT'): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_get_offset_shell_partitions(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --partitions argument correctly :return: None @@ -151,7 +156,8 @@ def fetch_and_sum_partitions_separately(): timeout_sec=10, err_msg="Timed out waiting to reach expected offset.") @cluster(num_nodes=4) - def test_get_offset_shell_topic_partitions(self, security_protocol='PLAINTEXT'): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_get_offset_shell_topic_partitions(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --topic-partitions argument correctly :return: None @@ -202,7 +208,8 @@ def test_get_offset_shell_topic_partitions(self, security_protocol='PLAINTEXT'): assert 0 == filtered_partitions.count("%s:%s" % (TOPIC_TEST_TOPIC_PARTITIONS2, 1)) @cluster(num_nodes=4) - def test_get_offset_shell_internal_filter(self, security_protocol='PLAINTEXT'): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_get_offset_shell_internal_filter(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if GetOffsetShell handles --exclude-internal-topics flag correctly :return: None diff --git a/tests/kafkatest/tests/core/group_mode_transactions_test.py b/tests/kafkatest/tests/core/group_mode_transactions_test.py index 141c613730d72..e9638d50ef4f1 100644 --- a/tests/kafkatest/tests/core/group_mode_transactions_test.py +++ b/tests/kafkatest/tests/core/group_mode_transactions_test.py @@ -14,7 +14,7 @@ # limitations under the License. from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.transactional_message_copier import TransactionalMessageCopier @@ -25,6 +25,7 @@ from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until +import time class GroupModeTransactionsTest(Test): """Essentially testing the same functionality as TransactionsTest by transactionally copying data @@ -60,13 +61,14 @@ def __init__(self, test_context): self.progress_timeout_sec = 60 self.consumer_group = "grouped-transactions-test-consumer-group" - self.zk = ZookeeperService(test_context, num_nodes=1) + self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=self.num_brokers, - zk=self.zk) + zk=self.zk, controller_num_nodes_override=1) def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def seed_messages(self, topic, num_seed_messages): seed_timeout_sec = 10000 @@ -95,10 +97,17 @@ def bounce_brokers(self, clean_shutdown): self.kafka.restart_node(node, clean_shutdown = True) else: self.kafka.stop_node(node, clean_shutdown = False) - wait_until(lambda: len(self.kafka.pids(node)) == 0 and not self.kafka.is_registered(node), - timeout_sec=self.kafka.zk_session_timeout + 5, - err_msg="Failed to see timely deregistration of \ - hard-killed broker %s" % str(node.account)) + gracePeriodSecs = 5 + if self.zk: + wait_until(lambda: len(self.kafka.pids(node)) == 0 and not self.kafka.is_registered(node), + timeout_sec=self.kafka.zk_session_timeout + gracePeriodSecs, + err_msg="Failed to see timely deregistration of hard-killed broker %s" % str(node.account)) + else: + brokerSessionTimeoutSecs = 18 + wait_until(lambda: len(self.kafka.pids(node)) == 0, + timeout_sec=brokerSessionTimeoutSecs + gracePeriodSecs, + err_msg="Failed to see timely disappearance of process for hard-killed broker %s" % str(node.account)) + time.sleep(brokerSessionTimeoutSecs + gracePeriodSecs) self.kafka.start_node(node) def create_and_start_message_copier(self, input_topic, output_topic, transactional_id): @@ -260,8 +269,8 @@ def setup_topics(self): @cluster(num_nodes=10) @matrix(failure_mode=["hard_bounce", "clean_bounce"], - bounce_target=["brokers", "clients"]) - def test_transactions(self, failure_mode, bounce_target): + bounce_target=["brokers", "clients"], metadata_quorum=quorum.all_non_upgrade) + def test_transactions(self, failure_mode, bounce_target, metadata_quorum=quorum.zk): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol diff --git a/tests/kafkatest/tests/core/produce_bench_test.py b/tests/kafkatest/tests/core/produce_bench_test.py index a316520335baf..734dfb580b8df 100644 --- a/tests/kafkatest/tests/core/produce_bench_test.py +++ b/tests/kafkatest/tests/core/produce_bench_test.py @@ -14,19 +14,20 @@ # limitations under the License. import json +from ducktape.mark import matrix +from ducktape.mark.resource import cluster from ducktape.tests.test import Test -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.trogdor.produce_bench_workload import ProduceBenchWorkloadService, ProduceBenchWorkloadSpec from kafkatest.services.trogdor.task_spec import TaskSpec from kafkatest.services.trogdor.trogdor import TrogdorService from kafkatest.services.zookeeper import ZookeeperService - class ProduceBenchTest(Test): def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ProduceBenchTest, self).__init__(test_context) - self.zk = ZookeeperService(test_context, num_nodes=3) + self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=3, zk=self.zk) self.workload_service = ProduceBenchWorkloadService(test_context, self.kafka) self.trogdor = TrogdorService(context=self.test_context, @@ -36,15 +37,19 @@ def __init__(self, test_context): def setUp(self): self.trogdor.start() - self.zk.start() + if self.zk: + self.zk.start() self.kafka.start() def teardown(self): self.trogdor.stop() self.kafka.stop() - self.zk.stop() + if self.zk: + self.zk.stop() - def test_produce_bench(self): + @cluster(num_nodes=8) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_produce_bench(self, metadata_quorum=quorum.zk): spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, self.workload_service.producer_node, self.workload_service.bootstrap_servers, @@ -60,7 +65,9 @@ def test_produce_bench(self): tasks = self.trogdor.tasks() self.logger.info("TASKS: %s\n" % json.dumps(tasks, sort_keys=True, indent=2)) - def test_produce_bench_transactions(self): + @cluster(num_nodes=8) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_produce_bench_transactions(self, metadata_quorum=quorum.zk): spec = ProduceBenchWorkloadSpec(0, TaskSpec.MAX_DURATION_MS, self.workload_service.producer_node, self.workload_service.bootstrap_servers, diff --git a/tests/kafkatest/tests/core/replica_scale_test.py b/tests/kafkatest/tests/core/replica_scale_test.py index 8541d398d390d..f8d0c8379bdf7 100644 --- a/tests/kafkatest/tests/core/replica_scale_test.py +++ b/tests/kafkatest/tests/core/replica_scale_test.py @@ -14,13 +14,13 @@ # limitations under the License. from ducktape.mark.resource import cluster -from ducktape.mark import parametrize +from ducktape.mark import matrix from ducktape.tests.test import Test from kafkatest.services.trogdor.produce_bench_workload import ProduceBenchWorkloadService, ProduceBenchWorkloadSpec from kafkatest.services.trogdor.consume_bench_workload import ConsumeBenchWorkloadService, ConsumeBenchWorkloadSpec from kafkatest.services.trogdor.task_spec import TaskSpec -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.trogdor.trogdor import TrogdorService from kafkatest.services.zookeeper import ZookeeperService @@ -31,11 +31,12 @@ class ReplicaScaleTest(Test): def __init__(self, test_context): super(ReplicaScaleTest, self).__init__(test_context=test_context) self.test_context = test_context - self.zk = ZookeeperService(test_context, num_nodes=1) - self.kafka = KafkaService(self.test_context, num_nodes=8, zk=self.zk) + self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None + self.kafka = KafkaService(self.test_context, num_nodes=8, zk=self.zk, controller_num_nodes_override=1) def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() self.kafka.start() def teardown(self): @@ -43,11 +44,12 @@ def teardown(self): for node in self.kafka.nodes: self.kafka.stop_node(node, clean_shutdown=False, timeout_sec=60) self.kafka.stop() - self.zk.stop() + if self.zk: + self.zk.stop() @cluster(num_nodes=12) - @parametrize(topic_count=50, partition_count=34, replication_factor=3) - def test_produce_consume(self, topic_count, partition_count, replication_factor): + @matrix(topic_count=[50], partition_count=[34], replication_factor=[3], metadata_quorum=quorum.all_non_upgrade) + def test_produce_consume(self, topic_count, partition_count, replication_factor, metadata_quorum=quorum.zk): topics_create_start_time = time.time() for i in range(topic_count): topic = "replicas_produce_consume_%d" % i @@ -101,8 +103,8 @@ def test_produce_consume(self, topic_count, partition_count, replication_factor) trogdor.stop() @cluster(num_nodes=12) - @parametrize(topic_count=50, partition_count=34, replication_factor=3) - def test_clean_bounce(self, topic_count, partition_count, replication_factor): + @matrix(topic_count=[50], partition_count=[34], replication_factor=[3], metadata_quorum=quorum.all_non_upgrade) + def test_clean_bounce(self, topic_count, partition_count, replication_factor, metadata_quorum=quorum.zk): topics_create_start_time = time.time() for i in range(topic_count): topic = "topic-%04d" % i diff --git a/tests/kafkatest/tests/core/replication_test.py b/tests/kafkatest/tests/core/replication_test.py index 01ef34f318390..a0c01567d4e9b 100644 --- a/tests/kafkatest/tests/core/replication_test.py +++ b/tests/kafkatest/tests/core/replication_test.py @@ -19,9 +19,11 @@ from ducktape.mark import parametrize from ducktape.mark.resource import cluster +from kafkatest.services.kafka import quorum from kafkatest.tests.end_to_end import EndToEndTest import signal +import time def broker_node(test, broker_type): """ Discover node of requested type. For leader type, discovers leader for our topic and partition 0 @@ -63,10 +65,19 @@ def hard_bounce(test, broker_type): # Since this is a hard kill, we need to make sure the process is down and that # zookeeper has registered the loss by expiring the broker's session timeout. - - wait_until(lambda: len(test.kafka.pids(prev_broker_node)) == 0 and not test.kafka.is_registered(prev_broker_node), - timeout_sec=test.kafka.zk_session_timeout + 5, - err_msg="Failed to see timely deregistration of hard-killed broker %s" % str(prev_broker_node.account)) + # Or, for a Raft-based quorum, we simply wait at least 18 seconds (the default for broker.session.timeout.ms) + + gracePeriodSecs = 5 + if test.zk: + wait_until(lambda: len(test.kafka.pids(prev_broker_node)) == 0 and not test.kafka.is_registered(prev_broker_node), + timeout_sec=test.kafka.zk_session_timeout + gracePeriodSecs, + err_msg="Failed to see timely deregistration of hard-killed broker %s" % str(prev_broker_node.account)) + else: + brokerSessionTimeoutSecs = 18 + wait_until(lambda: len(test.kafka.pids(prev_broker_node)) == 0, + timeout_sec=brokerSessionTimeoutSecs + gracePeriodSecs, + err_msg="Failed to see timely disappearance of process for hard-killed broker %s" % str(prev_broker_node.account)) + time.sleep(brokerSessionTimeoutSecs + gracePeriodSecs) test.kafka.start_node(prev_broker_node) @@ -98,11 +109,11 @@ class ReplicationTest(EndToEndTest): "replication-factor": 3, "configs": {"min.insync.replicas": 2} } - + def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(ReplicationTest, self).__init__(test_context=test_context, topic_config=self.TOPIC_CONFIG) - + def min_cluster_size(self): """Override this since we're adding services outside of the constructor""" return super(ReplicationTest, self).min_cluster_size() + self.num_producers + self.num_consumers @@ -111,29 +122,34 @@ def min_cluster_size(self): @matrix(failure_mode=["clean_shutdown", "hard_shutdown", "clean_bounce", "hard_bounce"], broker_type=["leader"], security_protocol=["PLAINTEXT"], - enable_idempotence=[True]) + enable_idempotence=[True], + metadata_quorum=quorum.all_non_upgrade) @matrix(failure_mode=["clean_shutdown", "hard_shutdown", "clean_bounce", "hard_bounce"], broker_type=["leader"], - security_protocol=["PLAINTEXT", "SASL_SSL"]) + security_protocol=["PLAINTEXT", "SASL_SSL"], + metadata_quorum=quorum.all_non_upgrade) @matrix(failure_mode=["clean_shutdown", "hard_shutdown", "clean_bounce", "hard_bounce"], broker_type=["controller"], security_protocol=["PLAINTEXT", "SASL_SSL"]) @matrix(failure_mode=["hard_bounce"], broker_type=["leader"], - security_protocol=["SASL_SSL"], client_sasl_mechanism=["PLAIN"], interbroker_sasl_mechanism=["PLAIN", "GSSAPI"]) + security_protocol=["SASL_SSL"], client_sasl_mechanism=["PLAIN"], interbroker_sasl_mechanism=["PLAIN", "GSSAPI"], + metadata_quorum=quorum.all_non_upgrade) @parametrize(failure_mode="hard_bounce", broker_type="leader", security_protocol="SASL_SSL", client_sasl_mechanism="SCRAM-SHA-256", interbroker_sasl_mechanism="SCRAM-SHA-512") @matrix(failure_mode=["clean_shutdown", "hard_shutdown", "clean_bounce", "hard_bounce"], - security_protocol=["PLAINTEXT"], broker_type=["leader"], compression_type=["gzip"], tls_version=["TLSv1.2", "TLSv1.3"]) + security_protocol=["PLAINTEXT"], broker_type=["leader"], compression_type=["gzip"], tls_version=["TLSv1.2", "TLSv1.3"], + metadata_quorum=quorum.all_non_upgrade) def test_replication_with_broker_failure(self, failure_mode, security_protocol, broker_type, client_sasl_mechanism="GSSAPI", interbroker_sasl_mechanism="GSSAPI", - compression_type=None, enable_idempotence=False, tls_version=None): + compression_type=None, enable_idempotence=False, tls_version=None, + metadata_quorum=quorum.zk): """Replication tests. These tests verify that replication provides simple durability guarantees by checking that data acked by brokers is still available for consumption in the face of various failure scenarios. - Setup: 1 zk, 3 kafka nodes, 1 topic with partitions=3, replication-factor=3, and min.insync.replicas=2 + Setup: 1 zk/Raft-based controller, 3 kafka nodes, 1 topic with partitions=3, replication-factor=3, and min.insync.replicas=2 - Produce messages in the background - Consume messages in the background @@ -142,15 +158,19 @@ def test_replication_with_broker_failure(self, failure_mode, security_protocol, - Validate that every acked message was consumed """ - self.create_zookeeper() - self.zk.start() + if failure_mode == "controller" and metadata_quorum != quorum.zk: + raise Exception("There is no controller broker when using a Raft-based metadata quorum") + self.create_zookeeper_if_necessary() + if self.zk: + self.zk.start() self.create_kafka(num_nodes=3, security_protocol=security_protocol, interbroker_security_protocol=security_protocol, client_sasl_mechanism=client_sasl_mechanism, interbroker_sasl_mechanism=interbroker_sasl_mechanism, - tls_version=tls_version) + tls_version=tls_version, + controller_num_nodes_override = 1) self.kafka.start() compression_types = None if not compression_type else [compression_type] diff --git a/tests/kafkatest/tests/core/round_trip_fault_test.py b/tests/kafkatest/tests/core/round_trip_fault_test.py index a0ce5aef54a3c..b9085cb8b5b71 100644 --- a/tests/kafkatest/tests/core/round_trip_fault_test.py +++ b/tests/kafkatest/tests/core/round_trip_fault_test.py @@ -14,10 +14,12 @@ # limitations under the License. import time +from ducktape.mark import matrix +from ducktape.mark.resource import cluster from ducktape.tests.test import Test from kafkatest.services.trogdor.network_partition_fault_spec import NetworkPartitionFaultSpec from kafkatest.services.trogdor.degraded_network_fault_spec import DegradedNetworkFaultSpec -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.trogdor.process_stop_fault_spec import ProcessStopFaultSpec from kafkatest.services.trogdor.round_trip_workload import RoundTripWorkloadService, RoundTripWorkloadSpec from kafkatest.services.trogdor.task_spec import TaskSpec @@ -31,11 +33,17 @@ class RoundTripFaultTest(Test): def __init__(self, test_context): """:type test_context: ducktape.tests.test.TestContext""" super(RoundTripFaultTest, self).__init__(test_context) - self.zk = ZookeeperService(test_context, num_nodes=3) + self.zk = ZookeeperService(test_context, num_nodes=3) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=4, zk=self.zk) self.workload_service = RoundTripWorkloadService(test_context, self.kafka) + if quorum.for_test(test_context) == quorum.zk: + trogdor_client_services = [self.zk, self.kafka, self.workload_service] + elif quorum.for_test(test_context) == quorum.remote_raft: + trogdor_client_services = [self.kafka.controller_quorum, self.kafka, self.workload_service] + else: #co-located case, which we currently don't test but handle here for completeness in case we do test it + trogdor_client_services = [self.kafka, self.workload_service] self.trogdor = TrogdorService(context=self.test_context, - client_services=[self.zk, self.kafka, self.workload_service]) + client_services=trogdor_client_services) topic_name = "round_trip_topic%d" % RoundTripFaultTest.topic_name_index RoundTripFaultTest.topic_name_index = RoundTripFaultTest.topic_name_index + 1 active_topics={topic_name : {"partitionAssignments":{"0": [0,1,2]}}} @@ -47,24 +55,38 @@ def __init__(self, test_context): active_topics=active_topics) def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() self.kafka.start() self.trogdor.start() def teardown(self): self.trogdor.stop() self.kafka.stop() - self.zk.stop() + if self.zk: + self.zk.stop() - def test_round_trip_workload(self): + def remote_quorum_nodes(self): + if quorum.for_test(self.test_context) == quorum.zk: + return self.zk.nodes + elif quorum.for_test(self.test_context) == quorum.remote_raft: + return self.kafka.controller_quorum.nodes + else: # co-located case, which we currently don't test but handle here for completeness in case we do test it + return [] + + @cluster(num_nodes=9) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_round_trip_workload(self, metadata_quorum=quorum.zk): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) workload1.wait_for_done(timeout_sec=600) - def test_round_trip_workload_with_broker_partition(self): + @cluster(num_nodes=9) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_round_trip_workload_with_broker_partition(self, metadata_quorum=quorum.zk): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) time.sleep(2) part1 = [self.kafka.nodes[0]] - part2 = self.kafka.nodes[1:] + [self.workload_service.nodes[0]] + self.zk.nodes + part2 = self.kafka.nodes[1:] + [self.workload_service.nodes[0]] + self.remote_quorum_nodes() partition1_spec = NetworkPartitionFaultSpec(0, TaskSpec.MAX_DURATION_MS, [part1, part2]) partition1 = self.trogdor.create_task("partition1", partition1_spec) @@ -72,7 +94,9 @@ def test_round_trip_workload_with_broker_partition(self): partition1.stop() partition1.wait_for_done() - def test_produce_consume_with_broker_pause(self): + @cluster(num_nodes=9) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_produce_consume_with_broker_pause(self, metadata_quorum=quorum.zk): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) time.sleep(2) stop1_spec = ProcessStopFaultSpec(0, TaskSpec.MAX_DURATION_MS, [self.kafka.nodes[0]], @@ -83,22 +107,26 @@ def test_produce_consume_with_broker_pause(self): stop1.wait_for_done() self.kafka.stop_node(self.kafka.nodes[0], False) - def test_produce_consume_with_client_partition(self): + @cluster(num_nodes=9) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_produce_consume_with_client_partition(self, metadata_quorum=quorum.zk): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) time.sleep(2) part1 = [self.workload_service.nodes[0]] - part2 = self.kafka.nodes + self.zk.nodes + part2 = self.kafka.nodes + self.remote_quorum_nodes() partition1_spec = NetworkPartitionFaultSpec(0, 60000, [part1, part2]) stop1 = self.trogdor.create_task("stop1", partition1_spec) workload1.wait_for_done(timeout_sec=600) stop1.stop() stop1.wait_for_done() - def test_produce_consume_with_latency(self): + @cluster(num_nodes=9) + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_produce_consume_with_latency(self, metadata_quorum=quorum.zk): workload1 = self.trogdor.create_task("workload1", self.round_trip_spec) time.sleep(2) spec = DegradedNetworkFaultSpec(0, 60000) - for node in self.kafka.nodes + self.zk.nodes: + for node in self.kafka.nodes + self.remote_quorum_nodes(): spec.add_node_spec(node.name, "eth0", latencyMs=100, rateLimitKbit=3000) slow1 = self.trogdor.create_task("slow1", spec) workload1.wait_for_done(timeout_sec=600) diff --git a/tests/kafkatest/tests/core/security_test.py b/tests/kafkatest/tests/core/security_test.py index 7339873ed9ffe..5d1d88651fe73 100644 --- a/tests/kafkatest/tests/core/security_test.py +++ b/tests/kafkatest/tests/core/security_test.py @@ -14,11 +14,12 @@ # limitations under the License. from ducktape.cluster.remoteaccount import RemoteCommandError -from ducktape.mark import parametrize +from ducktape.mark import matrix from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until from ducktape.errors import TimeoutError +from kafkatest.services.kafka import quorum from kafkatest.services.security.security_config import SecurityConfig from kafkatest.services.security.security_config import SslStores from kafkatest.tests.end_to_end import EndToEndTest @@ -57,9 +58,9 @@ def producer_consumer_have_expected_error(self, error): return True @cluster(num_nodes=7) - @parametrize(security_protocol='PLAINTEXT', interbroker_security_protocol='SSL') - @parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT') - def test_client_ssl_endpoint_validation_failure(self, security_protocol, interbroker_security_protocol): + @matrix(security_protocol='PLAINTEXT', interbroker_security_protocol='SSL', metadata_quorum=quorum.all_non_upgrade) + @matrix(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT', metadata_quorum=quorum.all_non_upgrade) + def test_client_ssl_endpoint_validation_failure(self, security_protocol, interbroker_security_protocol, metadata_quorum=quorum.zk): """ Test that invalid hostname in certificate results in connection failures. When security_protocol=SSL, client SSL handshakes are expected to fail due to hostname verification failure. @@ -71,8 +72,9 @@ def test_client_ssl_endpoint_validation_failure(self, security_protocol, interbr SecurityConfig.ssl_stores = TestSslStores(self.test_context.local_scratch_dir, valid_hostname=True) - self.create_zookeeper() - self.zk.start() + self.create_zookeeper_if_necessary() + if self.zk: + self.zk.start() self.create_kafka(security_protocol=security_protocol, interbroker_security_protocol=interbroker_security_protocol) diff --git a/tests/kafkatest/tests/core/transactions_test.py b/tests/kafkatest/tests/core/transactions_test.py index ad8d0a7dd01f5..2891e70ff19a0 100644 --- a/tests/kafkatest/tests/core/transactions_test.py +++ b/tests/kafkatest/tests/core/transactions_test.py @@ -14,7 +14,7 @@ # limitations under the License. from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.transactional_message_copier import TransactionalMessageCopier @@ -25,6 +25,7 @@ from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until +import time class TransactionsTest(Test): """Tests transactions by transactionally copying data from a source topic to @@ -58,13 +59,15 @@ def __init__(self, test_context): self.progress_timeout_sec = 60 self.consumer_group = "transactions-test-consumer-group" - self.zk = ZookeeperService(test_context, num_nodes=1) + self.zk = ZookeeperService(test_context, num_nodes=1) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService(test_context, num_nodes=self.num_brokers, - zk=self.zk) + zk=self.zk, + controller_num_nodes_override=1) def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def seed_messages(self, topic, num_seed_messages): seed_timeout_sec = 10000 @@ -92,10 +95,17 @@ def bounce_brokers(self, clean_shutdown): self.kafka.restart_node(node, clean_shutdown = True) else: self.kafka.stop_node(node, clean_shutdown = False) - wait_until(lambda: len(self.kafka.pids(node)) == 0 and not self.kafka.is_registered(node), - timeout_sec=self.kafka.zk_session_timeout + 5, - err_msg="Failed to see timely deregistration of \ - hard-killed broker %s" % str(node.account)) + gracePeriodSecs = 5 + if self.zk: + wait_until(lambda: len(self.kafka.pids(node)) == 0 and not self.kafka.is_registered(node), + timeout_sec=self.kafka.zk_session_timeout + gracePeriodSecs, + err_msg="Failed to see timely deregistration of hard-killed broker %s" % str(node.account)) + else: + brokerSessionTimeoutSecs = 18 + wait_until(lambda: len(self.kafka.pids(node)) == 0, + timeout_sec=brokerSessionTimeoutSecs + gracePeriodSecs, + err_msg="Failed to see timely disappearance of process for hard-killed broker %s" % str(node.account)) + time.sleep(brokerSessionTimeoutSecs + gracePeriodSecs) self.kafka.start_node(node) def create_and_start_message_copier(self, input_topic, input_partition, output_topic, transactional_id, use_group_metadata): @@ -234,8 +244,9 @@ def setup_topics(self): @matrix(failure_mode=["hard_bounce", "clean_bounce"], bounce_target=["brokers", "clients"], check_order=[True, False], - use_group_metadata=[True, False]) - def test_transactions(self, failure_mode, bounce_target, check_order, use_group_metadata): + use_group_metadata=[True, False], + metadata_quorum=quorum.all_non_upgrade) + def test_transactions(self, failure_mode, bounce_target, check_order, use_group_metadata, metadata_quorum=quorum.zk): security_protocol = 'PLAINTEXT' self.kafka.security_protocol = security_protocol self.kafka.interbroker_security_protocol = security_protocol diff --git a/tests/kafkatest/tests/core/upgrade_test.py b/tests/kafkatest/tests/core/upgrade_test.py index ae0b3e7d276ff..183e4900e8c1d 100644 --- a/tests/kafkatest/tests/core/upgrade_test.py +++ b/tests/kafkatest/tests/core/upgrade_test.py @@ -24,7 +24,7 @@ from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest from kafkatest.utils import is_int from kafkatest.utils.remote_account import java_version -from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, LATEST_2_5, LATEST_2_6, LATEST_2_7, V_0_9_0_0, V_0_11_0_0, V_2_8_0, DEV_BRANCH, KafkaVersion +from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, LATEST_2_5, LATEST_2_6, LATEST_2_7, V_0_11_0_0, V_2_8_0, DEV_BRANCH, KafkaVersion from kafkatest.services.kafka.util import new_jdk_not_supported class TestUpgrade(ProduceConsumeValidateTest): @@ -171,7 +171,7 @@ def test_upgrade(self, from_kafka_version, to_message_format_version, compressio # after leader change. Tolerate limited data loss for this case to avoid transient test failures. self.may_truncate_acked_records = False if from_kafka_version >= V_0_11_0_0 else True - new_consumer = from_kafka_version >= V_0_9_0_0 + new_consumer = from_kafka_version.consumer_supports_bootstrap_server() # TODO - reduce the timeout self.consumer = ConsoleConsumer(self.test_context, self.num_consumers, self.kafka, self.topic, new_consumer=new_consumer, consumer_timeout_ms=30000, diff --git a/tests/kafkatest/tests/end_to_end.py b/tests/kafkatest/tests/end_to_end.py index 7ef6b974f6905..bfc316eeea17b 100644 --- a/tests/kafkatest/tests/end_to_end.py +++ b/tests/kafkatest/tests/end_to_end.py @@ -16,7 +16,7 @@ from ducktape.tests.test import Test from ducktape.utils.util import wait_until -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.kafka import TopicPartition from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.verifiable_consumer import VerifiableConsumer @@ -41,8 +41,8 @@ def __init__(self, test_context, topic="test_topic", topic_config=DEFAULT_TOPIC_ self.records_consumed = [] self.last_consumed_offsets = {} - def create_zookeeper(self, num_nodes=1, **kwargs): - self.zk = ZookeeperService(self.test_context, num_nodes=num_nodes, **kwargs) + def create_zookeeper_if_necessary(self, num_nodes=1, **kwargs): + self.zk = ZookeeperService(self.test_context, num_nodes=num_nodes, **kwargs) if quorum.for_test(self.test_context) == quorum.zk else None def create_kafka(self, num_nodes=1, **kwargs): group_metadata_config = { diff --git a/tests/kafkatest/tests/kafka_test.py b/tests/kafkatest/tests/kafka_test.py index 7118721b5ded2..7852768a5f651 100644 --- a/tests/kafkatest/tests/kafka_test.py +++ b/tests/kafkatest/tests/kafka_test.py @@ -17,7 +17,7 @@ from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum class KafkaTest(Test): @@ -34,12 +34,14 @@ def __init__(self, test_context, num_zk, num_brokers, topics=None): self.num_brokers = num_brokers self.topics = topics - self.zk = ZookeeperService(test_context, self.num_zk) + self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None self.kafka = KafkaService( test_context, self.num_brokers, - self.zk, topics=self.topics) + self.zk, topics=self.topics, + controller_num_nodes_override=self.num_zk) def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() self.kafka.start() \ No newline at end of file diff --git a/tests/kafkatest/tests/streams/streams_application_upgrade_test.py b/tests/kafkatest/tests/streams/streams_application_upgrade_test.py index 4ac2795875882..ca07828442bc7 100644 --- a/tests/kafkatest/tests/streams/streams_application_upgrade_test.py +++ b/tests/kafkatest/tests/streams/streams_application_upgrade_test.py @@ -15,6 +15,7 @@ import random from ducktape.mark import matrix +from ducktape.mark.resource import cluster from ducktape.tests.test import Test from ducktape.utils.util import wait_until from kafkatest.services.kafka import KafkaService @@ -50,6 +51,7 @@ def perform_broker_upgrade(self, to_version): node.version = KafkaVersion(to_version) self.kafka.start_node(node) + @cluster(num_nodes=6) @matrix(from_version=smoke_test_versions, to_version=dev_version, bounce_type=["full"]) def test_app_upgrade(self, from_version, to_version, bounce_type): """ diff --git a/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py b/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py index 6ebd1a5e3557e..69fc3500ef380 100644 --- a/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py +++ b/tests/kafkatest/tests/streams/streams_broker_compatibility_test.py @@ -14,6 +14,7 @@ # limitations under the License. from ducktape.mark import parametrize +from ducktape.mark.resource import cluster from ducktape.tests.test import Test from ducktape.utils.util import wait_until from kafkatest.services.kafka import KafkaService @@ -61,6 +62,7 @@ def setUp(self): self.zk.start() + @cluster(num_nodes=4) @parametrize(broker_version=str(LATEST_2_4)) @parametrize(broker_version=str(LATEST_2_3)) @parametrize(broker_version=str(LATEST_2_2)) @@ -85,6 +87,7 @@ def test_compatible_brokers_eos_disabled(self, broker_version): self.consumer.stop() self.kafka.stop() + @cluster(num_nodes=4) @parametrize(broker_version=str(LATEST_2_6)) @parametrize(broker_version=str(LATEST_2_5)) @parametrize(broker_version=str(LATEST_2_4)) @@ -129,6 +132,7 @@ def test_compatible_brokers_eos_alpha_enabled(self, broker_version): # self.consumer.stop() # self.kafka.stop() + @cluster(num_nodes=4) @parametrize(broker_version=str(LATEST_0_10_2)) @parametrize(broker_version=str(LATEST_0_10_1)) @parametrize(broker_version=str(LATEST_0_10_0)) @@ -146,6 +150,7 @@ def test_fail_fast_on_incompatible_brokers(self, broker_version): self.kafka.stop() + @cluster(num_nodes=4) @parametrize(broker_version=str(LATEST_2_4)) @parametrize(broker_version=str(LATEST_2_3)) @parametrize(broker_version=str(LATEST_2_2)) diff --git a/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py b/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py index 8fcf14a3fcc6d..5026d7a23d6ea 100644 --- a/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py +++ b/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py @@ -14,6 +14,7 @@ # limitations under the License. import time +from ducktape.mark.resource import cluster from kafkatest.services.streams import StreamsBrokerDownResilienceService from kafkatest.tests.streams.base_streams_test import BaseStreamsTest @@ -40,6 +41,7 @@ def __init__(self, test_context): def setUp(self): self.zk.start() + @cluster(num_nodes=5) def test_streams_resilient_to_broker_down(self): self.kafka.start() @@ -75,6 +77,7 @@ def test_streams_resilient_to_broker_down(self): self.kafka.stop() + @cluster(num_nodes=7) def test_streams_runs_with_broker_down_initially(self): self.kafka.start() node = self.kafka.leader(self.inputTopic) @@ -141,6 +144,7 @@ def test_streams_runs_with_broker_down_initially(self): self.kafka.stop() + @cluster(num_nodes=7) def test_streams_should_scale_in_while_brokers_down(self): self.kafka.start() @@ -218,6 +222,7 @@ def test_streams_should_scale_in_while_brokers_down(self): self.kafka.stop() + @cluster(num_nodes=7) def test_streams_should_failover_while_brokers_down(self): self.kafka.start() diff --git a/tests/kafkatest/tests/streams/streams_cooperative_rebalance_upgrade_test.py b/tests/kafkatest/tests/streams/streams_cooperative_rebalance_upgrade_test.py index 461573a0a9e1a..4658a5326018a 100644 --- a/tests/kafkatest/tests/streams/streams_cooperative_rebalance_upgrade_test.py +++ b/tests/kafkatest/tests/streams/streams_cooperative_rebalance_upgrade_test.py @@ -15,6 +15,7 @@ import time from ducktape.mark import matrix +from ducktape.mark.resource import cluster from ducktape.tests.test import Test from kafkatest.services.kafka import KafkaService from kafkatest.services.verifiable_producer import VerifiableProducer @@ -66,6 +67,7 @@ def __init__(self, test_context): throughput=1000, acks=1) + @cluster(num_nodes=8) @matrix(upgrade_from_version=streams_eager_rebalance_upgrade_versions) def test_upgrade_to_cooperative_rebalance(self, upgrade_from_version): self.zookeeper.start() diff --git a/tests/kafkatest/tests/streams/streams_optimized_test.py b/tests/kafkatest/tests/streams/streams_optimized_test.py index 3209b2536963f..b96ec10d6ba45 100644 --- a/tests/kafkatest/tests/streams/streams_optimized_test.py +++ b/tests/kafkatest/tests/streams/streams_optimized_test.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time +from ducktape.mark.resource import cluster from ducktape.tests.test import Test from ducktape.utils.util import wait_until from kafkatest.services.kafka import KafkaService @@ -56,6 +56,7 @@ def __init__(self, test_context): throughput=1000, acks=1) + @cluster(num_nodes=9) def test_upgrade_optimized_topology(self): self.zookeeper.start() self.kafka.start() diff --git a/tests/kafkatest/tests/streams/streams_shutdown_deadlock_test.py b/tests/kafkatest/tests/streams/streams_shutdown_deadlock_test.py index 482da9c5d85f7..d190a1c311935 100644 --- a/tests/kafkatest/tests/streams/streams_shutdown_deadlock_test.py +++ b/tests/kafkatest/tests/streams/streams_shutdown_deadlock_test.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ducktape.mark.resource import cluster from kafkatest.tests.kafka_test import KafkaTest from kafkatest.services.streams import StreamsSmokeTestShutdownDeadlockService @@ -29,6 +30,7 @@ def __init__(self, test_context): self.driver = StreamsSmokeTestShutdownDeadlockService(test_context, self.kafka) + @cluster(num_nodes=3) def test_shutdown_wont_deadlock(self): """ Start ShutdownDeadLockTest, wait for upt to 1 minute, and check that the process exited. diff --git a/tests/kafkatest/tests/streams/streams_smoke_test.py b/tests/kafkatest/tests/streams/streams_smoke_test.py index 1a4f296eb2a72..b1f908ddcf3b2 100644 --- a/tests/kafkatest/tests/streams/streams_smoke_test.py +++ b/tests/kafkatest/tests/streams/streams_smoke_test.py @@ -16,6 +16,7 @@ from ducktape.mark import matrix from ducktape.mark.resource import cluster +from kafkatest.services.kafka import quorum from kafkatest.tests.kafka_test import KafkaTest from kafkatest.services.streams import StreamsSmokeTestDriverService, StreamsSmokeTestJobRunnerService @@ -46,8 +47,8 @@ def __init__(self, test_context): self.driver = StreamsSmokeTestDriverService(test_context, self.kafka) @cluster(num_nodes=8) - @matrix(processing_guarantee=['at_least_once', 'exactly_once', 'exactly_once_beta'], crash=[True, False]) - def test_streams(self, processing_guarantee, crash): + @matrix(processing_guarantee=['at_least_once', 'exactly_once', 'exactly_once_beta'], crash=[True, False], metadata_quorum=quorum.all_non_upgrade) + def test_streams(self, processing_guarantee, crash, metadata_quorum=quorum.zk): processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee) processor2 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee) processor3 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka, processing_guarantee) diff --git a/tests/kafkatest/tests/streams/streams_standby_replica_test.py b/tests/kafkatest/tests/streams/streams_standby_replica_test.py index e847c3ebf9d90..a8c07513c1c2e 100644 --- a/tests/kafkatest/tests/streams/streams_standby_replica_test.py +++ b/tests/kafkatest/tests/streams/streams_standby_replica_test.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until from kafkatest.services.streams import StreamsStandbyTaskService from kafkatest.tests.streams.base_streams_test import BaseStreamsTest @@ -43,6 +44,7 @@ def __init__(self, test_context): 'replication-factor': 1} }) + @cluster(num_nodes=10) def test_standby_tasks_rebalance(self): # TODO KIP-441: consider rewriting the test for HighAvailabilityTaskAssignor configs = self.get_configs( diff --git a/tests/kafkatest/tests/streams/streams_static_membership_test.py b/tests/kafkatest/tests/streams/streams_static_membership_test.py index e6072f4b3dc42..f31c38a75d488 100644 --- a/tests/kafkatest/tests/streams/streams_static_membership_test.py +++ b/tests/kafkatest/tests/streams/streams_static_membership_test.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ducktape.mark.resource import cluster from ducktape.tests.test import Test from kafkatest.services.kafka import KafkaService from kafkatest.services.streams import StaticMemberTestService @@ -48,6 +49,7 @@ def __init__(self, test_context): throughput=1000, acks=1) + @cluster(num_nodes=8) def test_rolling_bounces_will_not_trigger_rebalance_under_static_membership(self): self.zookeeper.start() self.kafka.start() diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py index 78f171aa83bdd..9aff673349c8d 100644 --- a/tests/kafkatest/tests/streams/streams_upgrade_test.py +++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py @@ -186,6 +186,7 @@ def test_upgrade_downgrade_brokers(self, from_version, to_version): processor.stop() processor.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % processor.STDOUT_FILE, allow_fail=False) + @cluster(num_nodes=6) @matrix(from_version=metadata_1_versions, to_version=[str(DEV_VERSION)]) @matrix(from_version=metadata_2_versions, to_version=[str(DEV_VERSION)]) def test_metadata_upgrade(self, from_version, to_version): @@ -238,6 +239,7 @@ def test_metadata_upgrade(self, from_version, to_version): timeout_sec=60, err_msg="Never saw output 'UPGRADE-TEST-CLIENT-CLOSED' on" + str(node.account)) + @cluster(num_nodes=6) def test_version_probing_upgrade(self): """ Starts 3 KafkaStreams instances, and upgrades one-by-one to "future version" diff --git a/tests/kafkatest/tests/tools/log4j_appender_test.py b/tests/kafkatest/tests/tools/log4j_appender_test.py index 61a5d2a8932ba..0287f2f4d0e83 100644 --- a/tests/kafkatest/tests/tools/log4j_appender_test.py +++ b/tests/kafkatest/tests/tools/log4j_appender_test.py @@ -20,7 +20,7 @@ from ducktape.mark.resource import cluster from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.services.kafka_log4j_appender import KafkaLog4jAppender @@ -41,16 +41,18 @@ def __init__(self, test_context): TOPIC: {'partitions': 1, 'replication-factor': 1} } - self.zk = ZookeeperService(test_context, self.num_zk) + self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, - interbroker_security_protocol=interbroker_security_protocol, topics=self.topics) + interbroker_security_protocol=interbroker_security_protocol, topics=self.topics, + controller_num_nodes_override=self.num_zk) self.kafka.start() def start_appender(self, security_protocol): @@ -70,10 +72,10 @@ def start_consumer(self): self.consumer.start() @cluster(num_nodes=4) - @matrix(security_protocol=['PLAINTEXT', 'SSL']) + @matrix(security_protocol=['PLAINTEXT', 'SSL'], metadata_quorum=quorum.all_non_upgrade) @cluster(num_nodes=5) - @matrix(security_protocol=['SASL_PLAINTEXT', 'SASL_SSL']) - def test_log4j_appender(self, security_protocol='PLAINTEXT'): + @matrix(security_protocol=['SASL_PLAINTEXT', 'SASL_SSL'], metadata_quorum=quorum.all_non_upgrade) + def test_log4j_appender(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests if KafkaLog4jAppender is producing to Kafka topic :return: None diff --git a/tests/kafkatest/tests/tools/log_compaction_test.py b/tests/kafkatest/tests/tools/log_compaction_test.py index 338060f72175b..a91a976550a9f 100644 --- a/tests/kafkatest/tests/tools/log_compaction_test.py +++ b/tests/kafkatest/tests/tools/log_compaction_test.py @@ -14,13 +14,14 @@ # limitations under the License. +from ducktape.mark import matrix from ducktape.utils.util import wait_until from ducktape.tests.test import Test from ducktape.mark.resource import cluster from kafkatest.services.kafka import config_property from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.log_compaction_tester import LogCompactionTester class LogCompactionTest(Test): @@ -33,12 +34,13 @@ def __init__(self, test_context): self.num_zk = 1 self.num_brokers = 1 - self.zk = ZookeeperService(test_context, self.num_zk) + self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None self.kafka = None self.compaction_verifier = None def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( @@ -49,7 +51,8 @@ def start_kafka(self, security_protocol, interbroker_security_protocol): interbroker_security_protocol=interbroker_security_protocol, server_prop_overides=[ [config_property.LOG_SEGMENT_BYTES, LogCompactionTest.LOG_SEGMENT_BYTES], - ]) + ], + controller_num_nodes_override=self.num_zk) self.kafka.start() def start_test_log_compaction_tool(self, security_protocol): @@ -57,7 +60,8 @@ def start_test_log_compaction_tool(self, security_protocol): self.compaction_verifier.start() @cluster(num_nodes=4) - def test_log_compaction(self, security_protocol='PLAINTEXT'): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_log_compaction(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): self.start_kafka(security_protocol, security_protocol) self.start_test_log_compaction_tool(security_protocol) diff --git a/tests/kafkatest/tests/tools/replica_verification_test.py b/tests/kafkatest/tests/tools/replica_verification_test.py index f296c73b76455..baa0536f218d0 100644 --- a/tests/kafkatest/tests/tools/replica_verification_test.py +++ b/tests/kafkatest/tests/tools/replica_verification_test.py @@ -14,13 +14,14 @@ # limitations under the License. +from ducktape.mark import matrix +from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until from ducktape.tests.test import Test -from ducktape.mark.resource import cluster from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.services.kafka import KafkaService +from kafkatest.services.kafka import KafkaService, quorum from kafkatest.services.replica_verification_tool import ReplicaVerificationTool TOPIC = "topic-replica-verification" @@ -39,19 +40,21 @@ def __init__(self, test_context): TOPIC: {'partitions': 1, 'replication-factor': 2} } - self.zk = ZookeeperService(test_context, self.num_zk) + self.zk = ZookeeperService(test_context, self.num_zk) if quorum.for_test(test_context) == quorum.zk else None self.kafka = None self.producer = None self.replica_verifier = None def setUp(self): - self.zk.start() + if self.zk: + self.zk.start() def start_kafka(self, security_protocol, interbroker_security_protocol): self.kafka = KafkaService( self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, - interbroker_security_protocol=interbroker_security_protocol, topics=self.topics) + interbroker_security_protocol=interbroker_security_protocol, topics=self.topics, + controller_num_nodes_override=self.num_zk) self.kafka.start() def start_replica_verification_tool(self, security_protocol): @@ -70,7 +73,8 @@ def stop_producer(self): self.producer.stop() @cluster(num_nodes=6) - def test_replica_lags(self, security_protocol='PLAINTEXT'): + @matrix(metadata_quorum=quorum.all_non_upgrade) + def test_replica_lags(self, security_protocol='PLAINTEXT', metadata_quorum=quorum.zk): """ Tests ReplicaVerificationTool :return: None diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py index dc6083e01f170..566dba5fe30ad 100644 --- a/tests/kafkatest/version.py +++ b/tests/kafkatest/version.py @@ -62,6 +62,18 @@ def _cmp(self, other): return LooseVersion._cmp(self, other) + def consumer_supports_bootstrap_server(self): + """ + Kafka supported a new consumer beginning with v0.9.0 where + we can specify --bootstrap-server instead of --zookeeper. + + This version also allowed a --consumer-config file where we could specify + a security protocol other than PLAINTEXT. + + :return: true if the version of Kafka supports a new consumer with --bootstrap-server + """ + return self >= V_0_9_0_0 + def supports_named_listeners(self): return self >= V_0_10_2_0 From 9e799cb23ca512c47db3e41e4d552e4e0095a012 Mon Sep 17 00:00:00 2001 From: Ron Dagostino Date: Mon, 22 Feb 2021 18:09:25 -0500 Subject: [PATCH 041/243] MINOR: fix some ducktape test issues (#10181) Reviewers: Colin P. McCabe --- tests/kafkatest/tests/core/security_test.py | 4 ++-- .../tests/streams/streams_broker_down_resilience_test.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/kafkatest/tests/core/security_test.py b/tests/kafkatest/tests/core/security_test.py index 5d1d88651fe73..8dcc264d831c0 100644 --- a/tests/kafkatest/tests/core/security_test.py +++ b/tests/kafkatest/tests/core/security_test.py @@ -58,8 +58,8 @@ def producer_consumer_have_expected_error(self, error): return True @cluster(num_nodes=7) - @matrix(security_protocol='PLAINTEXT', interbroker_security_protocol='SSL', metadata_quorum=quorum.all_non_upgrade) - @matrix(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT', metadata_quorum=quorum.all_non_upgrade) + @matrix(security_protocol=['PLAINTEXT'], interbroker_security_protocol=['SSL'], metadata_quorum=quorum.all_non_upgrade) + @matrix(security_protocol=['SSL'], interbroker_security_protocol=['PLAINTEXT'], metadata_quorum=quorum.all_non_upgrade) def test_client_ssl_endpoint_validation_failure(self, security_protocol, interbroker_security_protocol, metadata_quorum=quorum.zk): """ Test that invalid hostname in certificate results in connection failures. diff --git a/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py b/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py index 5026d7a23d6ea..6957d69961702 100644 --- a/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py +++ b/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py @@ -41,7 +41,7 @@ def __init__(self, test_context): def setUp(self): self.zk.start() - @cluster(num_nodes=5) + @cluster(num_nodes=7) def test_streams_resilient_to_broker_down(self): self.kafka.start() @@ -144,7 +144,7 @@ def test_streams_runs_with_broker_down_initially(self): self.kafka.stop() - @cluster(num_nodes=7) + @cluster(num_nodes=9) def test_streams_should_scale_in_while_brokers_down(self): self.kafka.start() @@ -222,7 +222,7 @@ def test_streams_should_scale_in_while_brokers_down(self): self.kafka.stop() - @cluster(num_nodes=7) + @cluster(num_nodes=9) def test_streams_should_failover_while_brokers_down(self): self.kafka.start() From 9728b4ff9218249a5d44b47d395a1d9d33759677 Mon Sep 17 00:00:00 2001 From: Jim Galasyn Date: Mon, 22 Feb 2021 17:06:25 -0800 Subject: [PATCH 042/243] KAFKA-12160: Remove max.poll.interval.ms from config docs (#10182) Reviewers: Matthias J. Sax --- docs/streams/developer-guide/config-streams.html | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/streams/developer-guide/config-streams.html b/docs/streams/developer-guide/config-streams.html index fe350260c6463..11d825d5592f4 100644 --- a/docs/streams/developer-guide/config-streams.html +++ b/docs/streams/developer-guide/config-streams.html @@ -865,10 +865,6 @@

    Default ValuesProducer

    		 100
    max.poll.interval.msConsumerInteger.MAX_VALUE
    max.poll.records Consumer 1000100
    max.poll.interval.msConsumer300000
    max.poll.records Consumer 1000