feature: Build schema on upstream version 3.8.0 (#206)

codegen/__init__.py

@@ -2,4 +2,4 @@
 
 # Note: the kafka-clients dependency of the Java tester also needs updating when
 # this is bumped (in java_tester/build.gradle).
-build_tag: Final = "3.7.0"
+build_tag: Final = "3.8.0"

compose.yml

@@ -1,7 +1,7 @@
 name: kio
 services:
   kafka:
-    image: apache/kafka:3.7.0
+    image: apache/kafka:3.8.0
     hostname: broker
     container_name: broker
     ports:
@@ -26,8 +26,6 @@ services:
       KAFKA_BUFFER_MEMORY: 104857600
       KAFKA_MAX_REQUEST_SIZE: 104857600
       KAFKA_REPLICA_FETCH_MAX_BYTES: 104857600
-    tmpfs:
-      - /tmp
 
   java_tester:
     build:

java_tester/build.gradle

@@ -13,7 +13,7 @@ repositories {
 }
 
 dependencies {
-    implementation 'org.apache.kafka:kafka-clients:3.7.0'
+    implementation 'org.apache.kafka:kafka-clients:3.8.0'
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
     implementation 'org.apache.commons:commons-text:1.9'
 }

java_tester/src/main/java/io/aiven/kio/java_tester/RootMessageInfo.java

@@ -57,6 +57,8 @@ private static EntityClass<ApiMessage> getRootClass(String shortClassName) throw
             && !shortClassName.equals("ConsumerProtocolSubscription")
             && !shortClassName.equals("LeaderChangeMessage")
             && !shortClassName.equals("DefaultPrincipalData")
+            && !shortClassName.equals("KRaftVersionRecord")
+            && !shortClassName.equals("VotersRecord")
         ) {
             className += "Data";
         }

src/kio/schema/add_offsets_to_txn/v4/__init__.py

@@ -0,0 +1,7 @@
+from .request import AddOffsetsToTxnRequest
+from .response import AddOffsetsToTxnResponse
+
+__all__ = (
+    "AddOffsetsToTxnRequest",
+    "AddOffsetsToTxnResponse",
+)

src/kio/schema/add_offsets_to_txn/v4/request.py

@@ -0,0 +1,31 @@
+"""
+Generated from ``clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json``.
+"""
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import ClassVar
+
+from kio.schema.request_header.v2.header import RequestHeader
+from kio.schema.types import GroupId
+from kio.schema.types import ProducerId
+from kio.schema.types import TransactionalId
+from kio.static.constants import EntityType
+from kio.static.primitive import i16
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddOffsetsToTxnRequest:
+    __type__: ClassVar = EntityType.request
+    __version__: ClassVar[i16] = i16(4)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(25)
+    __header_schema__: ClassVar[type[RequestHeader]] = RequestHeader
+    transactional_id: TransactionalId = field(metadata={"kafka_type": "string"})
+    """The transactional id corresponding to the transaction."""
+    producer_id: ProducerId = field(metadata={"kafka_type": "int64"})
+    """Current producer id in use by the transactional id."""
+    producer_epoch: i16 = field(metadata={"kafka_type": "int16"})
+    """Current epoch associated with the producer id."""
+    group_id: GroupId = field(metadata={"kafka_type": "string"})
+    """The unique group identifier."""

src/kio/schema/add_offsets_to_txn/v4/response.py

@@ -0,0 +1,26 @@
+"""
+Generated from ``clients/src/main/resources/common/message/AddOffsetsToTxnResponse.json``.
+"""
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import ClassVar
+
+from kio.schema.errors import ErrorCode
+from kio.schema.response_header.v1.header import ResponseHeader
+from kio.static.constants import EntityType
+from kio.static.primitive import i16
+from kio.static.primitive import i32Timedelta
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddOffsetsToTxnResponse:
+    __type__: ClassVar = EntityType.response
+    __version__: ClassVar[i16] = i16(4)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(25)
+    __header_schema__: ClassVar[type[ResponseHeader]] = ResponseHeader
+    throttle_time: i32Timedelta = field(metadata={"kafka_type": "timedelta_i32"})
+    """Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota."""
+    error_code: ErrorCode = field(metadata={"kafka_type": "error_code"})
+    """The response error code, or 0 if there was no error."""

src/kio/schema/add_partitions_to_txn/v5/__init__.py

@@ -0,0 +1,7 @@
+from .request import AddPartitionsToTxnRequest
+from .response import AddPartitionsToTxnResponse
+
+__all__ = (
+    "AddPartitionsToTxnRequest",
+    "AddPartitionsToTxnResponse",
+)

src/kio/schema/add_partitions_to_txn/v5/request.py

@@ -0,0 +1,58 @@
+"""
+Generated from ``clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json``.
+"""
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import ClassVar
+
+from kio.schema.request_header.v2.header import RequestHeader
+from kio.schema.types import ProducerId
+from kio.schema.types import TopicName
+from kio.schema.types import TransactionalId
+from kio.static.constants import EntityType
+from kio.static.primitive import i16
+from kio.static.primitive import i32
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddPartitionsToTxnTopic:
+    __type__: ClassVar = EntityType.nested
+    __version__: ClassVar[i16] = i16(5)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(24)
+    __header_schema__: ClassVar[type[RequestHeader]] = RequestHeader
+    name: TopicName = field(metadata={"kafka_type": "string"})
+    """The name of the topic."""
+    partitions: tuple[i32, ...] = field(metadata={"kafka_type": "int32"}, default=())
+    """The partition indexes to add to the transaction"""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddPartitionsToTxnTransaction:
+    __type__: ClassVar = EntityType.nested
+    __version__: ClassVar[i16] = i16(5)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(24)
+    __header_schema__: ClassVar[type[RequestHeader]] = RequestHeader
+    transactional_id: TransactionalId = field(metadata={"kafka_type": "string"})
+    """The transactional id corresponding to the transaction."""
+    producer_id: ProducerId = field(metadata={"kafka_type": "int64"})
+    """Current producer id in use by the transactional id."""
+    producer_epoch: i16 = field(metadata={"kafka_type": "int16"})
+    """Current epoch associated with the producer id."""
+    verify_only: bool = field(metadata={"kafka_type": "bool"}, default=False)
+    """Boolean to signify if we want to check if the partition is in the transaction rather than add it."""
+    topics: tuple[AddPartitionsToTxnTopic, ...]
+    """The partitions to add to the transaction."""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddPartitionsToTxnRequest:
+    __type__: ClassVar = EntityType.request
+    __version__: ClassVar[i16] = i16(5)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(24)
+    __header_schema__: ClassVar[type[RequestHeader]] = RequestHeader
+    transactions: tuple[AddPartitionsToTxnTransaction, ...]
+    """List of transactions to add partitions to."""

src/kio/schema/add_partitions_to_txn/v5/response.py

@@ -0,0 +1,70 @@
+"""
+Generated from ``clients/src/main/resources/common/message/AddPartitionsToTxnResponse.json``.
+"""
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import ClassVar
+
+from kio.schema.errors import ErrorCode
+from kio.schema.response_header.v1.header import ResponseHeader
+from kio.schema.types import TopicName
+from kio.schema.types import TransactionalId
+from kio.static.constants import EntityType
+from kio.static.primitive import i16
+from kio.static.primitive import i32
+from kio.static.primitive import i32Timedelta
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddPartitionsToTxnPartitionResult:
+    __type__: ClassVar = EntityType.nested
+    __version__: ClassVar[i16] = i16(5)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(24)
+    __header_schema__: ClassVar[type[ResponseHeader]] = ResponseHeader
+    partition_index: i32 = field(metadata={"kafka_type": "int32"})
+    """The partition indexes."""
+    partition_error_code: ErrorCode = field(metadata={"kafka_type": "error_code"})
+    """The response error code."""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddPartitionsToTxnTopicResult:
+    __type__: ClassVar = EntityType.nested
+    __version__: ClassVar[i16] = i16(5)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(24)
+    __header_schema__: ClassVar[type[ResponseHeader]] = ResponseHeader
+    name: TopicName = field(metadata={"kafka_type": "string"})
+    """The topic name."""
+    results_by_partition: tuple[AddPartitionsToTxnPartitionResult, ...]
+    """The results for each partition"""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddPartitionsToTxnResult:
+    __type__: ClassVar = EntityType.nested
+    __version__: ClassVar[i16] = i16(5)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(24)
+    __header_schema__: ClassVar[type[ResponseHeader]] = ResponseHeader
+    transactional_id: TransactionalId = field(metadata={"kafka_type": "string"})
+    """The transactional id corresponding to the transaction."""
+    topic_results: tuple[AddPartitionsToTxnTopicResult, ...]
+    """The results for each topic."""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AddPartitionsToTxnResponse:
+    __type__: ClassVar = EntityType.response
+    __version__: ClassVar[i16] = i16(5)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(24)
+    __header_schema__: ClassVar[type[ResponseHeader]] = ResponseHeader
+    throttle_time: i32Timedelta = field(metadata={"kafka_type": "timedelta_i32"})
+    """Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota."""
+    error_code: ErrorCode = field(metadata={"kafka_type": "error_code"})
+    """The response top level error code."""
+    results_by_transaction: tuple[AddPartitionsToTxnResult, ...]
+    """Results categorized by transactional ID."""

src/kio/schema/consumer_group_describe/v0/response.py

@@ -13,7 +13,6 @@
 from kio.schema.types import GroupId
 from kio.schema.types import TopicName
 from kio.static.constants import EntityType
-from kio.static.primitive import i8
 from kio.static.primitive import i16
 from kio.static.primitive import i32
 from kio.static.primitive import i32Timedelta
@@ -43,12 +42,6 @@ class Assignment:
     __header_schema__: ClassVar[type[ResponseHeader]] = ResponseHeader
     topic_partitions: tuple[TopicPartitions, ...]
     """The assigned topic-partitions to the member."""
-    error: i8 = field(metadata={"kafka_type": "int8"})
-    """The assigned error."""
-    metadata_version: i32 = field(metadata={"kafka_type": "int32"})
-    """The assignor metadata version."""
-    metadata_bytes: bytes = field(metadata={"kafka_type": "bytes"})
-    """The assignor metadata bytes."""
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)

src/kio/schema/describe_topic_partitions/v0/__init__.py

@@ -0,0 +1,7 @@
+from .request import DescribeTopicPartitionsRequest
+from .response import DescribeTopicPartitionsResponse
+
+__all__ = (
+    "DescribeTopicPartitionsRequest",
+    "DescribeTopicPartitionsResponse",
+)

src/kio/schema/describe_topic_partitions/v0/request.py

@@ -0,0 +1,54 @@
+"""
+Generated from ``clients/src/main/resources/common/message/DescribeTopicPartitionsRequest.json``.
+"""
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import ClassVar
+
+from kio.schema.request_header.v2.header import RequestHeader
+from kio.schema.types import TopicName
+from kio.static.constants import EntityType
+from kio.static.primitive import i16
+from kio.static.primitive import i32
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TopicRequest:
+    __type__: ClassVar = EntityType.nested
+    __version__: ClassVar[i16] = i16(0)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(75)
+    __header_schema__: ClassVar[type[RequestHeader]] = RequestHeader
+    name: TopicName = field(metadata={"kafka_type": "string"})
+    """The topic name"""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Cursor:
+    __type__: ClassVar = EntityType.nested
+    __version__: ClassVar[i16] = i16(0)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(75)
+    __header_schema__: ClassVar[type[RequestHeader]] = RequestHeader
+    topic_name: TopicName = field(metadata={"kafka_type": "string"})
+    """The name for the first topic to process"""
+    partition_index: i32 = field(metadata={"kafka_type": "int32"})
+    """The partition index to start with"""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class DescribeTopicPartitionsRequest:
+    __type__: ClassVar = EntityType.request
+    __version__: ClassVar[i16] = i16(0)
+    __flexible__: ClassVar[bool] = True
+    __api_key__: ClassVar[i16] = i16(75)
+    __header_schema__: ClassVar[type[RequestHeader]] = RequestHeader
+    topics: tuple[TopicRequest, ...]
+    """The topics to fetch details for."""
+    response_partition_limit: i32 = field(
+        metadata={"kafka_type": "int32"}, default=i32(2000)
+    )
+    """The maximum number of partitions included in the response."""
+    cursor: Cursor | None = field(default=None)
+    """The first topic and partition index to fetch details for."""