cbor: check inline value classes if marked as @ByteString (fixes #2187)

Kotlin · Oct 6, 2023 · dfee01c · dfee01c
1 parent 7b75e39
commit dfee01c
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 1 deletion.
diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt
@@ -70,6 +70,8 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb
         if (encodeByteArrayAsByteString && serializer.descriptor == ByteArraySerializer().descriptor) {
             encoder.encodeByteString(value as ByteArray)
         } else {
+            encodeByteArrayAsByteString = encodeByteArrayAsByteString || serializer.descriptor.isInlineByteString()
+
             super.encodeSerializableValue(serializer, value)
         }
     }
@@ -278,6 +280,7 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb
             @Suppress("UNCHECKED_CAST")
             decoder.nextByteString() as T
         } else {
+            decodeByteArrayAsByteString = decodeByteArrayAsByteString || deserializer.descriptor.isInlineByteString()
             super.decodeSerializableValue(deserializer)
         }
     }
@@ -636,6 +639,11 @@ private fun SerialDescriptor.isByteString(index: Int): Boolean {
     return getElementAnnotations(index).find { it is ByteString } != null
 }
 
+private fun SerialDescriptor.isInlineByteString(): Boolean {
+    // inline item classes should only have 1 item
+    return isInline && isByteString(0)
+}
+
 
 private val normalizeBaseBits = SINGLE_PRECISION_NORMALIZE_BASE.toBits()
 

diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt
@@ -633,6 +633,22 @@ class CborReaderTest {
         )
     }
 
+    @Test
+    fun testReadValueClassWithByteString() {
+        assertContentEquals(
+            expected = byteArrayOf(0x11, 0x22, 0x33),
+            actual = Cbor.decodeFromHexString<ValueClassWithByteString>("43112233").x
+        )
+    }
+
+    @Test
+    fun testReadValueClassCustomByteString() {
+        assertEquals(
+            expected = ValueClassWithCustomByteString(CustomByteString(0x11, 0x22, 0x33)),
+            actual = Cbor.decodeFromHexString("43112233")
+        )
+    }
+
     @Test
     fun testIgnoresTagsOnStrings() {
         /*

diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt
@@ -122,4 +122,20 @@ class CbrWriterTest {
                 actual = Cbor.encodeToHexString(TypeWithNullableCustomByteString(null))
         )
     }
+
+    @Test
+    fun testWriteValueClassWithByteString() {
+        assertEquals(
+            expected = "43112233",
+            actual = Cbor.encodeToHexString(ValueClassWithByteString(byteArrayOf(0x11, 0x22, 0x33)))
+        )
+    }
+
+    @Test
+    fun testWriteValueClassCustomByteString() {
+        assertEquals(
+            expected = "43112233",
+            actual = Cbor.encodeToHexString(ValueClassWithCustomByteString(CustomByteString(0x11, 0x22, 0x33)))
+        )
+    }
 }
diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt
@@ -8,6 +8,7 @@ import kotlinx.serialization.*
 import kotlinx.serialization.builtins.*
 import kotlinx.serialization.descriptors.*
 import kotlinx.serialization.encoding.*
+import kotlin.jvm.*
 
 @Serializable
 data class Simple(val a: String)
@@ -110,4 +111,12 @@ class CustomByteStringSerializer : KSerializer<CustomByteString> {
 data class TypeWithCustomByteString(@ByteString val x: CustomByteString)
 
 @Serializable
-data class TypeWithNullableCustomByteString(@ByteString val x: CustomByteString?)
+data class TypeWithNullableCustomByteString(@ByteString val x: CustomByteString?)
+
+@JvmInline
+@Serializable
+value class ValueClassWithByteString(@ByteString val x: ByteArray)
+
+@JvmInline
+@Serializable
+value class ValueClassWithCustomByteString(@ByteString val x: CustomByteString)