Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: revamp the schema generation #190

Merged
merged 4 commits into from
Apr 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
- name: Checkout the repo
uses: actions/checkout@v4
- name: Setup Gradle
uses: gradle/gradle-build-action@v3
uses: gradle/actions/setup-gradle@v3

- name: Run tests
run: ./gradlew check
Expand All @@ -38,7 +38,7 @@ jobs:
- name: Checkout the repo
uses: actions/checkout@v4
- name: Setup Gradle
uses: gradle/gradle-build-action@v3
uses: gradle/actions/setup-gradle@v3

- name: deploy to sonatype snapshots
run: ./gradlew publish
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
- name: Checkout the repo
uses: actions/checkout@v4
- name: Setup Gradle
uses: gradle/gradle-build-action@v3
uses: gradle/actions/setup-gradle@v3

- name: Run tests
run: ./gradlew check
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
- name: Checkout the repo
uses: actions/checkout@v4
- name: Setup Gradle
uses: gradle/gradle-build-action@v3
uses: gradle/actions/setup-gradle@v3

- name: publish release
run: ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository
Expand Down
5 changes: 3 additions & 2 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ plugins {
id("maven-publish")
signing
alias(libs.plugins.dokka)
alias(libs.plugins.kover)
alias(libs.plugins.kotest)
alias(libs.plugins.github.versions)
alias(libs.plugins.nexus.publish)
Expand All @@ -35,7 +36,7 @@ dependencies {
api(libs.apache.avro)
api(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json)
implementation(libs.xerial.snappy)
implementation(kotlin("reflect"))
testImplementation(libs.kotest.junit5)
testImplementation(libs.kotest.core)
testImplementation(libs.kotest.json)
Expand All @@ -46,7 +47,7 @@ tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.jvmTarget = "1.8"
kotlinOptions.apiVersion = "1.6"
kotlinOptions.languageVersion = "1.6"
kotlinOptions.freeCompilerArgs += "-opt-in=kotlin.RequiresOptIn"
kotlinOptions.freeCompilerArgs += listOf("-opt-in=kotlinx.serialization.ExperimentalSerializationApi", "-opt-in=kotlin.RequiresOptIn", "-Xcontext-receivers")
}
java {
sourceCompatibility = JavaVersion.VERSION_1_8
Expand Down
2 changes: 1 addition & 1 deletion settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ dependencyResolutionManagement {
version("kotlin", "1.9.22")
version("jvm", "18")

library("xerial-snappy", "org.xerial.snappy", "snappy-java").version("1.1.10.1")
library("apache-avro", "org.apache.avro", "avro").version("1.11.3")

val kotlinxSerialization = "1.6.2"
Expand All @@ -31,6 +30,7 @@ dependencyResolutionManagement {
plugin("github-versions", "com.github.ben-manes.versions").version("0.46.0")
plugin("nexus-publish", "io.github.gradle-nexus.publish-plugin").version("1.3.0")
plugin("spotless", "com.diffplug.spotless").version("6.25.0")
plugin("kover", "org.jetbrains.kotlinx.kover").version("0.7.6")
}
}
@Suppress("UnstableApiUsage")
Expand Down
41 changes: 22 additions & 19 deletions src/main/kotlin/com/github/avrokotlin/avro4k/Avro.kt
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ import com.github.avrokotlin.avro4k.io.AvroEncodeFormat
import com.github.avrokotlin.avro4k.io.AvroFormat
import com.github.avrokotlin.avro4k.io.AvroInputStream
import com.github.avrokotlin.avro4k.io.AvroOutputStream
import com.github.avrokotlin.avro4k.schema.schemaFor
import com.github.avrokotlin.avro4k.schema.ValueVisitor
import com.github.avrokotlin.avro4k.serializer.BigDecimalAsStringSerializer
import com.github.avrokotlin.avro4k.serializer.BigIntegerSerializer
import com.github.avrokotlin.avro4k.serializer.URLSerializer
import com.github.avrokotlin.avro4k.serializer.UUIDSerializer
import kotlinx.serialization.BinaryFormat
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.SerialFormat
import kotlinx.serialization.SerializationStrategy
import kotlinx.serialization.descriptors.SerialDescriptor
Expand All @@ -31,6 +33,7 @@ import java.nio.ByteBuffer
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.ConcurrentHashMap

open class AvroInputStreamBuilder<T>(
private val converter: (Any) -> T,
Expand Down Expand Up @@ -60,10 +63,12 @@ open class AvroInputStreamBuilder<T>(
val wschema = writerSchema ?: error("Writer schema needs to be supplied for Json format")
AvroDecodeFormat.Json(wschema, readerSchema ?: wschema)
}

is AvroFormat.BinaryFormat -> {
val wschema = writerSchema ?: error("Writer schema needs to be supplied for Binary format")
AvroDecodeFormat.Binary(wschema, readerSchema ?: wschema)
}

is AvroFormat.DataFormat -> AvroDecodeFormat.Data(writerSchema, readerSchema)
}
}
Expand Down Expand Up @@ -152,29 +157,28 @@ class AvroOutputStreamBuilder<T>(
}
}

@OptIn(ExperimentalSerializationApi::class)
class Avro internal constructor(
internal val configuration: AvroInternalConfiguration,
override val serializersModule: SerializersModule,
override val serializersModule: SerializersModule = defaultModule,
internal val configuration: AvroConfiguration = AvroConfiguration(),
) : SerialFormat, BinaryFormat {
constructor(
serializersModule: SerializersModule = defaultModule,
configuration: AvroConfiguration = AvroConfiguration(),
) : this(AvroInternalConfiguration(configuration), serializersModule)
internal val schemaCache: MutableMap<SerialDescriptor, Schema> = ConcurrentHashMap()

constructor(configuration: AvroConfiguration) : this(defaultModule, configuration)

companion object {
val defaultModule =
SerializersModule {
contextual(UUIDSerializer())
contextual(UUIDSerializer)
contextual(BigDecimalAsStringSerializer)
contextual(BigIntegerSerializer)
contextual(URLSerializer)
}
val default = Avro(defaultModule)

/**
* Use this constant if you want to explicitly set a default value of a field to avro null
*/
const val NULL = "com.github.avrokotlin.avro4k.Avro.AVRO_NULL_DEFAULT"
const val NULL = "null"
}

/**
Expand Down Expand Up @@ -303,14 +307,13 @@ class Avro internal constructor(
)
}

fun schema(descriptor: SerialDescriptor): Schema =
schemaFor(
serializersModule,
descriptor,
descriptor.annotations,
configuration,
mutableMapOf()
).schema()
fun schema(descriptor: SerialDescriptor): Schema {
return schemaCache.getOrPut(descriptor) {
lateinit var output: Schema
ValueVisitor(this) { output = it }.visitValue(descriptor)
return output
}
}

fun <T> schema(serializer: SerializationStrategy<T>): Schema {
return schema(serializer.descriptor)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
package com.github.avrokotlin.avro4k

import com.github.avrokotlin.avro4k.schema.FieldNamingStrategy
import com.github.avrokotlin.avro4k.schema.RecordName
import com.github.avrokotlin.avro4k.schema.RecordNamingStrategy
import kotlinx.serialization.descriptors.SerialDescriptor
import java.util.concurrent.ConcurrentHashMap

data class AvroConfiguration(
/**
Expand All @@ -24,57 +21,4 @@ data class AvroConfiguration(
* When set to `true`, the nullable fields that haven't any default value are set as null if the value is missing. It also adds `"default": null` to those fields when generating schema using avro4k.
*/
val implicitNulls: Boolean = false,
/**
* Enable caching of resolved names.
*
* Default: `true`
*/
val namingCacheEnabled: Boolean = true,
)

class AvroInternalConfiguration private constructor(
val recordNamingStrategy: RecordNamingStrategy,
val fieldNamingStrategy: FieldNamingStrategy,
val implicitNulls: Boolean,
) {
constructor(configuration: AvroConfiguration) : this(
recordNamingStrategy = configuration.recordNamingStrategy.cachedIfNecessary(configuration.namingCacheEnabled),
fieldNamingStrategy = configuration.fieldNamingStrategy.cachedIfNecessary(configuration.namingCacheEnabled),
implicitNulls = configuration.implicitNulls
)
}

internal fun RecordNamingStrategy.cachedIfNecessary(cacheEnabled: Boolean): RecordNamingStrategy =
if (!cacheEnabled) {
this
} else {
object : RecordNamingStrategy {
private val cache = ConcurrentHashMap<SerialDescriptor, RecordName>()

override fun resolve(
descriptor: SerialDescriptor,
serialName: String,
): RecordName =
cache.getOrPut(descriptor) {
this@cachedIfNecessary.resolve(descriptor, serialName)
}
}
}

internal fun FieldNamingStrategy.cachedIfNecessary(cacheEnabled: Boolean): FieldNamingStrategy =
if (!cacheEnabled) {
this
} else {
object : FieldNamingStrategy {
private val cache = ConcurrentHashMap<Pair<SerialDescriptor, Int>, String>()

override fun resolve(
descriptor: SerialDescriptor,
elementIndex: Int,
serialName: String,
): String =
cache.getOrPut(descriptor to elementIndex) {
this@cachedIfNecessary.resolve(descriptor, elementIndex, serialName)
}
}
}
)
Loading