Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds Scala 3 enum serializer #278

Merged
merged 8 commits into from
Mar 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ akka-kryo-serialization - kryo-based serializers for Scala and Akka
[![Full test prior to release](https://github.com/altoo-ag/akka-kryo-serialization/actions/workflows/fullTest.yml/badge.svg)](https://github.com/altoo-ag/akka-kryo-serialization/actions/workflows/fullTest.yml)
[![Latest version](https://index.scala-lang.org/altoo-ag/akka-kryo-serialization/akka-kryo-serialization/latest.svg)](https://index.scala-lang.org/altoo-ag/akka-kryo-serialization/akka-kryo-serialization)

:warning: **We found issues serializing Scala 3 Enums. If you use akka-kryo-serialization with Scala 3 you should upgrade to 2.4.1 asap.**

This library provides custom Kryo-based serializers for Scala and Akka. It can be
used for more efficient akka actor's remoting.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ private[kryo] object ScalaVersionSerializers {
def iterable(kryo: Kryo) = {
kryo.addDefaultSerializer(classOf[scala.collection.Traversable[_]], classOf[ScalaCollectionSerializer])
}

def enums(kryo: Kryo): Unit = () // Scala 3 only
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,6 @@ private[kryo] object ScalaVersionSerializers {
def iterable(kryo: Kryo): Unit = {
kryo.addDefaultSerializer(classOf[scala.collection.Iterable[_]], classOf[ScalaCollectionSerializer])
}

def enums(kryo: Kryo): Unit = () // Scala 3 only
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package io.altoo.akka.serialization.kryo

import com.esotericsoftware.kryo.Kryo
import io.altoo.akka.serialization.kryo.serializer.scala.{ScalaCollectionSerializer, ScalaEnumNameSerializer, ScalaImmutableMapSerializer}

private[kryo] object ScalaVersionSerializers {
def mapAndSet(kryo: Kryo): Unit = {
kryo.addDefaultSerializer(classOf[scala.collection.MapFactory[_root_.scala.collection.Map]], classOf[ScalaImmutableMapSerializer])
}

def iterable(kryo: Kryo): Unit = {
kryo.addDefaultSerializer(classOf[scala.collection.Iterable[_]], classOf[ScalaCollectionSerializer])
}

def enums(kryo: Kryo): Unit = {
kryo.addDefaultSerializer(classOf[scala.runtime.EnumValue], classOf[ScalaEnumNameSerializer[scala.runtime.EnumValue]])
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* *****************************************************************************
* Copyright 2012 Roman Levenstein
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ****************************************************************************
*/

package io.altoo.akka.serialization.kryo.serializer.scala

import com.esotericsoftware.kryo.io.{Input, Output}
import com.esotericsoftware.kryo.{Kryo, Serializer}

/**
* Generic serializer for traversable collections
*
* @author romix
nvollmar marked this conversation as resolved.
Show resolved Hide resolved
*/
class ScalaCollectionSerializer() extends Serializer[Iterable[_]] {

override def read(kryo: Kryo, input: Input, typ: Class[_ <: Iterable[_]]): Iterable[_] = {
val len = input.readInt(true)
val inst = kryo.newInstance(typ)
val coll = inst.iterableFactory.newBuilder[Any]

var i = 0
while (i < len) {
coll += kryo.readClassAndObject(input)
i += 1
}
coll.result()
}

override def write(kryo: Kryo, output: Output, obj: Iterable[_]): Unit = {
val collection: Iterable[_] = obj
val len = collection.size
output.writeInt(len, true)
collection.foreach { (e: Any) => kryo.writeClassAndObject(output, e) }
}
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package io.altoo.akka.serialization.kryo.serializer.scala

import com.esotericsoftware.kryo.{Kryo, Serializer}
import com.esotericsoftware.kryo.io.{Input, Output}

import scala.runtime.EnumValue

/** Serializes enums using the enum's name. This prevents invalidating previously serialized bytes when the enum order changes */
class ScalaEnumNameSerializer[T <: EnumValue] extends Serializer[T] {

def read(kryo: Kryo, input: Input, typ: Class[_ <: T]): T = {
val clazz = kryo.readClass(input).getType
val name = input.readString()
// using value instead of ordinal to make serialization more stable, e.g. allowing reordering without breaking compatibility
clazz.getDeclaredMethod("valueOf", classOf[String]).invoke(null, name).asInstanceOf[T]
}

def write(kryo: Kryo, output: Output, obj: T): Unit = {
val enumClass = obj.getClass.getSuperclass
val productPrefixMethod = obj.getClass.getDeclaredMethod("productPrefix")
if (!productPrefixMethod.canAccess(obj)) productPrefixMethod.setAccessible(true)
val name = productPrefixMethod.invoke(obj).asInstanceOf[String]
kryo.writeClass(output, enumClass)
output.writeString(name)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ class DefaultKryoInitializer {
// Map/Set Factories
ScalaVersionSerializers.mapAndSet(kryo)
ScalaVersionSerializers.iterable(kryo)
ScalaVersionSerializers.enums(kryo)
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import com.esotericsoftware.kryo.serializers.FieldSerializer
class ScalaKryo(classResolver: ClassResolver, referenceResolver: ReferenceResolver)
extends Kryo(classResolver, referenceResolver) {

lazy val objSer = new ObjectSerializer[AnyRef]
lazy val objSer = new ScalaObjectSerializer[AnyRef]

override def getDefaultSerializer(typ: Class[_]): Serializer[_] = {
if(isSingleton(typ)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import scala.collection.mutable.{Map => MMap}
import scala.util.control.Exception.allCatch

// Stolen with pride from Chill ;-)
class ObjectSerializer[T] extends Serializer[T] {
class ScalaObjectSerializer[T] extends Serializer[T] {
private val cachedObj = MMap[Class[_], Option[T]]()

// Does nothing
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package io.altoo.akka.serialization.kryo.serializer.scala

import com.esotericsoftware.kryo.util.{DefaultClassResolver, ListReferenceResolver}
import io.altoo.akka.serialization.kryo.testkit.{AbstractKryoTest, KryoSerializationTesting}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import io.altoo.akka.serialization.kryo.ScalaVersionSerializers

object ScalaEnumSerializationTest {
enum Sample(val name: String, val value: Int) {
case A extends Sample("a", 1)
case B extends Sample("b", 2)
case C extends Sample("c", 3)
}

case class EmbeddedEnum(sample: Sample) {
def this() = this(null)
}
}

class ScalaEnumSerializationTest extends AnyFlatSpec with Matchers with KryoSerializationTesting {
import ScalaEnumSerializationTest._

val kryo = new ScalaKryo(new DefaultClassResolver(), new ListReferenceResolver())
kryo.setRegistrationRequired(false)
kryo.addDefaultSerializer(classOf[scala.runtime.EnumValue], new ScalaEnumNameSerializer[scala.runtime.EnumValue])


behavior of "Kryo serialization"

it should "reoundtrip enum" in {
kryo.setRegistrationRequired(false)

testSerializationOf(Sample.B)
}

it should "reoundtrip external enum" in {
kryo.setRegistrationRequired(false)

testSerializationOf(io.altoo.external.ExternalEnum.A)
}

it should "reoundtrip embedded enum" in {
kryo.setRegistrationRequired(false)
kryo.register(classOf[EmbeddedEnum], 46)

testSerializationOf(EmbeddedEnum(Sample.C))
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package io.altoo.akka.serialization.kryo.serializer.scala

import com.esotericsoftware.kryo.Kryo

object ScalaVersionRegistry {
final val immutableHashMapImpl = "scala.collection.immutable.HashMap"
final val immutableHashSetImpl = "scala.collection.immutable.HashSet"

def registerHashMap(kryo: Kryo): Unit = {
kryo.register(classOf[scala.collection.immutable.HashMap[_, _]], 40)
}

def registerHashSet(kryo: Kryo): Unit = {
kryo.register(classOf[scala.collection.immutable.HashSet[_]], 41)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package io.altoo.external

import io.altoo.akka.serialization.kryo.serializer.scala.ScalaEnumSerializationTest.Sample

enum ExternalEnum(val name: String) {
case A extends ExternalEnum("a")
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ class TupleSerializationTest extends AbstractKryoTest {

type IntTuple6 = (Int, Int, Int, Int, Int, Int)

"Kryo" should "roundtrip tuples" in {
behavior of "Kryo serialization"

it should "roundtrip tuples" in {
kryo.setRegistrationRequired(false)
kryo.register(classOf[scala.Tuple1[Any]], 45)
kryo.register(classOf[scala.Tuple2[Any, Any]], 46)
Expand Down
10 changes: 6 additions & 4 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ val typesafe = "Typesafe Repository" at "https://repo.typesafe.com/typesafe/rele
val typesafeSnapshot = "Typesafe Snapshots Repository" at "https://repo.typesafe.com/typesafe/snapshots/"
val sonatypeSnapshot = "Sonatype Snapshots Repository" at "https://oss.sonatype.org/content/repositories/snapshots/"

val mainScalaVersion = "2.13.8"
val secondayScalaVersions = Seq("2.12.15", "3.0.2") // note: Scala 3.1 is not forward compatible - publishing with 3.1 would force users to Scala 3.1
val mainScalaVersion = "3.0.2" // note: Scala 3.1 is not forward compatible - publishing with 3.1 would force users to Scala 3.1
val secondayScalaVersions = Seq("2.12.15", "2.13.8")

val kryoVersion = "5.3.0"
val defaultAkkaVersion = "2.6.18"
Expand Down Expand Up @@ -41,13 +41,15 @@ lazy val core: Project = Project("akka-kryo-serialization", file("akka-kryo-seri
.settings(Compile / unmanagedSourceDirectories += {
scalaBinaryVersion.value match {
case "2.12" => baseDirectory.value / "src" / "main" / "scala-2.12"
case _ => baseDirectory.value / "src" / "main" / "scala-2.13"
case "2.13" => baseDirectory.value / "src" / "main" / "scala-2.13"
case _ => baseDirectory.value / "src" / "main" / "scala-3"
}
})
.settings(Test / unmanagedSourceDirectories += {
scalaBinaryVersion.value match {
case "2.12" => baseDirectory.value / "src" / "test" / "scala-2.12"
case _ => baseDirectory.value / "src" / "test" / "scala-2.13"
case "2.13" => baseDirectory.value / "src" / "test" / "scala-2.13"
case _ => baseDirectory.value / "src" / "test" / "scala-3"
}
})

Expand Down