Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support nesting Options in TypeDescriptor #1387

Merged
merged 2 commits into from
Jul 31, 2015
Merged

Support nesting Options in TypeDescriptor #1387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
afc06bb
Support nesting Options in TypeDescriptor
johnynek Jul 27, 2015
7593968
Fix DB macro breakage
johnynek Jul 27, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 84 additions & 100 deletions scalding-core/src/main/scala/com/twitter/scalding/macros/impl/CaseClassBasedSetterImpl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,113 +32,97 @@ object CaseClassBasedSetterImpl {
fsetter: CaseClassFieldSetter)(implicit T: c.WeakTypeTag[T]): (Int, c.Tree) = {
import c.universe._

val maybeHandlePrimitive: Option[(Int, Tree)] = {
def innerLoop(outerTpe: Type, pTree: Tree): Option[Tree] = {
val typedSetter: scala.util.Try[c.Tree] = fsetter.from(c)(outerTpe, 0, container, pTree)

(outerTpe, typedSetter) match {
case (_, Success(setter)) => Some(setter)
case (tpe, _) if tpe.erasure =:= typeOf[Option[Any]] =>
val cacheName = newTermName(c.fresh(s"optiIndx"))
innerLoop(tpe.asInstanceOf[TypeRefApi].args.head, q"$cacheName").map { subTree =>
//pTree is a tree of an Option, so we get and recurse, or write absent
q"""
if($pTree.isDefined) {
val $cacheName = $pTree.get
$subTree
} else {
${fsetter.absent(c)(0, container)}
}
"""
}
case _ => None
}
}

// in TupleSetterImpl, the outer-most input val is called t, so we pass that in here:
innerLoop(T.tpe, q"t").map { resTree =>
(1, resTree)
sealed trait SetterBuilder {
def columns: Int
/**
* This Tree assumes that "val $value = ..." has been set
*/
def setTree(value: Tree, offset: Int): Tree
}
case class PrimitiveSetter(tpe: Type) extends SetterBuilder {
def columns = 1
def setTree(value: Tree, offset: Int) = fsetter.from(c)(tpe, offset, container, value) match {
case Success(tree) => tree
case Failure(e) => c.abort(c.enclosingPosition,
s"Case class ${T} is supported. Error on $tpe, ${e.getMessage}")
}
}

maybeHandlePrimitive.getOrElse {
if (!IsCaseClassImpl.isCaseClassType(c)(T.tpe))
c.abort(c.enclosingPosition,
s"""|We cannot enforce ${T.tpe} is a case class, either it is not a
|case class or this macro call is possibly enclosed in a class.
|This will mean the macro is operating on a non-resolved type.
|Issue when building Setter.""".stripMargin)

@annotation.tailrec
def normalized(tpe: Type): Type = {
val norm = tpe.normalize
if (!(norm =:= tpe))
normalized(norm)
else
tpe
case object DefaultSetter extends SetterBuilder {
def columns = 1
def setTree(value: Tree, offset: Int) = fsetter.default(c)(offset, container, value)
}
case class OptionSetter(inner: SetterBuilder) extends SetterBuilder {
def columns = inner.columns
def setTree(value: Tree, offset: Int) = {
val someVal = newTermName(c.fresh(s"someVal"))
val someValTree = q"$someVal"
q"""if($value.isDefined) {
val $someVal = $value.get
${inner.setTree(someValTree, offset)}
} else {
${fsetter.absent(c)(offset, container)}
}"""
}

/*
* For a given outerType to be written at position idx, that has been stored in val named
* pTree, return the next position to write into, and the Tree to do the writing of the Type
*/
def matchField(outerTpe: Type, idx: Int, pTree: Tree): (Int, Tree) = {
val typedSetter: scala.util.Try[c.Tree] = fsetter.from(c)(outerTpe, idx, container, pTree)
(outerTpe, typedSetter) match {
case (_, Success(setter)) =>
// use type-specific setter if present
(idx + 1, setter)
case (tpe, _) if tpe.erasure =:= typeOf[Option[Any]] =>
val cacheName = newTermName(c.fresh(s"optiIndx"))
// Recurse on the inner type
val (newIdx, subTree) = matchField(tpe.asInstanceOf[TypeRefApi].args.head, idx, q"$cacheName")
// If we are absent, we use these setters to mark all fields null
val nullSetters = (idx until newIdx).map { curIdx =>
fsetter.absent(c)(idx, container)
}

(newIdx, q"""
if($pTree.isDefined) {
val $cacheName = $pTree.get
$subTree
} else {
..$nullSetters
}
""")

case (tpe, _) if (tpe.typeSymbol.isClass && tpe.typeSymbol.asClass.isCaseClass) =>
expandMethod(normalized(tpe), idx, pTree)
case (tpe, _) if allowUnknownTypes =>
// This just puts the value in directly
(idx + 1, fsetter.default(c)(idx, container, pTree))
case _ =>
c.abort(c.enclosingPosition,
s"Case class ${T} is not pure primitives, Option of a primitive nested case classes, when building Setter")
}
case class CaseClassSetter(members: Vector[(Tree => Tree, SetterBuilder)]) extends SetterBuilder {
val columns = members.map(_._2.columns).sum
def setTree(value: Tree, offset: Int) = {
val setters = members.scanLeft((offset, Option.empty[Tree])) {
case ((off, _), (access, sb)) =>
val cca = newTermName(c.fresh(s"access"))
val ccaT = q"$cca"
(off + sb.columns, Some(q"val $cca = ${access(value)}; ${sb.setTree(ccaT, off)}"))
}
.collect { case (_, Some(tree)) => tree }
q"""..$setters"""
}
}

/*
* For a given outerType to be written at position parentIdx, that has been stored in val named
* pTree, return the next position to write into, and the Tree to do the writing of the Type
*/
def expandMethod(outerTpe: Type, parentIdx: Int, pTree: Tree): (Int, Tree) =
outerTpe
.declarations
.collect { case m: MethodSymbol if m.isCaseAccessor => m }
.foldLeft((parentIdx, q"")) {
case ((idx, existingTree), accessorMethod) =>
val fieldType = normalized(accessorMethod.returnType.asSeenFrom(outerTpe, outerTpe.typeSymbol.asClass))

val (newIdx, subTree) = matchField(fieldType, idx, q"""$pTree.$accessorMethod""")
(newIdx, q"""
$existingTree
$subTree""")
}
@annotation.tailrec
def normalized(tpe: Type): Type = {
val norm = tpe.normalize
if (!(norm =:= tpe))
normalized(norm)
else
tpe
}

// in TupleSetterImpl, the outer-most input val is called t, so we pass that in here:
val (finalIdx, set) = expandMethod(normalized(T.tpe), 0, q"t")
if (finalIdx == 0) c.abort(c.enclosingPosition, "Didn't consume any elements in the tuple, possibly empty case class?")
(finalIdx, set)
def matchField(outerType: Type): SetterBuilder = {
// we do this just to see if the setter matches.
val dummyIdx = 0
val dummyTree = q"t"
outerType match {
case tpe if fsetter.from(c)(tpe, dummyIdx, container, dummyTree).isSuccess =>
PrimitiveSetter(tpe)
case tpe if tpe.erasure =:= typeOf[Option[Any]] =>
val innerType = tpe.asInstanceOf[TypeRefApi].args.head
OptionSetter(matchField(innerType))
case tpe if (tpe.typeSymbol.isClass && tpe.typeSymbol.asClass.isCaseClass) =>
CaseClassSetter(expandMethod(normalized(tpe)).map {
case (fn, tpe) =>
(fn, matchField(tpe))
})
case tpe if allowUnknownTypes =>
DefaultSetter
case _ =>
c.abort(c.enclosingPosition,
s"Case class ${T.tpe} is not supported at type: $outerType")
}
}
def expandMethod(outerTpe: Type): Vector[(Tree => Tree, Type)] =
outerTpe
.declarations
.collect { case m: MethodSymbol if m.isCaseAccessor => m }
.map { accessorMethod =>
val fieldType = normalized(accessorMethod.returnType.asSeenFrom(outerTpe, outerTpe.typeSymbol.asClass))

({ pTree: Tree => q"""$pTree.$accessorMethod""" }, fieldType)
}
.toVector

// in TupleSetterImpl, the outer-most input val is called t, so we pass that in here:
val sb = matchField(normalized(T.tpe))
if (sb.columns == 0) c.abort(c.enclosingPosition, "Didn't consume any elements in the tuple, possibly empty case class?")
(sb.columns, sb.setTree(q"t", 0))
}
}
189 changes: 87 additions & 102 deletions scalding-core/src/main/scala/com/twitter/scalding/macros/impl/FieldsProviderImpl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,112 +80,97 @@ object FieldsProviderImpl {
def toFieldsCommonImpl[T](c: Context, namingScheme: NamingScheme, allowUnknownTypes: Boolean)(implicit T: c.WeakTypeTag[T]): c.Expr[cascading.tuple.Fields] = {
import c.universe._

/**
* This returns the a Tree expressing the Class[_] to use for T if T is primitive or Option of
* primitive
* If we are in an Option, we return classOf[Object] since otherwise cascading will convert
* nulls or empty strings to numeric zeros.
* This is only applied
*/
val maybeHandlePrimitive: Option[Tree] = {
def innerLoop(tpe: Type, inOption: Boolean): Option[Tree] = {
val returningType = if (inOption) q"""classOf[java.lang.Object]""" else q"""classOf[${T.tpe}]"""
val simpleRet = Some(returningType)

tpe match {
case tpe if tpe =:= typeOf[String] => simpleRet
case tpe if tpe =:= typeOf[Boolean] => simpleRet
case tpe if tpe =:= typeOf[Short] => simpleRet
case tpe if tpe =:= typeOf[Int] => simpleRet
case tpe if tpe =:= typeOf[Long] => simpleRet
case tpe if tpe =:= typeOf[Float] => simpleRet
case tpe if tpe =:= typeOf[Double] => simpleRet
case tpe if tpe.erasure =:= typeOf[Option[Any]] && inOption == true =>
c.abort(c.enclosingPosition, s"Case class ${T} has nested options, not supported currently.")
case tpe if tpe.erasure =:= typeOf[Option[Any]] =>
val innerType = tpe.asInstanceOf[TypeRefApi].args.head
innerLoop(innerType, true)

case tpe => None
}
}
innerLoop(T.tpe, false)
}

val flattened: Either[List[(Tree, String)], List[(Tree, Int)]] =
maybeHandlePrimitive
.map { t => Right(List((t, 0))) }
.getOrElse {
if (!IsCaseClassImpl.isCaseClassType(c)(T.tpe))
c.abort(c.enclosingPosition,
s"""|We cannot enforce ${T.tpe} is a case class, either it is not a case class or this macro call
|is possibly enclosed in a class.
|This will mean the macro is operating on a non-resolved type. Issue when building Fields Provider.""".stripMargin)

/**
* This returns a List of pairs which flatten fieldType into (class, name) pairs
*/
def matchField(fieldType: Type, outerName: Option[String], fieldName: String, isOption: Boolean): List[(Tree, String)] = {
val returningType = if (isOption) q"""classOf[java.lang.Object]""" else q"""classOf[$fieldType]"""
val simpleRet = outerName match {
case Some(outer) => List((returningType, s"$outer$fieldName"))
case None => List((returningType, s"$fieldName"))
}
fieldType match {
case tpe if tpe =:= typeOf[String] => simpleRet
case tpe if tpe =:= typeOf[Boolean] => simpleRet
case tpe if tpe =:= typeOf[Short] => simpleRet
case tpe if tpe =:= typeOf[Int] => simpleRet
case tpe if tpe =:= typeOf[Long] => simpleRet
case tpe if tpe =:= typeOf[Float] => simpleRet
case tpe if tpe =:= typeOf[Double] => simpleRet
case tpe if tpe.erasure =:= typeOf[Option[Any]] && isOption == true =>
c.abort(c.enclosingPosition, s"Case class ${T} has nested options, not supported currently.")
case tpe if tpe.erasure =:= typeOf[Option[Any]] =>
val innerType = tpe.asInstanceOf[TypeRefApi].args.head
matchField(innerType, outerName, fieldName, true)
case tpe if (tpe.typeSymbol.isClass && tpe.typeSymbol.asClass.isCaseClass) =>
val prefix = outerName.map(pre => s"$pre$fieldName.")
expandMethod(tpe, prefix, isOption)
case tpe if allowUnknownTypes => simpleRet
case _ =>
c.abort(c.enclosingPosition, s"Case class ${T} is not pure primitives or nested case classes")
}
import TypeDescriptorProviderImpl.{ optionInner, evidentColumn }

def isNumbered(t: Type): Boolean =
t match {
case tpe if tpe =:= typeOf[Boolean] => true
case tpe if tpe =:= typeOf[Short] => true
case tpe if tpe =:= typeOf[Int] => true
case tpe if tpe =:= typeOf[Long] => true
case tpe if tpe =:= typeOf[Float] => true
case tpe if tpe =:= typeOf[Double] => true
case tpe if tpe =:= typeOf[String] => true
case tpe =>
optionInner(c)(tpe) match {
case Some(t) => isNumbered(t)
case None => false
}
}

def expandMethod(outerTpe: Type, outerName: Option[String], isOption: Boolean): List[(Tree, String)] =
outerTpe
.declarations
.collect { case m: MethodSymbol if m.isCaseAccessor => m }
.flatMap { accessorMethod =>
val fieldName = accessorMethod.name.toTermName.toString
val fieldType = accessorMethod.returnType.asSeenFrom(outerTpe, outerTpe.typeSymbol.asClass)

matchField(fieldType, outerName, fieldName, isOption)
}.toList

val prefix = if (namingScheme == NamedNoPrefix) None else Some("")
val expanded = expandMethod(T.tpe, prefix, false)
if (expanded.isEmpty) c.abort(c.enclosingPosition, s"Case class ${T} has no primitive types we were able to extract")

Left(expanded)
object FieldBuilder {
// This is method on the object to work around this compiler bug: SI-6231
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That bug was fixed for 2.10.0 by the link ?

def toFieldsTree(fb: FieldBuilder, scheme: NamingScheme): Tree = {
val nameTree = scheme match {
case Indexed =>
val indices = fb.names.zipWithIndex.map(_._2)
q"""_root_.scala.Array.apply[_root_.java.lang.Comparable[_]](..$indices)"""
case _ =>
q"""_root_.scala.Array.apply[_root_.java.lang.Comparable[_]](..${fb.names})"""
}

val typeTrees = flattened.fold({ list => list.map(_._1) }, { list => list.map(_._1) })
val namesOrIds = if (namingScheme == NamedWithPrefix || namingScheme == NamedNoPrefix) {
flattened match {
case Left(fieldNames) =>
q"""_root_.scala.Array.apply[_root_.java.lang.Comparable[_]](..${fieldNames.map(_._2)})"""
case Right(fieldIds) =>
q"""_root_.scala.Array.apply[_root_.java.lang.Comparable[_]](..${fieldIds.map(_._2)})"""
q"""new _root_.cascading.tuple.Fields($nameTree,
_root_.scala.Array.apply[_root_.java.lang.reflect.Type](..${fb.columnTypes}))
"""
}
} else {
val indices = typeTrees.zipWithIndex.map(_._2)
q"""_root_.scala.Array.apply[_root_.java.lang.Comparable[_]](..$indices)"""
}
c.Expr[cascading.tuple.Fields](q"""
new _root_.cascading.tuple.Fields($namesOrIds,
_root_.scala.Array.apply[_root_.java.lang.reflect.Type](..$typeTrees))
""")
sealed trait FieldBuilder {
def columnTypes: Vector[Tree]
def names: Vector[String]
}
case class Primitive(name: String, tpe: Type) extends FieldBuilder {
def columnTypes = Vector(q"""_root_.scala.Predef.classOf[$tpe]""")
def names = Vector(name)
}
case class OptionBuilder(of: FieldBuilder) extends FieldBuilder {
// Options just use Object as the type, due to the way cascading works on number types
def columnTypes = of.columnTypes.map(_ => q"""_root_.scala.Predef.classOf[_root_.java.lang.Object]""")
def names = of.names
}
case class CaseClassBuilder(prefix: String, members: Vector[FieldBuilder]) extends FieldBuilder {
def columnTypes = members.flatMap(_.columnTypes)
def names = for {
member <- members
name <- member.names
} yield if (namingScheme == NamedWithPrefix && prefix.nonEmpty) s"$prefix.$name" else name
}

/**
* This returns a List of pairs which flatten fieldType into (class, name) pairs
*/
def matchField(fieldType: Type, name: String): FieldBuilder =
fieldType match {
case tpe if tpe =:= typeOf[String] => Primitive(name, tpe)
case tpe if tpe =:= typeOf[Boolean] => Primitive(name, tpe)
case tpe if tpe =:= typeOf[Short] => Primitive(name, tpe)
case tpe if tpe =:= typeOf[Int] => Primitive(name, tpe)
case tpe if tpe =:= typeOf[Long] => Primitive(name, tpe)
case tpe if tpe =:= typeOf[Float] => Primitive(name, tpe)
case tpe if tpe =:= typeOf[Double] => Primitive(name, tpe)
case tpe if tpe.erasure =:= typeOf[Option[Any]] =>
val innerType = tpe.asInstanceOf[TypeRefApi].args.head
OptionBuilder(matchField(innerType, name))
case tpe if (tpe.typeSymbol.isClass && tpe.typeSymbol.asClass.isCaseClass) =>
CaseClassBuilder(name, expandMethod(tpe).map { case (t, s) => matchField(t, s) })
case tpe if allowUnknownTypes => Primitive(name, tpe)
case tpe =>
c.abort(c.enclosingPosition, s"${T.tpe} is unsupported at $tpe")
}

def expandMethod(outerTpe: Type): Vector[(Type, String)] =
outerTpe
.declarations
.collect { case m: MethodSymbol if m.isCaseAccessor => m }
.map { accessorMethod =>
val fieldName = accessorMethod.name.toTermName.toString
val fieldType = accessorMethod.returnType.asSeenFrom(outerTpe, outerTpe.typeSymbol.asClass)
(fieldType, fieldName)
}.toVector

val builder = matchField(T.tpe, "")
if (builder.columnTypes.isEmpty)
c.abort(c.enclosingPosition, s"Case class ${T.tpe} has no primitive types we were able to extract")
val scheme = if (isNumbered(T.tpe)) Indexed else namingScheme
val tree = FieldBuilder.toFieldsTree(builder, scheme)
c.Expr[cascading.tuple.Fields](tree)
}
}
Loading