Merge pull request #29 in BACIBBD/spline from feature/SL-62-rest-laye…

…r to feature/SL-43-new-data-model

* commit '2b75005016c6c9c0d5f059a93b2b5935dd3cbdf7':
  SL-62 + scala doc
  SL-62 Fix REST serialization by switching from Salat JSON to Json4s impl

commons/pom.xml

@@ -33,6 +33,10 @@
             <groupId>commons-configuration</groupId>
             <artifactId>commons-configuration</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-reflect</artifactId>
+        </dependency>
     </dependencies>
 
 </project>

web/src/main/scala/za/co/absa/spline/web/salat/JSONSalatContext.scala → commons/src/main/scala/za/co/absa/spline/common/ReflectionUtils.scala

@@ -14,22 +14,26 @@
  * limitations under the License.
  */
 
-package za.co.absa.spline.web.salat
+package za.co.absa.spline.common
 
-import java.util.UUID
+import scala.reflect.runtime.{universe => ru}
 
-import salat.transformers.CustomTransformer
-import za.co.absa.spline.persistence.mongo.serialization.CommonSalatContext
+/**
+  * Reflection utils
+  */
+object ReflectionUtils {
 
-object JSONSalatContext {
-  implicit val ctx = new salat.Context with CommonSalatContext {
-    override val name: String = "JSON Salat Context"
+  private val mirror: ru.Mirror = ru.runtimeMirror(getClass.getClassLoader)
 
-    registerCustomTransformer(new CustomTransformer[UUID, String]() {
-      override def serialize(uuid: UUID): String = uuid.toString
-
-      override def deserialize(str: String): UUID = UUID.fromString(str)
-    })
+  /**
+    * Lists all direct sub-classes of the given trait T
+    *
+    * @tparam T sealed trait type
+    * @return List of Class[_] instances
+    */
+  def subClassesOf[T: ru.TypeTag]: List[Class[_]] = {
+    val clazz: ru.ClassSymbol = ru.typeOf[T].typeSymbol.asClass
+    require(clazz.isTrait && clazz.isSealed)
+    clazz.knownDirectSubclasses.toList map ((s: ru.Symbol) => mirror runtimeClass s.asClass)
   }
 }
-

model/src/main/scala/za/co/absa/spline/model/expr/Expression.scala

@@ -83,7 +83,7 @@ case class Alias
   dataType: DataType,
   children: Seq[Expression]
 ) extends Expression {
-  @Persist
+
   val exprType: String = "Alias"
 
   override def outputAttributeNames: Seq[String] = Seq(alias)
@@ -121,7 +121,6 @@ case class AttributeRemoval
   children: Seq[Expression]
 ) extends Expression {
 
-  @Persist
   val exprType: String = "AttributeRemoval"
 }
 
@@ -156,10 +155,8 @@ case class AttributeReference
   dataType: DataType
 ) extends Expression {
 
-  @Persist
   val exprType: String = "AttributeReference"
 
-  @Persist
   val children: Seq[Expression] = Seq.empty
 
   override def inputAttributeNames: Seq[String] = Seq(name)
@@ -206,6 +203,5 @@ case class UserDefinedFunction
   children: Seq[Expression]
 ) extends Expression {
 
-  @Persist
   val exprType: String = "UserDefinedFunction"
 }

parent/pom.xml

@@ -70,7 +70,7 @@
         <spark.version>2.2.0</spark.version>
         <logback.version>1.2.2</logback.version>
         <slf4j.version>1.7.25</slf4j.version>
-        <salat.version>1.10.0</salat.version>
+        <salat.version>1.11.2</salat.version>
         <!-- Don't upgrade json4s.version to newer version. An upgrade would cause binary incompatibility of json4s between Spark and Spline. -->
         <json4s.version>3.2.11</json4s.version>
     </properties>
@@ -197,7 +197,6 @@
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>
-            <version>${scala.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
@@ -209,6 +208,26 @@
 
     <dependencyManagement>
         <dependencies>
+            <dependency>
+                <groupId>org.scala-lang</groupId>
+                <artifactId>scala-compiler</artifactId>
+                <version>${scala.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.scala-lang</groupId>
+                <artifactId>scala-reflect</artifactId>
+                <version>${scala.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.scala-lang</groupId>
+                <artifactId>scala-library</artifactId>
+                <version>${scala.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.scala-lang</groupId>
+                <artifactId>scalap</artifactId>
+                <version>${scala.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.apache.spark</groupId>
                 <artifactId>spark-core_${scala.compat.version}</artifactId>

persistence/hdfs/src/main/scala/za/co/absa/spline/persistence/hdfs/serialization/JSONSerialization.scala

@@ -21,9 +21,11 @@ import org.json4s.jackson.Serialization
 import org.json4s.jackson.Serialization.write
 
 object JSONSerialization {
-  implicit val formats = Serialization.formats(NoTypeHints) ++ org.json4s.ext.JavaTypesSerializers.all
+
+  implicit val formats: Formats = Serialization.formats(NoTypeHints) ++ org.json4s.ext.JavaTypesSerializers.all
 
   implicit class EntityToJson[T <: AnyRef](entity: T) {
     def toJson: String = write(entity)
   }
+
 }

web/pom.xml

@@ -63,9 +63,12 @@
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>
-            <version>${scala.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>org.json4s</groupId>
+            <artifactId>json4s-ext_${scala.compat.version}</artifactId>
+        </dependency>
         <dependency>
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>

web/src/main/scala/za/co/absa/spline/web/html/controller/MainController.scala

@@ -25,7 +25,7 @@ import za.co.absa.spline.common.ARMImplicits
 @Controller
 class MainController {
 
-  @RequestMapping(path = Array("/", "/lineage/**"), method = Array(GET, HEAD))
+  @RequestMapping(path = Array("/", "/lineage/**", "/dashboard/**"), method = Array(GET, HEAD))
   def index = "index"
 
   @RequestMapping(path = Array("/build-info"), method = Array(GET), produces = Array("text/x-java-properties"))

web/src/main/scala/za/co/absa/spline/web/json/StringJSONConverters.scala

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2017 Barclays Africa Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.spline.web.json
+
+import java.net.URI
+
+import org.json4s.ext.UUIDSerializer
+import org.json4s.native.Serialization
+import org.json4s.{CustomSerializer, DateFormat, DefaultFormats, Formats, FullTypeHints, JNull, JString, Serializer}
+import za.co.absa.spline.common.ReflectionUtils.subClassesOf
+import za.co.absa.spline.common.TypeFreaks._
+import za.co.absa.spline.model._
+
+/**
+  * Implicit JSON serializer/deserializer
+  * <p>
+  * Usage examples:
+  * <code><pre>
+  * import StringJSONConverters._
+  *
+  * // Serialize object to JSON
+  * val myObject: FooBar = ???
+  * val json:String == myObject.toJson
+  *
+  * // Deserialize from JSON
+  * val jsonString: String = """{ "foo":42, "bar":777 }"""
+  * val myObject: FooBar = jsonString.fromJson[FooBar]
+  * </pre></code>
+  */
+object StringJSONConverters {
+
+  object URISerializer extends CustomSerializer[URI](_ =>
+    ( {
+      case JString(s) => new URI(s)
+      case JNull => null
+    }, {
+      case uri: URI => JString(uri.toString)
+    }))
+
+  object SplineFormats extends Formats {
+    val dateFormat: DateFormat = DefaultFormats.lossless.dateFormat
+
+    override def typeHintFieldName: String = "_typeHint"
+
+    override val typeHints = FullTypeHints(
+      subClassesOf[op.Operation]
+        ++ subClassesOf[expr.Expression]
+        ++ subClassesOf[dt.DataType])
+
+    override val customSerializers: List[Serializer[_]] =
+      UUIDSerializer :: URISerializer :: super.customSerializers
+  }
+
+  implicit val formats: Formats = SplineFormats
+
+  implicit class JsonToModel(json: String) {
+
+    def fromJson[T <: AnyRef : Manifest]: T = Serialization.read(json)
+
+    def fromJsonArray[T <: AnyRef : Manifest]: Seq[T] = Serialization.read[Seq[T]](json)
+  }
+
+  implicit class CollectionToJson[T <: AnyRef : Manifest](xs: Traversable[T]) {
+    def toJsonArray: String = Serialization.write(xs)
+  }
+
+  implicit class EntityToJson[T <: AnyRef : `not a subtype of`[Traversable[_]]#λ : Manifest](entity: T) {
+    def toJson: String = Serialization.write(entity)
+  }
+
+
+}

web/src/main/scala/za/co/absa/spline/web/rest/controller/ErrorControllerAdvice.scala

@@ -17,8 +17,7 @@
 package za.co.absa.spline.web.rest.controller
 
 import za.co.absa.spline.web.logging.ErrorCode
-import za.co.absa.spline.web.salat.JSONSalatContext._
-import za.co.absa.spline.web.salat.StringJSONConverters.EntityToJson
+import za.co.absa.spline.web.json.StringJSONConverters.EntityToJson
 import org.springframework.http.HttpStatus.{INTERNAL_SERVER_ERROR, NOT_FOUND}
 import org.springframework.http.ResponseEntity
 import org.springframework.web.bind.annotation.{ControllerAdvice, ExceptionHandler}

web/src/main/scala/za/co/absa/spline/web/rest/controller/LineageController.scala

@@ -17,9 +17,7 @@
 package za.co.absa.spline.web.rest.controller
 
 import java.util.UUID
-
-import za.co.absa.spline.web.salat.JSONSalatContext._
-import za.co.absa.spline.web.salat.StringJSONConverters
+import za.co.absa.spline.web.json.StringJSONConverters
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Controller
 import org.springframework.web.bind.annotation.RequestMethod._
@@ -28,6 +26,7 @@ import za.co.absa.spline.persistence.api.DataLineageReader
 
 import scala.concurrent.Await
 import scala.concurrent.duration._
+import scala.language.postfixOps
 
 @Controller
 class LineageController @Autowired()
@@ -37,7 +36,7 @@ class LineageController @Autowired()
 
   import StringJSONConverters._
 
-  @RequestMapping(path = Array("/lineage/descriptors"), method = Array(GET))
+  @RequestMapping(path = Array("/dataset/descriptors"), method = Array(GET))
   @ResponseBody
   def lineageDescriptors: String = Await.result(reader.list(), 10 seconds).toSeq.toJsonArray