Merge pull request #25 in BACIBBD/spline from feature/SL-61 to featur…

…e/SL-43-new-data-model * commit '990b5674374999f8e2c8fff8c7c7a319f4f73fc8': SL-61 Angular/material: Replace 'MD' suffixes to 'MAT' according to the recent change in the library (see angular/components#7241) SL-61 Redo graph visualization + fix types + fix server side unit tests
AbsaOSS · Oct 9, 2017 · 40358ee · 40358ee
2 parents 3545f3c + 990b567
commit 40358ee
Show file tree

Hide file tree

Showing 60 changed files with 2,039 additions and 1,526 deletions.
diff --git a/model/src/main/scala/za/co/absa/spline/model/PersistedDatasetDescriptor.scala b/model/src/main/scala/za/co/absa/spline/model/PersistedDatasetDescriptor.scala
@@ -16,7 +16,7 @@
 
 package za.co.absa.spline.model
 
-import java.net.URL
+import java.net.URI
 import java.util.UUID
 
 /**
@@ -35,6 +35,6 @@ case class PersistedDatasetDescriptor
   appId: String,
   appName: String,
   lineageId: UUID,
-  path: URL,
+  path: URI,
   timestamp: Long
 )
diff --git a/model/src/main/scala/za/co/absa/spline/model/expr/Expression.scala b/model/src/main/scala/za/co/absa/spline/model/expr/Expression.scala
@@ -56,7 +56,7 @@ sealed trait Expression {
   * The case class represents Spark expressions for which a dedicated expression node type hasn't been created yet.
   *
   * @param exprType see [[za.co.absa.spline.model.expr.Expression#exprType Expression.exprType]]
-  * @param text     see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
+  * @param text     see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
   * @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
   * @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
   */
@@ -72,7 +72,7 @@ case class Generic
   * The case class represents renaming of an underlying expression to a specific alias.
   *
   * @param alias    A final name of the expression
-  * @param text     see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
+  * @param text     see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
   * @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
   * @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
   */
@@ -94,7 +94,7 @@ case class Alias
   *
   * @param exprType see [[za.co.absa.spline.model.expr.Expression#exprType Expression.exprType]]
   * @param symbol   A symbol expressing the operation (+, -, *, /, etc. )
-  * @param text     see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
+  * @param text     see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
   * @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
   * @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
   */
@@ -110,7 +110,7 @@ case class Binary
 /**
   * The case class represents a special expression for removing an attribute from a data set.
   *
-  * @param text     see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
+  * @param text     see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
   * @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
   * @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
   */
@@ -145,7 +145,7 @@ object AttributeRemoval {
   *
   * @param refId    An unique of a referenced attribute
   * @param name     A name of a referenced attribute
-  * @param text     see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
+  * @param text     see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
   * @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
   */
 case class AttributeReference
@@ -194,7 +194,7 @@ object AttributeReference {
   * The case class represents a special expression describing an user-defined function of Spark.
   *
   * @param name     A name assigned to an user-defined function
-  * @param text     see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
+  * @param text     see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
   * @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
   * @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
   */

diff --git a/...api/src/main/scala/za/co/absa/spline/persistence/api/serialization/BSONSalatContext.scala b/...api/src/main/scala/za/co/absa/spline/persistence/api/serialization/BSONSalatContext.scala
@@ -28,6 +28,5 @@ object BSONSalatContext {
     override val name: String = "BSON Salat Context"
 
     registerGlobalKeyOverride("id", "_id")
-    registerGlobalKeyOverride("dataLineageID", "_id")
   }
 }
diff --git a/persistence/atlas/src/main/atlas/spline-meta-model.json b/persistence/atlas/src/main/atlas/spline-meta-model.json
@@ -387,7 +387,7 @@
           "isUnique": false
         },
         {
-          "name": "textualRepresentation",
+          "name": "text",
           "typeName": "string",
           "cardinality": "SINGLE",
           "isIndexable": false,

diff --git a/persistence/atlas/src/main/scala/za/co/absa/spline/persistence/atlas/model/Expression.scala b/persistence/atlas/src/main/scala/za/co/absa/spline/persistence/atlas/model/Expression.scala
@@ -27,15 +27,15 @@ import scala.collection.JavaConverters._
 /**
   * The case case represents common properties for all expression types
   * @param qualifiedName An unique expression
-  * @param textualRepresentation A textual representation of the expression
+  * @param text A textual representation of the expression
   * @param expressionType A type of the expression
   * @param dataType A data type associated with the expression
   * @param children A sequence of sub-expressions
   */
 case class ExpressionCommonProperties
 (
   qualifiedName : String,
-  textualRepresentation: String,
+  text: String,
   expressionType: String,
   dataType: DataType,
   children: Seq[Expression]
@@ -54,9 +54,9 @@ class Expression(
 ) extends Referenceable (
   entityType,
   new java.util.HashMap[String, Object]{
-    put(AtlasClient.NAME, commonProperties.textualRepresentation)
+    put(AtlasClient.NAME, commonProperties.text)
     put(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, commonProperties.qualifiedName)
-    put("textualRepresentation", commonProperties.textualRepresentation)
+    put("text", commonProperties.text)
     put("expressionType", commonProperties.expressionType)
     put("dataType", commonProperties.dataType)
     put("children", commonProperties.children.asJava)

diff --git a/...nce/mongo/src/main/scala/za/co/absa/spline/persistence/mongo/MongoDataLineageReader.scala b/...nce/mongo/src/main/scala/za/co/absa/spline/persistence/mongo/MongoDataLineageReader.scala
@@ -16,15 +16,16 @@
 
 package za.co.absa.spline.persistence.mongo
 
+import java.util.Arrays.asList
 import java.util.UUID
 
-import com.mongodb.casbah.Imports._
 import _root_.salat._
+import com.mongodb.casbah.Imports._
+import za.co.absa.spline.common.FutureImplicits._
 import za.co.absa.spline.model.{DataLineage, PersistedDatasetDescriptor}
 import za.co.absa.spline.persistence.api.DataLineageReader
-import scala.collection.JavaConverters._
-import za.co.absa.spline.common.FutureImplicits._
 
+import scala.collection.JavaConverters._
 import scala.concurrent.Future
 
 /**
@@ -52,9 +53,24 @@ class MongoDataLineageReader(connection: MongoConnection) extends DataLineageRea
     * @return Descriptors of all data lineages
     */
   override def list(): Future[Iterator[PersistedDatasetDescriptor]] = Future {
+    val caseClassFields = classOf[PersistedDatasetDescriptor].getDeclaredFields map (_.getName)
+    val auxiliaryFields = Array("_ver")
+    val fieldsToFetch = caseClassFields ++ auxiliaryFields map (_ -> 1)
+
     connection.dataLineageCollection
-      .find(DBObject(), DBObject("_id" -> 1, "_ver" -> 1, "appId" -> 1, "appName" -> 1, "timestamp" -> 1))
-      .iterator.asScala
+      .aggregate(asList(
+        DBObject("$addFields" → DBObject(
+          "___rootDS" → DBObject("$arrayElemAt" → Array("$datasets", 0)),
+          "___rootOP" → DBObject("$arrayElemAt" → Array("$operations", 0))
+        )),
+        DBObject("$addFields" → DBObject(
+          "lineageId" → "$_id",
+          "datasetId" → "$___rootDS._id",
+          "path" → "$___rootOP.path"
+        )),
+        DBObject("$project" → DBObject(fieldsToFetch: _*))
+      ))
+      .results.iterator.asScala
       .map(withVersionCheck(grater[PersistedDatasetDescriptor].asObject(_)))
   }
 

diff --git a/.../test/scala/za/co/absa/spline/persistence/mongo/MongoDataLineagePersistenceSpecBase.scala b/.../test/scala/za/co/absa/spline/persistence/mongo/MongoDataLineagePersistenceSpecBase.scala
@@ -20,7 +20,7 @@ import java.util.UUID.randomUUID
 
 import org.scalatest.{AsyncFlatSpec, BeforeAndAfterEach, Matchers}
 import za.co.absa.spline.model.dt.Simple
-import za.co.absa.spline.model.op.{Generic, OperationProps}
+import za.co.absa.spline.model.op.{Destination, Generic, OperationProps}
 import za.co.absa.spline.model.{Attribute, Schema, _}
 
 abstract class MongoDataLineagePersistenceSpecBase extends AsyncFlatSpec with Matchers with BeforeAndAfterEach {
@@ -49,6 +49,7 @@ abstract class MongoDataLineagePersistenceSpecBase extends AsyncFlatSpec with Ma
       appName,
       123L,
       Seq(
+        Destination(OperationProps(randomUUID, "Destination", Seq(md1.id), md1.id), "parquet", "hdfs://foo/bar/path"),
         Generic(OperationProps(randomUUID, "Union", Seq(md1.id, md2.id), md3.id), "rawString1"),
         Generic(OperationProps(randomUUID, "Filter", Seq(md4.id), md2.id), "rawString2"),
         Generic(OperationProps(randomUUID, "LogicalRDD", Seq.empty, md4.id), "rawString3"),

diff --git a/...mongo/src/test/scala/za/co/absa/spline/persistence/mongo/MongoDataLineageReaderSpec.scala b/...mongo/src/test/scala/za/co/absa/spline/persistence/mongo/MongoDataLineageReaderSpec.scala
@@ -16,7 +16,7 @@
 
 package za.co.absa.spline.persistence.mongo
 
-import java.net.URL
+import java.net.URI
 
 import za.co.absa.spline.model.PersistedDatasetDescriptor
 import za.co.absa.spline.model.op.Destination
@@ -35,7 +35,7 @@ class MongoDataLineageReaderSpec extends MongoDataLineagePersistenceSpecBase{
       appId = l.appId,
       appName = l.appName,
       lineageId = l.id,
-      path = new URL(l.rootNode.asInstanceOf[Destination].path),
+      path = new URI(l.rootNode.asInstanceOf[Destination].path),
       timestamp = l.timestamp))
 
     val descriptions = Future.sequence(testLineages.map(i => mongoWriter.store(i))).flatMap(_ => mongoReader.list().map(_.toSeq))

diff --git a/web/pom.xml b/web/pom.xml
@@ -103,46 +103,119 @@
                 <version>1.5.0</version>
                 <executions>
                     <execution>
+                        <id>gen-ts-model-lineage</id>
                         <phase>process-classes</phase>
                         <goals>
                             <goal>java</goal>
                         </goals>
+                        <configuration>
+                            <includeProjectDependencies>true</includeProjectDependencies>
+                            <includePluginDependencies>true</includePluginDependencies>
+                            <executableDependency>
+                                <groupId>com.github.wajda</groupId>
+                                <artifactId>scala-ts_${scala.compat.version}</artifactId>
+                            </executableDependency>
+                            <mainClass>com.mpc.scalats.CLI</mainClass>
+                            <arguments>
+                                <argument>--out</argument>
+                                <argument>${project.basedir}/ui/src/generated-ts/lineage-model.ts</argument>
+                                <argument>--emit-interfaces</argument>
+                                <argument>--option-to-nullable</argument>
+                                <argument>za.co.absa.spline.model.PersistedDatasetDescriptor</argument>
+                                <argument>za.co.absa.spline.model.DataLineage</argument>
+                                <!-- workaround for a scala-ts bug -->
+                                <argument>za.co.absa.spline.model.op.OperationProps</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>gen-ts-model-operation</id>
+                        <phase>process-classes</phase>
+                        <goals>
+                            <goal>java</goal>
+                        </goals>
+                        <configuration>
+                            <includeProjectDependencies>true</includeProjectDependencies>
+                            <includePluginDependencies>true</includePluginDependencies>
+                            <executableDependency>
+                                <groupId>com.github.wajda</groupId>
+                                <artifactId>scala-ts_${scala.compat.version}</artifactId>
+                            </executableDependency>
+                            <mainClass>com.mpc.scalats.CLI</mainClass>
+                            <arguments>
+                                <argument>--out</argument>
+                                <argument>${project.basedir}/ui/src/generated-ts/operation-model.ts</argument>
+                                <argument>--emit-interfaces</argument>
+                                <argument>--option-to-nullable</argument>
+                                <argument>za.co.absa.spline.model.op.Operation</argument>
+                                <argument>za.co.absa.spline.model.op.Alias</argument>
+                                <argument>za.co.absa.spline.model.op.Destination</argument>
+                                <argument>za.co.absa.spline.model.op.Filter</argument>
+                                <argument>za.co.absa.spline.model.op.Generic</argument>
+                                <argument>za.co.absa.spline.model.op.Join</argument>
+                                <argument>za.co.absa.spline.model.op.Projection</argument>
+                                <argument>za.co.absa.spline.model.op.Source</argument>
+                                <!-- workaround for a scala-ts bug -->
+                                <argument>za.co.absa.spline.model.dt.DataType</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>gen-ts-model-expression</id>
+                        <phase>process-classes</phase>
+                        <goals>
+                            <goal>java</goal>
+                        </goals>
+                        <configuration>
+                            <includeProjectDependencies>true</includeProjectDependencies>
+                            <includePluginDependencies>true</includePluginDependencies>
+                            <executableDependency>
+                                <groupId>com.github.wajda</groupId>
+                                <artifactId>scala-ts_${scala.compat.version}</artifactId>
+                            </executableDependency>
+                            <mainClass>com.mpc.scalats.CLI</mainClass>
+                            <arguments>
+                                <argument>--out</argument>
+                                <argument>${project.basedir}/ui/src/generated-ts/expression-model.ts</argument>
+                                <argument>--emit-interfaces</argument>
+                                <argument>--option-to-nullable</argument>
+                                <argument>za.co.absa.spline.model.expr.Expression</argument>
+                                <argument>za.co.absa.spline.model.expr.AttributeRemoval</argument>
+                                <argument>za.co.absa.spline.model.expr.AttributeReference</argument>
+                                <argument>za.co.absa.spline.model.expr.Generic</argument>
+                                <argument>za.co.absa.spline.model.expr.Alias</argument>
+                                <argument>za.co.absa.spline.model.expr.Binary</argument>
+                                <argument>za.co.absa.spline.model.expr.UserDefinedFunction</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>gen-ts-model-datatype</id>
+                        <phase>process-classes</phase>
+                        <goals>
+                            <goal>java</goal>
+                        </goals>
+                        <configuration>
+                            <includeProjectDependencies>true</includeProjectDependencies>
+                            <includePluginDependencies>true</includePluginDependencies>
+                            <executableDependency>
+                                <groupId>com.github.wajda</groupId>
+                                <artifactId>scala-ts_${scala.compat.version}</artifactId>
+                            </executableDependency>
+                            <mainClass>com.mpc.scalats.CLI</mainClass>
+                            <arguments>
+                                <argument>--out</argument>
+                                <argument>${project.basedir}/ui/src/generated-ts/datatype-model.ts</argument>
+                                <argument>--emit-interfaces</argument>
+                                <argument>--option-to-nullable</argument>
+                                <argument>za.co.absa.spline.model.dt.DataType</argument>
+                                <argument>za.co.absa.spline.model.dt.Simple</argument>
+                                <argument>za.co.absa.spline.model.dt.Struct</argument>
+                                <argument>za.co.absa.spline.model.dt.Array</argument>
+                            </arguments>
+                        </configuration>
                     </execution>
                 </executions>
-                <configuration>
-                    <includeProjectDependencies>true</includeProjectDependencies>
-                    <includePluginDependencies>true</includePluginDependencies>
-                    <executableDependency>
-                        <groupId>com.github.wajda</groupId>
-                        <artifactId>scala-ts_${scala.compat.version}</artifactId>
-                    </executableDependency>
-                    <mainClass>com.mpc.scalats.CLI</mainClass>
-                    <arguments>
-                        <argument>--out</argument>
-                        <argument>${project.basedir}/ui/src/generated-ts/lineage-model.ts</argument>
-                        <argument>--emit-interfaces</argument>
-                        <argument>--option-to-nullable</argument>
-                        <argument>za.co.absa.spline.model.PersistedDatasetDescriptor</argument>
-                        <argument>za.co.absa.spline.model.DataLineage</argument>
-                        <argument>za.co.absa.spline.model.expr.AttributeRemoval</argument>
-                        <argument>za.co.absa.spline.model.expr.AttrRef</argument>
-                        <argument>za.co.absa.spline.model.expr.Generic</argument>
-                        <argument>za.co.absa.spline.model.expr.Alias</argument>
-                        <argument>za.co.absa.spline.model.expr.Binary</argument>
-                        <argument>za.co.absa.spline.model.expr.UDF</argument>
-                        <argument>za.co.absa.spline.model.op.Alias</argument>
-                        <argument>za.co.absa.spline.model.op.Destination</argument>
-                        <argument>za.co.absa.spline.model.op.Filter</argument>
-                        <argument>za.co.absa.spline.model.op.Generic</argument>
-                        <argument>za.co.absa.spline.model.op.Join</argument>
-                        <argument>za.co.absa.spline.model.op.Projection</argument>
-                        <argument>za.co.absa.spline.model.op.Source</argument>
-                        <argument>za.co.absa.spline.model.dt.DataType</argument>
-                        <argument>za.co.absa.spline.model.dt.Simple</argument>
-                        <argument>za.co.absa.spline.model.dt.Struct</argument>
-                        <argument>za.co.absa.spline.model.dt.Array</argument>
-                    </arguments>
-                </configuration>
                 <dependencies>
                     <dependency>
                         <groupId>com.github.wajda</groupId>