-
Notifications
You must be signed in to change notification settings - Fork 155
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #30 in BACIBBD/spline from feature/SL-43-new-data-…
…model to release/0.2 * commit '9c5cd9d43b8866d7be9646b8a8c66b8195107a7f': SL-62 + scala doc SL-62 Fix REST serialization by switching from Salat JSON to Json4s impl SL-61 Redo loading indicator SL-61 Redo nodes highlighting SL-61 Redo attribute selection SL-61 Redo full schema view SL-60 - Moving reference to salat-core from api to mongo module. SL-60 - Fixing binary incompatibility of json4s-native between hdfs and atlas persistence layers. SL-59 Updating comments of Dataset converter. SL-59 Fixing Atlas persistence layer SL-61 Angular/material: Replace 'MD' suffixes to 'MAT' according to the recent change in the library (see angular/components#7241) SL-61 Redo graph visualization + fix types + fix server side unit tests SL-59 Adding comment parameters SL-59 Refactoring Atlas persistence layer according to the latest Splline data model. SL-57 Refactoring harvester according the latest Spline data model SL-61 Start updating Web UI layer according to the new data model Fix SL-58 compilation errors SL-59 Renaming AtlasDataLineagePersistor to AtlasDataLineageWriter SL-58 Changing property name according to the naming convention for ids (yyyID -> yyyId) SL-58 Updating Mongo persistence layer according to the new data model SL-43 Data Set Oriented Data Model
- Loading branch information
Showing
149 changed files
with
9,439 additions
and
100,401 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
39 changes: 39 additions & 0 deletions
39
commons/src/main/scala/za/co/absa/spline/common/ReflectionUtils.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| /* | ||
| * Copyright 2017 Barclays Africa Group Limited | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package za.co.absa.spline.common | ||
|
|
||
| import scala.reflect.runtime.{universe => ru} | ||
|
|
||
| /** | ||
| * Reflection utils | ||
| */ | ||
| object ReflectionUtils { | ||
|
|
||
| private val mirror: ru.Mirror = ru.runtimeMirror(getClass.getClassLoader) | ||
|
|
||
| /** | ||
| * Lists all direct sub-classes of the given trait T | ||
| * | ||
| * @tparam T sealed trait type | ||
| * @return List of Class[_] instances | ||
| */ | ||
| def subClassesOf[T: ru.TypeTag]: List[Class[_]] = { | ||
| val clazz: ru.ClassSymbol = ru.typeOf[T].typeSymbol.asClass | ||
| require(clazz.isTrait && clazz.isSealed) | ||
| clazz.knownDirectSubclasses.toList map ((s: ru.Symbol) => mirror runtimeClass s.asClass) | ||
| } | ||
| } |
88 changes: 88 additions & 0 deletions
88
core/src/main/scala/za/co/absa/spline/core/AttributeFactory.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| * Copyright 2017 Barclays Africa Group Limited | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package za.co.absa.spline.core | ||
|
|
||
| import java.util.UUID | ||
| import java.util.UUID.randomUUID | ||
|
|
||
| import org.apache.spark.sql | ||
| import za.co.absa.spline.model.Attribute | ||
|
|
||
| import scala.collection.mutable.{Map, ListBuffer} | ||
|
|
||
| /** | ||
| * The class is responsible for creation of [[za.co.absa.spline.model.Attribute attributes]] and assigning them unique identifiers. | ||
| */ | ||
| class AttributeFactory extends DataTypeMapper{ | ||
| private val mapById : Map[UUID, Attribute] = Map() | ||
| private val mapBySparkId : Map[Long, Attribute] = Map() | ||
| private val allAttributes : ListBuffer[Attribute] = ListBuffer[Attribute]() | ||
|
|
||
| /** | ||
| * The method fills up the internal collection with initial sequence of attributes. | ||
| * @param sparkIds A sequence of unique identifiers provided by Spark | ||
| * @param attributes A sequence of attributes | ||
| */ | ||
| def initialize(sparkIds: Seq[Long], attributes: Seq[Attribute]) : Unit = | ||
| mapById.synchronized { | ||
| mapById.clear() | ||
| mapBySparkId.clear() | ||
| sparkIds.zip(attributes).foreach { | ||
| case (k, a) => { | ||
| mapBySparkId.put(k, a) | ||
| mapById.put(a.id, a) | ||
| allAttributes ++= attributes | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * The method creates an attribute if does not exist. Returns identifier to the attribute matching the input criteria. | ||
| * @param sparkAttributeId An unique identifier of the attribute assigned by Spark | ||
| * @param name A name of the attribute | ||
| * @param sparkDataType A Spark dataType related to the attribute | ||
| * @param nullable A flag expressing whether the attribute is nullable or not | ||
| * @return An unique identifier of the created attribute | ||
| */ | ||
| def getOrCreate(sparkAttributeId : Long, name : String, sparkDataType: sql.types.DataType, nullable: Boolean) : UUID = | ||
| mapById.synchronized( | ||
| mapBySparkId.get(sparkAttributeId) match { | ||
| case Some(x) => x.id | ||
| case None => { | ||
| val a = Attribute(randomUUID, name, fromSparkDataType(sparkDataType, nullable)) | ||
| mapBySparkId.put(sparkAttributeId, a) | ||
| mapById.put(a.id, a) | ||
| allAttributes += a | ||
| a.id | ||
| } | ||
| } | ||
| ) | ||
|
|
||
| /** | ||
| * The method returns an attribute for a specific identifier if has already been created by the factory. Otherwise, returns None. | ||
| * @param id An identifier of the attribute | ||
| * @return An option | ||
| */ | ||
| def getById(id : UUID) : Option[Attribute] = mapById.synchronized(mapById.get(id)) | ||
|
|
||
| /** | ||
| * The method returns all attributes created by the factory. | ||
| * @return A sequence of attributes | ||
| */ | ||
| def getAll(): Seq[Attribute] = mapById.synchronized(allAttributes.toList) | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
60 changes: 60 additions & 0 deletions
60
core/src/main/scala/za/co/absa/spline/core/MetaDatasetFactory.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| /* | ||
| * Copyright 2017 Barclays Africa Group Limited | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package za.co.absa.spline.core | ||
|
|
||
| import java.util.UUID | ||
| import java.util.UUID.randomUUID | ||
|
|
||
| import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan | ||
| import za.co.absa.spline.model.{MetaDataset, Schema} | ||
|
|
||
| import scala.collection.mutable.{ListBuffer, Map} | ||
|
|
||
| /** | ||
| * The class is responsible for creation of [[za.co.absa.spline.model.MetaDataset meta data sets]] and assigning them unique identifiers. | ||
| * @param attributeFactory An attribute factory | ||
| */ | ||
| class MetaDatasetFactory(val attributeFactory: AttributeFactory) { | ||
| private val datasets : Map[UUID,MetaDataset] = Map() | ||
| private val allDatasets : ListBuffer[MetaDataset] = ListBuffer[MetaDataset]() | ||
|
|
||
| /** | ||
| * The method crates a meta data set for a specific Spark operation and returns its identifier. | ||
| * @param operation A Spark operation | ||
| * @return An identifier of created meta data set | ||
| */ | ||
| def create(operation: LogicalPlan) : UUID = datasets.synchronized{ | ||
| val attributeIds = operation.output.map(i => attributeFactory.getOrCreate(i.exprId.id, i.name, i.dataType, i.nullable)) | ||
| val metaDataset = MetaDataset(randomUUID, Schema(attributeIds)) | ||
| datasets.put(metaDataset.id, metaDataset) | ||
| allDatasets += metaDataset | ||
| metaDataset.id | ||
| } | ||
|
|
||
| /** | ||
| * The method returns a meta data set for a specific identifier if has already been created by the factory. Otherwise, returns None. | ||
| * @param id An identifier of the meta data set | ||
| * @return An option | ||
| */ | ||
| def getById(id: UUID) : Option[MetaDataset] = datasets.synchronized(datasets.get(id)) | ||
|
|
||
| /** | ||
| * The method returns all meta data sets created by the factory. | ||
| * @return A sequence of meta data sets | ||
| */ | ||
| def getAll() : Seq[MetaDataset] = datasets.synchronized(allDatasets) | ||
| } |
Oops, something went wrong.