-
Notifications
You must be signed in to change notification settings - Fork 324
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Pawel Leszczynski <leszczynski.pawel@gmail.com>
- Loading branch information
1 parent
e18c13e
commit 5260505
Showing
21 changed files
with
1,174 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
/* | ||
* Copyright 2018-2022 contributors to the Marquez project | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package marquez.api; | ||
|
||
import static javax.ws.rs.core.MediaType.APPLICATION_JSON; | ||
|
||
import com.codahale.metrics.annotation.ExceptionMetered; | ||
import com.codahale.metrics.annotation.ResponseMetered; | ||
import com.codahale.metrics.annotation.Timed; | ||
import java.time.Instant; | ||
import java.util.concurrent.ExecutionException; | ||
import javax.validation.constraints.NotNull; | ||
import javax.ws.rs.Consumes; | ||
import javax.ws.rs.DefaultValue; | ||
import javax.ws.rs.GET; | ||
import javax.ws.rs.Path; | ||
import javax.ws.rs.Produces; | ||
import javax.ws.rs.QueryParam; | ||
import javax.ws.rs.core.Response; | ||
import lombok.NonNull; | ||
import lombok.extern.slf4j.Slf4j; | ||
import marquez.service.ServiceFactory; | ||
import marquez.service.models.NodeId; | ||
|
||
@Slf4j | ||
@Path("/api/v1/column-lineage") | ||
public class ColumnLineageResource extends BaseResource { | ||
|
||
private static final String DEFAULT_DEPTH = "20"; | ||
|
||
public ColumnLineageResource(@NonNull final ServiceFactory serviceFactory) { | ||
super(serviceFactory); | ||
} | ||
|
||
@Timed | ||
@ResponseMetered | ||
@ExceptionMetered | ||
@GET | ||
@Consumes(APPLICATION_JSON) | ||
@Produces(APPLICATION_JSON) | ||
public Response getLineage( | ||
@QueryParam("nodeId") @NotNull NodeId nodeId, | ||
@QueryParam("depth") @DefaultValue(DEFAULT_DEPTH) int depth) | ||
throws ExecutionException, InterruptedException { | ||
return Response.ok(columnLineageService.lineage(nodeId, depth, Instant.now())).build(); | ||
} | ||
|
||
// TODO: endpoint tests | ||
} |
27 changes: 27 additions & 0 deletions
27
api/src/main/java/marquez/common/models/DatasetFieldId.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
/* | ||
* Copyright 2018-2022 contributors to the Marquez project | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package marquez.common.models; | ||
|
||
import lombok.AllArgsConstructor; | ||
import lombok.EqualsAndHashCode; | ||
import lombok.Getter; | ||
import lombok.ToString; | ||
|
||
/** ID for {@code DatasetField}. */ | ||
@EqualsAndHashCode | ||
@AllArgsConstructor | ||
@ToString | ||
public class DatasetFieldId { | ||
|
||
@Getter private final DatasetId datasetId; | ||
@Getter private final FieldName fieldName; | ||
|
||
public static DatasetFieldId of(String namespace, String datasetName, String field) { | ||
return new DatasetFieldId( | ||
new DatasetId(NamespaceName.of(namespace), DatasetName.of(datasetName)), | ||
FieldName.of(field)); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
api/src/main/java/marquez/db/mappers/ColumnLineageNodeDataMapper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
package marquez.db.mappers; | ||
|
||
import static marquez.db.Columns.TRANSFORMATION_DESCRIPTION; | ||
import static marquez.db.Columns.TRANSFORMATION_TYPE; | ||
import static marquez.db.Columns.stringOrThrow; | ||
|
||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import com.google.common.collect.ImmutableList; | ||
import java.sql.ResultSet; | ||
import java.sql.SQLException; | ||
import java.util.Arrays; | ||
import java.util.List; | ||
import java.util.stream.Collectors; | ||
import lombok.extern.slf4j.Slf4j; | ||
import marquez.common.Utils; | ||
import marquez.db.Columns; | ||
import marquez.db.models.ColumnLineageNodeData; | ||
import org.jdbi.v3.core.mapper.RowMapper; | ||
import org.jdbi.v3.core.statement.StatementContext; | ||
import org.postgresql.jdbc.PgArray; | ||
|
||
@Slf4j | ||
public class ColumnLineageNodeDataMapper implements RowMapper<ColumnLineageNodeData> { | ||
|
||
private static final ObjectMapper MAPPER = Utils.getMapper(); | ||
|
||
@Override | ||
public ColumnLineageNodeData map(ResultSet results, StatementContext ctx) throws SQLException { | ||
return new ColumnLineageNodeData( | ||
stringOrThrow(results, Columns.NAMESPACE_NAME), | ||
stringOrThrow(results, Columns.DATASET_NAME), | ||
stringOrThrow(results, Columns.FIELD_NAME), | ||
stringOrThrow(results, Columns.TYPE), | ||
stringOrThrow(results, TRANSFORMATION_DESCRIPTION), | ||
stringOrThrow(results, TRANSFORMATION_TYPE), | ||
toInputFields(results, "inputFields")); | ||
} | ||
|
||
public static ImmutableList<List<String>> toInputFields(ResultSet results, String column) | ||
throws SQLException { | ||
if (results.getObject(column) == null) { | ||
return ImmutableList.of(); | ||
} | ||
|
||
PgArray pgArray = (PgArray) results.getObject(column); | ||
Object[] deserializedArray = (Object[]) pgArray.getArray(); | ||
|
||
return ImmutableList.copyOf( | ||
Arrays.asList(deserializedArray).stream() | ||
.map(o -> (String[]) o) | ||
.map( | ||
arr -> | ||
Arrays.asList( | ||
arr[0], arr[1], | ||
arr[2])) // TODO: add check array size and write unit test for this | ||
.collect(Collectors.toList())); | ||
} | ||
} |
23 changes: 23 additions & 0 deletions
23
api/src/main/java/marquez/db/models/ColumnLineageNodeData.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
/* | ||
* Copyright 2018-2022 contributors to the Marquez project | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package marquez.db.models; | ||
|
||
import java.util.List; | ||
import lombok.AllArgsConstructor; | ||
import lombok.Getter; | ||
import lombok.NonNull; | ||
|
||
@Getter | ||
@AllArgsConstructor | ||
public class ColumnLineageNodeData implements NodeData { | ||
@NonNull String namespace; | ||
@NonNull String name; | ||
@NonNull String field; | ||
@NonNull String type; | ||
@NonNull String transformationDescription; | ||
@NonNull String transformationType; | ||
@NonNull List<List<String>> inputFields; | ||
} |
Oops, something went wrong.