Skip to content

Commit

Permalink
Merge branch 'feature/doc-operations' of https://github.com/MarquezPr…
Browse files Browse the repository at this point in the history
…oject/marquez into feature/doc-operations
  • Loading branch information
Peter Hicks committed May 6, 2022
2 parents 6c58862 + 2618862 commit 7792fd5
Show file tree
Hide file tree
Showing 43 changed files with 912 additions and 235 deletions.
15 changes: 9 additions & 6 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
build-api:
working_directory: ~/marquez
machine:
image: ubuntu-2004:202010-01
image: ubuntu-2004:current
environment:
TESTCONTAINERS_RYUK_DISABLED: true
steps:
Expand All @@ -44,7 +44,8 @@ jobs:

build-image-api:
working_directory: ~/marquez
machine: true
machine:
image: ubuntu-2004:current
steps:
- checkout
- run: docker build --no-cache --tag "marquezproject/marquez:${CIRCLE_SHA1}" .
Expand All @@ -54,7 +55,8 @@ jobs:

build-image-web:
working_directory: ~/marquez/web
machine: true
machine:
image: ubuntu-2004:current
steps:
- *checkout_project_root
- run: docker build --no-cache --tag "marquezproject/marquez-web:${CIRCLE_SHA1}" .
Expand Down Expand Up @@ -85,7 +87,7 @@ jobs:
build-client-java:
working_directory: ~/marquez
machine:
image: ubuntu-2004:202010-01
image: ubuntu-2004:current
steps:
- checkout
- restore_cache:
Expand Down Expand Up @@ -142,7 +144,7 @@ jobs:
release-java:
working_directory: ~/marquez
machine:
image: ubuntu-2004:202010-01
image: ubuntu-2004:current
steps:
- checkout
- run: ./.circleci/get-jdk17.sh
Expand All @@ -165,7 +167,8 @@ jobs:

release-docker:
working_directory: ~/marquez
machine: true
machine:
image: ubuntu-2004:current
steps:
- checkout
- run: ./docker/login.sh
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test-chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
fetch-depth: 0

- name: Setup Helm
uses: azure/setup-helm@v2.0
uses: azure/setup-helm@v2.1

- name: Setup Python
uses: actions/setup-python@v3
Expand Down
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@

* Add support for `LifecycleStateChangeFacet` with an ability to softly delete datasets [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
* Enable pod specific annotations in Marquez Helm Chart via `marquez.podAnnotations` [@wslulciuc](https://github.com/wslulciuc)
* Add support for job renaming/redirection via symlink [@collado-mike](https://github.com/collado-mike)

### Changed

* Upgrade Flyway to v7.6.0 [@dakshin-k](https://github.com/dakshin-k)

## [0.21.0](https://github.com/MarquezProject/marquez/compare/0.20.0...0.21.0) - 2022-03-03

Expand Down
14 changes: 13 additions & 1 deletion RELEASING.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,16 @@
6. Before closing the project board for the _current_ release, move any open issues to the project board created in **step 5**
7. Draft a [new release](https://github.com/MarquezProject/marquez/releases/new) using the release notes for `X.Y.Z` in **step 1** as the release description:

![](./docs/assets/images/new-release.png)
![](./docs/assets/images/new-release.png)

# Voting on Releases

Anyone may request a new release of the project in the #general Slack channel.

After one is proposed, committers have 48 hours to give a +1 or -1.

A total of three +1s, taking into account -1s and excluding votes by the proposer, authorize the release.

Alternatively, if after 2 days the release has received at least one +1 and no -1s, the release is also authorized.

If the proposed release receives no +1s in two days, it is not authorized and the proposer must make a new request to reset the clock.
8 changes: 4 additions & 4 deletions api/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ plugins {
ext {
jdbi3Version = '3.28.0'
prometheusVersion = '0.15.0'
testcontainersVersion = '1.16.3'
sentryVersion = '5.7.2'
testcontainersVersion = '1.17.1'
sentryVersion = '5.7.3'
}

dependencies {
Expand All @@ -43,9 +43,9 @@ dependencies {
implementation "org.jdbi:jdbi3-postgres:${jdbi3Version}"
implementation "org.jdbi:jdbi3-sqlobject:${jdbi3Version}"
implementation 'com.google.guava:guava:31.1-jre'
implementation 'org.dhatim:dropwizard-sentry:2.0.28-10'
implementation 'org.dhatim:dropwizard-sentry:2.0.29'
implementation "io.sentry:sentry:${sentryVersion}"
implementation 'org.flywaydb:flyway-core:6.5.7'
implementation 'org.flywaydb:flyway-core:8.5.10'
implementation "org.postgresql:postgresql:${postgresqlVersion}"
implementation 'com.graphql-java:graphql-java:18.0'
implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'
Expand Down
42 changes: 26 additions & 16 deletions api/src/main/java/marquez/MarquezApp.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import javax.sql.DataSource;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import marquez.api.filter.JobRedirectFilter;
import marquez.cli.SeedCommand;
import marquez.common.Utils;
import marquez.db.DbMigration;
Expand Down Expand Up @@ -115,9 +116,10 @@ public void run(@NonNull MarquezConfig config, @NonNull Environment env) {
env.jersey().register(new TracingContainerResponseFilter());
}

registerResources(config, env, source);
MarquezContext marquezContext = buildMarquezContext(config, env, (ManagedDataSource) source);
registerResources(config, env, marquezContext);
registerServlets(env);
registerFilters(env);
registerFilters(env, marquezContext);
}

private boolean isSentryEnabled(MarquezConfig config) {
Expand All @@ -126,11 +128,26 @@ private boolean isSentryEnabled(MarquezConfig config) {
}

public void registerResources(
@NonNull MarquezConfig config, @NonNull Environment env, @NonNull DataSource source) {
@NonNull MarquezConfig config, @NonNull Environment env, MarquezContext context) {

if (config.getGraphql().isEnabled()) {
env.servlets()
.addServlet("api/v1-beta/graphql", context.getGraphqlServlet())
.addMapping("/api/v1-beta/graphql", "/api/v1/schema.json");
}

log.debug("Registering resources...");
for (final Object resource : context.getResources()) {
env.jersey().register(resource);
}
}

private MarquezContext buildMarquezContext(
MarquezConfig config, Environment env, ManagedDataSource source) {
final JdbiFactory factory = new JdbiFactory();
final Jdbi jdbi =
factory
.build(env, config.getDataSourceFactory(), (ManagedDataSource) source, DB_POSTGRES)
.build(env, config.getDataSourceFactory(), source, DB_POSTGRES)
.installPlugin(new SqlObjectPlugin())
.installPlugin(new PostgresPlugin());
SqlLogger sqlLogger = new InstrumentedSqlLogger(env.metrics());
Expand All @@ -141,17 +158,7 @@ public void registerResources(

final MarquezContext context =
MarquezContext.builder().jdbi(jdbi).tags(config.getTags()).build();

if (config.getGraphql().isEnabled()) {
env.servlets()
.addServlet("api/v1-beta/graphql", context.getGraphqlServlet())
.addMapping("/api/v1-beta/graphql", "/api/v1/schema.json");
}

log.debug("Registering resources...");
for (final Object resource : context.getResources()) {
env.jersey().register(resource);
}
return context;
}

private void registerServlets(@NonNull Environment env) {
Expand All @@ -161,7 +168,10 @@ private void registerServlets(@NonNull Environment env) {
env.servlets().addServlet(PROMETHEUS, new MetricsServlet()).addMapping(PROMETHEUS_ENDPOINT);
}

private void registerFilters(@NonNull Environment env) {
private void registerFilters(@NonNull Environment env, MarquezContext marquezContext) {
env.jersey().getResourceConfig().register(new LoggingMdcFilter());
env.jersey()
.getResourceConfig()
.register(new JobRedirectFilter(marquezContext.getJobService()));
}
}
92 changes: 92 additions & 0 deletions api/src/main/java/marquez/api/filter/JobRedirectFilter.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package marquez.api.filter;

import java.io.IOException;
import java.net.URI;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;
import lombok.extern.slf4j.Slf4j;
import marquez.common.models.JobId;
import marquez.db.models.JobRow;
import marquez.service.JobService;
import marquez.service.models.Job;
import org.glassfish.jersey.server.ExtendedUriInfo;
import org.glassfish.jersey.uri.UriComponent;
import org.glassfish.jersey.uri.UriComponent.Type;
import org.glassfish.jersey.uri.UriTemplate;

/**
* Filters requests that reference a job that has been symlinked to another job. This filter
* redirects such requests to the URL with the symlink target's name using a 301 status code.
*/
@Slf4j
public class JobRedirectFilter implements ContainerRequestFilter {

public static final String JOB_PATH_PARAM = "job";
public static final String NAMESPACE_PATH_PARAM = "namespace";
private final JobService jobService;

public JobRedirectFilter(JobService jobService) {
this.jobService = jobService;
}

@Override
public void filter(ContainerRequestContext requestContext) throws IOException {
MultivaluedMap<String, String> pathParams = requestContext.getUriInfo().getPathParameters();
if (!pathParams.containsKey(NAMESPACE_PATH_PARAM) || !pathParams.containsKey(JOB_PATH_PARAM)) {
return;
}
List<String> namespaceParams = pathParams.get(NAMESPACE_PATH_PARAM);
List<String> jobParams = pathParams.get(JOB_PATH_PARAM);
if (namespaceParams.isEmpty() || jobParams.isEmpty()) {
return;
}
Optional<Job> job = jobService.findJobByName(namespaceParams.get(0), jobParams.get(0));
job.ifPresent(
j -> {
if (!j.getName().getValue().equals(jobParams.get(0))) {
log.info(
"Job {}.{} has been redirected to {}.{}",
namespaceParams.get(0),
jobParams.get(0),
j.getNamespace().getValue(),
j.getName().getValue());
URI location = buildLocationFor(requestContext, j.getId());
log.debug("Redirecting to url {}", location);
requestContext.abortWith(Response.status(301).location(location).build());
}
});
}

/**
* Construct a URI from a Request's matched resource, replacing the {@value #JOB_PATH_PARAM} and
* {@value #NAMESPACE_PATH_PARAM} parameters with the fully-qualified values from the provided
* {@link JobRow}.
*
* @param ctx
* @param jobId
* @return
*/
private URI buildLocationFor(ContainerRequestContext ctx, JobId jobId) {
Object resource = ctx.getUriInfo().getMatchedResources().get(0);
MultivaluedMap<String, String> pathParameters = ctx.getUriInfo().getPathParameters();
MultivaluedHashMap<String, String> copy = new MultivaluedHashMap<>(pathParameters);
copy.putSingle(
JOB_PATH_PARAM, UriComponent.encode(jobId.getName().getValue(), Type.PATH_SEGMENT));
copy.putSingle(
NAMESPACE_PATH_PARAM,
UriComponent.encode(jobId.getNamespace().getValue(), Type.PATH_SEGMENT));
Map<String, String> singletonMap = new HashMap<>();
copy.forEach((k, v) -> singletonMap.put(k, v.get(0)));
UriTemplate pathTemplate = ((ExtendedUriInfo) ctx.getUriInfo()).getMatchedTemplates().get(0);
String newPath = pathTemplate.createURI(singletonMap);
return UriBuilder.fromResource(resource.getClass()).path(newPath).buildFromEncodedMap(copy);
}
}
4 changes: 2 additions & 2 deletions api/src/main/java/marquez/common/models/NamespaceName.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ public final class NamespaceName {
private static final int MIN_SIZE = 1;
private static final int MAX_SIZE = 1024;
private static final Pattern PATTERN =
Pattern.compile(String.format("^[a-zA-Z:;=/0-9_\\-\\.]{%d,%d}$", MIN_SIZE, MAX_SIZE));
Pattern.compile(String.format("^[a-zA-Z:;=/0-9_\\-\\.@]{%d,%d}$", MIN_SIZE, MAX_SIZE));

@Getter private final String value;

public NamespaceName(@NonNull final String value) {
checkArgument(
PATTERN.matcher(value).matches(),
"namespace '%s' must contain only letters (a-z, A-Z), numbers (0-9), "
+ "underscores (_), dashes (-), colons (:), equals (=), semicolons (;), slashes (/) "
+ "underscores (_), at (@), dashes (-), colons (:), equals (=), semicolons (;), slashes (/) "
+ "or dots (.) with a maximum length of %s characters.",
value,
MAX_SIZE);
Expand Down
4 changes: 4 additions & 0 deletions api/src/main/java/marquez/db/Columns.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

@Slf4j
public final class Columns {

private Columns() {}

private static final ObjectMapper MAPPER = Utils.getMapper();
Expand Down Expand Up @@ -78,6 +79,9 @@ private Columns() {}
/* STREAM VERSION ROW COLUMNS */
public static final String SCHEMA_LOCATION = "schema_location";

/* JOB ROW COLUMNS */
public static final String SYMLINK_TARGET_UUID = "symlink_target_uuid";

/* JOB VERSION I/O ROW COLUMNS */
public static final String INPUT_UUIDS = "input_uuids";
public static final String OUTPUT_UUIDS = "output_uuids";
Expand Down
6 changes: 4 additions & 2 deletions api/src/main/java/marquez/db/DbMigration.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import lombok.extern.slf4j.Slf4j;
import org.flywaydb.core.Flyway;
import org.flywaydb.core.api.FlywayException;
import org.flywaydb.core.api.output.MigrateResult;

@Slf4j
public final class DbMigration {
Expand All @@ -31,8 +32,9 @@ public static void migrateDbOrError(
// issues before app termination.
try {
log.info("Migrating database...");
final int migrations = flyway.migrate();
log.info("Successfully applied '{}' migrations to database.", migrations);
final MigrateResult migrateResult = flyway.migrate();
log.info(
"Successfully applied '{}' migrations to database.", migrateResult.migrationsExecuted);
} catch (FlywayException errorOnDbMigrate) {
log.error("Failed to apply migration to database.", errorOnDbMigrate);
try {
Expand Down
Loading

0 comments on commit 7792fd5

Please sign in to comment.