Merge branch 'main' into feature/update-lineage-events-paging

.circleci/api-load-test.sh

@@ -14,7 +14,7 @@
 set -e
 
 # Build version of Marquez
-readonly MARQUEZ_VERSION=0.39.0-SNAPSHOT
+readonly MARQUEZ_VERSION=0.40.0-SNAPSHOT
 # Fully qualified path to marquez.jar
 readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar"
 

.circleci/db-migration.sh

@@ -13,7 +13,7 @@
 # Version of PostgreSQL
 readonly POSTGRES_VERSION="12.1"
 # Version of Marquez
-readonly MARQUEZ_VERSION=0.38.0
+readonly MARQUEZ_VERSION=0.39.0
 # Build version of Marquez
 readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1
 

.env.example

@@ -1,4 +1,4 @@
 API_PORT=5000
 API_ADMIN_PORT=5001
 WEB_PORT=3000
-TAG=0.38.0
+TAG=0.39.0

.github/boring-cyborg.yml

@@ -55,4 +55,4 @@ firstPRMergeComment: >
 
 # Comment to be posted to on first time issues
 firstIssueWelcomeComment: >
-  Thanks for opening your first issue in the Marquez project! Please be sure to follow the issue template!
+  Thanks for opening your first issue in the Marquez project! Please be sure to follow the issue template!

.github/pull_request_template.md

@@ -20,4 +20,4 @@ One-line summary:
 - [ ] You've updated any relevant documentation (_if relevant_)
 - [ ] You've included a one-line summary of your change for the [`CHANGELOG.md`](https://github.com/MarquezProject/marquez/blob/main/CHANGELOG.md#unreleased) (_Depending on the change, this may not be necessary_).
 - [ ] You've versioned your `.sql` database schema migration according to [Flyway's naming convention](https://flywaydb.org/documentation/concepts/migrations#naming) (_if relevant_)
-- [ ] You've included a [header](https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md#copyright--license) in any source code files (_if relevant_)
+- [ ] You've included a [header](https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md#copyright--license) in any source code files (_if relevant_)

.github/workflows/headerchecker.yml

@@ -32,7 +32,7 @@ jobs:
         run: |
           ok=1
           readarray -t files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.added_modified }}')"
-          for file in ${files[@]}; do 
+          for file in ${files[@]}; do
             if [[ ($file == *".java") ]]; then
               if ! grep -q Copyright "$file"; then
                 ok=0
@@ -45,4 +45,4 @@ jobs:
           else
             GREEN="\e[32m"
             echo -e "${GREEN}All changed & added files have been scanned. Result: no headers are missing.${ENDCOLOR}"
-          fi
+          fi

.gitpod.yml

@@ -14,4 +14,3 @@ ports:
 vscode:
   extensions:
     - ms-azuretools.vscode-docker
-

.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
         -   id: trailing-whitespace
         -   id: end-of-file-fixer
         -   id: check-yaml
+            exclude: ^chart/
         -   id: check-added-large-files
 
   -   repo: https://github.com/jguttman94/pre-commit-gradle

CODE_OF_CONDUCT.md

@@ -117,11 +117,11 @@ This Code of Conduct is adapted from the [Contributor Covenant][homepage],
 version 2.0, available at
 [https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
 
-Community Impact Guidelines were inspired by 
+Community Impact Guidelines were inspired by
 [Mozilla's code of conduct enforcement ladder][Mozilla CoC].
 
 For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available 
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available
 at [https://www.contributor-covenant.org/translations][translations].
 
 [homepage]: https://www.contributor-covenant.org
@@ -131,5 +131,5 @@ at [https://www.contributor-covenant.org/translations][translations].
 [translations]: https://www.contributor-covenant.org/translations
 
 ----
-SPDX-License-Identifier: Apache-2.0 
+SPDX-License-Identifier: Apache-2.0
 Copyright 2018-2023 contributors to the Marquez project.

CODE_QUALITY_AND_SECURITY.md

@@ -2,12 +2,12 @@
 
 The authors of Marquez are committed to providing secure software of the highest quality possible. To this end, we employ a number of tools and methodologies to ensure that our design, build, maintenance and testing practices maximize efficiency and minimize risk.
 
-The specific security and analysis methodologies that we employ include but are not limited to: 
+The specific security and analysis methodologies that we employ include but are not limited to:
 
 ## Security
 
 - Participation in the [OpenSSF Best Practices Badge Program](https://bestpractices.coreinfrastructure.org/en/projects/5106) for Free/Libre and FLOSS projects to ensure that we follow current best practices for quality and security
-- Use of [HTTPS](https://en.wikipedia.org/wiki/HTTPS) for network communication 
+- Use of [HTTPS](https://en.wikipedia.org/wiki/HTTPS) for network communication
 - Support for multiple cryptographic algorithms (through the use of HTTPS)
 - Separate storage of authentication credentials according to best practices
 - Use of secure protocols for network communication (through the use of HTTPS)
@@ -30,5 +30,5 @@ For more information about our approach to quality and security, feel free to re
 - Twitter: [@MarquezProject](https://twitter.com/MarquezProject)
 
 ----
-SPDX-License-Identifier: Apache-2.0 
-Copyright 2018-2023 contributors to the Marquez project.
+SPDX-License-Identifier: Apache-2.0
+Copyright 2018-2023 contributors to the Marquez project.

COMMITTERS.md

@@ -25,7 +25,7 @@ They take responsibility for guiding new pull requests into the main branch.
 | Ross Turk          | [@rossturk](https://github.com/rossturk)             |
 | Minkyu Park        | [@fm100](https://github.com/fm100)                   |
 | Paweł Leszczyński  | [@pawel-big-lebowski](https://github.com/pawel-big-lebowski) |
-     
+
 ## Emeritus
 
 The following people are no longer working on the Marquez project.
@@ -41,5 +41,5 @@ A Contributor may become a Committer by the approval of a majority of the
 existing Committers (as per the project [charter](https://wiki.lfaidata.foundation/download/attachments/18481434/Marquez%20Project%20Technical%20Charter%20Final_Adopted%2005.21.20.pdf?version=1&modificationDate=1591718661000&api=v2)).
 
 ----
-SPDX-License-Identifier: Apache-2.0 
-Copyright 2018-2023 contributors to the Marquez project.
+SPDX-License-Identifier: Apache-2.0
+Copyright 2018-2023 contributors to the Marquez project.

CONTRIBUTING.md

@@ -249,5 +249,5 @@ We use [SPDX](https://spdx.dev) for copyright and license information. The follo
 * [Signing Commits](https://docs.github.com/en/github/authenticating-to-github/signing-commits)
 
 ----
-SPDX-License-Identifier: Apache-2.0 
-Copyright 2018-2023 contributors to the Marquez project.
+SPDX-License-Identifier: Apache-2.0
+Copyright 2018-2023 contributors to the Marquez project.

GOVERNANCE.md

@@ -89,7 +89,7 @@ Marquez is currently led by Willy Lulciuc.
 
 ## Marquez project meetings
 
-Some meetings are face-to-face, but most are conference calls.  
+Some meetings are face-to-face, but most are conference calls.
 Attendance at meetings is open to all.  Conference calls can be joined without an explicit invitation.
 However, due to physical security requirements at some of the venues we use,
 it is necessary to ensure you are added to the invitee list of any face-to-face meetings
@@ -106,7 +106,7 @@ This creates a recorded discussion of design decisions and discussions that comp
 Follow the link above and register with the Slack service using your email address.
 Once signed in you can see all of the active Slack channels.
 
-Additional channels are added from time to time as new workgroups and discussion topics are established. 
+Additional channels are added from time to time as new workgroups and discussion topics are established.
 
 ## Marquez email
 
@@ -142,5 +142,5 @@ be resolved by voting. The voting process is a simple majority in which each com
 
 
 ----
-SPDX-License-Identifier: Apache-2.0 
+SPDX-License-Identifier: Apache-2.0
 Copyright 2018-2023 contributors to the Marquez project.

METRICS.md

@@ -12,5 +12,5 @@
 | `marquez_job_runs_completed`     | _gauge_ |                                                            | Total number of completed job runs. |
 
 ----
-SPDX-License-Identifier: Apache-2.0 
-Copyright 2018-2023 contributors to the Marquez project.
+SPDX-License-Identifier: Apache-2.0
+Copyright 2018-2023 contributors to the Marquez project.

api/build.gradle

@@ -24,7 +24,7 @@ ext {
     jdbi3Version = '3.37.1'
     prometheusVersion = '0.16.0'
     testcontainersVersion = '1.17.6'
-    sentryVersion = '6.21.0'
+    sentryVersion = '6.28.0'
 }
 
 dependencies {
@@ -44,11 +44,11 @@ dependencies {
     implementation "org.jdbi:jdbi3-postgres:${jdbi3Version}"
     implementation "org.jdbi:jdbi3-sqlobject:${jdbi3Version}"
     implementation 'com.google.guava:guava:31.1-jre'
-    implementation 'org.dhatim:dropwizard-sentry:2.1.2-4'
+    implementation 'org.dhatim:dropwizard-sentry:2.1.6'
     implementation "io.sentry:sentry:${sentryVersion}"
     implementation 'org.flywaydb:flyway-core:8.5.13'
     implementation "org.postgresql:postgresql:${postgresqlVersion}"
-    implementation 'com.graphql-java:graphql-java:20.0'
+    implementation 'com.graphql-java:graphql-java:20.4'
     implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'
 
     testImplementation "io.dropwizard:dropwizard-testing:${dropwizardVersion}"

api/src/main/java/marquez/service/RunTransitionListener.java

@@ -87,8 +87,10 @@ class RunOutput {
   class RunTransition {
     /** the unique ID of the run. */
     @NonNull RunId runId;
+
     /** The old state of the run. */
     @Nullable RunState oldState;
+
     /** The new state of the run. */
     @NonNull RunState newState;
 

api/src/main/resources/assets/graphql-playground/index.htm

@@ -1,6 +1,6 @@
 <!--
 Copyright 2018-2023 contributors to the Marquez project
-SPDX-License-Identifier: Apache-2.0 
+SPDX-License-Identifier: Apache-2.0
 -->
 
 <!DOCTYPE html>
@@ -542,4 +542,4 @@
   })
 </script>
 </body>
-</html>
+</html>

api/src/main/resources/banner.txt

@@ -1,7 +1,6 @@
-    __  ___                               
+    __  ___
    /  |/  /___ __________ ___  _____  ____
   / /|_/ / __ `/ ___/ __ `/ / / / _ \/_  /
  / /  / / /_/ / /  / /_/ / /_/ /  __/ / /_
 /_/  /_/\__,_/_/   \__, /\__,_/\___/ /___/
-                     /_/                  
-
+                     /_/

api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql

@@ -20,4 +20,4 @@ FROM datasets d
 JOIN dataset_symlinks symlinks ON d.uuid = symlinks.dataset_uuid
 INNER JOIN namespaces ON symlinks.namespace_uuid = namespaces.uuid
 WHERE d.is_hidden IS FALSE
-GROUP BY d.uuid;
+GROUP BY d.uuid;

api/src/main/resources/marquez/db/migration/V13__alter_run_add_start_end_state.sql

@@ -1,4 +1,4 @@
 /* SPDX-License-Identifier: Apache-2.0 */
 
 ALTER TABLE runs ADD start_run_state_uuid UUID REFERENCES run_states(uuid);
-ALTER TABLE runs ADD end_run_state_uuid UUID REFERENCES run_states(uuid);
+ALTER TABLE runs ADD end_run_state_uuid UUID REFERENCES run_states(uuid);

api/src/main/resources/marquez/db/migration/V14__index_datasetversion_datasetid.sql

@@ -1,3 +1,3 @@
 /* SPDX-License-Identifier: Apache-2.0 */
 
-CREATE INDEX datasetversion_datasetid_idx ON dataset_versions (dataset_uuid);
+CREATE INDEX datasetversion_datasetid_idx ON dataset_versions (dataset_uuid);

api/src/main/resources/marquez/db/migration/V17.1__unique_version_constraint.sql

@@ -1,4 +1,4 @@
 /* SPDX-License-Identifier: Apache-2.0 */
 
 ALTER TABLE dataset_versions ADD CONSTRAINT dataset_versions_version UNIQUE(version);
-ALTER TABLE job_versions ADD CONSTRAINT job_versions_version UNIQUE(version);
+ALTER TABLE job_versions ADD CONSTRAINT job_versions_version UNIQUE(version);

api/src/main/resources/marquez/db/migration/V18__drop_dataset_constraint.sql

@@ -1,3 +1,3 @@
 /* SPDX-License-Identifier: Apache-2.0 */
 
-alter table datasets drop constraint datasets_source_uuid_physical_name_key;
+alter table datasets drop constraint datasets_source_uuid_physical_name_key;

api/src/main/resources/marquez/db/migration/V1__initial_schema.sql

@@ -76,7 +76,7 @@ CREATE TABLE job_versions_io_mapping (
   job_version_uuid UUID REFERENCES job_versions(uuid),
   dataset_uuid     UUID REFERENCES datasets(uuid),
   io_type          VARCHAR(64) NOT NULL,
-  PRIMARY KEY (job_version_uuid, dataset_uuid, io_type) 
+  PRIMARY KEY (job_version_uuid, dataset_uuid, io_type)
 );
 
 CREATE TABLE run_args (
@@ -121,5 +121,5 @@ CREATE TABLE stream_versions (
 CREATE TABLE runs_input_mapping (
   run_uuid             UUID REFERENCES runs(uuid),
   dataset_version_uuid UUID REFERENCES dataset_versions(uuid),
-  PRIMARY KEY (run_uuid, dataset_version_uuid) 
+  PRIMARY KEY (run_uuid, dataset_version_uuid)
 );

api/src/main/resources/marquez/db/migration/V20__drop_lineage_pk.sql

@@ -1,3 +1,3 @@
 /* SPDX-License-Identifier: Apache-2.0 */
 
-ALTER TABLE lineage_events DROP CONSTRAINT lineage_event_pk;
+ALTER TABLE lineage_events DROP CONSTRAINT lineage_event_pk;

api/src/main/resources/marquez/db/migration/V21__alter_jobs_to_add_namespace_name.sql

@@ -1,4 +1,4 @@
 /* SPDX-License-Identifier: Apache-2.0 */
 
 ALTER TABLE jobs ADD namespace_name VARCHAR(255);
-UPDATE jobs SET namespace_name = namespaces.name FROM namespaces WHERE jobs.namespace_uuid = namespaces.uuid;
+UPDATE jobs SET namespace_name = namespaces.name FROM namespaces WHERE jobs.namespace_uuid = namespaces.uuid;

api/src/main/resources/marquez/db/migration/V22__alter_job_versions_to_add_namespace.sql

@@ -12,4 +12,4 @@ FROM jobs
 WHERE job_versions.job_uuid = jobs.uuid;
 
 CREATE INDEX job_versions_selector
-    ON job_versions (job_name, namespace_name);
+    ON job_versions (job_name, namespace_name);

api/src/main/resources/marquez/db/migration/V24__alter_jobs.sql

@@ -19,4 +19,4 @@ UPDATE jobs SET (current_inputs) = (select jsonb_agg(query) from
 UPDATE jobs SET (current_outputs) = (select jsonb_agg(query) from
   (select jv.namespace_name as namespaceName, ds.name as datasetName
    from job_versions_io_mapping m inner join job_versions jv on m.job_version_uuid = jv.uuid inner join datasets ds on m.dataset_uuid = ds.uuid
-   where m.io_type = 'OUTPUT' and jobs.uuid = jv.job_uuid) query);
+   where m.io_type = 'OUTPUT' and jobs.uuid = jv.job_uuid) query);

api/src/main/resources/marquez/db/migration/V25__alter_datasets.sql

@@ -12,4 +12,3 @@ UPDATE datasets SET
     source_name = s.name
 FROM sources s
 WHERE s.uuid = datasets.source_uuid;
-

api/src/main/resources/marquez/db/migration/V26__alter_jobs_change_type.sql

@@ -7,4 +7,4 @@ ALTER TABLE jobs
 ALTER TABLE job_versions
     ALTER COLUMN namespace_name TYPE VARCHAR;
 ALTER TABLE job_versions
-    ALTER COLUMN job_name TYPE VARCHAR;
+    ALTER COLUMN job_name TYPE VARCHAR;

api/src/main/resources/marquez/db/migration/V27__alter_runs_add_context.sql

@@ -5,4 +5,4 @@ ALTER TABLE runs ADD job_context_uuid uuid;
 UPDATE runs SET
     job_context_uuid = jv.job_context_uuid
 FROM job_versions jv
-WHERE jv.uuid = runs.job_version_uuid;
+WHERE jv.uuid = runs.job_version_uuid;

api/src/main/resources/marquez/db/migration/V28__update_jobs_fix_inputs.sql

@@ -5,4 +5,4 @@ UPDATE jobs SET (current_inputs) = (select jsonb_agg(query) from
      from job_versions_io_mapping m inner join job_versions jv on m.job_version_uuid = jv.uuid inner join datasets ds on m.dataset_uuid = ds.uuid
      where m.io_type = 'INPUT' and jobs.uuid = jv.job_uuid) query);
 
-alter table jobs drop column current_outputs;
+alter table jobs drop column current_outputs;

api/src/main/resources/marquez/db/migration/V30__alter_runs_change_transitioned_at_type.sql

@@ -1,4 +1,4 @@
 /* SPDX-License-Identifier: Apache-2.0 */
 
 alter table runs alter column transitioned_at type timestamp without time zone
-    USING transitioned_at::timestamp without time zone;
+    USING transitioned_at::timestamp without time zone;

api/src/main/resources/marquez/db/migration/V32__index_runs_created_at.sql

@@ -2,4 +2,4 @@
 
 create index runs_created_at_by_name_index
     on runs(job_name, namespace_name, created_at DESC)
-    include (uuid, created_at, updated_at, nominal_start_time, nominal_end_time, current_run_state, started_at, ended_at, namespace_name, job_name, location);
+    include (uuid, created_at, updated_at, nominal_start_time, nominal_end_time, current_run_state, started_at, ended_at, namespace_name, job_name, location);

api/src/main/resources/marquez/db/migration/V40__alter_tables_resize_namespace_and_connection_url.sql

@@ -7,4 +7,4 @@ ALTER TABLE namespaces ALTER COLUMN name TYPE VARCHAR;
 ALTER TABLE runs ALTER COLUMN external_id TYPE VARCHAR;
 ALTER TABLE sources ALTER COLUMN name TYPE VARCHAR;
 ALTER TABLE sources ALTER COLUMN connection_url TYPE VARCHAR;
-ALTER TABLE tags ALTER COLUMN name TYPE VARCHAR;
+ALTER TABLE tags ALTER COLUMN name TYPE VARCHAR;

api/src/main/resources/marquez/db/migration/V41__add_operation_to_dataset_versions.sql

@@ -1,2 +1,2 @@
 alter table dataset_versions add column lifecycle_state VARCHAR(63);
-alter table datasets add column is_deleted BOOLEAN DEFAULT FALSE;
+alter table datasets add column is_deleted BOOLEAN DEFAULT FALSE;

api/src/main/resources/marquez/db/migration/V42__add_job_symlink_target.sql

@@ -1,4 +1,4 @@
 ALTER TABLE jobs ADD COLUMN symlink_target_uuid uuid REFERENCES jobs (uuid);
 CREATE INDEX jobs_symlinks ON jobs (symlink_target_uuid)
     INCLUDE (uuid, namespace_name, name)
-    WHERE symlink_target_uuid IS NOT NULL;
+    WHERE symlink_target_uuid IS NOT NULL;

api/src/main/resources/marquez/db/migration/V45__update_jobs_view_rule.sql

@@ -75,4 +75,4 @@ FROM jobs j
          INNER JOIN fqn jf ON jf.uuid = COALESCE(js.link_target_uuid, j.uuid)
 ON CONFLICT (uuid) DO UPDATE
     SET job_fqn=EXCLUDED.job_fqn,
-        aliases = jobs_fqn.aliases || EXCLUDED.aliases;
+        aliases = jobs_fqn.aliases || EXCLUDED.aliases;

api/src/main/resources/marquez/db/migration/V46__add_hidden_column_to_jobs_and_datasets.sql

@@ -1,3 +1,3 @@
 ALTER TABLE datasets ADD COLUMN is_hidden BOOLEAN DEFAULT FALSE;
 
-ALTER TABLE jobs ADD COLUMN is_hidden BOOLEAN DEFAULT FALSE;
+ALTER TABLE jobs ADD COLUMN is_hidden BOOLEAN DEFAULT FALSE;

api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql

@@ -21,4 +21,3 @@ CREATE TYPE DATASET_NAME AS (
     namespace       VARCHAR(255),
     name            VARCHAR(255)
 );
-

api/src/main/resources/marquez/db/migration/V49__column_lineage.sql

@@ -10,4 +10,4 @@ CREATE TABLE column_lineage (
   created_at                    TIMESTAMP NOT NULL,
   updated_at                    TIMESTAMP NOT NULL,
   UNIQUE (output_dataset_version_uuid, output_dataset_field_uuid, input_dataset_version_uuid, input_dataset_field_uuid)
-);
+);

api/src/main/resources/marquez/db/migration/V50__index_dataset_fields.sql

@@ -1,4 +1,4 @@
 /* SPDX-License-Identifier: Apache-2.0 */
 
 create index dataset_fields_dataset_uuid
-    on dataset_fields (dataset_uuid);
+    on dataset_fields (dataset_uuid);