Merge branch 'main' into web/softdelete

.circleci/config.yml

@@ -148,6 +148,15 @@ jobs:
       - run: npm install --prefix=${HOME}/.local --global redoc-cli
       - run: redoc-cli bundle spec/openapi.yml
 
+  migrate-db:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:current
+    steps:
+      - checkout
+      - run: ./.circleci/get-docker-compose.sh
+      - run: ./.circleci/db-migration.sh
+
   release-java:
     working_directory: ~/marquez
     machine:
@@ -195,6 +204,9 @@ workflows:
       - unit-test-web
       - unit-test-client-python
       - lint-spec-api
+      - migrate-db:
+          requires:
+            - build-api
   release:
     jobs:
       - build-client-python:

.circleci/db-migration.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Copyright 2018-2022 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
+#
+# A script used in CI to test database migrations by:
+#   (1) Applying db migrations on latest Marquez release
+#   (2) Take a backup of db from Step 1
+#   (3) Applying db migrations on latest Marquez build using backup
+#
+# Usage: $ ./db-migration.sh
+
+# Version of PostgreSQL
+readonly POSTGRES_VERSION="12.1"
+# Version of Marquez
+readonly MARQUEZ_VERSION="0.29.0"
+# Build version of Marquez
+readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1
+
+readonly DB_MIGRATION_VOLUME="marquez_db-backup"
+readonly DB_MIGRATION_BACKUP="db-migration-backup"
+readonly DB_MIGRATION_QUERY=$(cat <<-END
+  SELECT version,installed_on,checksum
+    FROM flyway_schema_history
+   WHERE version IS NOT NULL
+   ORDER BY installed_on DESC LIMIT 1;
+END
+)
+
+log() {
+  echo -e "\033[1m>>\033[0m ${1}"
+}
+
+error() {
+  echo -e "\033[0;31merror: ${1}\033[0m"
+}
+
+exit_with_cause() {
+  log "please view container logs for more details on cause:"
+  docker-compose logs
+  exit 1
+}
+
+query_db_migration() {
+  # Start db using backup
+  [[ $(docker ps -f "name=${DB_MIGRATION_BACKUP}" --format '{{.Names}}') == "${DB_MIGRATION_BACKUP}" ]] || \
+    docker run -d --name "${DB_MIGRATION_BACKUP}" \
+      -v "${DB_MIGRATION_VOLUME}:/var/lib/postgresql/data" \
+      "postgres:${POSTGRES_VERSION}"
+  # Query applied db migrations
+  log "latest migration applied to db:"
+  docker exec "${DB_MIGRATION_BACKUP}" \
+    psql -U marquez -c "${DB_MIGRATION_QUERY}"
+}
+
+# Change working directory to project root
+project_root=$(git rev-parse --show-toplevel)
+cd "${project_root}/"
+
+# (1) Apply db migrations on latest Marquez release
+log "start db with latest migrations (marquez=${MARQUEZ_VERSION}):"
+if ! ./docker/up.sh \
+  --args "--exit-code-from seed_marquez" \
+  --tag "${MARQUEZ_VERSION}" \
+  --no-web \
+  --seed > /dev/null; then
+  error "failed to start db using backup!"
+  exit_with_cause
+fi
+
+# Query, then display schema migration applied
+query_db_migration
+
+# (2) Apply db migrations on latest Marquez build using backup
+log "start db using backup (marquez=${MARQUEZ_BUILD_VERSION}):"
+if ! ./docker/up.sh \
+  --args "--exit-code-from seed_marquez" \
+  --no-web \
+  --no-volumes \
+  --build \
+  --seed > /dev/null; then
+  error "failed to start db using backup!"
+  exit_with_cause
+fi
+
+# Query, then display additional schema migration applied on backup (if any)
+query_db_migration
+
+log "DONE!"

.circleci/get-docker-compose.sh

@@ -1,24 +1,18 @@
 #!/bin/bash
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Copyright 2018-2022 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
 #
 # Usage: $ ./get-docker-compose.sh
 
 set -e
 
-curl -L https://github.com/docker/compose/releases/download/1.25.3/docker-compose-`uname -s`-`uname -m` > ~/docker-compose
-chmod +x ~/docker-compose
-sudo mv ~/docker-compose /usr/local/bin/docker-compose
-docker-compose --version
+# Download docker compose
+curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-`uname -s`-`uname -m` > ~/docker-compose
+
+# Change permissions, relocate docker compose, then verify
+chmod +x ~/docker-compose && \
+  sudo mv ~/docker-compose /usr/local/bin/docker-compose && \
+  docker-compose --version
 
-echo "DONE!"
+echo "DONE!"

CHANGELOG.md

@@ -894,3 +894,7 @@
 ## [0.1.0](https://github.com/MarquezProject/marquez/releases/tag/0.1.0) - 2018-12-18
 
 * Marquez initial public release.
+
+----
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

CODE_OF_CONDUCT.md

@@ -129,3 +129,7 @@ at [https://www.contributor-covenant.org/translations][translations].
 [Mozilla CoC]: https://github.com/mozilla/diversity
 [FAQ]: https://www.contributor-covenant.org/faq
 [translations]: https://www.contributor-covenant.org/translations
+
+----
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

CODE_QUALITY_AND_SECURITY.md

@@ -27,4 +27,8 @@ The specific security and analysis methodologies that we employ include but are
 For more information about our approach to quality and security, feel free to reach out to the Marquez development team:
 
 - Slack: [Marquezproject.slack.com](http://bit.ly/MarquezSlack)
-- Twitter: [@MarquezProject](https://twitter.com/MarquezProject)
+- Twitter: [@MarquezProject](https://twitter.com/MarquezProject)
+
+----
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

COMMITTERS.md

@@ -36,3 +36,7 @@ contributions, we have a strong foundation to build on.
 
 A Contributor may become a Committer by the approval of a majority of the
 existing Committers (as per the project [charter](https://wiki.lfaidata.foundation/download/attachments/18481434/Marquez%20Project%20Technical%20Charter%20Final_Adopted%2005.21.20.pdf?version=1&modificationDate=1591718661000&api=v2)).
+
+----
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

CONTRIBUTING.md

@@ -247,3 +247,7 @@ We use [SPDX](https://spdx.dev) for copyright and license information. The follo
 * [Keeping a Changelog](https://keepachangelog.com)
 * [Code Review Developer Guide](https://google.github.io/eng-practices/review)
 * [Signing Commits](https://docs.github.com/en/github/authenticating-to-github/signing-commits)
+
+----
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

GOVERNANCE.md

@@ -142,5 +142,5 @@ be resolved by voting. The voting process is a simple majority in which each com
 
 
 ----
-License: Apache-2.0, 
-Copyright 2018-2022 contributors to the Marquez project.
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

METRICS.md

@@ -9,4 +9,8 @@
 | `marquez_job_total`              | _count_ | `namespace_name`, <br> `job_type`                          | Total number of jobs.               |
 | `marquez_job_versions_total`     | _count_ | `namespace_name`, <br> `job_type`, <br> `job_name`         | Total number of job versions.       |
 | `marquez_job_runs_active`        | _gauge_ |                                                            | Total number of active job runs.    |
-| `marquez_job_runs_completed`     | _gauge_ |                                                            | Total number of completed job runs. |
+| `marquez_job_runs_completed`     | _gauge_ |                                                            | Total number of completed job runs. |
+
+----
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

README.md

@@ -156,3 +156,7 @@ See [CONTRIBUTING.md](https://github.com/MarquezProject/marquez/blob/main/CONTRI
 ## Reporting a Vulnerability
 
 If you discover a vulnerability in the project, please open an issue and attach the "security" label.
+
+----
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

RELEASING.md

@@ -32,3 +32,7 @@ Alternatively, if after 2 days the release has received at least one +1 and no -
 If the proposed release receives no +1s in two days, it is not authorized and the proposer must make a new request to reset the clock.
 
 Once a release is authorized, it will be initiated within two business days. Releases will not be made on a Friday unless doing so will address an important defect, an issue with project infrastructure, or a security vulnerability.
+
+----
+SPDX-License-Identifier: Apache-2.0 
+Copyright 2018-2023 contributors to the Marquez project.

api/build.gradle

@@ -21,10 +21,10 @@ plugins {
 }
 
 ext {
-    jdbi3Version = '3.35.0'
+    jdbi3Version = '3.36.0'
     prometheusVersion = '0.16.0'
     testcontainersVersion = '1.17.6'
-    sentryVersion = '6.10.0'
+    sentryVersion = '6.11.0'
 }
 
 dependencies {

api/src/main/java/marquez/MarquezApp.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/MarquezConfig.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/MarquezContext.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/BaseResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/ColumnLineageResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/DatasetResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/JobResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/NamespaceResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/OpenLineageResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/RunResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/SearchResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/SourceResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/TagResource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/exceptions/DatasetNotFoundException.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/exceptions/DatasetVersionNotFoundException.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/exceptions/FieldNotFoundException.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/exceptions/JdbiExceptionExceptionMapper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/exceptions/JobNotFoundException.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/exceptions/JobVersionNotFoundException.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/exceptions/JsonProcessingExceptionMapper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */
 

api/src/main/java/marquez/api/exceptions/NamespaceNotFoundException.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2022 contributors to the Marquez project
+ * Copyright 2018-2023 contributors to the Marquez project
  * SPDX-License-Identifier: Apache-2.0
  */