From 1330831c2767c0b2675ba47700426bae6e873b3a Mon Sep 17 00:00:00 2001 From: Michael Collado <40346148+collado-mike@users.noreply.github.com> Date: Tue, 9 Aug 2022 15:22:45 -0700 Subject: [PATCH 1/4] Update FlywayFactory to support custom schema (#2055) Signed-off-by: Michael Collado Co-authored-by: Willy Lulciuc --- api/src/main/java/marquez/db/FlywayFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/src/main/java/marquez/db/FlywayFactory.java b/api/src/main/java/marquez/db/FlywayFactory.java index b7b48a0f1d..16bcce6336 100644 --- a/api/src/main/java/marquez/db/FlywayFactory.java +++ b/api/src/main/java/marquez/db/FlywayFactory.java @@ -74,6 +74,8 @@ public final class FlywayFactory { @Getter @Setter private String repeatableSqlMigrationPrefix = DEFAULT_REPEATABLE_SQL_MIGRATION_PREFIX; + @Getter @Setter private String schema; + public Flyway build(@NonNull DataSource source) { return Flyway.configure() .dataSource(source) @@ -102,6 +104,7 @@ public Flyway build(@NonNull DataSource source) { .placeholderSuffix(placeholderSuffix) .sqlMigrationPrefix(sqlMigrationPrefix) .repeatableSqlMigrationPrefix(repeatableSqlMigrationPrefix) + .defaultSchema(schema) .load(); } } From dbdea1a1448a2e4fbf01e3e6084e0e61c8f09f4c Mon Sep 17 00:00:00 2001 From: Willy Lulciuc Date: Wed, 10 Aug 2022 10:11:27 -0700 Subject: [PATCH 2/4] Add steps on proposing changes to Marquez (#2065) * Add steps for proposal review Signed-off-by: wslulciuc Signed-off-by: wslulciuc --- proposals/README.md | 21 +++++++++++++++++++++ proposals/TEMPLATE.md | 25 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 proposals/README.md create mode 100644 proposals/TEMPLATE.md diff --git a/proposals/README.md b/proposals/README.md new file mode 100644 index 0000000000..08a2a407d1 --- /dev/null +++ b/proposals/README.md @@ -0,0 +1,21 @@ +# Proposals + +Marquez uses a _multi_-project structure and contains the modules [`api`](https://github.com/MarquezProject/marquez/tree/main/api), [`web`](https://github.com/MarquezProject/marquez/tree/main/web), [`clients`](https://github.com/MarquezProject/marquez/tree/main/clients), and [`chart`](https://github.com/MarquezProject/marquez/tree/main/chart). Below, we describe the process for proposing, documenting, and implementing changes to Marquez. + + +## Submitting a Proposal for Review + +1. Open a [new issue](https://github.com/MarquezProject/marquez/issues/new) briefly describing your proposal +2. Work with Marquez [committers](https://github.com/MarquezProject/marquez/blob/main/COMMITTERS.md) to get your proposal reviewed + + > **Note:** Your proposal can have one of two outcomes: _`accepted`_ or _`decline`_; if the proposal is _`declined`_, the process is done. + +3. If the proposal is _`accepted`_, the proposal author **must** write a design doc using our [template](https://github.com/MarquezProject/marquez/blob/main/proposals/TEMPLATE.md) for new proposals + + > **Note:** Your proposal **must** be added under [`proposals/`]() using the naming convention: `[#ISSUE]-[SHORT-NAME].md`, where `[#ISSUE]` is the GitHub issue in **Step 1**, and `[SHORT-NAME]` is a shortened name of your proposal (each word seperated by dashes(`-`)). + +4. Open a pull request with your proposal for final discussion (please follow our [contributing](https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md) guide) + +## Questions? + +If you need help with the proposal process, please reach out to us on our [slack](http://bit.ly/MarquezSlack) channel. diff --git a/proposals/TEMPLATE.md b/proposals/TEMPLATE.md new file mode 100644 index 0000000000..b272c120c6 --- /dev/null +++ b/proposals/TEMPLATE.md @@ -0,0 +1,25 @@ +[_Template for proposing changes to Marquez as documented [here](https://github.com/MarquezProject/marquez/blob/main/proposals/README.md)._] + +# Proposal: [_Your Title Here_] + +Author(s): [_Author, Co-author(s)_] + +Created: [_YYYY-MM-DD_] + +Dicussion: [_Link to issue with dicussion on the change being proposed._] + +## Overview + +[_A short summary of the proposal, background, and the problem being solved._] + +## Proposal + +[_The change being proposed._] + +## Implementation + +[_A description of implementation steps. We recommend you include any dependencies. If your change requires a database schema migration, please describe the schema modification(s) and whether it's a backwards-incompatible or backwards-compatible change._] + +## Next Steps + +[_A discussion of related issue(s) for this proposal, who will be assigned to work on them, timeline for when the change being proposed will be worked on, etc. **You may omit this section if there are none.**_] From 3dba9957cd41f284f93438ef83ead35b904de77e Mon Sep 17 00:00:00 2001 From: wslulciuc Date: Wed, 10 Aug 2022 13:13:38 -0700 Subject: [PATCH 3/4] More info on proposal acceptance Signed-off-by: wslulciuc --- proposals/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/proposals/README.md b/proposals/README.md index 08a2a407d1..1e1ec55bcc 100644 --- a/proposals/README.md +++ b/proposals/README.md @@ -10,11 +10,13 @@ Marquez uses a _multi_-project structure and contains the modules [`api`](https: > **Note:** Your proposal can have one of two outcomes: _`accepted`_ or _`decline`_; if the proposal is _`declined`_, the process is done. -3. If the proposal is _`accepted`_, the proposal author **must** write a design doc using our [template](https://github.com/MarquezProject/marquez/blob/main/proposals/TEMPLATE.md) for new proposals +3. If the proposal is _`accepted`_, it will be added to our [Backlog](https://github.com/orgs/MarquezProject/projects/1); the proposal author **must** write a design doc using our [template](https://github.com/MarquezProject/marquez/blob/main/proposals/TEMPLATE.md) for new proposals - > **Note:** Your proposal **must** be added under [`proposals/`]() using the naming convention: `[#ISSUE]-[SHORT-NAME].md`, where `[#ISSUE]` is the GitHub issue in **Step 1**, and `[SHORT-NAME]` is a shortened name of your proposal (each word seperated by dashes(`-`)). + > **Note:** Your proposal **must** be added under [`proposals/`]() using the naming convention: `[#ISSUE]-[SHORT-NAME].md`, where `[#ISSUE]` is the GitHub issue in **Step 1**, and `[SHORT-NAME]` is a shortened name of your proposal (each word seperated by dashes (`-`)). -4. Open a pull request with your proposal for final discussion (please follow our [contributing](https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md) guide) +4. Open a [pull request](https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md#submitting-a-pull-request) with your proposal for final discussion; once Marquez [committers](https://github.com/MarquezProject/marquez/blob/main/COMMITTERS.md) reach a general consensus on your proposal, it will be added to a [Milestone](https://github.com/MarquezProject/marquez/milestones) + +Once your proposal has been _`accepted`_, and has been associated with a milestone, you can begin implementation work by following our [contributing](https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md) guide for Marquez. ## Questions? From 476e4722b965293d4b8168c3480bb3ccc556ed4a Mon Sep 17 00:00:00 2001 From: Michael Collado <40346148+collado-mike@users.noreply.github.com> Date: Wed, 10 Aug 2022 14:32:01 -0700 Subject: [PATCH 4/4] Fix/rewrite jobs fqn locks (#2067) * Update rewrite_jobs_fqn_table function to correctly ignore existing symlinked jobs Signed-off-by: Michael Collado * Update rewrite_jobs_fqn_table to avoid updating jobs_fqn table when nothing has changed Signed-off-by: Michael Collado Signed-off-by: Michael Collado --- .../R__1_Jobs_view_and_rewrite_function.sql | 24 ++++++++++++------- api/src/test/java/marquez/db/JobDaoTest.java | 13 ++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql b/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql index 12a1273657..6225963adb 100644 --- a/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql +++ b/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql @@ -27,6 +27,8 @@ CREATE OR REPLACE FUNCTION rewrite_jobs_fqn_table() RETURNS TRIGGER AS $$ DECLARE job_uuid uuid; + new_symlink_target_uuid uuid; + old_symlink_target_uuid uuid; inserted_job jobs_view%rowtype; BEGIN INSERT INTO jobs (uuid, type, created_at, updated_at, namespace_uuid, name, description, @@ -55,13 +57,19 @@ BEGIN current_job_context_uuid = EXCLUDED.current_job_context_uuid, current_location = EXCLUDED.current_location, current_inputs = EXCLUDED.current_inputs, - -- update the symlink target if not null. otherwise, keep the old value - symlink_target_uuid = COALESCE( - EXCLUDED.symlink_target_uuid, - jobs.symlink_target_uuid) - RETURNING uuid INTO job_uuid; - IF TG_OP = 'INSERT' OR - (TG_OP = 'UPDATE' AND OLD.symlink_target_uuid IS DISTINCT FROM NEW.symlink_target_uuid) THEN + -- update the symlink target if null. otherwise, keep the old value + symlink_target_uuid = COALESCE(jobs.symlink_target_uuid, + EXCLUDED.symlink_target_uuid) + -- the SELECT statement below will get the OLD symlink_target_uuid in case of update and the NEW + -- version in case of insert + RETURNING uuid, symlink_target_uuid, (SELECT symlink_target_uuid FROM jobs j2 WHERE j2.uuid=jobs.uuid) + INTO job_uuid, new_symlink_target_uuid, old_symlink_target_uuid; + + -- update the jobs_fqn table only when inserting a new record (NEW.uuid will equal the job_uuid + -- when inserting a new record) or when the symlink_target_uuid is being updated. + IF NEW.uuid = job_uuid OR + (new_symlink_target_uuid IS DISTINCT FROM old_symlink_target_uuid) THEN + RAISE LOG 'Updating jobs_fqn due to % to job % (%)', TG_OP, NEW.name, job_uuid; WITH RECURSIVE jobs_symlink AS (SELECT uuid, uuid AS link_target_uuid, symlink_target_uuid FROM jobs j @@ -124,4 +132,4 @@ CREATE TRIGGER update_symlinks INSTEAD OF UPDATE OR INSERT ON jobs_view FOR EACH ROW -EXECUTE FUNCTION rewrite_jobs_fqn_table(); \ No newline at end of file +EXECUTE FUNCTION rewrite_jobs_fqn_table(); diff --git a/api/src/test/java/marquez/db/JobDaoTest.java b/api/src/test/java/marquez/db/JobDaoTest.java index bd1ef46c7f..09ddb29b96 100644 --- a/api/src/test/java/marquez/db/JobDaoTest.java +++ b/api/src/test/java/marquez/db/JobDaoTest.java @@ -171,6 +171,19 @@ public void testUpsertJobWithNewSymlink() { jobDao.findJobByName(symlinkJob.getNamespaceName(), symlinkJob.getName()), targetJob.getNamespaceName(), targetJob.getName()); + + // try to update the symlink target - it should be ignored + JobRow anotherTargetJob = + createJobWithoutSymlinkTarget( + jdbi, namespace, "anotherTarget", "we'll attempt to update the symlink"); + createJobWithSymlinkTarget( + jdbi, namespace, symlinkJobName, anotherTargetJob.getUuid(), "the symlink job"); + + // the original symlink target should be returned + assertJobIdEquals( + jobDao.findJobByName(symlinkJob.getNamespaceName(), symlinkJob.getName()), + targetJob.getNamespaceName(), + targetJob.getName()); } public void testSymlinkParentJobRenamesChildren() throws SQLException {