Merge branch 'main' into fix/lineage_query_perf

api/src/main/java/marquez/db/FlywayFactory.java

@@ -74,6 +74,8 @@ public final class FlywayFactory {
   @Getter @Setter
   private String repeatableSqlMigrationPrefix = DEFAULT_REPEATABLE_SQL_MIGRATION_PREFIX;
 
+  @Getter @Setter private String schema;
+
   public Flyway build(@NonNull DataSource source) {
     return Flyway.configure()
         .dataSource(source)
@@ -102,6 +104,7 @@ public Flyway build(@NonNull DataSource source) {
         .placeholderSuffix(placeholderSuffix)
         .sqlMigrationPrefix(sqlMigrationPrefix)
         .repeatableSqlMigrationPrefix(repeatableSqlMigrationPrefix)
+        .defaultSchema(schema)
         .load();
   }
 }

api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql

@@ -27,6 +27,8 @@ CREATE OR REPLACE FUNCTION rewrite_jobs_fqn_table() RETURNS TRIGGER AS
 $$
 DECLARE
     job_uuid uuid;
+    new_symlink_target_uuid uuid;
+    old_symlink_target_uuid uuid;
     inserted_job jobs_view%rowtype;
 BEGIN
     INSERT INTO jobs (uuid, type, created_at, updated_at, namespace_uuid, name, description,
@@ -55,13 +57,19 @@ BEGIN
                       current_job_context_uuid = EXCLUDED.current_job_context_uuid,
                       current_location         = EXCLUDED.current_location,
                       current_inputs           = EXCLUDED.current_inputs,
-                      -- update the symlink target if not null. otherwise, keep the old value
-                      symlink_target_uuid      = COALESCE(
-                              EXCLUDED.symlink_target_uuid,
-                              jobs.symlink_target_uuid)
-    RETURNING uuid INTO job_uuid;
-    IF TG_OP = 'INSERT' OR
-       (TG_OP = 'UPDATE' AND OLD.symlink_target_uuid IS DISTINCT FROM NEW.symlink_target_uuid) THEN
+                      -- update the symlink target if null. otherwise, keep the old value
+                      symlink_target_uuid      = COALESCE(jobs.symlink_target_uuid,
+                              EXCLUDED.symlink_target_uuid)
+    -- the SELECT statement below will get the OLD symlink_target_uuid in case of update and the NEW
+    -- version in case of insert
+    RETURNING uuid, symlink_target_uuid, (SELECT symlink_target_uuid FROM jobs j2 WHERE j2.uuid=jobs.uuid)
+        INTO job_uuid, new_symlink_target_uuid, old_symlink_target_uuid;
+
+    -- update the jobs_fqn table only when inserting a new record (NEW.uuid will equal the job_uuid
+    -- when inserting a new record) or when the symlink_target_uuid is being updated.
+    IF NEW.uuid = job_uuid OR
+       (new_symlink_target_uuid IS DISTINCT FROM old_symlink_target_uuid) THEN
+        RAISE LOG 'Updating jobs_fqn due to % to job % (%)', TG_OP, NEW.name, job_uuid;
         WITH RECURSIVE
             jobs_symlink AS (SELECT uuid, uuid AS link_target_uuid, symlink_target_uuid
                              FROM jobs j
@@ -124,4 +132,4 @@ CREATE TRIGGER update_symlinks
     INSTEAD OF UPDATE OR INSERT
     ON jobs_view
     FOR EACH ROW
-EXECUTE FUNCTION rewrite_jobs_fqn_table();
+EXECUTE FUNCTION rewrite_jobs_fqn_table();

api/src/test/java/marquez/db/JobDaoTest.java

@@ -171,6 +171,19 @@ public void testUpsertJobWithNewSymlink() {
         jobDao.findJobByName(symlinkJob.getNamespaceName(), symlinkJob.getName()),
         targetJob.getNamespaceName(),
         targetJob.getName());
+
+    // try to update the symlink target - it should be ignored
+    JobRow anotherTargetJob =
+        createJobWithoutSymlinkTarget(
+            jdbi, namespace, "anotherTarget", "we'll attempt to update the symlink");
+    createJobWithSymlinkTarget(
+        jdbi, namespace, symlinkJobName, anotherTargetJob.getUuid(), "the symlink job");
+
+    // the original symlink target should be returned
+    assertJobIdEquals(
+        jobDao.findJobByName(symlinkJob.getNamespaceName(), symlinkJob.getName()),
+        targetJob.getNamespaceName(),
+        targetJob.getName());
   }
 
   public void testSymlinkParentJobRenamesChildren() throws SQLException {

proposals/README.md

@@ -0,0 +1,23 @@
+# Proposals
+
+Marquez uses a _multi_-project structure and contains the modules [`api`](https://github.com/MarquezProject/marquez/tree/main/api), [`web`](https://github.com/MarquezProject/marquez/tree/main/web), [`clients`](https://github.com/MarquezProject/marquez/tree/main/clients), and [`chart`](https://github.com/MarquezProject/marquez/tree/main/chart). Below, we describe the process for proposing, documenting, and implementing changes to Marquez.
+
+
+## Submitting a Proposal for Review
+
+1. Open a [new issue](https://github.com/MarquezProject/marquez/issues/new) briefly describing your proposal
+2. Work with Marquez [committers](https://github.com/MarquezProject/marquez/blob/main/COMMITTERS.md) to get your proposal reviewed
+
+   > **Note:** Your proposal can have one of two outcomes: _`accepted`_ or _`decline`_; if the proposal is _`declined`_, the process is done.
+
+3. If the proposal is _`accepted`_, it will be added to our [Backlog](https://github.com/orgs/MarquezProject/projects/1); the proposal author **must** write a design doc using our [template](https://github.com/MarquezProject/marquez/blob/main/proposals/TEMPLATE.md) for new proposals
+
+   > **Note:** Your proposal **must** be added under [`proposals/`]() using the naming convention: `[#ISSUE]-[SHORT-NAME].md`, where `[#ISSUE]` is the GitHub issue in **Step 1**, and `[SHORT-NAME]` is a shortened name of your proposal (each word seperated by dashes (`-`)).
+
+4. Open a [pull request](https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md#submitting-a-pull-request) with your proposal for final discussion; once Marquez [committers](https://github.com/MarquezProject/marquez/blob/main/COMMITTERS.md) reach a general consensus on your proposal, it will be added to a [Milestone](https://github.com/MarquezProject/marquez/milestones)
+
+Once your proposal has been _`accepted`_, and has been associated with a milestone, you can begin implementation work by following our [contributing](https://github.com/MarquezProject/marquez/blob/main/CONTRIBUTING.md) guide for Marquez.
+
+## Questions?
+
+If you need help with the proposal process, please reach out to us on our [slack](http://bit.ly/MarquezSlack) channel.

proposals/TEMPLATE.md

@@ -0,0 +1,25 @@
+[_Template for proposing changes to Marquez as documented [here](https://github.com/MarquezProject/marquez/blob/main/proposals/README.md)._]
+
+# Proposal: [_Your Title Here_]
+
+Author(s): [_Author, Co-author(s)_]
+
+Created: [_YYYY-MM-DD_]
+
+Dicussion: [_Link to issue with dicussion on the change being proposed._]
+
+## Overview
+
+[_A short summary of the proposal, background, and the problem being solved._]
+
+## Proposal
+
+[_The change being proposed._]
+
+## Implementation
+
+[_A description of implementation steps. We recommend you include any dependencies. If your change requires a database schema migration, please describe the schema modification(s) and whether it's a backwards-incompatible or backwards-compatible change._]
+
+## Next Steps
+
+[_A discussion of related issue(s) for this proposal, who will be assigned to work on them, timeline for when the change being proposed will be worked on, etc. **You may omit this section if there are none.**_]