From 99f4546eddb102d4b59f07272861e87c1c70e2cf Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 10:56:51 -0700
Subject: [PATCH 01/32] Add the log source to the MDP

---
 .../commons/logging/LoggingHelper.java        | 40 +++++++++++++++++++
 airbyte-commons/src/main/resources/log4j2.xml |  2 +-
 .../airbyte/DefaultAirbyteDestination.java    |  4 ++
 .../airbyte/DefaultAirbyteSource.java         |  8 ++++
 4 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100644 airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
new file mode 100644
index 000000000000..387884f495d7
--- /dev/null
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.commons.logging;
+
+public class LoggingHelper {
+
+  public enum Color {
+
+    BLACK("\u001b[30m"),
+    RED("\u001b[31m"),
+    GREEN("\u001b[32m"),
+    YELLOW("\u001b[33m"),
+    BLUE("\u001b[34m"),
+    MAGENTA("\u001b[35m"),
+    CYAN("\u001b[36m"),
+    WHITE("\u001b[37m");
+
+    private final String ainsiCode;
+
+    Color(final String ainsiCode) {
+      this.ainsiCode = ainsiCode;
+    }
+
+    public String getCode() {
+      return ainsiCode;
+    }
+
+  }
+
+  public static final String LOG_SOURCE_MDC_KEY = "source";
+
+  private static final String RESET = "\u001B[0m";
+
+  public static String applyColor(final Color color, final String msg) {
+    return color.getCode() + msg + RESET;
+  }
+
+}
diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-commons/src/main/resources/log4j2.xml
index b7e7eafe2433..b3f6b955243f 100644
--- a/airbyte-commons/src/main/resources/log4j2.xml
+++ b/airbyte-commons/src/main/resources/log4j2.xml
@@ -4,7 +4,7 @@
 
         <!-- The following patterns mask the string apikey=<string> to apikey=***** to prevent secrets leaking. -->
         <Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %X - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
-        <Property name="docker-worker-file-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %p (%X{job_root}) %C{1}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
+        <Property name="docker-worker-file-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %p (%X{job_root}) %C{1}(%M):%L - %X - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
         <!-- Remove paths from logs on Cloud as users aren't able to access file paths and these end up being noise. -->
         <Property name="cloud-worker-file-pattern">%d{yyyy-MM-dd HH:mm:ss} %-5p %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index 9bf76989307a..da50c60dfa13 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -9,6 +9,8 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.LoggingHelper.Color;
 import io.airbyte.config.WorkerDestinationConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -26,6 +28,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
 
 public class DefaultAirbyteDestination implements AirbyteDestination {
 
@@ -54,6 +57,7 @@ public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher,
   @Override
   public void start(final WorkerDestinationConfig destinationConfig, final Path jobRoot) throws IOException, WorkerException {
     Preconditions.checkState(destinationProcess == null);
+    MDC.put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.CYAN, "airbyte-destination"));
 
     LOGGER.info("Running destination...");
     destinationProcess = integrationLauncher.write(
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index 2b17b4050fdc..7b449f7810c2 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -9,6 +9,8 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.LoggingHelper.Color;
 import io.airbyte.config.WorkerSourceConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -23,6 +25,7 @@
 import java.util.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
 
 public class DefaultAirbyteSource implements AirbyteSource {
 
@@ -57,6 +60,7 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
 
   @Override
   public void start(final WorkerSourceConfig sourceConfig, final Path jobRoot) throws Exception {
+    MDC.put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.BLUE, "airbyte-source"));
     Preconditions.checkState(sourceProcess == null);
 
     sourceProcess = integrationLauncher.read(jobRoot,
@@ -110,6 +114,8 @@ public void close() throws Exception {
         CHECK_HEARTBEAT_DURATION,
         FORCED_SHUTDOWN_DURATION);
 
+    MDC.remove(LoggingHelper.LOG_SOURCE_MDC_KEY);
+
     if (sourceProcess.isAlive() || sourceProcess.exitValue() != 0) {
       final String message = sourceProcess.isAlive() ? "Source has not terminated " : "Source process exit with code " + sourceProcess.exitValue();
       throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
@@ -127,6 +133,8 @@ public void cancel() throws Exception {
       WorkerUtils.cancelProcess(sourceProcess);
       LOGGER.info("Cancelled source process!");
     }
+
+    MDC.remove(LoggingHelper.LOG_SOURCE_MDC_KEY);
   }
 
 }

From ed4bcc60b45d0c7aa98010e9a04b59d3bbb4f70b Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 15:03:04 -0700
Subject: [PATCH 02/32] Add helper to safely update the MDC

---
 .../commons/logging/LoggingHelper.java        |  2 +-
 .../commons/logging/ScopedMDCChange.java      | 37 ++++++++++++
 airbyte-commons/src/main/resources/log4j2.xml |  2 +-
 .../commons/logging/ScopedMDCChangeTest.java  | 60 +++++++++++++++++++
 .../airbyte/DefaultAirbyteDestination.java    |  4 --
 .../airbyte/DefaultAirbyteSource.java         | 15 +----
 6 files changed, 102 insertions(+), 18 deletions(-)
 create mode 100644 airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java
 create mode 100644 airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
index 387884f495d7..26f6fc6695bb 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
@@ -29,7 +29,7 @@ public String getCode() {
 
   }
 
-  public static final String LOG_SOURCE_MDC_KEY = "source";
+  public static final String LOG_SOURCE_MDC_KEY = "log_source";
 
   private static final String RESET = "\u001B[0m";
 
diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java
new file mode 100644
index 000000000000..752e707661b0
--- /dev/null
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java
@@ -0,0 +1,37 @@
+package io.airbyte.commons.logging;
+
+import java.util.Map;
+import org.slf4j.MDC;
+
+/**
+ * This class is an autoClosable class that will add some specific values into the log MDC. When being close, it will restore the orginal MDC. It is
+ * advise to use it like that:
+ * <pre>
+ *   <code>
+ *     try(final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(
+ *      new HashMap<String, String>() {{
+ *        put("my", "value");
+ *      }}
+ *     )) {
+ *        ...
+ *     }
+ *   </code>
+ * </pre>
+ */
+public class ScopedMDCChange implements AutoCloseable {
+
+  private final Map<String, String> originalContextMap;
+
+  public ScopedMDCChange(final Map<String, String> keyValuesToAdd) {
+    originalContextMap = MDC.getCopyOfContextMap();
+
+    keyValuesToAdd.forEach(
+        (key, value) -> MDC.put(key, value)
+    );
+  }
+
+  @Override public void close() throws Exception {
+    MDC.clear();
+    MDC.setContextMap(originalContextMap);
+  }
+}
diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-commons/src/main/resources/log4j2.xml
index b3f6b955243f..91ab632ea1d3 100644
--- a/airbyte-commons/src/main/resources/log4j2.xml
+++ b/airbyte-commons/src/main/resources/log4j2.xml
@@ -4,7 +4,7 @@
 
         <!-- The following patterns mask the string apikey=<string> to apikey=***** to prevent secrets leaking. -->
         <Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %X - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
-        <Property name="docker-worker-file-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %p (%X{job_root}) %C{1}(%M):%L - %X - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
+        <Property name="docker-worker-file-pattern">%replace{%X{log_source} - }{^ - }{}%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %p (%X{job_root}) %C{1}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
         <!-- Remove paths from logs on Cloud as users aren't able to access file paths and these end up being noise. -->
         <Property name="cloud-worker-file-pattern">%d{yyyy-MM-dd HH:mm:ss} %-5p %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
 
diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java
new file mode 100644
index 000000000000..39f76197a71b
--- /dev/null
+++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java
@@ -0,0 +1,60 @@
+package io.airbyte.commons.logging;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.slf4j.MDC;
+
+public class ScopedMDCChangeTest {
+
+  private static final Map<String, String> originalMap = new HashMap<>() {{
+    put("test", "entry");
+    put("testOverride", "should be overrided");
+  }};
+
+  private static final Map<String, String> modificationInMDC = new HashMap<>() {{
+    put("new", "will be added");
+    put("testOverride", "will override");
+  }};
+
+  @BeforeEach
+  public void init() {
+    MDC.clear();
+    MDC.setContextMap(originalMap);
+  }
+
+  @Test
+  @DisplayName("The MDC context is properly overrided")
+  public void testMDCModified() {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(modificationInMDC)) {
+      final Map<String, String> mdcState = MDC.getCopyOfContextMap();
+
+      Assertions.assertThat(mdcState).containsExactlyInAnyOrderEntriesOf(
+          new HashMap<String, String>() {{
+            put("test", "entry");
+            put("new", "will be added");
+            put("testOverride", "will override");
+          }}
+      );
+
+    } catch (final Exception e) {
+      e.printStackTrace();
+    }
+  }
+
+  @Test
+  @DisplayName("The MDC context is properly restored")
+  public void testMDCRestore() {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(modificationInMDC)) {
+    } catch (final Exception e) {
+      e.printStackTrace();
+    }
+    final Map<String, String> mdcState = MDC.getCopyOfContextMap();
+
+    Assertions.assertThat(mdcState).containsAllEntriesOf(originalMap);
+    Assertions.assertThat(mdcState).doesNotContainKey("new");
+  }
+}
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index da50c60dfa13..9bf76989307a 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -9,8 +9,6 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.LoggingHelper.Color;
 import io.airbyte.config.WorkerDestinationConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -28,7 +26,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
 
 public class DefaultAirbyteDestination implements AirbyteDestination {
 
@@ -57,7 +54,6 @@ public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher,
   @Override
   public void start(final WorkerDestinationConfig destinationConfig, final Path jobRoot) throws IOException, WorkerException {
     Preconditions.checkState(destinationProcess == null);
-    MDC.put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.CYAN, "airbyte-destination"));
 
     LOGGER.info("Running destination...");
     destinationProcess = integrationLauncher.write(
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index 7b449f7810c2..c0441f19110d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -9,8 +9,6 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.LoggingHelper.Color;
 import io.airbyte.config.WorkerSourceConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -25,7 +23,6 @@
 import java.util.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
 
 public class DefaultAirbyteSource implements AirbyteSource {
 
@@ -49,10 +46,9 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
     this(integrationLauncher, new DefaultAirbyteStreamFactory(), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION));
   }
 
-  @VisibleForTesting
-  DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
-                       final AirbyteStreamFactory streamFactory,
-                       final HeartbeatMonitor heartbeatMonitor) {
+  @VisibleForTesting DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
+                                          final AirbyteStreamFactory streamFactory,
+                                          final HeartbeatMonitor heartbeatMonitor) {
     this.integrationLauncher = integrationLauncher;
     this.streamFactory = streamFactory;
     this.heartbeatMonitor = heartbeatMonitor;
@@ -60,7 +56,6 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
 
   @Override
   public void start(final WorkerSourceConfig sourceConfig, final Path jobRoot) throws Exception {
-    MDC.put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.BLUE, "airbyte-source"));
     Preconditions.checkState(sourceProcess == null);
 
     sourceProcess = integrationLauncher.read(jobRoot,
@@ -114,8 +109,6 @@ public void close() throws Exception {
         CHECK_HEARTBEAT_DURATION,
         FORCED_SHUTDOWN_DURATION);
 
-    MDC.remove(LoggingHelper.LOG_SOURCE_MDC_KEY);
-
     if (sourceProcess.isAlive() || sourceProcess.exitValue() != 0) {
       final String message = sourceProcess.isAlive() ? "Source has not terminated " : "Source process exit with code " + sourceProcess.exitValue();
       throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
@@ -133,8 +126,6 @@ public void cancel() throws Exception {
       WorkerUtils.cancelProcess(sourceProcess);
       LOGGER.info("Cancelled source process!");
     }
-
-    MDC.remove(LoggingHelper.LOG_SOURCE_MDC_KEY);
   }
 
 }

From 85b81a2041c5688fd31b825061e0134ffbf1f89a Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 15:05:48 -0700
Subject: [PATCH 03/32] Format

---
 .../commons/logging/ScopedMDCChange.java      | 16 +++++--
 .../commons/logging/ScopedMDCChangeTest.java  | 47 ++++++++++++-------
 .../airbyte/DefaultAirbyteSource.java         |  7 +--
 3 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java
index 752e707661b0..b9be48b55ae6 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java
@@ -1,11 +1,16 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
 package io.airbyte.commons.logging;
 
 import java.util.Map;
 import org.slf4j.MDC;
 
 /**
- * This class is an autoClosable class that will add some specific values into the log MDC. When being close, it will restore the orginal MDC. It is
- * advise to use it like that:
+ * This class is an autoClosable class that will add some specific values into the log MDC. When
+ * being close, it will restore the orginal MDC. It is advise to use it like that:
+ *
  * <pre>
  *   <code>
  *     try(final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(
@@ -26,12 +31,13 @@ public ScopedMDCChange(final Map<String, String> keyValuesToAdd) {
     originalContextMap = MDC.getCopyOfContextMap();
 
     keyValuesToAdd.forEach(
-        (key, value) -> MDC.put(key, value)
-    );
+        (key, value) -> MDC.put(key, value));
   }
 
-  @Override public void close() throws Exception {
+  @Override
+  public void close() throws Exception {
     MDC.clear();
     MDC.setContextMap(originalContextMap);
   }
+
 }
diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java
index 39f76197a71b..c3a83c22b209 100644
--- a/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java
+++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java
@@ -1,3 +1,7 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
 package io.airbyte.commons.logging;
 
 import java.util.HashMap;
@@ -10,15 +14,23 @@
 
 public class ScopedMDCChangeTest {
 
-  private static final Map<String, String> originalMap = new HashMap<>() {{
-    put("test", "entry");
-    put("testOverride", "should be overrided");
-  }};
+  private static final Map<String, String> originalMap = new HashMap<>() {
+
+    {
+      put("test", "entry");
+      put("testOverride", "should be overrided");
+    }
+
+  };
+
+  private static final Map<String, String> modificationInMDC = new HashMap<>() {
+
+    {
+      put("new", "will be added");
+      put("testOverride", "will override");
+    }
 
-  private static final Map<String, String> modificationInMDC = new HashMap<>() {{
-    put("new", "will be added");
-    put("testOverride", "will override");
-  }};
+  };
 
   @BeforeEach
   public void init() {
@@ -33,12 +45,15 @@ public void testMDCModified() {
       final Map<String, String> mdcState = MDC.getCopyOfContextMap();
 
       Assertions.assertThat(mdcState).containsExactlyInAnyOrderEntriesOf(
-          new HashMap<String, String>() {{
-            put("test", "entry");
-            put("new", "will be added");
-            put("testOverride", "will override");
-          }}
-      );
+          new HashMap<String, String>() {
+
+            {
+              put("test", "entry");
+              put("new", "will be added");
+              put("testOverride", "will override");
+            }
+
+          });
 
     } catch (final Exception e) {
       e.printStackTrace();
@@ -48,8 +63,7 @@ public void testMDCModified() {
   @Test
   @DisplayName("The MDC context is properly restored")
   public void testMDCRestore() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(modificationInMDC)) {
-    } catch (final Exception e) {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(modificationInMDC)) {} catch (final Exception e) {
       e.printStackTrace();
     }
     final Map<String, String> mdcState = MDC.getCopyOfContextMap();
@@ -57,4 +71,5 @@ public void testMDCRestore() {
     Assertions.assertThat(mdcState).containsAllEntriesOf(originalMap);
     Assertions.assertThat(mdcState).doesNotContainKey("new");
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index c0441f19110d..2b17b4050fdc 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -46,9 +46,10 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
     this(integrationLauncher, new DefaultAirbyteStreamFactory(), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION));
   }
 
-  @VisibleForTesting DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
-                                          final AirbyteStreamFactory streamFactory,
-                                          final HeartbeatMonitor heartbeatMonitor) {
+  @VisibleForTesting
+  DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
+                       final AirbyteStreamFactory streamFactory,
+                       final HeartbeatMonitor heartbeatMonitor) {
     this.integrationLauncher = integrationLauncher;
     this.streamFactory = streamFactory;
     this.heartbeatMonitor = heartbeatMonitor;

From 3b1a65a0a6d3005387c270cee67f9b37bf574c0e Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 15:38:54 -0700
Subject: [PATCH 04/32] Add an application name

---
 .../java/io/airbyte/workers/Application.java  |  9 +++++++++
 .../workers/DbtTransformationRunner.java      | 20 ++++++++++---------
 .../workers/DbtTransformationWorker.java      |  5 ++++-
 .../workers/DefaultCheckConnectionWorker.java |  3 +++
 .../workers/DefaultDiscoverCatalogWorker.java |  3 +++
 .../airbyte/workers/DefaultGetSpecWorker.java |  3 +++
 .../workers/DefaultNormalizationWorker.java   |  3 +++
 .../workers/DefaultReplicationWorker.java     | 14 +++++++------
 .../java/io/airbyte/workers/EchoWorker.java   |  6 +++++-
 .../main/java/io/airbyte/workers/Worker.java  |  9 ++++-----
 .../workers/protocols/Destination.java        |  3 ++-
 .../io/airbyte/workers/protocols/Mapper.java  |  3 ++-
 .../workers/protocols/MessageTracker.java     |  3 ++-
 .../io/airbyte/workers/protocols/Source.java  |  3 ++-
 .../airbyte/AirbyteMessageTracker.java        |  3 +++
 .../airbyte/AirbyteStreamFactory.java         |  3 ++-
 .../airbyte/DefaultAirbyteDestination.java    |  3 +++
 .../airbyte/DefaultAirbyteSource.java         | 10 ++++++----
 .../airbyte/DefaultAirbyteStreamFactory.java  | 13 ++++++------
 .../protocols/airbyte/EmptyAirbyteSource.java |  6 ++++--
 .../protocols/airbyte/NamespacingMapper.java  | 10 ++++++----
 21 files changed, 92 insertions(+), 43 deletions(-)
 create mode 100644 airbyte-workers/src/main/java/io/airbyte/workers/Application.java

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/Application.java b/airbyte-workers/src/main/java/io/airbyte/workers/Application.java
new file mode 100644
index 000000000000..f8a80107c7ce
--- /dev/null
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/Application.java
@@ -0,0 +1,9 @@
+package io.airbyte.workers;
+
+public interface Application {
+
+  default String getApplicationName() {
+    // This value should only be used in the U-Test, it is an empty string instead of airbyte-test in order to avoid displaying airbyte-test in prod
+    return "";
+  }
+}
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index 6e09ef9bceb4..c30364d888cf 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -24,7 +24,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DbtTransformationRunner implements AutoCloseable {
+public class DbtTransformationRunner implements AutoCloseable, Application {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DbtTransformationRunner.class);
   private static final String DBT_ENTRYPOINT_SH = "entrypoint.sh";
@@ -43,14 +43,13 @@ public void start() throws Exception {
   }
 
   /**
-   * The docker image used by the DbtTransformationRunner is provided by the User, so we can't ensure
-   * to have the right python, dbt, dependencies etc software installed to successfully run our
-   * transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend
-   * on the NormalizationRunner to configure the dbt project with the appropriate destination settings
-   * and pull the custom git repository into the workspace.
-   *
-   * Once the workspace folder/files is setup to run, we invoke the custom transformation command as
-   * provided by the user to execute whatever extra transformation has been implemented.
+   * The docker image used by the DbtTransformationRunner is provided by the User, so we can't ensure to have the right python, dbt, dependencies etc
+   * software installed to successfully run our transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend on
+   * the NormalizationRunner to configure the dbt project with the appropriate destination settings and pull the custom git repository into the
+   * workspace.
+   * <p>
+   * Once the workspace folder/files is setup to run, we invoke the custom transformation command as provided by the user to execute whatever extra
+   * transformation has been implemented.
    */
   public boolean run(final String jobId,
                      final int attempt,
@@ -117,4 +116,7 @@ public void close() throws Exception {
     }
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-dbt-transformation-runner";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
index 1e8f4b697ede..34c017a70d5d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
@@ -13,7 +13,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DbtTransformationWorker implements Worker<OperatorDbtInput, Void> {
+public class DbtTransformationWorker implements Worker<OperatorDbtInput, Void>, Application {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DbtTransformationWorker.class);
 
@@ -77,4 +77,7 @@ public void cancel() {
     }
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-dbt-transformation-worker";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
index c4e32d16079f..e06371fb1958 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
@@ -86,4 +86,7 @@ public void cancel() {
     WorkerUtils.cancelProcess(process);
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-connection-checker";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
index bf3dd1a371b4..e879ae28c144 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
@@ -81,4 +81,7 @@ public void cancel() {
     WorkerUtils.cancelProcess(process);
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-catalog-discovery";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
index 82265aa9370c..812118c89994 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
@@ -81,4 +81,7 @@ public void cancel() {
     WorkerUtils.cancelProcess(process);
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-spec-getter";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
index 2ce6c078f0fa..72de2ba7b27e 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
@@ -82,4 +82,7 @@ public void cancel() {
     }
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-normalization";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
index 280cb30f9875..2999c9d92d2f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
@@ -66,14 +66,13 @@ public DefaultReplicationWorker(final String jobId,
   }
 
   /**
-   * Run executes two threads. The first pipes data from STDOUT of the source to STDIN of the
-   * destination. The second listen on STDOUT of the destination. The goal of this second thread is to
-   * detect when the destination emits state messages. Only state messages emitted by the destination
-   * should be treated as state that is safe to return from run. In the case when the destination
-   * emits no state, we fall back on whatever state is pass in as an argument to this method.
+   * Run executes two threads. The first pipes data from STDOUT of the source to STDIN of the destination. The second listen on STDOUT of the
+   * destination. The goal of this second thread is to detect when the destination emits state messages. Only state messages emitted by the
+   * destination should be treated as state that is safe to return from run. In the case when the destination emits no state, we fall back on whatever
+   * state is pass in as an argument to this method.
    *
    * @param syncInput all configuration for running replication
-   * @param jobRoot file root that worker is allowed to use
+   * @param jobRoot   file root that worker is allowed to use
    * @return output of the replication attempt (including state)
    * @throws WorkerException
    */
@@ -271,4 +270,7 @@ public void cancel() {
 
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-replication";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
index e968e70ff095..5c9e18727107 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
@@ -12,7 +12,8 @@ public class EchoWorker implements Worker<String, String> {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(EchoWorker.class);
 
-  public EchoWorker() {}
+  public EchoWorker() {
+  }
 
   @Override
   public String run(final String string, final Path jobRoot) {
@@ -25,4 +26,7 @@ public void cancel() {
     // no-op
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-echo-worker";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java b/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
index ffa64ac35399..6f0b297970ca 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
@@ -6,17 +6,16 @@
 
 import java.nio.file.Path;
 
-public interface Worker<InputType, OutputType> {
+public interface Worker<InputType, OutputType> extends Application {
 
   /**
-   * Blocking call to run the worker's workflow. Once this is complete, getStatus should return either
-   * COMPLETE, FAILED, or CANCELLED.
+   * Blocking call to run the worker's workflow. Once this is complete, getStatus should return either COMPLETE, FAILED, or CANCELLED.
    */
   OutputType run(InputType inputType, Path jobRoot) throws WorkerException;
 
   /**
-   * Cancels in-progress workers. Although all workers support cancel, in reality only the
-   * asynchronous {@link DefaultReplicationWorker}'s cancel is used.
+   * Cancels in-progress workers. Although all workers support cancel, in reality only the asynchronous {@link DefaultReplicationWorker}'s cancel is
+   * used.
    */
   void cancel();
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
index c874e108cfbc..d3246acc4ed7 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
@@ -6,10 +6,11 @@
 
 import io.airbyte.commons.functional.CheckedConsumer;
 import io.airbyte.config.WorkerDestinationConfig;
+import io.airbyte.workers.Application;
 import java.nio.file.Path;
 import java.util.Optional;
 
-public interface Destination<T> extends CheckedConsumer<T, Exception>, AutoCloseable {
+public interface Destination<T> extends CheckedConsumer<T, Exception>, AutoCloseable, Application {
 
   void start(WorkerDestinationConfig destinationConfig, Path jobRoot) throws Exception;
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
index 32a9d1858580..d8ef8cdb9849 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
@@ -5,8 +5,9 @@
 package io.airbyte.workers.protocols;
 
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
+import io.airbyte.workers.Application;
 
-public interface Mapper<T> {
+public interface Mapper<T> extends Application {
 
   ConfiguredAirbyteCatalog mapCatalog(ConfiguredAirbyteCatalog catalog);
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
index d48679339994..10f480c42182 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
@@ -5,10 +5,11 @@
 package io.airbyte.workers.protocols;
 
 import io.airbyte.config.State;
+import io.airbyte.workers.Application;
 import java.util.Optional;
 import java.util.function.Consumer;
 
-public interface MessageTracker<T> extends Consumer<T> {
+public interface MessageTracker<T> extends Consumer<T>, Application {
 
   @Override
   void accept(T message);
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
index 8612ab290e04..1f1b176a9645 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
@@ -5,10 +5,11 @@
 package io.airbyte.workers.protocols;
 
 import io.airbyte.config.WorkerSourceConfig;
+import io.airbyte.workers.Application;
 import java.nio.file.Path;
 import java.util.Optional;
 
-public interface Source<T> extends AutoCloseable {
+public interface Source<T> extends AutoCloseable, Application {
 
   void start(WorkerSourceConfig sourceConfig, Path jobRoot) throws Exception;
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
index 34e12a068b32..7dd2493de66c 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
@@ -52,4 +52,7 @@ public Optional<State> getOutputState() {
     return Optional.ofNullable(outputState.get());
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-message-tracker";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
index 757f7e4e2519..1a876d5ac50d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
@@ -5,10 +5,11 @@
 package io.airbyte.workers.protocols.airbyte;
 
 import io.airbyte.protocol.models.AirbyteMessage;
+import io.airbyte.workers.Application;
 import java.io.BufferedReader;
 import java.util.stream.Stream;
 
-public interface AirbyteStreamFactory {
+public interface AirbyteStreamFactory extends Application {
 
   Stream<AirbyteMessage> create(BufferedReader bufferedReader);
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index 9bf76989307a..92cf1a4d03bb 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -142,4 +142,7 @@ public Optional<AirbyteMessage> attemptRead() {
     return Optional.ofNullable(messageIterator.hasNext() ? messageIterator.next() : null);
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-destination";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index 2b17b4050fdc..46be24d916c2 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -46,10 +46,9 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
     this(integrationLauncher, new DefaultAirbyteStreamFactory(), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION));
   }
 
-  @VisibleForTesting
-  DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
-                       final AirbyteStreamFactory streamFactory,
-                       final HeartbeatMonitor heartbeatMonitor) {
+  @VisibleForTesting DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
+                                          final AirbyteStreamFactory streamFactory,
+                                          final HeartbeatMonitor heartbeatMonitor) {
     this.integrationLauncher = integrationLauncher;
     this.streamFactory = streamFactory;
     this.heartbeatMonitor = heartbeatMonitor;
@@ -129,4 +128,7 @@ public void cancel() throws Exception {
     }
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-source";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index 69b32f64be92..c57d205cd45f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -15,14 +15,12 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Creates a stream from an input stream. The produced stream attempts to parse each line of the
- * InputStream into a AirbyteMessage. If the line cannot be parsed into a AirbyteMessage it is
- * dropped. Each record MUST be new line separated.
+ * Creates a stream from an input stream. The produced stream attempts to parse each line of the InputStream into a AirbyteMessage. If the line cannot
+ * be parsed into a AirbyteMessage it is dropped. Each record MUST be new line separated.
  *
  * <p>
- * If a line starts with a AirbyteMessage and then has other characters after it, that
- * AirbyteMessage will still be parsed. If there are multiple AirbyteMessage records on the same
- * line, only the first will be parsed.
+ * If a line starts with a AirbyteMessage and then has other characters after it, that AirbyteMessage will still be parsed. If there are multiple
+ * AirbyteMessage records on the same line, only the first will be parsed.
  */
 public class DefaultAirbyteStreamFactory implements AirbyteStreamFactory {
 
@@ -89,4 +87,7 @@ private void internalLog(final AirbyteLogMessage logMessage) {
     }
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-stream-factory";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
index 59c331be6b28..d13ed57c51dc 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
@@ -14,8 +14,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
- * This source will never emit any messages. It can be used in cases where that is helpful (hint:
- * reset connection jobs).
+ * This source will never emit any messages. It can be used in cases where that is helpful (hint: reset connection jobs).
  */
 public class EmptyAirbyteSource implements AirbyteSource {
 
@@ -56,4 +55,7 @@ public void cancel() throws Exception {
     // no op.
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-empty-source";
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
index c3f88c2e76ca..c1bcac44806c 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
@@ -17,10 +17,9 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * We apply some transformations on the fly on the catalog (same should be done on records too) from
- * the source before it reaches the destination. One of the transformation is to define the
- * destination namespace where data will be stored and how to mirror (or not) the namespace used in
- * the source (if any). This is configured in the UI through the syncInput.
+ * We apply some transformations on the fly on the catalog (same should be done on records too) from the source before it reaches the destination. One
+ * of the transformation is to define the destination namespace where data will be stored and how to mirror (or not) the namespace used in the source
+ * (if any). This is configured in the UI through the syncInput.
  */
 public class NamespacingMapper implements Mapper<AirbyteMessage> {
 
@@ -97,4 +96,7 @@ private static String transformStreamName(final String streamName, final String
     }
   }
 
+  @Override public String getApplicationName() {
+    return "airbyte-namespacing-mapper";
+  }
 }

From 0b27e0221aa0b793e162353da8a197f8acb8259e Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 16:13:29 -0700
Subject: [PATCH 05/32] Tmp

---
 .../workers/DbtTransformationWorker.java      | 70 ++++++++++++-------
 1 file changed, 44 insertions(+), 26 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
index 34c017a70d5d..7ba8d3c2309c 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
@@ -4,11 +4,16 @@
 
 package io.airbyte.workers;
 
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.LoggingHelper.Color;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.OperatorDbtInput;
 import io.airbyte.config.ResourceRequirements;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -21,6 +26,7 @@ public class DbtTransformationWorker implements Worker<OperatorDbtInput, Void>,
   private final int attempt;
   private final DbtTransformationRunner dbtTransformationRunner;
   private final ResourceRequirements resourceRequirements;
+  private final Map<String, String> newMDCValue;
 
   private final AtomicBoolean cancelled;
 
@@ -34,44 +40,56 @@ public DbtTransformationWorker(final String jobId,
     this.resourceRequirements = resourceRequirements;
 
     this.cancelled = new AtomicBoolean(false);
+
+    this.newMDCValue = new HashMap<>() {{
+      put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.GREEN, getApplicationName()));
+    }};
   }
 
   @Override
   public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) throws WorkerException {
-    final long startTime = System.currentTimeMillis();
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(newMDCValue)) {
+      final long startTime = System.currentTimeMillis();
 
-    try (dbtTransformationRunner) {
-      LOGGER.info("Running dbt transformation.");
-      dbtTransformationRunner.start();
-      final Path transformRoot = Files.createDirectories(jobRoot.resolve("transform"));
-      if (!dbtTransformationRunner.run(
-          jobId,
-          attempt,
-          transformRoot,
-          operatorDbtInput.getDestinationConfiguration(),
-          resourceRequirements,
-          operatorDbtInput.getOperatorDbt())) {
-        throw new WorkerException("DBT Transformation Failed.");
+      try (dbtTransformationRunner) {
+        LOGGER.info("Running dbt transformation.");
+        dbtTransformationRunner.start();
+        final Path transformRoot = Files.createDirectories(jobRoot.resolve("transform"));
+        if (!dbtTransformationRunner.run(
+            jobId,
+            attempt,
+            transformRoot,
+            operatorDbtInput.getDestinationConfiguration(),
+            resourceRequirements,
+            operatorDbtInput.getOperatorDbt())) {
+          throw new WorkerException("DBT Transformation Failed.");
+        }
+      } catch (final Exception e) {
+        throw new WorkerException("Dbt Transformation Failed.", e);
+      }
+      if (cancelled.get()) {
+        LOGGER.info("Dbt Transformation was cancelled.");
       }
-    } catch (final Exception e) {
-      throw new WorkerException("Dbt Transformation Failed.", e);
-    }
-    if (cancelled.get()) {
-      LOGGER.info("Dbt Transformation was cancelled.");
-    }
 
-    final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime);
-    LOGGER.info("Dbt Transformation executed in {}.", duration.toMinutesPart());
+      final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime);
+      LOGGER.info("Dbt Transformation executed in {}.", duration.toMinutesPart());
 
-    return null;
+      return null;
+    } catch (final Exception e) {
+      throw new WorkerException(getApplicationName() + " failed", e);
+    }
   }
 
   @Override
   public void cancel() {
-    LOGGER.info("Cancelling Dbt Transformation runner...");
-    try {
-      cancelled.set(true);
-      dbtTransformationRunner.close();
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(newMDCValue)) {
+      LOGGER.info("Cancelling Dbt Transformation runner...");
+      try {
+        cancelled.set(true);
+        dbtTransformationRunner.close();
+      } catch (final Exception e) {
+        e.printStackTrace();
+      }
     } catch (final Exception e) {
       e.printStackTrace();
     }

From 2e794b17139961ff4510a69ce447c8877c8e804f Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 16:50:58 -0700
Subject: [PATCH 06/32] Log the application name

---
 .../commons/application}/Application.java     |  2 +-
 .../commons/logging/LoggingHelper.java        |  9 ++
 .../workers/DbtTransformationRunner.java      | 35 ++++---
 .../workers/DbtTransformationWorker.java      | 13 +--
 .../workers/DefaultDiscoverCatalogWorker.java |  9 +-
 .../airbyte/workers/DefaultGetSpecWorker.java |  9 +-
 .../workers/DefaultNormalizationWorker.java   | 31 ++++---
 .../workers/DefaultReplicationWorker.java     | 56 +++++------
 .../main/java/io/airbyte/workers/Worker.java  |  1 +
 .../workers/protocols/Destination.java        |  2 +-
 .../io/airbyte/workers/protocols/Mapper.java  |  2 +-
 .../workers/protocols/MessageTracker.java     |  2 +-
 .../io/airbyte/workers/protocols/Source.java  |  2 +-
 .../airbyte/AirbyteMessageTracker.java        | 19 ++--
 .../airbyte/AirbyteStreamFactory.java         |  2 +-
 .../airbyte/DefaultAirbyteDestination.java    | 92 +++++++++++--------
 .../airbyte/DefaultAirbyteSource.java         | 88 ++++++++++--------
 .../airbyte/DefaultAirbyteStreamFactory.java  | 78 +++++++++-------
 .../protocols/airbyte/NamespacingMapper.java  | 66 +++++++------
 19 files changed, 295 insertions(+), 223 deletions(-)
 rename {airbyte-workers/src/main/java/io/airbyte/workers => airbyte-commons/src/main/java/io/airbyte/commons/application}/Application.java (85%)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/Application.java b/airbyte-commons/src/main/java/io/airbyte/commons/application/Application.java
similarity index 85%
rename from airbyte-workers/src/main/java/io/airbyte/workers/Application.java
rename to airbyte-commons/src/main/java/io/airbyte/commons/application/Application.java
index f8a80107c7ce..48c36c14248c 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/Application.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/application/Application.java
@@ -1,4 +1,4 @@
-package io.airbyte.workers;
+package io.airbyte.commons.application;
 
 public interface Application {
 
diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
index 26f6fc6695bb..1e81e3cd94b0 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
@@ -4,6 +4,10 @@
 
 package io.airbyte.commons.logging;
 
+import io.airbyte.commons.application.Application;
+import java.util.HashMap;
+import java.util.Map;
+
 public class LoggingHelper {
 
   public enum Color {
@@ -37,4 +41,9 @@ public static String applyColor(final Color color, final String msg) {
     return color.getCode() + msg + RESET;
   }
 
+  public static Map<String, String> getExtraMDCEntries(final Application application) {
+    return new HashMap<>() {{
+      put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.GREEN, application.getApplicationName()));
+    }};
+  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index c30364d888cf..858b9fd4ab74 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -7,7 +7,10 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
+import io.airbyte.commons.application.Application;
 import io.airbyte.commons.io.LineGobbler;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.config.OperatorDbt;
 import io.airbyte.config.ResourceRequirements;
@@ -39,7 +42,9 @@ public DbtTransformationRunner(final ProcessFactory processFactory, final Normal
   }
 
   public void start() throws Exception {
-    normalizationRunner.start();
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      normalizationRunner.start();
+    }
   }
 
   /**
@@ -58,10 +63,12 @@ public boolean run(final String jobId,
                      final ResourceRequirements resourceRequirements,
                      final OperatorDbt dbtConfig)
       throws Exception {
-    if (!normalizationRunner.configureDbt(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig)) {
-      return false;
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      if (!normalizationRunner.configureDbt(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig)) {
+        return false;
+      }
+      return transform(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig);
     }
-    return transform(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig);
   }
 
   public boolean transform(final String jobId,
@@ -71,7 +78,7 @@ public boolean transform(final String jobId,
                            final ResourceRequirements resourceRequirements,
                            final OperatorDbt dbtConfig)
       throws Exception {
-    try {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
       final Map<String, String> files = ImmutableMap.of(
           DBT_ENTRYPOINT_SH, MoreResources.readResource("dbt_transformation_entrypoint.sh"),
           "sshtunneling.sh", MoreResources.readResource("sshtunneling.sh"));
@@ -103,16 +110,18 @@ public boolean transform(final String jobId,
 
   @Override
   public void close() throws Exception {
-    normalizationRunner.close();
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      normalizationRunner.close();
 
-    if (process == null) {
-      return;
-    }
+      if (process == null) {
+        return;
+      }
 
-    LOGGER.debug("Closing dbt transformation process");
-    WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
-    if (process.isAlive() || process.exitValue() != 0) {
-      throw new WorkerException("Dbt transformation process wasn't successful");
+      LOGGER.debug("Closing dbt transformation process");
+      WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
+      if (process.isAlive() || process.exitValue() != 0) {
+        throw new WorkerException("Dbt transformation process wasn't successful");
+      }
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
index 7ba8d3c2309c..58b57fd908ef 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
@@ -4,16 +4,14 @@
 
 package io.airbyte.workers;
 
+import io.airbyte.commons.application.Application;
 import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.LoggingHelper.Color;
 import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.OperatorDbtInput;
 import io.airbyte.config.ResourceRequirements;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,7 +24,6 @@ public class DbtTransformationWorker implements Worker<OperatorDbtInput, Void>,
   private final int attempt;
   private final DbtTransformationRunner dbtTransformationRunner;
   private final ResourceRequirements resourceRequirements;
-  private final Map<String, String> newMDCValue;
 
   private final AtomicBoolean cancelled;
 
@@ -40,15 +37,11 @@ public DbtTransformationWorker(final String jobId,
     this.resourceRequirements = resourceRequirements;
 
     this.cancelled = new AtomicBoolean(false);
-
-    this.newMDCValue = new HashMap<>() {{
-      put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.GREEN, getApplicationName()));
-    }};
   }
 
   @Override
   public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) throws WorkerException {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(newMDCValue)) {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
       final long startTime = System.currentTimeMillis();
 
       try (dbtTransformationRunner) {
@@ -82,7 +75,7 @@ public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) thr
 
   @Override
   public void cancel() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(newMDCValue)) {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
       LOGGER.info("Cancelling Dbt Transformation runner...");
       try {
         cancelled.set(true);
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
index e879ae28c144..4999a8abfda2 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
@@ -7,6 +7,8 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.StandardDiscoverCatalogInput;
 import io.airbyte.protocol.models.AirbyteCatalog;
 import io.airbyte.protocol.models.AirbyteMessage;
@@ -41,7 +43,7 @@ public DefaultDiscoverCatalogWorker(final IntegrationLauncher integrationLaunche
 
   @Override
   public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput, final Path jobRoot) throws WorkerException {
-    try {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
       process = integrationLauncher.discover(
           jobRoot,
           WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
@@ -78,7 +80,10 @@ public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput
 
   @Override
   public void cancel() {
-    WorkerUtils.cancelProcess(process);
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      WorkerUtils.cancelProcess(process);
+    } catch (final Exception e) {
+    }
   }
 
   @Override public String getApplicationName() {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
index 812118c89994..c486bdc2c1a1 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
@@ -6,6 +6,8 @@
 
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.JobGetSpecConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -40,7 +42,7 @@ public DefaultGetSpecWorker(final IntegrationLauncher integrationLauncher) {
 
   @Override
   public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobRoot) throws WorkerException {
-    try {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
       process = integrationLauncher.spec(jobRoot);
 
       LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
@@ -78,7 +80,10 @@ public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobR
 
   @Override
   public void cancel() {
-    WorkerUtils.cancelProcess(process);
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      WorkerUtils.cancelProcess(process);
+    } catch (final Exception e) {
+    }
   }
 
   @Override public String getApplicationName() {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
index 72de2ba7b27e..6886160015c0 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
@@ -4,6 +4,8 @@
 
 package io.airbyte.workers;
 
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.Configs.WorkerEnvironment;
 import io.airbyte.config.NormalizationInput;
 import io.airbyte.workers.normalization.NormalizationRunner;
@@ -41,8 +43,8 @@ public DefaultNormalizationWorker(final String jobId,
   @Override
   public Void run(final NormalizationInput input, final Path jobRoot) throws WorkerException {
     final long startTime = System.currentTimeMillis();
-
-    try (normalizationRunner) {
+    try (normalizationRunner;
+        final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
       LOGGER.info("Running normalization.");
       normalizationRunner.start();
 
@@ -56,25 +58,26 @@ public Void run(final NormalizationInput input, final Path jobRoot) throws Worke
           input.getResourceRequirements())) {
         throw new WorkerException("Normalization Failed.");
       }
-    } catch (final Exception e) {
-      throw new WorkerException("Normalization Failed.", e);
-    }
 
-    if (cancelled.get()) {
-      LOGGER.info("Normalization was cancelled.");
-    }
+      if (cancelled.get()) {
+        LOGGER.info("Normalization was cancelled.");
+      }
+
+      final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime);
+      final String durationDescription = DurationFormatUtils.formatDurationWords(duration.toMillis(), true, true);
+      LOGGER.info("Normalization executed in {}.", durationDescription);
 
-    final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime);
-    final String durationDescription = DurationFormatUtils.formatDurationWords(duration.toMillis(), true, true);
-    LOGGER.info("Normalization executed in {}.", durationDescription);
+      return null;
 
-    return null;
+    } catch (final Exception e) {
+      throw new WorkerException("Normalization Failed.", e);
+    }
   }
 
   @Override
   public void cancel() {
-    LOGGER.info("Cancelling normalization runner...");
-    try {
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      LOGGER.info("Cancelling normalization runner...");
       cancelled.set(true);
       normalizationRunner.close();
     } catch (final Exception e) {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
index 2999c9d92d2f..9df5045109be 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
@@ -4,6 +4,8 @@
 
 package io.airbyte.workers;
 
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.ReplicationAttemptSummary;
 import io.airbyte.config.ReplicationOutput;
 import io.airbyte.config.StandardSyncInput;
@@ -78,15 +80,15 @@ public DefaultReplicationWorker(final String jobId,
    */
   @Override
   public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRoot) throws WorkerException {
-    LOGGER.info("start sync worker. job id: {} attempt id: {}", jobId, attempt);
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      LOGGER.info("start sync worker. job id: {} attempt id: {}", jobId, attempt);
 
-    // todo (cgardens) - this should not be happening in the worker. this is configuration information
-    // that is independent of workflow executions.
-    final WorkerDestinationConfig destinationConfig = WorkerUtils.syncToWorkerDestinationConfig(syncInput);
-    destinationConfig.setCatalog(mapper.mapCatalog(destinationConfig.getCatalog()));
+      // todo (cgardens) - this should not be happening in the worker. this is configuration information
+      // that is independent of workflow executions.
+      final WorkerDestinationConfig destinationConfig = WorkerUtils.syncToWorkerDestinationConfig(syncInput);
+      destinationConfig.setCatalog(mapper.mapCatalog(destinationConfig.getCatalog()));
 
-    final long startTime = System.currentTimeMillis();
-    try {
+      final long startTime = System.currentTimeMillis();
       LOGGER.info("configured sync modes: {}", syncInput.getCatalog().getStreams()
           .stream()
           .collect(Collectors.toMap(s -> s.getStream().getNamespace() + "." + s.getStream().getName(),
@@ -245,29 +247,31 @@ private static Runnable getDestinationOutputRunnable(final Destination<AirbyteMe
 
   @Override
   public void cancel() {
-    // Resources are closed in the opposite order they are declared.
-    LOGGER.info("Cancelling replication worker...");
-    try {
-      executors.awaitTermination(10, TimeUnit.SECONDS);
-    } catch (final InterruptedException e) {
-      e.printStackTrace();
-    }
-    cancelled.set(true);
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      // Resources are closed in the opposite order they are declared.
+      LOGGER.info("Cancelling replication worker...");
+      try {
+        executors.awaitTermination(10, TimeUnit.SECONDS);
+      } catch (final InterruptedException e) {
+        e.printStackTrace();
+      }
+      cancelled.set(true);
 
-    LOGGER.info("Cancelling destination...");
-    try {
-      destination.cancel();
-    } catch (final Exception e) {
-      LOGGER.info("Error cancelling destination: ", e);
-    }
+      LOGGER.info("Cancelling destination...");
+      try {
+        destination.cancel();
+      } catch (final Exception e) {
+        LOGGER.info("Error cancelling destination: ", e);
+      }
 
-    LOGGER.info("Cancelling source...");
-    try {
-      source.cancel();
+      LOGGER.info("Cancelling source...");
+      try {
+        source.cancel();
+      } catch (final Exception e) {
+        LOGGER.info("Error cancelling source: ", e);
+      }
     } catch (final Exception e) {
-      LOGGER.info("Error cancelling source: ", e);
     }
-
   }
 
   @Override public String getApplicationName() {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java b/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
index 6f0b297970ca..8f1139ad0857 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.workers;
 
+import io.airbyte.commons.application.Application;
 import java.nio.file.Path;
 
 public interface Worker<InputType, OutputType> extends Application {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
index d3246acc4ed7..5ba5462a526f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
@@ -4,9 +4,9 @@
 
 package io.airbyte.workers.protocols;
 
+import io.airbyte.commons.application.Application;
 import io.airbyte.commons.functional.CheckedConsumer;
 import io.airbyte.config.WorkerDestinationConfig;
-import io.airbyte.workers.Application;
 import java.nio.file.Path;
 import java.util.Optional;
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
index d8ef8cdb9849..f3a6de322490 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
@@ -4,8 +4,8 @@
 
 package io.airbyte.workers.protocols;
 
+import io.airbyte.commons.application.Application;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
-import io.airbyte.workers.Application;
 
 public interface Mapper<T> extends Application {
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
index 10f480c42182..6b5fbca34884 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
@@ -4,8 +4,8 @@
 
 package io.airbyte.workers.protocols;
 
+import io.airbyte.commons.application.Application;
 import io.airbyte.config.State;
-import io.airbyte.workers.Application;
 import java.util.Optional;
 import java.util.function.Consumer;
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
index 1f1b176a9645..d1ef55d458cf 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
@@ -4,8 +4,8 @@
 
 package io.airbyte.workers.protocols;
 
+import io.airbyte.commons.application.Application;
 import io.airbyte.config.WorkerSourceConfig;
-import io.airbyte.workers.Application;
 import java.nio.file.Path;
 import java.util.Optional;
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
index 7dd2493de66c..40903731bf1f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
@@ -6,6 +6,8 @@
 
 import com.google.common.base.Charsets;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.State;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.workers.protocols.MessageTracker;
@@ -27,13 +29,16 @@ public AirbyteMessageTracker() {
 
   @Override
   public void accept(final AirbyteMessage message) {
-    if (message.getType() == AirbyteMessage.Type.RECORD) {
-      recordCount.incrementAndGet();
-      // todo (cgardens) - pretty wasteful to do an extra serialization just to get size.
-      numBytes.addAndGet(Jsons.serialize(message.getRecord().getData()).getBytes(Charsets.UTF_8).length);
-    }
-    if (message.getType() == AirbyteMessage.Type.STATE) {
-      outputState.set(new State().withState(message.getState().getData()));
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      if (message.getType() == AirbyteMessage.Type.RECORD) {
+        recordCount.incrementAndGet();
+        // todo (cgardens) - pretty wasteful to do an extra serialization just to get size.
+        numBytes.addAndGet(Jsons.serialize(message.getRecord().getData()).getBytes(Charsets.UTF_8).length);
+      }
+      if (message.getType() == AirbyteMessage.Type.STATE) {
+        outputState.set(new State().withState(message.getState().getData()));
+      }
+    } catch (final Exception e) {
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
index 1a876d5ac50d..01d57e91266f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
@@ -4,8 +4,8 @@
 
 package io.airbyte.workers.protocols.airbyte;
 
+import io.airbyte.commons.application.Application;
 import io.airbyte.protocol.models.AirbyteMessage;
-import io.airbyte.workers.Application;
 import java.io.BufferedReader;
 import java.util.stream.Stream;
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index 92cf1a4d03bb..970185096bc5 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -9,6 +9,8 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.WorkerDestinationConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -53,23 +55,29 @@ public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher,
 
   @Override
   public void start(final WorkerDestinationConfig destinationConfig, final Path jobRoot) throws IOException, WorkerException {
-    Preconditions.checkState(destinationProcess == null);
-
-    LOGGER.info("Running destination...");
-    destinationProcess = integrationLauncher.write(
-        jobRoot,
-        WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
-        Jsons.serialize(destinationConfig.getDestinationConnectionConfiguration()),
-        WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
-        Jsons.serialize(destinationConfig.getCatalog()));
-    // stdout logs are logged elsewhere since stdout also contains data
-    LineGobbler.gobble(destinationProcess.getErrorStream(), LOGGER::error, "airbyte-destination");
-
-    writer = new BufferedWriter(new OutputStreamWriter(destinationProcess.getOutputStream(), Charsets.UTF_8));
-
-    messageIterator = streamFactory.create(IOs.newBufferedReader(destinationProcess.getInputStream()))
-        .filter(message -> message.getType() == Type.STATE)
-        .iterator();
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      Preconditions.checkState(destinationProcess == null);
+
+      LOGGER.info("Running destination...");
+      destinationProcess = integrationLauncher.write(
+          jobRoot,
+          WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
+          Jsons.serialize(destinationConfig.getDestinationConnectionConfiguration()),
+          WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
+          Jsons.serialize(destinationConfig.getCatalog()));
+      // stdout logs are logged elsewhere since stdout also contains data
+      LineGobbler.gobble(destinationProcess.getErrorStream(), LOGGER::error, "airbyte-destination");
+
+      writer = new BufferedWriter(new OutputStreamWriter(destinationProcess.getOutputStream(), Charsets.UTF_8));
+
+      messageIterator = streamFactory.create(IOs.newBufferedReader(destinationProcess.getInputStream()))
+          .filter(message -> message.getType() == Type.STATE)
+          .iterator();
+    } catch (final IOException | WorkerException e) {
+      throw e;
+    } catch (final Exception e) {
+      throw new WorkerException("Destination Failed");
+    }
   }
 
   @Override
@@ -91,34 +99,38 @@ public void notifyEndOfStream() throws IOException {
 
   @Override
   public void close() throws Exception {
-    if (destinationProcess == null) {
-      LOGGER.debug("Destination process already exited");
-      return;
-    }
-
-    if (!endOfStream.get()) {
-      notifyEndOfStream();
-    }
-
-    LOGGER.debug("Closing destination process");
-    WorkerUtils.gentleClose(destinationProcess, 10, TimeUnit.HOURS);
-    if (destinationProcess.isAlive() || destinationProcess.exitValue() != 0) {
-      final String message =
-          destinationProcess.isAlive() ? "Destination has not terminated " : "Destination process exit with code " + destinationProcess.exitValue();
-      throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      if (destinationProcess == null) {
+        LOGGER.debug("Destination process already exited");
+        return;
+      }
+
+      if (!endOfStream.get()) {
+        notifyEndOfStream();
+      }
+
+      LOGGER.debug("Closing destination process");
+      WorkerUtils.gentleClose(destinationProcess, 10, TimeUnit.HOURS);
+      if (destinationProcess.isAlive() || destinationProcess.exitValue() != 0) {
+        final String message =
+            destinationProcess.isAlive() ? "Destination has not terminated " : "Destination process exit with code " + destinationProcess.exitValue();
+        throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
+      }
     }
   }
 
   @Override
   public void cancel() throws Exception {
-    LOGGER.info("Attempting to cancel destination process...");
-
-    if (destinationProcess == null) {
-      LOGGER.info("Destination process no longer exists, cancellation is a no-op.");
-    } else {
-      LOGGER.info("Destination process exists, cancelling...");
-      WorkerUtils.cancelProcess(destinationProcess);
-      LOGGER.info("Cancelled destination process!");
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      LOGGER.info("Attempting to cancel destination process...");
+
+      if (destinationProcess == null) {
+        LOGGER.info("Destination process no longer exists, cancellation is a no-op.");
+      } else {
+        LOGGER.info("Destination process exists, cancelling...");
+        WorkerUtils.cancelProcess(destinationProcess);
+        LOGGER.info("Cancelled destination process!");
+      }
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index 46be24d916c2..6781703adbf7 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -9,6 +9,8 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.WorkerSourceConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -56,22 +58,24 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
 
   @Override
   public void start(final WorkerSourceConfig sourceConfig, final Path jobRoot) throws Exception {
-    Preconditions.checkState(sourceProcess == null);
-
-    sourceProcess = integrationLauncher.read(jobRoot,
-        WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
-        Jsons.serialize(sourceConfig.getSourceConnectionConfiguration()),
-        WorkerConstants.SOURCE_CATALOG_JSON_FILENAME,
-        Jsons.serialize(sourceConfig.getCatalog()),
-        sourceConfig.getState() == null ? null : WorkerConstants.INPUT_STATE_JSON_FILENAME,
-        sourceConfig.getState() == null ? null : Jsons.serialize(sourceConfig.getState().getState()));
-    // stdout logs are logged elsewhere since stdout also contains data
-    LineGobbler.gobble(sourceProcess.getErrorStream(), LOGGER::error, "airbyte-source");
-
-    messageIterator = streamFactory.create(IOs.newBufferedReader(sourceProcess.getInputStream()))
-        .peek(message -> heartbeatMonitor.beat())
-        .filter(message -> message.getType() == Type.RECORD || message.getType() == Type.STATE)
-        .iterator();
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      Preconditions.checkState(sourceProcess == null);
+
+      sourceProcess = integrationLauncher.read(jobRoot,
+          WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
+          Jsons.serialize(sourceConfig.getSourceConnectionConfiguration()),
+          WorkerConstants.SOURCE_CATALOG_JSON_FILENAME,
+          Jsons.serialize(sourceConfig.getCatalog()),
+          sourceConfig.getState() == null ? null : WorkerConstants.INPUT_STATE_JSON_FILENAME,
+          sourceConfig.getState() == null ? null : Jsons.serialize(sourceConfig.getState().getState()));
+      // stdout logs are logged elsewhere since stdout also contains data
+      LineGobbler.gobble(sourceProcess.getErrorStream(), LOGGER::error, "airbyte-source");
+
+      messageIterator = streamFactory.create(IOs.newBufferedReader(sourceProcess.getInputStream()))
+          .peek(message -> heartbeatMonitor.beat())
+          .filter(message -> message.getType() == Type.RECORD || message.getType() == Type.STATE)
+          .iterator();
+    }
   }
 
   @Override
@@ -96,35 +100,39 @@ public Optional<AirbyteMessage> attemptRead() {
 
   @Override
   public void close() throws Exception {
-    if (sourceProcess == null) {
-      LOGGER.debug("Source process already exited");
-      return;
-    }
-
-    LOGGER.debug("Closing source process");
-    WorkerUtils.gentleCloseWithHeartbeat(
-        sourceProcess,
-        heartbeatMonitor,
-        GRACEFUL_SHUTDOWN_DURATION,
-        CHECK_HEARTBEAT_DURATION,
-        FORCED_SHUTDOWN_DURATION);
-
-    if (sourceProcess.isAlive() || sourceProcess.exitValue() != 0) {
-      final String message = sourceProcess.isAlive() ? "Source has not terminated " : "Source process exit with code " + sourceProcess.exitValue();
-      throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      if (sourceProcess == null) {
+        LOGGER.debug("Source process already exited");
+        return;
+      }
+
+      LOGGER.debug("Closing source process");
+      WorkerUtils.gentleCloseWithHeartbeat(
+          sourceProcess,
+          heartbeatMonitor,
+          GRACEFUL_SHUTDOWN_DURATION,
+          CHECK_HEARTBEAT_DURATION,
+          FORCED_SHUTDOWN_DURATION);
+
+      if (sourceProcess.isAlive() || sourceProcess.exitValue() != 0) {
+        final String message = sourceProcess.isAlive() ? "Source has not terminated " : "Source process exit with code " + sourceProcess.exitValue();
+        throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
+      }
     }
   }
 
   @Override
   public void cancel() throws Exception {
-    LOGGER.info("Attempting to cancel source process...");
-
-    if (sourceProcess == null) {
-      LOGGER.info("Source process no longer exists, cancellation is a no-op.");
-    } else {
-      LOGGER.info("Source process exists, cancelling...");
-      WorkerUtils.cancelProcess(sourceProcess);
-      LOGGER.info("Cancelled source process!");
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      LOGGER.info("Attempting to cancel source process...");
+
+      if (sourceProcess == null) {
+        LOGGER.info("Source process no longer exists, cancellation is a no-op.");
+      } else {
+        LOGGER.info("Source process exists, cancelling...");
+        WorkerUtils.cancelProcess(sourceProcess);
+        LOGGER.info("Cancelled source process!");
+      }
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index c57d205cd45f..34eeff25074d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -6,6 +6,8 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.protocol.models.AirbyteLogMessage;
 import io.airbyte.protocol.models.AirbyteMessage;
 import java.io.BufferedReader;
@@ -40,41 +42,47 @@ public DefaultAirbyteStreamFactory() {
 
   @Override
   public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
-    return bufferedReader
-        .lines()
-        .flatMap(line -> {
-          final Optional<JsonNode> jsonLine = Jsons.tryDeserialize(line);
-          if (jsonLine.isEmpty()) {
-            // we log as info all the lines that are not valid json
-            // some sources actually log their process on stdout, we
-            // want to make sure this info is available in the logs.
-            logger.info(line);
-          }
-          return jsonLine.stream();
-        })
-        // filter invalid messages
-        .filter(jsonLine -> {
-          final boolean res = protocolValidator.test(jsonLine);
-          if (!res) {
-            logger.error("Validation failed: {}", Jsons.serialize(jsonLine));
-          }
-          return res;
-        })
-        .flatMap(jsonLine -> {
-          final Optional<AirbyteMessage> m = Jsons.tryObject(jsonLine, AirbyteMessage.class);
-          if (m.isEmpty()) {
-            logger.error("Deserialization failed: {}", Jsons.serialize(jsonLine));
-          }
-          return m.stream();
-        })
-        // filter logs
-        .filter(airbyteMessage -> {
-          final boolean isLog = airbyteMessage.getType() == AirbyteMessage.Type.LOG;
-          if (isLog) {
-            internalLog(airbyteMessage.getLog());
-          }
-          return !isLog;
-        });
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      return bufferedReader
+          .lines()
+          .flatMap(line -> {
+            final Optional<JsonNode> jsonLine = Jsons.tryDeserialize(line);
+            if (jsonLine.isEmpty()) {
+              // we log as info all the lines that are not valid json
+              // some sources actually log their process on stdout, we
+              // want to make sure this info is available in the logs.
+              logger.info(line);
+            }
+            return jsonLine.stream();
+          })
+          // filter invalid messages
+          .filter(jsonLine -> {
+            final boolean res = protocolValidator.test(jsonLine);
+            if (!res) {
+              logger.error("Validation failed: {}", Jsons.serialize(jsonLine));
+            }
+            return res;
+          })
+          .flatMap(jsonLine -> {
+            final Optional<AirbyteMessage> m = Jsons.tryObject(jsonLine, AirbyteMessage.class);
+            if (m.isEmpty()) {
+              logger.error("Deserialization failed: {}", Jsons.serialize(jsonLine));
+            }
+            return m.stream();
+          })
+          // filter logs
+          .filter(airbyteMessage -> {
+            final boolean isLog = airbyteMessage.getType() == AirbyteMessage.Type.LOG;
+            if (isLog) {
+              internalLog(airbyteMessage.getLog());
+            }
+            return !isLog;
+          });
+    } catch (final RuntimeException e) {
+      throw e;
+    } catch (final Exception e) {
+      return Stream.empty();
+    }
   }
 
   private void internalLog(final AirbyteLogMessage logMessage) {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
index c1bcac44806c..5b8d2e09e2de 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
@@ -5,6 +5,8 @@
 package io.airbyte.workers.protocols.airbyte;
 
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -37,43 +39,51 @@ public NamespacingMapper(final NamespaceDefinitionType namespaceDefinition, fina
 
   @Override
   public ConfiguredAirbyteCatalog mapCatalog(final ConfiguredAirbyteCatalog inputCatalog) {
-    final ConfiguredAirbyteCatalog catalog = Jsons.clone(inputCatalog);
-    catalog.getStreams().forEach(s -> {
-      final AirbyteStream stream = s.getStream();
-      // Default behavior if namespaceDefinition is not set is to follow SOURCE
-      if (namespaceDefinition != null) {
-        if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) {
-          stream.withNamespace(null);
-        } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) {
-          final String namespace = formatNamespace(stream.getNamespace(), namespaceFormat);
-          if (namespace == null) {
-            LOGGER.error("Namespace Format cannot be blank for Stream {}. Falling back to default namespace from destination settings",
-                stream.getName());
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      final ConfiguredAirbyteCatalog catalog = Jsons.clone(inputCatalog);
+      catalog.getStreams().forEach(s -> {
+        final AirbyteStream stream = s.getStream();
+        // Default behavior if namespaceDefinition is not set is to follow SOURCE
+        if (namespaceDefinition != null) {
+          if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) {
+            stream.withNamespace(null);
+          } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) {
+            final String namespace = formatNamespace(stream.getNamespace(), namespaceFormat);
+            if (namespace == null) {
+              LOGGER.error("Namespace Format cannot be blank for Stream {}. Falling back to default namespace from destination settings",
+                  stream.getName());
+            }
+            stream.withNamespace(namespace);
           }
-          stream.withNamespace(namespace);
         }
-      }
-      stream.withName(transformStreamName(stream.getName(), streamPrefix));
-    });
-    return catalog;
+        stream.withName(transformStreamName(stream.getName(), streamPrefix));
+      });
+      return catalog;
+    } catch (final Exception e) {
+      throw new RuntimeException(e);
+    }
   }
 
   @Override
   public AirbyteMessage mapMessage(final AirbyteMessage inputMessage) {
-    if (inputMessage.getType() == Type.RECORD) {
-      final AirbyteMessage message = Jsons.clone(inputMessage);
-      // Default behavior if namespaceDefinition is not set is to follow SOURCE
-      if (namespaceDefinition != null) {
-        if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) {
-          message.getRecord().withNamespace(null);
-        } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) {
-          message.getRecord().withNamespace(formatNamespace(message.getRecord().getNamespace(), namespaceFormat));
+    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+      if (inputMessage.getType() == Type.RECORD) {
+        final AirbyteMessage message = Jsons.clone(inputMessage);
+        // Default behavior if namespaceDefinition is not set is to follow SOURCE
+        if (namespaceDefinition != null) {
+          if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) {
+            message.getRecord().withNamespace(null);
+          } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) {
+            message.getRecord().withNamespace(formatNamespace(message.getRecord().getNamespace(), namespaceFormat));
+          }
         }
+        message.getRecord().setStream(transformStreamName(message.getRecord().getStream(), streamPrefix));
+        return message;
       }
-      message.getRecord().setStream(transformStreamName(message.getRecord().getStream(), streamPrefix));
-      return message;
+      return inputMessage;
+    } catch (final Exception e) {
+      throw new RuntimeException(e);
     }
-    return inputMessage;
   }
 
   private static String formatNamespace(final String sourceNamespace, final String namespaceFormat) {

From 110334883d5f65a681c75db5c24d7ca6f9bf4225 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 17:03:46 -0700
Subject: [PATCH 07/32] PR Comments

---
 .../java/io/airbyte/commons/logging/LoggingHelper.java   | 8 ++++----
 .../logging/{ScopedMDCChange.java => MdcScope.java}      | 9 ++++-----
 airbyte-commons/src/main/resources/log4j2.xml            | 1 +
 .../{ScopedMDCChangeTest.java => MdcScopeTest.java}      | 8 ++++----
 4 files changed, 13 insertions(+), 13 deletions(-)
 rename airbyte-commons/src/main/java/io/airbyte/commons/logging/{ScopedMDCChange.java => MdcScope.java} (74%)
 rename airbyte-commons/src/test/java/io/airbyte/commons/logging/{ScopedMDCChangeTest.java => MdcScopeTest.java} (86%)

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
index 26f6fc6695bb..1975461edbe4 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
@@ -17,14 +17,14 @@ public enum Color {
     CYAN("\u001b[36m"),
     WHITE("\u001b[37m");
 
-    private final String ainsiCode;
+    private final String ansi;
 
-    Color(final String ainsiCode) {
-      this.ainsiCode = ainsiCode;
+    Color(final String ansiCode) {
+      this.ansi = ansiCode;
     }
 
     public String getCode() {
-      return ainsiCode;
+      return ansi;
     }
 
   }
diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
similarity index 74%
rename from airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java
rename to airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
index b9be48b55ae6..9980e85585b5 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/ScopedMDCChange.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
@@ -8,8 +8,8 @@
 import org.slf4j.MDC;
 
 /**
- * This class is an autoClosable class that will add some specific values into the log MDC. When
- * being close, it will restore the orginal MDC. It is advise to use it like that:
+ * This class is an autoClosable class that will add some specific values into the log MDC. When being close, it will restore the orginal MDC. It is
+ * advise to use it like that:
  *
  * <pre>
  *   <code>
@@ -23,11 +23,11 @@
  *   </code>
  * </pre>
  */
-public class ScopedMDCChange implements AutoCloseable {
+public class MdcScope implements AutoCloseable {
 
   private final Map<String, String> originalContextMap;
 
-  public ScopedMDCChange(final Map<String, String> keyValuesToAdd) {
+  public MdcScope(final Map<String, String> keyValuesToAdd) {
     originalContextMap = MDC.getCopyOfContextMap();
 
     keyValuesToAdd.forEach(
@@ -36,7 +36,6 @@ public ScopedMDCChange(final Map<String, String> keyValuesToAdd) {
 
   @Override
   public void close() throws Exception {
-    MDC.clear();
     MDC.setContextMap(originalContextMap);
   }
 
diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-commons/src/main/resources/log4j2.xml
index 91ab632ea1d3..c07bd80c4b31 100644
--- a/airbyte-commons/src/main/resources/log4j2.xml
+++ b/airbyte-commons/src/main/resources/log4j2.xml
@@ -4,6 +4,7 @@
 
         <!-- The following patterns mask the string apikey=<string> to apikey=***** to prevent secrets leaking. -->
         <Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %X - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
+        <!-- The following patter log the application name in the beginning of the line, then a regular log line after a - separator. If the application name is not present in the context, the separator is removed -->
         <Property name="docker-worker-file-pattern">%replace{%X{log_source} - }{^ - }{}%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %p (%X{job_root}) %C{1}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
         <!-- Remove paths from logs on Cloud as users aren't able to access file paths and these end up being noise. -->
         <Property name="cloud-worker-file-pattern">%d{yyyy-MM-dd HH:mm:ss} %-5p %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
similarity index 86%
rename from airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java
rename to airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
index c3a83c22b209..b58733f751b9 100644
--- a/airbyte-commons/src/test/java/io/airbyte/commons/logging/ScopedMDCChangeTest.java
+++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
@@ -12,7 +12,7 @@
 import org.junit.jupiter.api.Test;
 import org.slf4j.MDC;
 
-public class ScopedMDCChangeTest {
+public class MdcScopeTest {
 
   private static final Map<String, String> originalMap = new HashMap<>() {
 
@@ -34,14 +34,13 @@ public class ScopedMDCChangeTest {
 
   @BeforeEach
   public void init() {
-    MDC.clear();
     MDC.setContextMap(originalMap);
   }
 
   @Test
   @DisplayName("The MDC context is properly overrided")
   public void testMDCModified() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(modificationInMDC)) {
+    try (final MdcScope mdcScope = new MdcScope(modificationInMDC)) {
       final Map<String, String> mdcState = MDC.getCopyOfContextMap();
 
       Assertions.assertThat(mdcState).containsExactlyInAnyOrderEntriesOf(
@@ -63,7 +62,8 @@ public void testMDCModified() {
   @Test
   @DisplayName("The MDC context is properly restored")
   public void testMDCRestore() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(modificationInMDC)) {} catch (final Exception e) {
+    try (final MdcScope mdcScope = new MdcScope(modificationInMDC)) {
+    } catch (final Exception e) {
       e.printStackTrace();
     }
     final Map<String, String> mdcState = MDC.getCopyOfContextMap();

From 20964f02f07622dff97d06e0e83966336e160dc5 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 17:12:58 -0700
Subject: [PATCH 08/32] PR Comments

---
 .../java/io/airbyte/workers/DbtTransformationRunner.java    | 6 +-----
 .../java/io/airbyte/workers/DbtTransformationWorker.java    | 2 +-
 .../io/airbyte/workers/DefaultCheckConnectionWorker.java    | 2 +-
 .../io/airbyte/workers/DefaultDiscoverCatalogWorker.java    | 2 +-
 .../main/java/io/airbyte/workers/DefaultGetSpecWorker.java  | 2 +-
 .../java/io/airbyte/workers/DefaultNormalizationWorker.java | 2 +-
 .../java/io/airbyte/workers/DefaultReplicationWorker.java   | 2 +-
 .../main/java/io/airbyte/workers/protocols/Destination.java | 3 +--
 .../src/main/java/io/airbyte/workers/protocols/Mapper.java  | 3 +--
 .../src/main/java/io/airbyte/workers/protocols/Source.java  | 3 +--
 .../workers/protocols/airbyte/AirbyteStreamFactory.java     | 3 +--
 .../protocols/airbyte/DefaultAirbyteDestination.java        | 3 ---
 .../workers/protocols/airbyte/DefaultAirbyteSource.java     | 3 ---
 .../protocols/airbyte/DefaultAirbyteStreamFactory.java      | 3 ---
 .../workers/protocols/airbyte/EmptyAirbyteSource.java       | 3 ---
 .../workers/protocols/airbyte/NamespacingMapper.java        | 3 ---
 16 files changed, 11 insertions(+), 34 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index c30364d888cf..ccc0ad81f31d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -24,7 +24,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DbtTransformationRunner implements AutoCloseable, Application {
+public class DbtTransformationRunner implements AutoCloseable {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DbtTransformationRunner.class);
   private static final String DBT_ENTRYPOINT_SH = "entrypoint.sh";
@@ -115,8 +115,4 @@ public void close() throws Exception {
       throw new WorkerException("Dbt transformation process wasn't successful");
     }
   }
-
-  @Override public String getApplicationName() {
-    return "airbyte-dbt-transformation-runner";
-  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
index 34c017a70d5d..2c0750dc8a2b 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
@@ -78,6 +78,6 @@ public void cancel() {
   }
 
   @Override public String getApplicationName() {
-    return "airbyte-dbt-transformation-worker";
+    return "normalization-worker";
   }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
index e06371fb1958..c7a15a433b59 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
@@ -87,6 +87,6 @@ public void cancel() {
   }
 
   @Override public String getApplicationName() {
-    return "airbyte-connection-checker";
+    return "check-connection-worker";
   }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
index e879ae28c144..a907e944a6ed 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
@@ -82,6 +82,6 @@ public void cancel() {
   }
 
   @Override public String getApplicationName() {
-    return "airbyte-catalog-discovery";
+    return "discover-worker";
   }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
index 812118c89994..b00ec64aa715 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
@@ -82,6 +82,6 @@ public void cancel() {
   }
 
   @Override public String getApplicationName() {
-    return "airbyte-spec-getter";
+    return "get-spec-worker";
   }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
index 72de2ba7b27e..c73aa7c586f4 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
@@ -83,6 +83,6 @@ public void cancel() {
   }
 
   @Override public String getApplicationName() {
-    return "airbyte-normalization";
+    return "normalization-worker";
   }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
index 2999c9d92d2f..9e0146c33020 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
@@ -271,6 +271,6 @@ public void cancel() {
   }
 
   @Override public String getApplicationName() {
-    return "airbyte-replication";
+    return "sync-worker";
   }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
index d3246acc4ed7..c874e108cfbc 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
@@ -6,11 +6,10 @@
 
 import io.airbyte.commons.functional.CheckedConsumer;
 import io.airbyte.config.WorkerDestinationConfig;
-import io.airbyte.workers.Application;
 import java.nio.file.Path;
 import java.util.Optional;
 
-public interface Destination<T> extends CheckedConsumer<T, Exception>, AutoCloseable, Application {
+public interface Destination<T> extends CheckedConsumer<T, Exception>, AutoCloseable {
 
   void start(WorkerDestinationConfig destinationConfig, Path jobRoot) throws Exception;
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
index d8ef8cdb9849..32a9d1858580 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
@@ -5,9 +5,8 @@
 package io.airbyte.workers.protocols;
 
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
-import io.airbyte.workers.Application;
 
-public interface Mapper<T> extends Application {
+public interface Mapper<T> {
 
   ConfiguredAirbyteCatalog mapCatalog(ConfiguredAirbyteCatalog catalog);
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
index 1f1b176a9645..8612ab290e04 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
@@ -5,11 +5,10 @@
 package io.airbyte.workers.protocols;
 
 import io.airbyte.config.WorkerSourceConfig;
-import io.airbyte.workers.Application;
 import java.nio.file.Path;
 import java.util.Optional;
 
-public interface Source<T> extends AutoCloseable, Application {
+public interface Source<T> extends AutoCloseable {
 
   void start(WorkerSourceConfig sourceConfig, Path jobRoot) throws Exception;
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
index 1a876d5ac50d..757f7e4e2519 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
@@ -5,11 +5,10 @@
 package io.airbyte.workers.protocols.airbyte;
 
 import io.airbyte.protocol.models.AirbyteMessage;
-import io.airbyte.workers.Application;
 import java.io.BufferedReader;
 import java.util.stream.Stream;
 
-public interface AirbyteStreamFactory extends Application {
+public interface AirbyteStreamFactory {
 
   Stream<AirbyteMessage> create(BufferedReader bufferedReader);
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index 92cf1a4d03bb..9bf76989307a 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -142,7 +142,4 @@ public Optional<AirbyteMessage> attemptRead() {
     return Optional.ofNullable(messageIterator.hasNext() ? messageIterator.next() : null);
   }
 
-  @Override public String getApplicationName() {
-    return "airbyte-destination";
-  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index 46be24d916c2..c0441f19110d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -128,7 +128,4 @@ public void cancel() throws Exception {
     }
   }
 
-  @Override public String getApplicationName() {
-    return "airbyte-source";
-  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index c57d205cd45f..456cd8277396 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -87,7 +87,4 @@ private void internalLog(final AirbyteLogMessage logMessage) {
     }
   }
 
-  @Override public String getApplicationName() {
-    return "airbyte-stream-factory";
-  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
index d13ed57c51dc..f027707cc0b9 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
@@ -55,7 +55,4 @@ public void cancel() throws Exception {
     // no op.
   }
 
-  @Override public String getApplicationName() {
-    return "airbyte-empty-source";
-  }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
index c1bcac44806c..c0f344f4f971 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
@@ -96,7 +96,4 @@ private static String transformStreamName(final String streamName, final String
     }
   }
 
-  @Override public String getApplicationName() {
-    return "airbyte-namespacing-mapper";
-  }
 }

From 80d5aab87edf869be93d65807b99af158d998223 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 17:25:09 -0700
Subject: [PATCH 09/32] Update related to merge

---
 .../workers/DbtTransformationRunner.java      | 75 ++++++---------
 .../workers/DbtTransformationWorker.java      | 10 +-
 .../workers/DefaultCheckConnectionWorker.java | 14 ++-
 .../workers/DefaultDiscoverCatalogWorker.java | 11 ++-
 .../airbyte/workers/DefaultGetSpecWorker.java | 11 ++-
 .../workers/DefaultNormalizationWorker.java   | 12 ++-
 .../workers/DefaultReplicationWorker.java     | 11 ++-
 .../airbyte/DefaultAirbyteDestination.java    | 92 ++++++++-----------
 .../airbyte/DefaultAirbyteSource.java         | 88 ++++++++----------
 .../airbyte/DefaultAirbyteStreamFactory.java  | 78 +++++++---------
 .../protocols/airbyte/NamespacingMapper.java  | 66 ++++++-------
 11 files changed, 223 insertions(+), 245 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index 30f88548469c..7a38c3e7e491 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -7,10 +7,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
-import io.airbyte.commons.application.Application;
 import io.airbyte.commons.io.LineGobbler;
-import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.config.OperatorDbt;
 import io.airbyte.config.ResourceRequirements;
@@ -42,9 +39,7 @@ public DbtTransformationRunner(final ProcessFactory processFactory, final Normal
   }
 
   public void start() throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      normalizationRunner.start();
-    }
+    normalizationRunner.start();
   }
 
   /**
@@ -63,12 +58,10 @@ public boolean run(final String jobId,
                      final ResourceRequirements resourceRequirements,
                      final OperatorDbt dbtConfig)
       throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      if (!normalizationRunner.configureDbt(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig)) {
-        return false;
-      }
-      return transform(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig);
+    if (!normalizationRunner.configureDbt(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig)) {
+      return false;
     }
+    return transform(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig);
   }
 
   public boolean transform(final String jobId,
@@ -78,50 +71,40 @@ public boolean transform(final String jobId,
                            final ResourceRequirements resourceRequirements,
                            final OperatorDbt dbtConfig)
       throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      final Map<String, String> files = ImmutableMap.of(
-          DBT_ENTRYPOINT_SH, MoreResources.readResource("dbt_transformation_entrypoint.sh"),
-          "sshtunneling.sh", MoreResources.readResource("sshtunneling.sh"));
-      final List<String> dbtArguments = new ArrayList<>();
-      dbtArguments.add(DBT_ENTRYPOINT_SH);
-      if (Strings.isNullOrEmpty(dbtConfig.getDbtArguments())) {
-        throw new WorkerException("Dbt Arguments are required");
-      }
-      Collections.addAll(dbtArguments, Commandline.translateCommandline(dbtConfig.getDbtArguments()));
-      process =
-          processFactory.create(jobId, attempt, jobRoot, dbtConfig.getDockerImage(), false, files, "/bin/bash", resourceRequirements,
-              Map.of(KubeProcessFactory.JOB_TYPE, KubeProcessFactory.SYNC_JOB, KubeProcessFactory.SYNC_STEP, KubeProcessFactory.CUSTOM_STEP),
-              dbtArguments);
+    final Map<String, String> files = ImmutableMap.of(
+        DBT_ENTRYPOINT_SH, MoreResources.readResource("dbt_transformation_entrypoint.sh"),
+        "sshtunneling.sh", MoreResources.readResource("sshtunneling.sh"));
+    final List<String> dbtArguments = new ArrayList<>();
+    dbtArguments.add(DBT_ENTRYPOINT_SH);
+    if (Strings.isNullOrEmpty(dbtConfig.getDbtArguments())) {
+      throw new WorkerException("Dbt Arguments are required");
+    }
+    Collections.addAll(dbtArguments, Commandline.translateCommandline(dbtConfig.getDbtArguments()));
+    process =
+        processFactory.create(jobId, attempt, jobRoot, dbtConfig.getDockerImage(), false, files, "/bin/bash", resourceRequirements,
+            Map.of(KubeProcessFactory.JOB_TYPE, KubeProcessFactory.SYNC_JOB, KubeProcessFactory.SYNC_STEP, KubeProcessFactory.CUSTOM_STEP),
+            dbtArguments);
 
-      LineGobbler.gobble(process.getInputStream(), LOGGER::info);
-      LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
+    LineGobbler.gobble(process.getInputStream(), LOGGER::info);
+    LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
 
-      WorkerUtils.wait(process);
+    WorkerUtils.wait(process);
 
-      return process.exitValue() == 0;
-    } catch (final Exception e) {
-      // make sure we kill the process on failure to avoid zombies.
-      if (process != null) {
-        WorkerUtils.cancelProcess(process);
-      }
-      throw e;
-    }
+    return process.exitValue() == 0;
   }
 
   @Override
   public void close() throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      normalizationRunner.close();
+    normalizationRunner.close();
 
-      if (process == null) {
-        return;
-      }
+    if (process == null) {
+      return;
+    }
 
-      LOGGER.debug("Closing dbt transformation process");
-      WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
-      if (process.isAlive() || process.exitValue() != 0) {
-        throw new WorkerException("Dbt transformation process wasn't successful");
-      }
+    LOGGER.debug("Closing dbt transformation process");
+    WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
+    if (process.isAlive() || process.exitValue() != 0) {
+      throw new WorkerException("Dbt transformation process wasn't successful");
     }
   }
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
index 33e10b3d754f..1e25cdc0cc94 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
@@ -6,7 +6,7 @@
 
 import io.airbyte.commons.application.Application;
 import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
+import io.airbyte.commons.logging.MdcScope;
 import io.airbyte.config.OperatorDbtInput;
 import io.airbyte.config.ResourceRequirements;
 import java.nio.file.Files;
@@ -41,7 +41,7 @@ public DbtTransformationWorker(final String jobId,
 
   @Override
   public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) throws WorkerException {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       final long startTime = System.currentTimeMillis();
 
       try (dbtTransformationRunner) {
@@ -68,6 +68,8 @@ public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) thr
       LOGGER.info("Dbt Transformation executed in {}.", duration.toMinutesPart());
 
       return null;
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
       throw new WorkerException(getApplicationName() + " failed", e);
     }
@@ -75,7 +77,7 @@ public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) thr
 
   @Override
   public void cancel() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       LOGGER.info("Cancelling Dbt Transformation runner...");
       try {
         cancelled.set(true);
@@ -83,6 +85,8 @@ public void cancel() {
       } catch (final Exception e) {
         e.printStackTrace();
       }
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
       e.printStackTrace();
     }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
index c7a15a433b59..14969cb281d0 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
@@ -8,6 +8,8 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper;
+import io.airbyte.commons.logging.MdcScope;
 import io.airbyte.config.StandardCheckConnectionInput;
 import io.airbyte.config.StandardCheckConnectionOutput;
 import io.airbyte.config.StandardCheckConnectionOutput.Status;
@@ -45,7 +47,7 @@ public DefaultCheckConnectionWorker(final IntegrationLauncher integrationLaunche
   @Override
   public StandardCheckConnectionOutput run(final StandardCheckConnectionInput input, final Path jobRoot) throws WorkerException {
 
-    try {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       process = integrationLauncher.check(
           jobRoot,
           WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
@@ -76,6 +78,8 @@ public StandardCheckConnectionOutput run(final StandardCheckConnectionInput inpu
         throw new WorkerException("Error while getting checking connection.");
       }
 
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
       throw new WorkerException("Error while getting checking connection.", e);
     }
@@ -83,7 +87,13 @@ public StandardCheckConnectionOutput run(final StandardCheckConnectionInput inpu
 
   @Override
   public void cancel() {
-    WorkerUtils.cancelProcess(process);
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+      WorkerUtils.cancelProcess(process);
+    } catch (final RuntimeException e) {
+      throw e;
+    } catch (final Exception e) {
+      throw new RuntimeException(e);
+    }
   }
 
   @Override public String getApplicationName() {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
index d487fa8209b0..27bffa581149 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
@@ -8,7 +8,7 @@
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
+import io.airbyte.commons.logging.MdcScope;
 import io.airbyte.config.StandardDiscoverCatalogInput;
 import io.airbyte.protocol.models.AirbyteCatalog;
 import io.airbyte.protocol.models.AirbyteMessage;
@@ -43,7 +43,7 @@ public DefaultDiscoverCatalogWorker(final IntegrationLauncher integrationLaunche
 
   @Override
   public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput, final Path jobRoot) throws WorkerException {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       process = integrationLauncher.discover(
           jobRoot,
           WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
@@ -73,6 +73,8 @@ public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput
       }
     } catch (final WorkerException e) {
       throw e;
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
       throw new WorkerException("Error while discovering schema", e);
     }
@@ -80,9 +82,12 @@ public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput
 
   @Override
   public void cancel() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       WorkerUtils.cancelProcess(process);
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
+      throw new RuntimeException(e);
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
index 8c53dc3dcb7e..ec394e0e34ab 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
@@ -7,7 +7,7 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
+import io.airbyte.commons.logging.MdcScope;
 import io.airbyte.config.JobGetSpecConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -42,7 +42,7 @@ public DefaultGetSpecWorker(final IntegrationLauncher integrationLauncher) {
 
   @Override
   public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobRoot) throws WorkerException {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       process = integrationLauncher.spec(jobRoot);
 
       LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
@@ -72,6 +72,8 @@ public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobR
       } else {
         throw new WorkerException(String.format("Spec job subprocess finished with exit code %s", exitCode));
       }
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
       throw new WorkerException(String.format("Error while getting spec from image %s", config.getDockerImage()), e);
     }
@@ -80,9 +82,12 @@ public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobR
 
   @Override
   public void cancel() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       WorkerUtils.cancelProcess(process);
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
+      throw new RuntimeException(e);
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
index 409b803f18ee..65ad1933cc82 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
@@ -5,7 +5,7 @@
 package io.airbyte.workers;
 
 import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
+import io.airbyte.commons.logging.MdcScope;
 import io.airbyte.config.Configs.WorkerEnvironment;
 import io.airbyte.config.NormalizationInput;
 import io.airbyte.workers.normalization.NormalizationRunner;
@@ -44,7 +44,7 @@ public DefaultNormalizationWorker(final String jobId,
   public Void run(final NormalizationInput input, final Path jobRoot) throws WorkerException {
     final long startTime = System.currentTimeMillis();
     try (normalizationRunner;
-        final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+        final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       LOGGER.info("Running normalization.");
       normalizationRunner.start();
 
@@ -69,6 +69,8 @@ public Void run(final NormalizationInput input, final Path jobRoot) throws Worke
 
       return null;
 
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
       throw new WorkerException("Normalization Failed.", e);
     }
@@ -76,12 +78,14 @@ public Void run(final NormalizationInput input, final Path jobRoot) throws Worke
 
   @Override
   public void cancel() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       LOGGER.info("Cancelling normalization runner...");
       cancelled.set(true);
       normalizationRunner.close();
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
-      e.printStackTrace();
+      throw new RuntimeException(e);
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
index 13de4b54c473..c7e05da3035f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
@@ -5,7 +5,7 @@
 package io.airbyte.workers;
 
 import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
+import io.airbyte.commons.logging.MdcScope;
 import io.airbyte.config.ReplicationAttemptSummary;
 import io.airbyte.config.ReplicationOutput;
 import io.airbyte.config.StandardSyncInput;
@@ -80,7 +80,7 @@ public DefaultReplicationWorker(final String jobId,
    */
   @Override
   public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRoot) throws WorkerException {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       LOGGER.info("start sync worker. job id: {} attempt id: {}", jobId, attempt);
 
       // todo (cgardens) - this should not be happening in the worker. this is configuration information
@@ -174,6 +174,8 @@ else if (hasFailed.get()) {
       }
 
       return output;
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
       throw new WorkerException("Sync failed", e);
     }
@@ -247,7 +249,7 @@ private static Runnable getDestinationOutputRunnable(final Destination<AirbyteMe
 
   @Override
   public void cancel() {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       // Resources are closed in the opposite order they are declared.
       LOGGER.info("Cancelling replication worker...");
       try {
@@ -270,7 +272,10 @@ public void cancel() {
       } catch (final Exception e) {
         LOGGER.info("Error cancelling source: ", e);
       }
+    } catch (final RuntimeException e) {
+      throw e;
     } catch (final Exception e) {
+      throw new RuntimeException(e);
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index b1d66938e041..9bf76989307a 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -9,8 +9,6 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.WorkerDestinationConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -55,29 +53,23 @@ public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher,
 
   @Override
   public void start(final WorkerDestinationConfig destinationConfig, final Path jobRoot) throws IOException, WorkerException {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      Preconditions.checkState(destinationProcess == null);
-
-      LOGGER.info("Running destination...");
-      destinationProcess = integrationLauncher.write(
-          jobRoot,
-          WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
-          Jsons.serialize(destinationConfig.getDestinationConnectionConfiguration()),
-          WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
-          Jsons.serialize(destinationConfig.getCatalog()));
-      // stdout logs are logged elsewhere since stdout also contains data
-      LineGobbler.gobble(destinationProcess.getErrorStream(), LOGGER::error, "airbyte-destination");
-
-      writer = new BufferedWriter(new OutputStreamWriter(destinationProcess.getOutputStream(), Charsets.UTF_8));
-
-      messageIterator = streamFactory.create(IOs.newBufferedReader(destinationProcess.getInputStream()))
-          .filter(message -> message.getType() == Type.STATE)
-          .iterator();
-    } catch (final IOException | WorkerException e) {
-      throw e;
-    } catch (final Exception e) {
-      throw new WorkerException("Destination Failed");
-    }
+    Preconditions.checkState(destinationProcess == null);
+
+    LOGGER.info("Running destination...");
+    destinationProcess = integrationLauncher.write(
+        jobRoot,
+        WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
+        Jsons.serialize(destinationConfig.getDestinationConnectionConfiguration()),
+        WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
+        Jsons.serialize(destinationConfig.getCatalog()));
+    // stdout logs are logged elsewhere since stdout also contains data
+    LineGobbler.gobble(destinationProcess.getErrorStream(), LOGGER::error, "airbyte-destination");
+
+    writer = new BufferedWriter(new OutputStreamWriter(destinationProcess.getOutputStream(), Charsets.UTF_8));
+
+    messageIterator = streamFactory.create(IOs.newBufferedReader(destinationProcess.getInputStream()))
+        .filter(message -> message.getType() == Type.STATE)
+        .iterator();
   }
 
   @Override
@@ -99,38 +91,34 @@ public void notifyEndOfStream() throws IOException {
 
   @Override
   public void close() throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      if (destinationProcess == null) {
-        LOGGER.debug("Destination process already exited");
-        return;
-      }
-
-      if (!endOfStream.get()) {
-        notifyEndOfStream();
-      }
-
-      LOGGER.debug("Closing destination process");
-      WorkerUtils.gentleClose(destinationProcess, 10, TimeUnit.HOURS);
-      if (destinationProcess.isAlive() || destinationProcess.exitValue() != 0) {
-        final String message =
-            destinationProcess.isAlive() ? "Destination has not terminated " : "Destination process exit with code " + destinationProcess.exitValue();
-        throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
-      }
+    if (destinationProcess == null) {
+      LOGGER.debug("Destination process already exited");
+      return;
+    }
+
+    if (!endOfStream.get()) {
+      notifyEndOfStream();
+    }
+
+    LOGGER.debug("Closing destination process");
+    WorkerUtils.gentleClose(destinationProcess, 10, TimeUnit.HOURS);
+    if (destinationProcess.isAlive() || destinationProcess.exitValue() != 0) {
+      final String message =
+          destinationProcess.isAlive() ? "Destination has not terminated " : "Destination process exit with code " + destinationProcess.exitValue();
+      throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
     }
   }
 
   @Override
   public void cancel() throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      LOGGER.info("Attempting to cancel destination process...");
-
-      if (destinationProcess == null) {
-        LOGGER.info("Destination process no longer exists, cancellation is a no-op.");
-      } else {
-        LOGGER.info("Destination process exists, cancelling...");
-        WorkerUtils.cancelProcess(destinationProcess);
-        LOGGER.info("Cancelled destination process!");
-      }
+    LOGGER.info("Attempting to cancel destination process...");
+
+    if (destinationProcess == null) {
+      LOGGER.info("Destination process no longer exists, cancellation is a no-op.");
+    } else {
+      LOGGER.info("Destination process exists, cancelling...");
+      WorkerUtils.cancelProcess(destinationProcess);
+      LOGGER.info("Cancelled destination process!");
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index 3144c896561a..c0441f19110d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -9,8 +9,6 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.WorkerSourceConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -58,24 +56,22 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
 
   @Override
   public void start(final WorkerSourceConfig sourceConfig, final Path jobRoot) throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      Preconditions.checkState(sourceProcess == null);
-
-      sourceProcess = integrationLauncher.read(jobRoot,
-          WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
-          Jsons.serialize(sourceConfig.getSourceConnectionConfiguration()),
-          WorkerConstants.SOURCE_CATALOG_JSON_FILENAME,
-          Jsons.serialize(sourceConfig.getCatalog()),
-          sourceConfig.getState() == null ? null : WorkerConstants.INPUT_STATE_JSON_FILENAME,
-          sourceConfig.getState() == null ? null : Jsons.serialize(sourceConfig.getState().getState()));
-      // stdout logs are logged elsewhere since stdout also contains data
-      LineGobbler.gobble(sourceProcess.getErrorStream(), LOGGER::error, "airbyte-source");
-
-      messageIterator = streamFactory.create(IOs.newBufferedReader(sourceProcess.getInputStream()))
-          .peek(message -> heartbeatMonitor.beat())
-          .filter(message -> message.getType() == Type.RECORD || message.getType() == Type.STATE)
-          .iterator();
-    }
+    Preconditions.checkState(sourceProcess == null);
+
+    sourceProcess = integrationLauncher.read(jobRoot,
+        WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
+        Jsons.serialize(sourceConfig.getSourceConnectionConfiguration()),
+        WorkerConstants.SOURCE_CATALOG_JSON_FILENAME,
+        Jsons.serialize(sourceConfig.getCatalog()),
+        sourceConfig.getState() == null ? null : WorkerConstants.INPUT_STATE_JSON_FILENAME,
+        sourceConfig.getState() == null ? null : Jsons.serialize(sourceConfig.getState().getState()));
+    // stdout logs are logged elsewhere since stdout also contains data
+    LineGobbler.gobble(sourceProcess.getErrorStream(), LOGGER::error, "airbyte-source");
+
+    messageIterator = streamFactory.create(IOs.newBufferedReader(sourceProcess.getInputStream()))
+        .peek(message -> heartbeatMonitor.beat())
+        .filter(message -> message.getType() == Type.RECORD || message.getType() == Type.STATE)
+        .iterator();
   }
 
   @Override
@@ -100,39 +96,35 @@ public Optional<AirbyteMessage> attemptRead() {
 
   @Override
   public void close() throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      if (sourceProcess == null) {
-        LOGGER.debug("Source process already exited");
-        return;
-      }
-
-      LOGGER.debug("Closing source process");
-      WorkerUtils.gentleCloseWithHeartbeat(
-          sourceProcess,
-          heartbeatMonitor,
-          GRACEFUL_SHUTDOWN_DURATION,
-          CHECK_HEARTBEAT_DURATION,
-          FORCED_SHUTDOWN_DURATION);
-
-      if (sourceProcess.isAlive() || sourceProcess.exitValue() != 0) {
-        final String message = sourceProcess.isAlive() ? "Source has not terminated " : "Source process exit with code " + sourceProcess.exitValue();
-        throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
-      }
+    if (sourceProcess == null) {
+      LOGGER.debug("Source process already exited");
+      return;
+    }
+
+    LOGGER.debug("Closing source process");
+    WorkerUtils.gentleCloseWithHeartbeat(
+        sourceProcess,
+        heartbeatMonitor,
+        GRACEFUL_SHUTDOWN_DURATION,
+        CHECK_HEARTBEAT_DURATION,
+        FORCED_SHUTDOWN_DURATION);
+
+    if (sourceProcess.isAlive() || sourceProcess.exitValue() != 0) {
+      final String message = sourceProcess.isAlive() ? "Source has not terminated " : "Source process exit with code " + sourceProcess.exitValue();
+      throw new WorkerException(message + ". This warning is normal if the job was cancelled.");
     }
   }
 
   @Override
   public void cancel() throws Exception {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      LOGGER.info("Attempting to cancel source process...");
-
-      if (sourceProcess == null) {
-        LOGGER.info("Source process no longer exists, cancellation is a no-op.");
-      } else {
-        LOGGER.info("Source process exists, cancelling...");
-        WorkerUtils.cancelProcess(sourceProcess);
-        LOGGER.info("Cancelled source process!");
-      }
+    LOGGER.info("Attempting to cancel source process...");
+
+    if (sourceProcess == null) {
+      LOGGER.info("Source process no longer exists, cancellation is a no-op.");
+    } else {
+      LOGGER.info("Source process exists, cancelling...");
+      WorkerUtils.cancelProcess(sourceProcess);
+      LOGGER.info("Cancelled source process!");
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index fb8cb9afb881..456cd8277396 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -6,8 +6,6 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.protocol.models.AirbyteLogMessage;
 import io.airbyte.protocol.models.AirbyteMessage;
 import java.io.BufferedReader;
@@ -42,47 +40,41 @@ public DefaultAirbyteStreamFactory() {
 
   @Override
   public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      return bufferedReader
-          .lines()
-          .flatMap(line -> {
-            final Optional<JsonNode> jsonLine = Jsons.tryDeserialize(line);
-            if (jsonLine.isEmpty()) {
-              // we log as info all the lines that are not valid json
-              // some sources actually log their process on stdout, we
-              // want to make sure this info is available in the logs.
-              logger.info(line);
-            }
-            return jsonLine.stream();
-          })
-          // filter invalid messages
-          .filter(jsonLine -> {
-            final boolean res = protocolValidator.test(jsonLine);
-            if (!res) {
-              logger.error("Validation failed: {}", Jsons.serialize(jsonLine));
-            }
-            return res;
-          })
-          .flatMap(jsonLine -> {
-            final Optional<AirbyteMessage> m = Jsons.tryObject(jsonLine, AirbyteMessage.class);
-            if (m.isEmpty()) {
-              logger.error("Deserialization failed: {}", Jsons.serialize(jsonLine));
-            }
-            return m.stream();
-          })
-          // filter logs
-          .filter(airbyteMessage -> {
-            final boolean isLog = airbyteMessage.getType() == AirbyteMessage.Type.LOG;
-            if (isLog) {
-              internalLog(airbyteMessage.getLog());
-            }
-            return !isLog;
-          });
-    } catch (final RuntimeException e) {
-      throw e;
-    } catch (final Exception e) {
-      return Stream.empty();
-    }
+    return bufferedReader
+        .lines()
+        .flatMap(line -> {
+          final Optional<JsonNode> jsonLine = Jsons.tryDeserialize(line);
+          if (jsonLine.isEmpty()) {
+            // we log as info all the lines that are not valid json
+            // some sources actually log their process on stdout, we
+            // want to make sure this info is available in the logs.
+            logger.info(line);
+          }
+          return jsonLine.stream();
+        })
+        // filter invalid messages
+        .filter(jsonLine -> {
+          final boolean res = protocolValidator.test(jsonLine);
+          if (!res) {
+            logger.error("Validation failed: {}", Jsons.serialize(jsonLine));
+          }
+          return res;
+        })
+        .flatMap(jsonLine -> {
+          final Optional<AirbyteMessage> m = Jsons.tryObject(jsonLine, AirbyteMessage.class);
+          if (m.isEmpty()) {
+            logger.error("Deserialization failed: {}", Jsons.serialize(jsonLine));
+          }
+          return m.stream();
+        })
+        // filter logs
+        .filter(airbyteMessage -> {
+          final boolean isLog = airbyteMessage.getType() == AirbyteMessage.Type.LOG;
+          if (isLog) {
+            internalLog(airbyteMessage.getLog());
+          }
+          return !isLog;
+        });
   }
 
   private void internalLog(final AirbyteLogMessage logMessage) {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
index c33b29fbbf12..c0f344f4f971 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
@@ -5,8 +5,6 @@
 package io.airbyte.workers.protocols.airbyte;
 
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
 import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -39,51 +37,43 @@ public NamespacingMapper(final NamespaceDefinitionType namespaceDefinition, fina
 
   @Override
   public ConfiguredAirbyteCatalog mapCatalog(final ConfiguredAirbyteCatalog inputCatalog) {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      final ConfiguredAirbyteCatalog catalog = Jsons.clone(inputCatalog);
-      catalog.getStreams().forEach(s -> {
-        final AirbyteStream stream = s.getStream();
-        // Default behavior if namespaceDefinition is not set is to follow SOURCE
-        if (namespaceDefinition != null) {
-          if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) {
-            stream.withNamespace(null);
-          } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) {
-            final String namespace = formatNamespace(stream.getNamespace(), namespaceFormat);
-            if (namespace == null) {
-              LOGGER.error("Namespace Format cannot be blank for Stream {}. Falling back to default namespace from destination settings",
-                  stream.getName());
-            }
-            stream.withNamespace(namespace);
+    final ConfiguredAirbyteCatalog catalog = Jsons.clone(inputCatalog);
+    catalog.getStreams().forEach(s -> {
+      final AirbyteStream stream = s.getStream();
+      // Default behavior if namespaceDefinition is not set is to follow SOURCE
+      if (namespaceDefinition != null) {
+        if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) {
+          stream.withNamespace(null);
+        } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) {
+          final String namespace = formatNamespace(stream.getNamespace(), namespaceFormat);
+          if (namespace == null) {
+            LOGGER.error("Namespace Format cannot be blank for Stream {}. Falling back to default namespace from destination settings",
+                stream.getName());
           }
+          stream.withNamespace(namespace);
         }
-        stream.withName(transformStreamName(stream.getName(), streamPrefix));
-      });
-      return catalog;
-    } catch (final Exception e) {
-      throw new RuntimeException(e);
-    }
+      }
+      stream.withName(transformStreamName(stream.getName(), streamPrefix));
+    });
+    return catalog;
   }
 
   @Override
   public AirbyteMessage mapMessage(final AirbyteMessage inputMessage) {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
-      if (inputMessage.getType() == Type.RECORD) {
-        final AirbyteMessage message = Jsons.clone(inputMessage);
-        // Default behavior if namespaceDefinition is not set is to follow SOURCE
-        if (namespaceDefinition != null) {
-          if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) {
-            message.getRecord().withNamespace(null);
-          } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) {
-            message.getRecord().withNamespace(formatNamespace(message.getRecord().getNamespace(), namespaceFormat));
-          }
+    if (inputMessage.getType() == Type.RECORD) {
+      final AirbyteMessage message = Jsons.clone(inputMessage);
+      // Default behavior if namespaceDefinition is not set is to follow SOURCE
+      if (namespaceDefinition != null) {
+        if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) {
+          message.getRecord().withNamespace(null);
+        } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) {
+          message.getRecord().withNamespace(formatNamespace(message.getRecord().getNamespace(), namespaceFormat));
         }
-        message.getRecord().setStream(transformStreamName(message.getRecord().getStream(), streamPrefix));
-        return message;
       }
-      return inputMessage;
-    } catch (final Exception e) {
-      throw new RuntimeException(e);
+      message.getRecord().setStream(transformStreamName(message.getRecord().getStream(), streamPrefix));
+      return message;
     }
+    return inputMessage;
   }
 
   private static String formatNamespace(final String sourceNamespace, final String namespaceFormat) {

From d53634272f44a7632c69037f4240436715e87e58 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 17:30:47 -0700
Subject: [PATCH 10/32] Format

---
 .../src/main/java/io/airbyte/commons/logging/MdcScope.java    | 4 ++--
 .../test/java/io/airbyte/commons/logging/MdcScopeTest.java    | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
index 9980e85585b5..82f71c987a9d 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
@@ -8,8 +8,8 @@
 import org.slf4j.MDC;
 
 /**
- * This class is an autoClosable class that will add some specific values into the log MDC. When being close, it will restore the orginal MDC. It is
- * advise to use it like that:
+ * This class is an autoClosable class that will add some specific values into the log MDC. When
+ * being close, it will restore the orginal MDC. It is advise to use it like that:
  *
  * <pre>
  *   <code>
diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
index b58733f751b9..4d985d5a6c6a 100644
--- a/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
+++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
@@ -62,8 +62,7 @@ public void testMDCModified() {
   @Test
   @DisplayName("The MDC context is properly restored")
   public void testMDCRestore() {
-    try (final MdcScope mdcScope = new MdcScope(modificationInMDC)) {
-    } catch (final Exception e) {
+    try (final MdcScope mdcScope = new MdcScope(modificationInMDC)) {} catch (final Exception e) {
       e.printStackTrace();
     }
     final Map<String, String> mdcState = MDC.getCopyOfContextMap();

From ee6b7e92e44c13d0618fbfac55d297b7c449ed8f Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 17:31:27 -0700
Subject: [PATCH 11/32] Format

---
 .../main/java/io/airbyte/workers/Application.java |  8 +++++++-
 .../airbyte/workers/DbtTransformationRunner.java  | 14 ++++++++------
 .../airbyte/workers/DbtTransformationWorker.java  |  4 +++-
 .../workers/DefaultCheckConnectionWorker.java     |  4 +++-
 .../workers/DefaultDiscoverCatalogWorker.java     |  4 +++-
 .../io/airbyte/workers/DefaultGetSpecWorker.java  |  4 +++-
 .../workers/DefaultNormalizationWorker.java       |  4 +++-
 .../airbyte/workers/DefaultReplicationWorker.java | 15 +++++++++------
 .../main/java/io/airbyte/workers/EchoWorker.java  |  7 ++++---
 .../src/main/java/io/airbyte/workers/Worker.java  |  7 ++++---
 .../protocols/airbyte/AirbyteMessageTracker.java  |  4 +++-
 .../protocols/airbyte/DefaultAirbyteSource.java   |  7 ++++---
 .../airbyte/DefaultAirbyteStreamFactory.java      | 10 ++++++----
 .../protocols/airbyte/EmptyAirbyteSource.java     |  3 ++-
 .../protocols/airbyte/NamespacingMapper.java      |  7 ++++---
 15 files changed, 66 insertions(+), 36 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/Application.java b/airbyte-workers/src/main/java/io/airbyte/workers/Application.java
index f8a80107c7ce..65573d0a111f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/Application.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/Application.java
@@ -1,9 +1,15 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
 package io.airbyte.workers;
 
 public interface Application {
 
   default String getApplicationName() {
-    // This value should only be used in the U-Test, it is an empty string instead of airbyte-test in order to avoid displaying airbyte-test in prod
+    // This value should only be used in the U-Test, it is an empty string instead of airbyte-test in
+    // order to avoid displaying airbyte-test in prod
     return "";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index ccc0ad81f31d..ec35dccdae76 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -43,13 +43,14 @@ public void start() throws Exception {
   }
 
   /**
-   * The docker image used by the DbtTransformationRunner is provided by the User, so we can't ensure to have the right python, dbt, dependencies etc
-   * software installed to successfully run our transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend on
-   * the NormalizationRunner to configure the dbt project with the appropriate destination settings and pull the custom git repository into the
-   * workspace.
+   * The docker image used by the DbtTransformationRunner is provided by the User, so we can't ensure
+   * to have the right python, dbt, dependencies etc software installed to successfully run our
+   * transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend
+   * on the NormalizationRunner to configure the dbt project with the appropriate destination settings
+   * and pull the custom git repository into the workspace.
    * <p>
-   * Once the workspace folder/files is setup to run, we invoke the custom transformation command as provided by the user to execute whatever extra
-   * transformation has been implemented.
+   * Once the workspace folder/files is setup to run, we invoke the custom transformation command as
+   * provided by the user to execute whatever extra transformation has been implemented.
    */
   public boolean run(final String jobId,
                      final int attempt,
@@ -115,4 +116,5 @@ public void close() throws Exception {
       throw new WorkerException("Dbt transformation process wasn't successful");
     }
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
index 2c0750dc8a2b..0a7e1da44889 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
@@ -77,7 +77,9 @@ public void cancel() {
     }
   }
 
-  @Override public String getApplicationName() {
+  @Override
+  public String getApplicationName() {
     return "normalization-worker";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
index c7a15a433b59..5a1abf740292 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
@@ -86,7 +86,9 @@ public void cancel() {
     WorkerUtils.cancelProcess(process);
   }
 
-  @Override public String getApplicationName() {
+  @Override
+  public String getApplicationName() {
     return "check-connection-worker";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
index a907e944a6ed..9790a634ae51 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
@@ -81,7 +81,9 @@ public void cancel() {
     WorkerUtils.cancelProcess(process);
   }
 
-  @Override public String getApplicationName() {
+  @Override
+  public String getApplicationName() {
     return "discover-worker";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
index b00ec64aa715..aa6951d7784f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
@@ -81,7 +81,9 @@ public void cancel() {
     WorkerUtils.cancelProcess(process);
   }
 
-  @Override public String getApplicationName() {
+  @Override
+  public String getApplicationName() {
     return "get-spec-worker";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
index c73aa7c586f4..5b6024b61825 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
@@ -82,7 +82,9 @@ public void cancel() {
     }
   }
 
-  @Override public String getApplicationName() {
+  @Override
+  public String getApplicationName() {
     return "normalization-worker";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
index 9e0146c33020..73b8678b0720 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
@@ -66,13 +66,14 @@ public DefaultReplicationWorker(final String jobId,
   }
 
   /**
-   * Run executes two threads. The first pipes data from STDOUT of the source to STDIN of the destination. The second listen on STDOUT of the
-   * destination. The goal of this second thread is to detect when the destination emits state messages. Only state messages emitted by the
-   * destination should be treated as state that is safe to return from run. In the case when the destination emits no state, we fall back on whatever
-   * state is pass in as an argument to this method.
+   * Run executes two threads. The first pipes data from STDOUT of the source to STDIN of the
+   * destination. The second listen on STDOUT of the destination. The goal of this second thread is to
+   * detect when the destination emits state messages. Only state messages emitted by the destination
+   * should be treated as state that is safe to return from run. In the case when the destination
+   * emits no state, we fall back on whatever state is pass in as an argument to this method.
    *
    * @param syncInput all configuration for running replication
-   * @param jobRoot   file root that worker is allowed to use
+   * @param jobRoot file root that worker is allowed to use
    * @return output of the replication attempt (including state)
    * @throws WorkerException
    */
@@ -270,7 +271,9 @@ public void cancel() {
 
   }
 
-  @Override public String getApplicationName() {
+  @Override
+  public String getApplicationName() {
     return "sync-worker";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
index 5c9e18727107..32ebd42e6522 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
@@ -12,8 +12,7 @@ public class EchoWorker implements Worker<String, String> {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(EchoWorker.class);
 
-  public EchoWorker() {
-  }
+  public EchoWorker() {}
 
   @Override
   public String run(final String string, final Path jobRoot) {
@@ -26,7 +25,9 @@ public void cancel() {
     // no-op
   }
 
-  @Override public String getApplicationName() {
+  @Override
+  public String getApplicationName() {
     return "airbyte-echo-worker";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java b/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
index 6f0b297970ca..a2fb2c3f8269 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
@@ -9,13 +9,14 @@
 public interface Worker<InputType, OutputType> extends Application {
 
   /**
-   * Blocking call to run the worker's workflow. Once this is complete, getStatus should return either COMPLETE, FAILED, or CANCELLED.
+   * Blocking call to run the worker's workflow. Once this is complete, getStatus should return either
+   * COMPLETE, FAILED, or CANCELLED.
    */
   OutputType run(InputType inputType, Path jobRoot) throws WorkerException;
 
   /**
-   * Cancels in-progress workers. Although all workers support cancel, in reality only the asynchronous {@link DefaultReplicationWorker}'s cancel is
-   * used.
+   * Cancels in-progress workers. Although all workers support cancel, in reality only the
+   * asynchronous {@link DefaultReplicationWorker}'s cancel is used.
    */
   void cancel();
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
index 7dd2493de66c..9d3c58d66a74 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
@@ -52,7 +52,9 @@ public Optional<State> getOutputState() {
     return Optional.ofNullable(outputState.get());
   }
 
-  @Override public String getApplicationName() {
+  @Override
+  public String getApplicationName() {
     return "airbyte-message-tracker";
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index c0441f19110d..2b17b4050fdc 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -46,9 +46,10 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
     this(integrationLauncher, new DefaultAirbyteStreamFactory(), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION));
   }
 
-  @VisibleForTesting DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
-                                          final AirbyteStreamFactory streamFactory,
-                                          final HeartbeatMonitor heartbeatMonitor) {
+  @VisibleForTesting
+  DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
+                       final AirbyteStreamFactory streamFactory,
+                       final HeartbeatMonitor heartbeatMonitor) {
     this.integrationLauncher = integrationLauncher;
     this.streamFactory = streamFactory;
     this.heartbeatMonitor = heartbeatMonitor;
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index 456cd8277396..69b32f64be92 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -15,12 +15,14 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Creates a stream from an input stream. The produced stream attempts to parse each line of the InputStream into a AirbyteMessage. If the line cannot
- * be parsed into a AirbyteMessage it is dropped. Each record MUST be new line separated.
+ * Creates a stream from an input stream. The produced stream attempts to parse each line of the
+ * InputStream into a AirbyteMessage. If the line cannot be parsed into a AirbyteMessage it is
+ * dropped. Each record MUST be new line separated.
  *
  * <p>
- * If a line starts with a AirbyteMessage and then has other characters after it, that AirbyteMessage will still be parsed. If there are multiple
- * AirbyteMessage records on the same line, only the first will be parsed.
+ * If a line starts with a AirbyteMessage and then has other characters after it, that
+ * AirbyteMessage will still be parsed. If there are multiple AirbyteMessage records on the same
+ * line, only the first will be parsed.
  */
 public class DefaultAirbyteStreamFactory implements AirbyteStreamFactory {
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
index f027707cc0b9..59c331be6b28 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java
@@ -14,7 +14,8 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
- * This source will never emit any messages. It can be used in cases where that is helpful (hint: reset connection jobs).
+ * This source will never emit any messages. It can be used in cases where that is helpful (hint:
+ * reset connection jobs).
  */
 public class EmptyAirbyteSource implements AirbyteSource {
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
index c0f344f4f971..c3f88c2e76ca 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/NamespacingMapper.java
@@ -17,9 +17,10 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * We apply some transformations on the fly on the catalog (same should be done on records too) from the source before it reaches the destination. One
- * of the transformation is to define the destination namespace where data will be stored and how to mirror (or not) the namespace used in the source
- * (if any). This is configured in the UI through the syncInput.
+ * We apply some transformations on the fly on the catalog (same should be done on records too) from
+ * the source before it reaches the destination. One of the transformation is to define the
+ * destination namespace where data will be stored and how to mirror (or not) the namespace used in
+ * the source (if any). This is configured in the UI through the syncInput.
  */
 public class NamespacingMapper implements Mapper<AirbyteMessage> {
 

From a041a7d6dfa9a2e055306675b72c5e7f721f9673 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Thu, 21 Oct 2021 17:33:35 -0700
Subject: [PATCH 12/32] Format

---
 .../io/airbyte/commons/logging/LoggingHelper.java     | 11 ++++++++---
 .../io/airbyte/workers/protocols/Destination.java     |  1 -
 .../java/io/airbyte/workers/protocols/Mapper.java     |  1 -
 .../java/io/airbyte/workers/protocols/Source.java     |  1 -
 .../protocols/airbyte/AirbyteMessageTracker.java      |  3 +--
 .../protocols/airbyte/AirbyteStreamFactory.java       |  1 -
 6 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
index f56715ea831e..cb5cdb4e272f 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
@@ -42,8 +42,13 @@ public static String applyColor(final Color color, final String msg) {
   }
 
   public static Map<String, String> getExtraMDCEntries(final Application application) {
-    return new HashMap<>() {{
-      put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.GREEN, application.getApplicationName()));
-    }};
+    return new HashMap<>() {
+
+      {
+        put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.GREEN, application.getApplicationName()));
+      }
+
+    };
   }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
index fa01ba6c8cca..c874e108cfbc 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java
@@ -4,7 +4,6 @@
 
 package io.airbyte.workers.protocols;
 
-import io.airbyte.commons.application.Application;
 import io.airbyte.commons.functional.CheckedConsumer;
 import io.airbyte.config.WorkerDestinationConfig;
 import java.nio.file.Path;
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
index e66c3be84a64..32a9d1858580 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Mapper.java
@@ -4,7 +4,6 @@
 
 package io.airbyte.workers.protocols;
 
-import io.airbyte.commons.application.Application;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
 
 public interface Mapper<T> {
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
index 41e7c7194893..8612ab290e04 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java
@@ -4,7 +4,6 @@
 
 package io.airbyte.workers.protocols;
 
-import io.airbyte.commons.application.Application;
 import io.airbyte.config.WorkerSourceConfig;
 import java.nio.file.Path;
 import java.util.Optional;
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
index fdebeba490bd..932ea6621f41 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
@@ -38,8 +38,7 @@ public void accept(final AirbyteMessage message) {
       if (message.getType() == AirbyteMessage.Type.STATE) {
         outputState.set(new State().withState(message.getState().getData()));
       }
-    } catch (final Exception e) {
-    }
+    } catch (final Exception e) {}
   }
 
   @Override
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
index 98d32c05e616..757f7e4e2519 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteStreamFactory.java
@@ -4,7 +4,6 @@
 
 package io.airbyte.workers.protocols.airbyte;
 
-import io.airbyte.commons.application.Application;
 import io.airbyte.protocol.models.AirbyteMessage;
 import java.io.BufferedReader;
 import java.util.stream.Stream;

From 988c9d888ef1e0ea57a38189b70d1952f3cf1e47 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 10:20:46 -0700
Subject: [PATCH 13/32] Fix one test

---
 .../airbyte/AirbyteMessageTracker.java        |  4 +-
 .../workers/DefaultGetSpecWorkerTest.java     | 55 +++++++++++--------
 2 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
index 932ea6621f41..6d864fcbe96a 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
@@ -7,7 +7,7 @@
 import com.google.common.base.Charsets;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.logging.LoggingHelper;
-import io.airbyte.commons.logging.ScopedMDCChange;
+import io.airbyte.commons.logging.MdcScope;
 import io.airbyte.config.State;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.workers.protocols.MessageTracker;
@@ -29,7 +29,7 @@ public AirbyteMessageTracker() {
 
   @Override
   public void accept(final AirbyteMessage message) {
-    try (final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(LoggingHelper.getExtraMDCEntries(this))) {
+    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
       if (message.getType() == AirbyteMessage.Type.RECORD) {
         recordCount.incrementAndGet();
         // todo (cgardens) - pretty wasteful to do an extra serialization just to get size.
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultGetSpecWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultGetSpecWorkerTest.java
index 4b927d023493..6cbb906987a8 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultGetSpecWorkerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultGetSpecWorkerTest.java
@@ -4,14 +4,6 @@
 
 package io.airbyte.workers;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
 import com.google.common.base.Charsets;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.resources.MoreResources;
@@ -24,8 +16,10 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
 
 class DefaultGetSpecWorkerTest {
 
@@ -42,10 +36,10 @@ class DefaultGetSpecWorkerTest {
   public void setup() throws IOException, WorkerException {
     jobRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "");
     config = new JobGetSpecConfig().withDockerImage(DUMMY_IMAGE_NAME);
-    integrationLauncher = mock(IntegrationLauncher.class, RETURNS_DEEP_STUBS);
-    process = mock(Process.class);
-    when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
-    when(integrationLauncher.spec(jobRoot)).thenReturn(process);
+    integrationLauncher = Mockito.mock(IntegrationLauncher.class, Mockito.RETURNS_DEEP_STUBS);
+    process = Mockito.mock(Process.class);
+    Mockito.when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+    Mockito.when(integrationLauncher.spec(jobRoot)).thenReturn(process);
 
     worker = new DefaultGetSpecWorker(integrationLauncher);
   }
@@ -58,32 +52,45 @@ public void testSuccessfulRun() throws IOException, InterruptedException, Worker
         .withType(Type.SPEC)
         .withSpec(Jsons.deserialize(expectedSpecString, io.airbyte.protocol.models.ConnectorSpecification.class));
 
-    when(process.getInputStream()).thenReturn(new ByteArrayInputStream(Jsons.serialize(message).getBytes(Charsets.UTF_8)));
-    when(process.waitFor(anyLong(), any())).thenReturn(true);
-    when(process.exitValue()).thenReturn(0);
+    Mockito.when(process.getInputStream()).thenReturn(new ByteArrayInputStream(Jsons.serialize(message).getBytes(Charsets.UTF_8)));
+    Mockito.when(process.waitFor(Mockito.anyLong(), Mockito.any())).thenReturn(true);
+    Mockito.when(process.exitValue()).thenReturn(0);
 
     final ConnectorSpecification actualOutput = worker.run(config, jobRoot);
     final ConnectorSpecification expectedOutput = Jsons.deserialize(expectedSpecString, ConnectorSpecification.class);
 
-    assertEquals(expectedOutput, actualOutput);
+    Assertions.assertThat(actualOutput).isEqualTo(expectedOutput);
   }
 
   @Test
   public void testFailureOnInvalidSpec() throws InterruptedException {
     final String expectedSpecString = "{\"key\":\"value\"}";
-    when(process.getInputStream()).thenReturn(new ByteArrayInputStream(expectedSpecString.getBytes()));
-    when(process.waitFor(anyLong(), any())).thenReturn(true);
-    when(process.exitValue()).thenReturn(0);
+    Mockito.when(process.getInputStream()).thenReturn(new ByteArrayInputStream(expectedSpecString.getBytes()));
+    Mockito.when(process.waitFor(Mockito.anyLong(), Mockito.any())).thenReturn(true);
+    Mockito.when(process.exitValue()).thenReturn(0);
 
-    assertThrows(WorkerException.class, () -> worker.run(config, jobRoot));
+    Assertions.assertThatThrownBy(() -> worker.run(config, jobRoot))
+        .hasCauseExactlyInstanceOf(WorkerException.class)
+        .hasRootCauseExactlyInstanceOf(WorkerException.class);
   }
 
   @Test
-  public void testFailureOnNonzeroExitCode() throws InterruptedException {
-    when(process.waitFor(anyLong(), any())).thenReturn(true);
-    when(process.exitValue()).thenReturn(1);
+  public void testFailureOnNonzeroExitCode() throws InterruptedException, IOException {
+    final String expectedSpecString = MoreResources.readResource("valid_spec.json");
+
+    final AirbyteMessage message = new AirbyteMessage()
+        .withType(Type.SPEC)
+        .withSpec(Jsons.deserialize(expectedSpecString, io.airbyte.protocol.models.ConnectorSpecification.class));
+
+    Mockito.when(process.getInputStream()).thenReturn(new ByteArrayInputStream(Jsons.serialize(message).getBytes(Charsets.UTF_8)));
+    Mockito.when(process.waitFor(Mockito.anyLong(), Mockito.any())).thenReturn(true);
+    Mockito.when(process.exitValue()).thenReturn(1);
 
-    assertThrows(WorkerException.class, () -> worker.run(config, jobRoot));
+    Assertions.assertThatThrownBy(() -> worker.run(config, jobRoot))
+        .hasCauseExactlyInstanceOf(WorkerException.class)
+        .hasRootCauseExactlyInstanceOf(WorkerException.class)
+        .getRootCause()
+        .hasNoCause();
   }
 
 }

From bbdb3e93236d97943f87817393df1d95c83586e5 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 11:53:54 -0700
Subject: [PATCH 14/32] Remove the need of an application interface

---
 .../commons/application/Application.java      | 15 ------
 .../commons/logging/LoggingHelper.java        | 14 ------
 .../io/airbyte/commons/logging/MdcScope.java  | 23 +++++++++
 .../workers/DbtTransformationWorker.java      | 20 ++++----
 .../workers/DefaultCheckConnectionWorker.java | 15 +++---
 .../workers/DefaultDiscoverCatalogWorker.java | 15 +++---
 .../airbyte/workers/DefaultGetSpecWorker.java | 15 +++---
 .../workers/DefaultNormalizationWorker.java   | 16 +++----
 .../workers/DefaultReplicationWorker.java     | 15 +++---
 .../java/io/airbyte/workers/EchoWorker.java   |  5 --
 .../main/java/io/airbyte/workers/Worker.java  |  3 +-
 .../workers/protocols/MessageTracker.java     |  3 +-
 .../airbyte/AirbyteMessageTracker.java        | 13 +++--
 .../DefaultCheckConnectionWorkerTest.java     |  2 +-
 .../DefaultDiscoverCatalogWorkerTest.java     | 47 +++++++++----------
 .../workers/DefaultReplicationWorkerTest.java |  2 +-
 16 files changed, 100 insertions(+), 123 deletions(-)
 delete mode 100644 airbyte-commons/src/main/java/io/airbyte/commons/application/Application.java

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/application/Application.java b/airbyte-commons/src/main/java/io/airbyte/commons/application/Application.java
deleted file mode 100644
index a360e559d6d0..000000000000
--- a/airbyte-commons/src/main/java/io/airbyte/commons/application/Application.java
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
- */
-
-package io.airbyte.commons.application;
-
-public interface Application {
-
-  default String getApplicationName() {
-    // This value should only be used in the U-Test, it is an empty string instead of airbyte-test in
-    // order to avoid displaying airbyte-test in prod
-    return "";
-  }
-
-}
diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
index cb5cdb4e272f..1975461edbe4 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
@@ -4,10 +4,6 @@
 
 package io.airbyte.commons.logging;
 
-import io.airbyte.commons.application.Application;
-import java.util.HashMap;
-import java.util.Map;
-
 public class LoggingHelper {
 
   public enum Color {
@@ -41,14 +37,4 @@ public static String applyColor(final Color color, final String msg) {
     return color.getCode() + msg + RESET;
   }
 
-  public static Map<String, String> getExtraMDCEntries(final Application application) {
-    return new HashMap<>() {
-
-      {
-        put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.GREEN, application.getApplicationName()));
-      }
-
-    };
-  }
-
 }
diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
index 82f71c987a9d..f71cae647547 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
@@ -4,7 +4,10 @@
 
 package io.airbyte.commons.logging;
 
+import io.airbyte.commons.logging.LoggingHelper.Color;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import org.slf4j.MDC;
 
 /**
@@ -39,4 +42,24 @@ public void close() throws Exception {
     MDC.setContextMap(originalContextMap);
   }
 
+  public static class MdcScopeBuilder {
+
+    private Optional<String> maybeLogPrefix = Optional.empty();
+
+    public MdcScopeBuilder setLogPrefix(final String logPrefix) {
+      this.maybeLogPrefix = Optional.ofNullable(logPrefix);
+
+      return this;
+    }
+
+    public MdcScope build() {
+      final Map<String, String> extraMdcEntries = new HashMap<>();
+
+      maybeLogPrefix.map(logPrefix -> extraMdcEntries.put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.GREEN, logPrefix)));
+
+      return new MdcScope(extraMdcEntries);
+    }
+
+  }
+
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
index 66dcaacc28b4..ac91507d5514 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
@@ -4,9 +4,8 @@
 
 package io.airbyte.workers;
 
-import io.airbyte.commons.application.Application;
-import io.airbyte.commons.logging.LoggingHelper;
 import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.OperatorDbtInput;
 import io.airbyte.config.ResourceRequirements;
 import java.nio.file.Files;
@@ -16,7 +15,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DbtTransformationWorker implements Worker<OperatorDbtInput, Void>, Application {
+public class DbtTransformationWorker implements Worker<OperatorDbtInput, Void> {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DbtTransformationWorker.class);
 
@@ -27,6 +26,10 @@ public class DbtTransformationWorker implements Worker<OperatorDbtInput, Void>,
 
   private final AtomicBoolean cancelled;
 
+  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
+      .setLogPrefix("sync-worker")
+      .build();
+
   public DbtTransformationWorker(final String jobId,
                                  final int attempt,
                                  final ResourceRequirements resourceRequirements,
@@ -41,7 +44,7 @@ public DbtTransformationWorker(final String jobId,
 
   @Override
   public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) throws WorkerException {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       final long startTime = System.currentTimeMillis();
 
       try (dbtTransformationRunner) {
@@ -71,13 +74,13 @@ public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) thr
     } catch (final RuntimeException e) {
       throw e;
     } catch (final Exception e) {
-      throw new WorkerException(getApplicationName() + " failed", e);
+      throw new WorkerException("Dbt Transformation failed", e);
     }
   }
 
   @Override
   public void cancel() {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       LOGGER.info("Cancelling Dbt Transformation runner...");
       try {
         cancelled.set(true);
@@ -92,9 +95,4 @@ public void cancel() {
     }
   }
 
-  @Override
-  public String getApplicationName() {
-    return "normalization-worker";
-  }
-
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
index d4e18a84ad97..c851b8cab3e4 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
@@ -8,8 +8,8 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
 import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.StandardCheckConnectionInput;
 import io.airbyte.config.StandardCheckConnectionOutput;
 import io.airbyte.config.StandardCheckConnectionOutput.Status;
@@ -33,6 +33,10 @@ public class DefaultCheckConnectionWorker implements CheckConnectionWorker {
   private final IntegrationLauncher integrationLauncher;
   private final AirbyteStreamFactory streamFactory;
 
+  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
+      .setLogPrefix("check-connection-worker")
+      .build();
+
   private Process process;
 
   public DefaultCheckConnectionWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) {
@@ -47,7 +51,7 @@ public DefaultCheckConnectionWorker(final IntegrationLauncher integrationLaunche
   @Override
   public StandardCheckConnectionOutput run(final StandardCheckConnectionInput input, final Path jobRoot) throws WorkerException {
 
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       process = integrationLauncher.check(
           jobRoot,
           WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
@@ -87,7 +91,7 @@ public StandardCheckConnectionOutput run(final StandardCheckConnectionInput inpu
 
   @Override
   public void cancel() {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       WorkerUtils.cancelProcess(process);
     } catch (final RuntimeException e) {
       throw e;
@@ -96,9 +100,4 @@ public void cancel() {
     }
   }
 
-  @Override
-  public String getApplicationName() {
-    return "check-connection-worker";
-  }
-
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
index 5df9d9957773..81b8927e9e84 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
@@ -7,8 +7,8 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
 import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.StandardDiscoverCatalogInput;
 import io.airbyte.protocol.models.AirbyteCatalog;
 import io.airbyte.protocol.models.AirbyteMessage;
@@ -30,6 +30,10 @@ public class DefaultDiscoverCatalogWorker implements DiscoverCatalogWorker {
   private final IntegrationLauncher integrationLauncher;
   private final AirbyteStreamFactory streamFactory;
 
+  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
+      .setLogPrefix("discover-worker")
+      .build();
+
   private volatile Process process;
 
   public DefaultDiscoverCatalogWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) {
@@ -43,7 +47,7 @@ public DefaultDiscoverCatalogWorker(final IntegrationLauncher integrationLaunche
 
   @Override
   public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput, final Path jobRoot) throws WorkerException {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       process = integrationLauncher.discover(
           jobRoot,
           WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
@@ -82,7 +86,7 @@ public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput
 
   @Override
   public void cancel() {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       WorkerUtils.cancelProcess(process);
     } catch (final RuntimeException e) {
       throw e;
@@ -91,9 +95,4 @@ public void cancel() {
     }
   }
 
-  @Override
-  public String getApplicationName() {
-    return "discover-worker";
-  }
-
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
index 2c35ec2873e4..0e52fab23078 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
@@ -6,8 +6,8 @@
 
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
-import io.airbyte.commons.logging.LoggingHelper;
 import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.JobGetSpecConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -29,6 +29,10 @@ public class DefaultGetSpecWorker implements GetSpecWorker {
   private final IntegrationLauncher integrationLauncher;
   private final AirbyteStreamFactory streamFactory;
 
+  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
+      .setLogPrefix("get-spec-worker")
+      .build();
+
   private Process process;
 
   public DefaultGetSpecWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) {
@@ -42,7 +46,7 @@ public DefaultGetSpecWorker(final IntegrationLauncher integrationLauncher) {
 
   @Override
   public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobRoot) throws WorkerException {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       process = integrationLauncher.spec(jobRoot);
 
       LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
@@ -82,7 +86,7 @@ public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobR
 
   @Override
   public void cancel() {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       WorkerUtils.cancelProcess(process);
     } catch (final RuntimeException e) {
       throw e;
@@ -91,9 +95,4 @@ public void cancel() {
     }
   }
 
-  @Override
-  public String getApplicationName() {
-    return "get-spec-worker";
-  }
-
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
index e0f0614a63cb..1499726e0233 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
@@ -4,8 +4,8 @@
 
 package io.airbyte.workers;
 
-import io.airbyte.commons.logging.LoggingHelper;
 import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.Configs.WorkerEnvironment;
 import io.airbyte.config.NormalizationInput;
 import io.airbyte.workers.normalization.NormalizationRunner;
@@ -26,6 +26,10 @@ public class DefaultNormalizationWorker implements NormalizationWorker {
   private final NormalizationRunner normalizationRunner;
   private final WorkerEnvironment workerEnvironment;
 
+  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
+      .setLogPrefix("normalization-worker")
+      .build();
+
   private final AtomicBoolean cancelled;
 
   public DefaultNormalizationWorker(final String jobId,
@@ -43,8 +47,7 @@ public DefaultNormalizationWorker(final String jobId,
   @Override
   public Void run(final NormalizationInput input, final Path jobRoot) throws WorkerException {
     final long startTime = System.currentTimeMillis();
-    try (normalizationRunner;
-        final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (normalizationRunner; extraMdcEntries) {
       LOGGER.info("Running normalization.");
       normalizationRunner.start();
 
@@ -78,7 +81,7 @@ public Void run(final NormalizationInput input, final Path jobRoot) throws Worke
 
   @Override
   public void cancel() {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       LOGGER.info("Cancelling normalization runner...");
       cancelled.set(true);
       normalizationRunner.close();
@@ -89,9 +92,4 @@ public void cancel() {
     }
   }
 
-  @Override
-  public String getApplicationName() {
-    return "normalization-worker";
-  }
-
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
index 57aa1feb0f5a..1430b17cf93a 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
@@ -4,8 +4,8 @@
 
 package io.airbyte.workers;
 
-import io.airbyte.commons.logging.LoggingHelper;
 import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.ReplicationAttemptSummary;
 import io.airbyte.config.ReplicationOutput;
 import io.airbyte.config.StandardSyncInput;
@@ -47,6 +47,10 @@ public class DefaultReplicationWorker implements ReplicationWorker {
   private final AtomicBoolean cancelled;
   private final AtomicBoolean hasFailed;
 
+  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
+      .setLogPrefix("sync-worker")
+      .build();
+
   public DefaultReplicationWorker(final String jobId,
                                   final int attempt,
                                   final Source<AirbyteMessage> source,
@@ -81,7 +85,7 @@ public DefaultReplicationWorker(final String jobId,
    */
   @Override
   public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRoot) throws WorkerException {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       LOGGER.info("start sync worker. job id: {} attempt id: {}", jobId, attempt);
 
       // todo (cgardens) - this should not be happening in the worker. this is configuration information
@@ -250,7 +254,7 @@ private static Runnable getDestinationOutputRunnable(final Destination<AirbyteMe
 
   @Override
   public void cancel() {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       // Resources are closed in the opposite order they are declared.
       LOGGER.info("Cancelling replication worker...");
       try {
@@ -280,9 +284,4 @@ public void cancel() {
     }
   }
 
-  @Override
-  public String getApplicationName() {
-    return "sync-worker";
-  }
-
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
index 32ebd42e6522..e968e70ff095 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/EchoWorker.java
@@ -25,9 +25,4 @@ public void cancel() {
     // no-op
   }
 
-  @Override
-  public String getApplicationName() {
-    return "airbyte-echo-worker";
-  }
-
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java b/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
index d08aea30a713..ffa64ac35399 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/Worker.java
@@ -4,10 +4,9 @@
 
 package io.airbyte.workers;
 
-import io.airbyte.commons.application.Application;
 import java.nio.file.Path;
 
-public interface Worker<InputType, OutputType> extends Application {
+public interface Worker<InputType, OutputType> {
 
   /**
    * Blocking call to run the worker's workflow. Once this is complete, getStatus should return either
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
index 6b5fbca34884..d48679339994 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/MessageTracker.java
@@ -4,12 +4,11 @@
 
 package io.airbyte.workers.protocols;
 
-import io.airbyte.commons.application.Application;
 import io.airbyte.config.State;
 import java.util.Optional;
 import java.util.function.Consumer;
 
-public interface MessageTracker<T> extends Consumer<T>, Application {
+public interface MessageTracker<T> extends Consumer<T> {
 
   @Override
   void accept(T message);
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
index 6d864fcbe96a..6ed52a0b1bca 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
@@ -6,8 +6,8 @@
 
 import com.google.common.base.Charsets;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper;
 import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.State;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.workers.protocols.MessageTracker;
@@ -21,6 +21,10 @@ public class AirbyteMessageTracker implements MessageTracker<AirbyteMessage> {
   private final AtomicLong numBytes;
   private final AtomicReference<State> outputState;
 
+  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
+      .setLogPrefix("airbyte-message-tracker")
+      .build();
+
   public AirbyteMessageTracker() {
     this.recordCount = new AtomicLong();
     this.numBytes = new AtomicLong();
@@ -29,7 +33,7 @@ public AirbyteMessageTracker() {
 
   @Override
   public void accept(final AirbyteMessage message) {
-    try (final MdcScope scopedMDCChange = new MdcScope(LoggingHelper.getExtraMDCEntries(this))) {
+    try (extraMdcEntries) {
       if (message.getType() == AirbyteMessage.Type.RECORD) {
         recordCount.incrementAndGet();
         // todo (cgardens) - pretty wasteful to do an extra serialization just to get size.
@@ -56,9 +60,4 @@ public Optional<State> getOutputState() {
     return Optional.ofNullable(outputState.get());
   }
 
-  @Override
-  public String getApplicationName() {
-    return "airbyte-message-tracker";
-  }
-
 }
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultCheckConnectionWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultCheckConnectionWorkerTest.java
index 30fbd03c7978..688de79c1efa 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultCheckConnectionWorkerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultCheckConnectionWorkerTest.java
@@ -107,7 +107,7 @@ public void testExceptionThrownInRun() throws WorkerException {
     doThrow(new RuntimeException()).when(integrationLauncher).check(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDS));
 
     final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(integrationLauncher, failureStreamFactory);
-    assertThrows(WorkerException.class, () -> worker.run(input, jobRoot));
+    assertThrows(RuntimeException.class, () -> worker.run(input, jobRoot));
   }
 
   @Test
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java
index 28f544f0734f..6c17eb99a19a 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java
@@ -4,13 +4,6 @@
 
 package io.airbyte.workers;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
@@ -31,9 +24,10 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
-import org.junit.jupiter.api.Assertions;
+import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
 
 public class DefaultDiscoverCatalogWorkerTest {
 
@@ -58,13 +52,14 @@ public class DefaultDiscoverCatalogWorkerTest {
   @BeforeEach
   public void setup() throws Exception {
     jobRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "");
-    integrationLauncher = mock(IntegrationLauncher.class, RETURNS_DEEP_STUBS);
-    process = mock(Process.class);
+    integrationLauncher = Mockito.mock(IntegrationLauncher.class, Mockito.RETURNS_DEEP_STUBS);
+    process = Mockito.mock(Process.class);
 
-    when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS))).thenReturn(process);
-    final InputStream inputStream = mock(InputStream.class);
-    when(process.getInputStream()).thenReturn(inputStream);
-    when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+    Mockito.when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS)))
+        .thenReturn(process);
+    final InputStream inputStream = Mockito.mock(InputStream.class);
+    Mockito.when(process.getInputStream()).thenReturn(inputStream);
+    Mockito.when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
 
     IOs.writeFile(jobRoot, WorkerConstants.SOURCE_CATALOG_JSON_FILENAME, MoreResources.readResource("airbyte_postgres_catalog.json"));
 
@@ -77,41 +72,45 @@ public void testDiscoverSchema() throws Exception {
     final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory);
     final AirbyteCatalog output = worker.run(INPUT, jobRoot);
 
-    assertEquals(CATALOG, output);
+    Assertions.assertThat(output).isEqualTo(CATALOG);
 
-    Assertions.assertTimeout(Duration.ofSeconds(5), () -> {
+    org.junit.jupiter.api.Assertions.assertTimeout(Duration.ofSeconds(5), () -> {
       while (process.getErrorStream().available() != 0) {
         Thread.sleep(50);
       }
     });
 
-    verify(process).exitValue();
+    Mockito.verify(process).exitValue();
   }
 
   @SuppressWarnings("BusyWait")
   @Test
   public void testDiscoverSchemaProcessFail() throws Exception {
-    when(process.exitValue()).thenReturn(1);
+    Mockito.when(process.exitValue()).thenReturn(1);
 
     final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory);
-    assertThrows(WorkerException.class, () -> worker.run(INPUT, jobRoot));
+    Assertions.assertThatThrownBy(() -> worker.run(INPUT, jobRoot))
+        .isInstanceOf(WorkerException.class)
+        .hasNoCause();
 
-    Assertions.assertTimeout(Duration.ofSeconds(5), () -> {
+    org.junit.jupiter.api.Assertions.assertTimeout(Duration.ofSeconds(5), () -> {
       while (process.getErrorStream().available() != 0) {
         Thread.sleep(50);
       }
     });
 
-    verify(process).exitValue();
+    Mockito.verify(process).exitValue();
   }
 
   @Test
   public void testDiscoverSchemaException() throws WorkerException {
-    when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS)))
+    Mockito.when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS)))
         .thenThrow(new RuntimeException());
 
     final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory);
-    assertThrows(WorkerException.class, () -> worker.run(INPUT, jobRoot));
+    Assertions.assertThatThrownBy(() -> worker.run(INPUT, jobRoot))
+        .isInstanceOf(RuntimeException.class)
+        .hasNoCause();
   }
 
   @Test
@@ -121,7 +120,7 @@ public void testCancel() throws WorkerException {
 
     worker.cancel();
 
-    verify(process).destroy();
+    Mockito.verify(process).destroy();
   }
 
 }
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java
index 23d1a358cd65..62dfe8c4da9b 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java
@@ -324,7 +324,7 @@ void testDoesNotPopulateOnIrrecoverableFailure() {
         destination,
         sourceMessageTracker,
         destinationMessageTracker);
-    assertThrows(WorkerException.class, () -> worker.run(syncInput, jobRoot));
+    assertThrows(IllegalStateException.class, () -> worker.run(syncInput, jobRoot));
   }
 
 }

From 7e645273f79e2df6a5b728936522c00b0eda0a81 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 12:01:53 -0700
Subject: [PATCH 15/32] restore dbt runner

---
 .../workers/DbtTransformationRunner.java      | 48 +++++++++++--------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index 441ec0bac892..6e09ef9bceb4 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -48,7 +48,7 @@ public void start() throws Exception {
    * transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend
    * on the NormalizationRunner to configure the dbt project with the appropriate destination settings
    * and pull the custom git repository into the workspace.
-   * <p>
+   *
    * Once the workspace folder/files is setup to run, we invoke the custom transformation command as
    * provided by the user to execute whatever extra transformation has been implemented.
    */
@@ -72,26 +72,34 @@ public boolean transform(final String jobId,
                            final ResourceRequirements resourceRequirements,
                            final OperatorDbt dbtConfig)
       throws Exception {
-    final Map<String, String> files = ImmutableMap.of(
-        DBT_ENTRYPOINT_SH, MoreResources.readResource("dbt_transformation_entrypoint.sh"),
-        "sshtunneling.sh", MoreResources.readResource("sshtunneling.sh"));
-    final List<String> dbtArguments = new ArrayList<>();
-    dbtArguments.add(DBT_ENTRYPOINT_SH);
-    if (Strings.isNullOrEmpty(dbtConfig.getDbtArguments())) {
-      throw new WorkerException("Dbt Arguments are required");
+    try {
+      final Map<String, String> files = ImmutableMap.of(
+          DBT_ENTRYPOINT_SH, MoreResources.readResource("dbt_transformation_entrypoint.sh"),
+          "sshtunneling.sh", MoreResources.readResource("sshtunneling.sh"));
+      final List<String> dbtArguments = new ArrayList<>();
+      dbtArguments.add(DBT_ENTRYPOINT_SH);
+      if (Strings.isNullOrEmpty(dbtConfig.getDbtArguments())) {
+        throw new WorkerException("Dbt Arguments are required");
+      }
+      Collections.addAll(dbtArguments, Commandline.translateCommandline(dbtConfig.getDbtArguments()));
+      process =
+          processFactory.create(jobId, attempt, jobRoot, dbtConfig.getDockerImage(), false, files, "/bin/bash", resourceRequirements,
+              Map.of(KubeProcessFactory.JOB_TYPE, KubeProcessFactory.SYNC_JOB, KubeProcessFactory.SYNC_STEP, KubeProcessFactory.CUSTOM_STEP),
+              dbtArguments);
+
+      LineGobbler.gobble(process.getInputStream(), LOGGER::info);
+      LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
+
+      WorkerUtils.wait(process);
+
+      return process.exitValue() == 0;
+    } catch (final Exception e) {
+      // make sure we kill the process on failure to avoid zombies.
+      if (process != null) {
+        WorkerUtils.cancelProcess(process);
+      }
+      throw e;
     }
-    Collections.addAll(dbtArguments, Commandline.translateCommandline(dbtConfig.getDbtArguments()));
-    process =
-        processFactory.create(jobId, attempt, jobRoot, dbtConfig.getDockerImage(), false, files, "/bin/bash", resourceRequirements,
-            Map.of(KubeProcessFactory.JOB_TYPE, KubeProcessFactory.SYNC_JOB, KubeProcessFactory.SYNC_STEP, KubeProcessFactory.CUSTOM_STEP),
-            dbtArguments);
-
-    LineGobbler.gobble(process.getInputStream(), LOGGER::info);
-    LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
-
-    WorkerUtils.wait(process);
-
-    return process.exitValue() == 0;
   }
 
   @Override

From 2c1d9fcce1b5a18ade78a3e683bb7a51e3633b57 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 14:01:48 -0700
Subject: [PATCH 16/32] Rollback changes non related to the issue.

---
 .../workers/DbtTransformationWorker.java      | 70 +++++++------------
 .../workers/DefaultCheckConnectionWorker.java | 18 +----
 .../workers/DefaultDiscoverCatalogWorker.java | 18 +----
 .../airbyte/workers/DefaultGetSpecWorker.java | 18 +----
 .../workers/DefaultNormalizationWorker.java   | 40 ++++-------
 .../workers/DefaultReplicationWorker.java     | 65 +++++++----------
 .../DefaultCheckConnectionWorkerTest.java     |  2 +-
 .../DefaultDiscoverCatalogWorkerTest.java     | 47 +++++++------
 .../workers/DefaultGetSpecWorkerTest.java     | 55 +++++++--------
 .../workers/DefaultReplicationWorkerTest.java |  2 +-
 10 files changed, 123 insertions(+), 212 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
index ac91507d5514..1e8f4b697ede 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationWorker.java
@@ -4,8 +4,6 @@
 
 package io.airbyte.workers;
 
-import io.airbyte.commons.logging.MdcScope;
-import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.OperatorDbtInput;
 import io.airbyte.config.ResourceRequirements;
 import java.nio.file.Files;
@@ -26,10 +24,6 @@ public class DbtTransformationWorker implements Worker<OperatorDbtInput, Void> {
 
   private final AtomicBoolean cancelled;
 
-  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
-      .setLogPrefix("sync-worker")
-      .build();
-
   public DbtTransformationWorker(final String jobId,
                                  final int attempt,
                                  final ResourceRequirements resourceRequirements,
@@ -44,52 +38,40 @@ public DbtTransformationWorker(final String jobId,
 
   @Override
   public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) throws WorkerException {
-    try (extraMdcEntries) {
-      final long startTime = System.currentTimeMillis();
+    final long startTime = System.currentTimeMillis();
 
-      try (dbtTransformationRunner) {
-        LOGGER.info("Running dbt transformation.");
-        dbtTransformationRunner.start();
-        final Path transformRoot = Files.createDirectories(jobRoot.resolve("transform"));
-        if (!dbtTransformationRunner.run(
-            jobId,
-            attempt,
-            transformRoot,
-            operatorDbtInput.getDestinationConfiguration(),
-            resourceRequirements,
-            operatorDbtInput.getOperatorDbt())) {
-          throw new WorkerException("DBT Transformation Failed.");
-        }
-      } catch (final Exception e) {
-        throw new WorkerException("Dbt Transformation Failed.", e);
-      }
-      if (cancelled.get()) {
-        LOGGER.info("Dbt Transformation was cancelled.");
+    try (dbtTransformationRunner) {
+      LOGGER.info("Running dbt transformation.");
+      dbtTransformationRunner.start();
+      final Path transformRoot = Files.createDirectories(jobRoot.resolve("transform"));
+      if (!dbtTransformationRunner.run(
+          jobId,
+          attempt,
+          transformRoot,
+          operatorDbtInput.getDestinationConfiguration(),
+          resourceRequirements,
+          operatorDbtInput.getOperatorDbt())) {
+        throw new WorkerException("DBT Transformation Failed.");
       }
-
-      final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime);
-      LOGGER.info("Dbt Transformation executed in {}.", duration.toMinutesPart());
-
-      return null;
-    } catch (final RuntimeException e) {
-      throw e;
     } catch (final Exception e) {
-      throw new WorkerException("Dbt Transformation failed", e);
+      throw new WorkerException("Dbt Transformation Failed.", e);
     }
+    if (cancelled.get()) {
+      LOGGER.info("Dbt Transformation was cancelled.");
+    }
+
+    final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime);
+    LOGGER.info("Dbt Transformation executed in {}.", duration.toMinutesPart());
+
+    return null;
   }
 
   @Override
   public void cancel() {
-    try (extraMdcEntries) {
-      LOGGER.info("Cancelling Dbt Transformation runner...");
-      try {
-        cancelled.set(true);
-        dbtTransformationRunner.close();
-      } catch (final Exception e) {
-        e.printStackTrace();
-      }
-    } catch (final RuntimeException e) {
-      throw e;
+    LOGGER.info("Cancelling Dbt Transformation runner...");
+    try {
+      cancelled.set(true);
+      dbtTransformationRunner.close();
     } catch (final Exception e) {
       e.printStackTrace();
     }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
index c851b8cab3e4..c4e32d16079f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultCheckConnectionWorker.java
@@ -8,8 +8,6 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.MdcScope;
-import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.StandardCheckConnectionInput;
 import io.airbyte.config.StandardCheckConnectionOutput;
 import io.airbyte.config.StandardCheckConnectionOutput.Status;
@@ -33,10 +31,6 @@ public class DefaultCheckConnectionWorker implements CheckConnectionWorker {
   private final IntegrationLauncher integrationLauncher;
   private final AirbyteStreamFactory streamFactory;
 
-  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
-      .setLogPrefix("check-connection-worker")
-      .build();
-
   private Process process;
 
   public DefaultCheckConnectionWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) {
@@ -51,7 +45,7 @@ public DefaultCheckConnectionWorker(final IntegrationLauncher integrationLaunche
   @Override
   public StandardCheckConnectionOutput run(final StandardCheckConnectionInput input, final Path jobRoot) throws WorkerException {
 
-    try (extraMdcEntries) {
+    try {
       process = integrationLauncher.check(
           jobRoot,
           WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
@@ -82,8 +76,6 @@ public StandardCheckConnectionOutput run(final StandardCheckConnectionInput inpu
         throw new WorkerException("Error while getting checking connection.");
       }
 
-    } catch (final RuntimeException e) {
-      throw e;
     } catch (final Exception e) {
       throw new WorkerException("Error while getting checking connection.", e);
     }
@@ -91,13 +83,7 @@ public StandardCheckConnectionOutput run(final StandardCheckConnectionInput inpu
 
   @Override
   public void cancel() {
-    try (extraMdcEntries) {
-      WorkerUtils.cancelProcess(process);
-    } catch (final RuntimeException e) {
-      throw e;
-    } catch (final Exception e) {
-      throw new RuntimeException(e);
-    }
+    WorkerUtils.cancelProcess(process);
   }
 
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
index 81b8927e9e84..bf3dd1a371b4 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultDiscoverCatalogWorker.java
@@ -7,8 +7,6 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.MdcScope;
-import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.StandardDiscoverCatalogInput;
 import io.airbyte.protocol.models.AirbyteCatalog;
 import io.airbyte.protocol.models.AirbyteMessage;
@@ -30,10 +28,6 @@ public class DefaultDiscoverCatalogWorker implements DiscoverCatalogWorker {
   private final IntegrationLauncher integrationLauncher;
   private final AirbyteStreamFactory streamFactory;
 
-  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
-      .setLogPrefix("discover-worker")
-      .build();
-
   private volatile Process process;
 
   public DefaultDiscoverCatalogWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) {
@@ -47,7 +41,7 @@ public DefaultDiscoverCatalogWorker(final IntegrationLauncher integrationLaunche
 
   @Override
   public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput, final Path jobRoot) throws WorkerException {
-    try (extraMdcEntries) {
+    try {
       process = integrationLauncher.discover(
           jobRoot,
           WorkerConstants.SOURCE_CONFIG_JSON_FILENAME,
@@ -77,8 +71,6 @@ public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput
       }
     } catch (final WorkerException e) {
       throw e;
-    } catch (final RuntimeException e) {
-      throw e;
     } catch (final Exception e) {
       throw new WorkerException("Error while discovering schema", e);
     }
@@ -86,13 +78,7 @@ public AirbyteCatalog run(final StandardDiscoverCatalogInput discoverSchemaInput
 
   @Override
   public void cancel() {
-    try (extraMdcEntries) {
-      WorkerUtils.cancelProcess(process);
-    } catch (final RuntimeException e) {
-      throw e;
-    } catch (final Exception e) {
-      throw new RuntimeException(e);
-    }
+    WorkerUtils.cancelProcess(process);
   }
 
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
index 0e52fab23078..82265aa9370c 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultGetSpecWorker.java
@@ -6,8 +6,6 @@
 
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
-import io.airbyte.commons.logging.MdcScope;
-import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.JobGetSpecConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -29,10 +27,6 @@ public class DefaultGetSpecWorker implements GetSpecWorker {
   private final IntegrationLauncher integrationLauncher;
   private final AirbyteStreamFactory streamFactory;
 
-  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
-      .setLogPrefix("get-spec-worker")
-      .build();
-
   private Process process;
 
   public DefaultGetSpecWorker(final IntegrationLauncher integrationLauncher, final AirbyteStreamFactory streamFactory) {
@@ -46,7 +40,7 @@ public DefaultGetSpecWorker(final IntegrationLauncher integrationLauncher) {
 
   @Override
   public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobRoot) throws WorkerException {
-    try (extraMdcEntries) {
+    try {
       process = integrationLauncher.spec(jobRoot);
 
       LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
@@ -76,8 +70,6 @@ public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobR
       } else {
         throw new WorkerException(String.format("Spec job subprocess finished with exit code %s", exitCode));
       }
-    } catch (final RuntimeException e) {
-      throw e;
     } catch (final Exception e) {
       throw new WorkerException(String.format("Error while getting spec from image %s", config.getDockerImage()), e);
     }
@@ -86,13 +78,7 @@ public ConnectorSpecification run(final JobGetSpecConfig config, final Path jobR
 
   @Override
   public void cancel() {
-    try (extraMdcEntries) {
-      WorkerUtils.cancelProcess(process);
-    } catch (final RuntimeException e) {
-      throw e;
-    } catch (final Exception e) {
-      throw new RuntimeException(e);
-    }
+    WorkerUtils.cancelProcess(process);
   }
 
 }
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
index 1499726e0233..2ce6c078f0fa 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java
@@ -4,8 +4,6 @@
 
 package io.airbyte.workers;
 
-import io.airbyte.commons.logging.MdcScope;
-import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.Configs.WorkerEnvironment;
 import io.airbyte.config.NormalizationInput;
 import io.airbyte.workers.normalization.NormalizationRunner;
@@ -26,10 +24,6 @@ public class DefaultNormalizationWorker implements NormalizationWorker {
   private final NormalizationRunner normalizationRunner;
   private final WorkerEnvironment workerEnvironment;
 
-  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
-      .setLogPrefix("normalization-worker")
-      .build();
-
   private final AtomicBoolean cancelled;
 
   public DefaultNormalizationWorker(final String jobId,
@@ -47,7 +41,8 @@ public DefaultNormalizationWorker(final String jobId,
   @Override
   public Void run(final NormalizationInput input, final Path jobRoot) throws WorkerException {
     final long startTime = System.currentTimeMillis();
-    try (normalizationRunner; extraMdcEntries) {
+
+    try (normalizationRunner) {
       LOGGER.info("Running normalization.");
       normalizationRunner.start();
 
@@ -61,34 +56,29 @@ public Void run(final NormalizationInput input, final Path jobRoot) throws Worke
           input.getResourceRequirements())) {
         throw new WorkerException("Normalization Failed.");
       }
-
-      if (cancelled.get()) {
-        LOGGER.info("Normalization was cancelled.");
-      }
-
-      final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime);
-      final String durationDescription = DurationFormatUtils.formatDurationWords(duration.toMillis(), true, true);
-      LOGGER.info("Normalization executed in {}.", durationDescription);
-
-      return null;
-
-    } catch (final RuntimeException e) {
-      throw e;
     } catch (final Exception e) {
       throw new WorkerException("Normalization Failed.", e);
     }
+
+    if (cancelled.get()) {
+      LOGGER.info("Normalization was cancelled.");
+    }
+
+    final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime);
+    final String durationDescription = DurationFormatUtils.formatDurationWords(duration.toMillis(), true, true);
+    LOGGER.info("Normalization executed in {}.", durationDescription);
+
+    return null;
   }
 
   @Override
   public void cancel() {
-    try (extraMdcEntries) {
-      LOGGER.info("Cancelling normalization runner...");
+    LOGGER.info("Cancelling normalization runner...");
+    try {
       cancelled.set(true);
       normalizationRunner.close();
-    } catch (final RuntimeException e) {
-      throw e;
     } catch (final Exception e) {
-      throw new RuntimeException(e);
+      e.printStackTrace();
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
index 1430b17cf93a..280cb30f9875 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultReplicationWorker.java
@@ -4,8 +4,6 @@
 
 package io.airbyte.workers;
 
-import io.airbyte.commons.logging.MdcScope;
-import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.ReplicationAttemptSummary;
 import io.airbyte.config.ReplicationOutput;
 import io.airbyte.config.StandardSyncInput;
@@ -47,10 +45,6 @@ public class DefaultReplicationWorker implements ReplicationWorker {
   private final AtomicBoolean cancelled;
   private final AtomicBoolean hasFailed;
 
-  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
-      .setLogPrefix("sync-worker")
-      .build();
-
   public DefaultReplicationWorker(final String jobId,
                                   final int attempt,
                                   final Source<AirbyteMessage> source,
@@ -85,15 +79,15 @@ public DefaultReplicationWorker(final String jobId,
    */
   @Override
   public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRoot) throws WorkerException {
-    try (extraMdcEntries) {
-      LOGGER.info("start sync worker. job id: {} attempt id: {}", jobId, attempt);
+    LOGGER.info("start sync worker. job id: {} attempt id: {}", jobId, attempt);
 
-      // todo (cgardens) - this should not be happening in the worker. this is configuration information
-      // that is independent of workflow executions.
-      final WorkerDestinationConfig destinationConfig = WorkerUtils.syncToWorkerDestinationConfig(syncInput);
-      destinationConfig.setCatalog(mapper.mapCatalog(destinationConfig.getCatalog()));
+    // todo (cgardens) - this should not be happening in the worker. this is configuration information
+    // that is independent of workflow executions.
+    final WorkerDestinationConfig destinationConfig = WorkerUtils.syncToWorkerDestinationConfig(syncInput);
+    destinationConfig.setCatalog(mapper.mapCatalog(destinationConfig.getCatalog()));
 
-      final long startTime = System.currentTimeMillis();
+    final long startTime = System.currentTimeMillis();
+    try {
       LOGGER.info("configured sync modes: {}", syncInput.getCatalog().getStreams()
           .stream()
           .collect(Collectors.toMap(s -> s.getStream().getNamespace() + "." + s.getStream().getName(),
@@ -179,8 +173,6 @@ else if (hasFailed.get()) {
       }
 
       return output;
-    } catch (final RuntimeException e) {
-      throw e;
     } catch (final Exception e) {
       throw new WorkerException("Sync failed", e);
     }
@@ -254,34 +246,29 @@ private static Runnable getDestinationOutputRunnable(final Destination<AirbyteMe
 
   @Override
   public void cancel() {
-    try (extraMdcEntries) {
-      // Resources are closed in the opposite order they are declared.
-      LOGGER.info("Cancelling replication worker...");
-      try {
-        executors.awaitTermination(10, TimeUnit.SECONDS);
-      } catch (final InterruptedException e) {
-        e.printStackTrace();
-      }
-      cancelled.set(true);
+    // Resources are closed in the opposite order they are declared.
+    LOGGER.info("Cancelling replication worker...");
+    try {
+      executors.awaitTermination(10, TimeUnit.SECONDS);
+    } catch (final InterruptedException e) {
+      e.printStackTrace();
+    }
+    cancelled.set(true);
 
-      LOGGER.info("Cancelling destination...");
-      try {
-        destination.cancel();
-      } catch (final Exception e) {
-        LOGGER.info("Error cancelling destination: ", e);
-      }
+    LOGGER.info("Cancelling destination...");
+    try {
+      destination.cancel();
+    } catch (final Exception e) {
+      LOGGER.info("Error cancelling destination: ", e);
+    }
 
-      LOGGER.info("Cancelling source...");
-      try {
-        source.cancel();
-      } catch (final Exception e) {
-        LOGGER.info("Error cancelling source: ", e);
-      }
-    } catch (final RuntimeException e) {
-      throw e;
+    LOGGER.info("Cancelling source...");
+    try {
+      source.cancel();
     } catch (final Exception e) {
-      throw new RuntimeException(e);
+      LOGGER.info("Error cancelling source: ", e);
     }
+
   }
 
 }
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultCheckConnectionWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultCheckConnectionWorkerTest.java
index 688de79c1efa..30fbd03c7978 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultCheckConnectionWorkerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultCheckConnectionWorkerTest.java
@@ -107,7 +107,7 @@ public void testExceptionThrownInRun() throws WorkerException {
     doThrow(new RuntimeException()).when(integrationLauncher).check(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDS));
 
     final DefaultCheckConnectionWorker worker = new DefaultCheckConnectionWorker(integrationLauncher, failureStreamFactory);
-    assertThrows(RuntimeException.class, () -> worker.run(input, jobRoot));
+    assertThrows(WorkerException.class, () -> worker.run(input, jobRoot));
   }
 
   @Test
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java
index 6c17eb99a19a..28f544f0734f 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java
@@ -4,6 +4,13 @@
 
 package io.airbyte.workers;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
@@ -24,10 +31,9 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
-import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
 
 public class DefaultDiscoverCatalogWorkerTest {
 
@@ -52,14 +58,13 @@ public class DefaultDiscoverCatalogWorkerTest {
   @BeforeEach
   public void setup() throws Exception {
     jobRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "");
-    integrationLauncher = Mockito.mock(IntegrationLauncher.class, Mockito.RETURNS_DEEP_STUBS);
-    process = Mockito.mock(Process.class);
+    integrationLauncher = mock(IntegrationLauncher.class, RETURNS_DEEP_STUBS);
+    process = mock(Process.class);
 
-    Mockito.when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS)))
-        .thenReturn(process);
-    final InputStream inputStream = Mockito.mock(InputStream.class);
-    Mockito.when(process.getInputStream()).thenReturn(inputStream);
-    Mockito.when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+    when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS))).thenReturn(process);
+    final InputStream inputStream = mock(InputStream.class);
+    when(process.getInputStream()).thenReturn(inputStream);
+    when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
 
     IOs.writeFile(jobRoot, WorkerConstants.SOURCE_CATALOG_JSON_FILENAME, MoreResources.readResource("airbyte_postgres_catalog.json"));
 
@@ -72,45 +77,41 @@ public void testDiscoverSchema() throws Exception {
     final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory);
     final AirbyteCatalog output = worker.run(INPUT, jobRoot);
 
-    Assertions.assertThat(output).isEqualTo(CATALOG);
+    assertEquals(CATALOG, output);
 
-    org.junit.jupiter.api.Assertions.assertTimeout(Duration.ofSeconds(5), () -> {
+    Assertions.assertTimeout(Duration.ofSeconds(5), () -> {
       while (process.getErrorStream().available() != 0) {
         Thread.sleep(50);
       }
     });
 
-    Mockito.verify(process).exitValue();
+    verify(process).exitValue();
   }
 
   @SuppressWarnings("BusyWait")
   @Test
   public void testDiscoverSchemaProcessFail() throws Exception {
-    Mockito.when(process.exitValue()).thenReturn(1);
+    when(process.exitValue()).thenReturn(1);
 
     final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory);
-    Assertions.assertThatThrownBy(() -> worker.run(INPUT, jobRoot))
-        .isInstanceOf(WorkerException.class)
-        .hasNoCause();
+    assertThrows(WorkerException.class, () -> worker.run(INPUT, jobRoot));
 
-    org.junit.jupiter.api.Assertions.assertTimeout(Duration.ofSeconds(5), () -> {
+    Assertions.assertTimeout(Duration.ofSeconds(5), () -> {
       while (process.getErrorStream().available() != 0) {
         Thread.sleep(50);
       }
     });
 
-    Mockito.verify(process).exitValue();
+    verify(process).exitValue();
   }
 
   @Test
   public void testDiscoverSchemaException() throws WorkerException {
-    Mockito.when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS)))
+    when(integrationLauncher.discover(jobRoot, WorkerConstants.SOURCE_CONFIG_JSON_FILENAME, Jsons.serialize(CREDENTIALS)))
         .thenThrow(new RuntimeException());
 
     final DefaultDiscoverCatalogWorker worker = new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory);
-    Assertions.assertThatThrownBy(() -> worker.run(INPUT, jobRoot))
-        .isInstanceOf(RuntimeException.class)
-        .hasNoCause();
+    assertThrows(WorkerException.class, () -> worker.run(INPUT, jobRoot));
   }
 
   @Test
@@ -120,7 +121,7 @@ public void testCancel() throws WorkerException {
 
     worker.cancel();
 
-    Mockito.verify(process).destroy();
+    verify(process).destroy();
   }
 
 }
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultGetSpecWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultGetSpecWorkerTest.java
index 6cbb906987a8..4b927d023493 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultGetSpecWorkerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultGetSpecWorkerTest.java
@@ -4,6 +4,14 @@
 
 package io.airbyte.workers;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 import com.google.common.base.Charsets;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.resources.MoreResources;
@@ -16,10 +24,8 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
 
 class DefaultGetSpecWorkerTest {
 
@@ -36,10 +42,10 @@ class DefaultGetSpecWorkerTest {
   public void setup() throws IOException, WorkerException {
     jobRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "");
     config = new JobGetSpecConfig().withDockerImage(DUMMY_IMAGE_NAME);
-    integrationLauncher = Mockito.mock(IntegrationLauncher.class, Mockito.RETURNS_DEEP_STUBS);
-    process = Mockito.mock(Process.class);
-    Mockito.when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
-    Mockito.when(integrationLauncher.spec(jobRoot)).thenReturn(process);
+    integrationLauncher = mock(IntegrationLauncher.class, RETURNS_DEEP_STUBS);
+    process = mock(Process.class);
+    when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(new byte[0]));
+    when(integrationLauncher.spec(jobRoot)).thenReturn(process);
 
     worker = new DefaultGetSpecWorker(integrationLauncher);
   }
@@ -52,45 +58,32 @@ public void testSuccessfulRun() throws IOException, InterruptedException, Worker
         .withType(Type.SPEC)
         .withSpec(Jsons.deserialize(expectedSpecString, io.airbyte.protocol.models.ConnectorSpecification.class));
 
-    Mockito.when(process.getInputStream()).thenReturn(new ByteArrayInputStream(Jsons.serialize(message).getBytes(Charsets.UTF_8)));
-    Mockito.when(process.waitFor(Mockito.anyLong(), Mockito.any())).thenReturn(true);
-    Mockito.when(process.exitValue()).thenReturn(0);
+    when(process.getInputStream()).thenReturn(new ByteArrayInputStream(Jsons.serialize(message).getBytes(Charsets.UTF_8)));
+    when(process.waitFor(anyLong(), any())).thenReturn(true);
+    when(process.exitValue()).thenReturn(0);
 
     final ConnectorSpecification actualOutput = worker.run(config, jobRoot);
     final ConnectorSpecification expectedOutput = Jsons.deserialize(expectedSpecString, ConnectorSpecification.class);
 
-    Assertions.assertThat(actualOutput).isEqualTo(expectedOutput);
+    assertEquals(expectedOutput, actualOutput);
   }
 
   @Test
   public void testFailureOnInvalidSpec() throws InterruptedException {
     final String expectedSpecString = "{\"key\":\"value\"}";
-    Mockito.when(process.getInputStream()).thenReturn(new ByteArrayInputStream(expectedSpecString.getBytes()));
-    Mockito.when(process.waitFor(Mockito.anyLong(), Mockito.any())).thenReturn(true);
-    Mockito.when(process.exitValue()).thenReturn(0);
+    when(process.getInputStream()).thenReturn(new ByteArrayInputStream(expectedSpecString.getBytes()));
+    when(process.waitFor(anyLong(), any())).thenReturn(true);
+    when(process.exitValue()).thenReturn(0);
 
-    Assertions.assertThatThrownBy(() -> worker.run(config, jobRoot))
-        .hasCauseExactlyInstanceOf(WorkerException.class)
-        .hasRootCauseExactlyInstanceOf(WorkerException.class);
+    assertThrows(WorkerException.class, () -> worker.run(config, jobRoot));
   }
 
   @Test
-  public void testFailureOnNonzeroExitCode() throws InterruptedException, IOException {
-    final String expectedSpecString = MoreResources.readResource("valid_spec.json");
-
-    final AirbyteMessage message = new AirbyteMessage()
-        .withType(Type.SPEC)
-        .withSpec(Jsons.deserialize(expectedSpecString, io.airbyte.protocol.models.ConnectorSpecification.class));
-
-    Mockito.when(process.getInputStream()).thenReturn(new ByteArrayInputStream(Jsons.serialize(message).getBytes(Charsets.UTF_8)));
-    Mockito.when(process.waitFor(Mockito.anyLong(), Mockito.any())).thenReturn(true);
-    Mockito.when(process.exitValue()).thenReturn(1);
+  public void testFailureOnNonzeroExitCode() throws InterruptedException {
+    when(process.waitFor(anyLong(), any())).thenReturn(true);
+    when(process.exitValue()).thenReturn(1);
 
-    Assertions.assertThatThrownBy(() -> worker.run(config, jobRoot))
-        .hasCauseExactlyInstanceOf(WorkerException.class)
-        .hasRootCauseExactlyInstanceOf(WorkerException.class)
-        .getRootCause()
-        .hasNoCause();
+    assertThrows(WorkerException.class, () -> worker.run(config, jobRoot));
   }
 
 }
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java
index 62dfe8c4da9b..23d1a358cd65 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultReplicationWorkerTest.java
@@ -324,7 +324,7 @@ void testDoesNotPopulateOnIrrecoverableFailure() {
         destination,
         sourceMessageTracker,
         destinationMessageTracker);
-    assertThrows(IllegalStateException.class, () -> worker.run(syncInput, jobRoot));
+    assertThrows(WorkerException.class, () -> worker.run(syncInput, jobRoot));
   }
 
 }

From 0c085366c9dba64fff068388cfcd852e30def9a2 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 14:03:49 -0700
Subject: [PATCH 17/32] Add missing file

---
 .../airbyte/AirbyteMessageTracker.java        | 24 +++++++------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
index 6ed52a0b1bca..34e12a068b32 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/AirbyteMessageTracker.java
@@ -6,8 +6,6 @@
 
 import com.google.common.base.Charsets;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.MdcScope;
-import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.config.State;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.workers.protocols.MessageTracker;
@@ -21,10 +19,6 @@ public class AirbyteMessageTracker implements MessageTracker<AirbyteMessage> {
   private final AtomicLong numBytes;
   private final AtomicReference<State> outputState;
 
-  private final MdcScope extraMdcEntries = new MdcScopeBuilder()
-      .setLogPrefix("airbyte-message-tracker")
-      .build();
-
   public AirbyteMessageTracker() {
     this.recordCount = new AtomicLong();
     this.numBytes = new AtomicLong();
@@ -33,16 +27,14 @@ public AirbyteMessageTracker() {
 
   @Override
   public void accept(final AirbyteMessage message) {
-    try (extraMdcEntries) {
-      if (message.getType() == AirbyteMessage.Type.RECORD) {
-        recordCount.incrementAndGet();
-        // todo (cgardens) - pretty wasteful to do an extra serialization just to get size.
-        numBytes.addAndGet(Jsons.serialize(message.getRecord().getData()).getBytes(Charsets.UTF_8).length);
-      }
-      if (message.getType() == AirbyteMessage.Type.STATE) {
-        outputState.set(new State().withState(message.getState().getData()));
-      }
-    } catch (final Exception e) {}
+    if (message.getType() == AirbyteMessage.Type.RECORD) {
+      recordCount.incrementAndGet();
+      // todo (cgardens) - pretty wasteful to do an extra serialization just to get size.
+      numBytes.addAndGet(Jsons.serialize(message.getRecord().getData()).getBytes(Charsets.UTF_8).length);
+    }
+    if (message.getType() == AirbyteMessage.Type.STATE) {
+      outputState.set(new State().withState(message.getState().getData()));
+    }
   }
 
   @Override

From 2f5e0dc393b1d7c3f6573af2e42b6d5d9b5d8582 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 14:24:37 -0700
Subject: [PATCH 18/32] limit the labelling to logs coming from a container and
 not a worker

---
 .../io/airbyte/commons/logging/MdcScope.java    |  7 +++++++
 .../airbyte/DefaultAirbyteStreamFactory.java    | 17 +++++++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
index f71cae647547..d905af6b610c 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
@@ -45,6 +45,7 @@ public void close() throws Exception {
   public static class MdcScopeBuilder {
 
     private Optional<String> maybeLogPrefix = Optional.empty();
+    private Optional<Color> maybePrefixColor = Optional.empty();
 
     public MdcScopeBuilder setLogPrefix(final String logPrefix) {
       this.maybeLogPrefix = Optional.ofNullable(logPrefix);
@@ -52,6 +53,12 @@ public MdcScopeBuilder setLogPrefix(final String logPrefix) {
       return this;
     }
 
+    public MdcScopeBuilder setPrefixColor(final Color color) {
+      this.maybePrefixColor = Optional.ofNullable(color);
+
+      return this;
+    }
+
     public MdcScope build() {
       final Map<String, String> extraMdcEntries = new HashMap<>();
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index 69b32f64be92..a7e0e4d426b9 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -6,6 +6,9 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper.Color;
+import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
 import io.airbyte.protocol.models.AirbyteLogMessage;
 import io.airbyte.protocol.models.AirbyteMessage;
 import java.io.BufferedReader;
@@ -27,6 +30,10 @@
 public class DefaultAirbyteStreamFactory implements AirbyteStreamFactory {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteStreamFactory.class);
+  private static final MdcScope containerLogMDC = new MdcScopeBuilder()
+      .setLogPrefix("container-log")
+      .setPrefixColor(Color.MAGENTA)
+      .build();
 
   private final AirbyteProtocolPredicate protocolValidator;
   private final Logger logger;
@@ -50,7 +57,9 @@ public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
             // we log as info all the lines that are not valid json
             // some sources actually log their process on stdout, we
             // want to make sure this info is available in the logs.
-            logger.info(line);
+            try (containerLogMDC) {
+              logger.info(line);
+            } catch (final Exception e) {}
           }
           return jsonLine.stream();
         })
@@ -73,7 +82,11 @@ public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
         .filter(airbyteMessage -> {
           final boolean isLog = airbyteMessage.getType() == AirbyteMessage.Type.LOG;
           if (isLog) {
-            internalLog(airbyteMessage.getLog());
+            try (containerLogMDC) {
+              internalLog(airbyteMessage.getLog());
+            } catch (final Exception e) {
+              e.printStackTrace();
+            }
           }
           return !isLog;
         });

From 734bc1abfe9dd24ce535fd679065fb48f842cb6d Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 15:09:23 -0700
Subject: [PATCH 19/32] Add the right colot to the prefix

---
 .../main/java/io/airbyte/commons/logging/MdcScope.java    | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
index d905af6b610c..7194863c714c 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
@@ -62,7 +62,13 @@ public MdcScopeBuilder setPrefixColor(final Color color) {
     public MdcScope build() {
       final Map<String, String> extraMdcEntries = new HashMap<>();
 
-      maybeLogPrefix.map(logPrefix -> extraMdcEntries.put(LoggingHelper.LOG_SOURCE_MDC_KEY, LoggingHelper.applyColor(Color.GREEN, logPrefix)));
+      maybeLogPrefix.stream().forEach(logPrefix -> {
+        final String potentiallyColoredLog = maybePrefixColor
+            .map(color -> LoggingHelper.applyColor(color, logPrefix))
+            .orElse(logPrefix);
+
+        extraMdcEntries.put(LoggingHelper.LOG_SOURCE_MDC_KEY, potentiallyColoredLog);
+      });
 
       return new MdcScope(extraMdcEntries);
     }

From 3fa1260231d9041bc679216a38aaa236fdd4ec69 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 15:13:11 -0700
Subject: [PATCH 20/32] Rm useless print of a stack trace

---
 .../airbyte/DefaultAirbyteStreamFactory.java       | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index a7e0e4d426b9..c3f3c84919c6 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -18,14 +18,12 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Creates a stream from an input stream. The produced stream attempts to parse each line of the
- * InputStream into a AirbyteMessage. If the line cannot be parsed into a AirbyteMessage it is
- * dropped. Each record MUST be new line separated.
+ * Creates a stream from an input stream. The produced stream attempts to parse each line of the InputStream into a AirbyteMessage. If the line cannot
+ * be parsed into a AirbyteMessage it is dropped. Each record MUST be new line separated.
  *
  * <p>
- * If a line starts with a AirbyteMessage and then has other characters after it, that
- * AirbyteMessage will still be parsed. If there are multiple AirbyteMessage records on the same
- * line, only the first will be parsed.
+ * If a line starts with a AirbyteMessage and then has other characters after it, that AirbyteMessage will still be parsed. If there are multiple
+ * AirbyteMessage records on the same line, only the first will be parsed.
  */
 public class DefaultAirbyteStreamFactory implements AirbyteStreamFactory {
 
@@ -59,7 +57,8 @@ public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
             // want to make sure this info is available in the logs.
             try (containerLogMDC) {
               logger.info(line);
-            } catch (final Exception e) {}
+            } catch (final Exception e) {
+            }
           }
           return jsonLine.stream();
         })
@@ -85,7 +84,6 @@ public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
             try (containerLogMDC) {
               internalLog(airbyteMessage.getLog());
             } catch (final Exception e) {
-              e.printStackTrace();
             }
           }
           return !isLog;

From 0001cc2f06cea275c90620af16bdaf1a677bd439 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Fri, 22 Oct 2021 15:13:51 -0700
Subject: [PATCH 21/32] Format

---
 .../airbyte/DefaultAirbyteStreamFactory.java     | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index c3f3c84919c6..e12beb961a5e 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -18,12 +18,14 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Creates a stream from an input stream. The produced stream attempts to parse each line of the InputStream into a AirbyteMessage. If the line cannot
- * be parsed into a AirbyteMessage it is dropped. Each record MUST be new line separated.
+ * Creates a stream from an input stream. The produced stream attempts to parse each line of the
+ * InputStream into a AirbyteMessage. If the line cannot be parsed into a AirbyteMessage it is
+ * dropped. Each record MUST be new line separated.
  *
  * <p>
- * If a line starts with a AirbyteMessage and then has other characters after it, that AirbyteMessage will still be parsed. If there are multiple
- * AirbyteMessage records on the same line, only the first will be parsed.
+ * If a line starts with a AirbyteMessage and then has other characters after it, that
+ * AirbyteMessage will still be parsed. If there are multiple AirbyteMessage records on the same
+ * line, only the first will be parsed.
  */
 public class DefaultAirbyteStreamFactory implements AirbyteStreamFactory {
 
@@ -57,8 +59,7 @@ public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
             // want to make sure this info is available in the logs.
             try (containerLogMDC) {
               logger.info(line);
-            } catch (final Exception e) {
-            }
+            } catch (final Exception e) {}
           }
           return jsonLine.stream();
         })
@@ -83,8 +84,7 @@ public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
           if (isLog) {
             try (containerLogMDC) {
               internalLog(airbyteMessage.getLog());
-            } catch (final Exception e) {
-            }
+            } catch (final Exception e) {}
           }
           return !isLog;
         });

From 1b313a1150a420931289beb01d7db70fa0f77c71 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Mon, 25 Oct 2021 15:11:22 -0700
Subject: [PATCH 22/32] Add a ScopeMdc to the line gobbler

---
 .../io/airbyte/commons/io/LineGobbler.java    | 30 +++++++++++++++----
 .../io/airbyte/commons/logging/MdcScope.java  | 10 ++++---
 .../airbyte/commons/logging/MdcScopeTest.java |  7 ++---
 .../airbyte/DefaultAirbyteDestination.java    | 11 +++++--
 .../airbyte/DefaultAirbyteSource.java         | 12 ++++++--
 .../airbyte/DefaultAirbyteStreamFactory.java  | 21 ++++++-------
 .../DefaultAirbyteStreamFactoryTest.java      |  3 +-
 7 files changed, 64 insertions(+), 30 deletions(-)

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java b/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java
index 29e055ce0de9..6e87ce1f4517 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java
@@ -5,6 +5,7 @@
 package io.airbyte.commons.io;
 
 import io.airbyte.commons.concurrency.VoidCallable;
+import io.airbyte.commons.logging.MdcScope;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -21,13 +22,17 @@ public class LineGobbler implements VoidCallable {
   private final static Logger LOGGER = LoggerFactory.getLogger(LineGobbler.class);
 
   public static void gobble(final InputStream is, final Consumer<String> consumer) {
-    gobble(is, consumer, "generic");
+    gobble(is, consumer, "generic", MdcScope.DEFAULT);
   }
 
-  public static void gobble(final InputStream is, final Consumer<String> consumer, final String caller) {
+  public static void gobble(final InputStream is, final Consumer<String> consumer, final MdcScope mdcScope) {
+    gobble(is, consumer, "generic", mdcScope);
+  }
+
+  public static void gobble(final InputStream is, final Consumer<String> consumer, final String caller, final MdcScope mdcScope) {
     final ExecutorService executor = Executors.newSingleThreadExecutor();
     final Map<String, String> mdc = MDC.getCopyOfContextMap();
-    final var gobbler = new LineGobbler(is, consumer, executor, mdc, caller);
+    final var gobbler = new LineGobbler(is, consumer, executor, mdc, caller, mdcScope);
     executor.submit(gobbler);
   }
 
@@ -36,24 +41,35 @@ public static void gobble(final InputStream is, final Consumer<String> consumer,
   private final ExecutorService executor;
   private final Map<String, String> mdc;
   private final String caller;
+  private final MdcScope containerLogMDC;
 
   LineGobbler(final InputStream is,
               final Consumer<String> consumer,
               final ExecutorService executor,
               final Map<String, String> mdc) {
-    this(is, consumer, executor, mdc, "generic");
+    this(is, consumer, executor, mdc, "generic", MdcScope.DEFAULT);
+  }
+
+  LineGobbler(final InputStream is,
+              final Consumer<String> consumer,
+              final ExecutorService executor,
+              final Map<String, String> mdc,
+              final MdcScope mdcScope) {
+    this(is, consumer, executor, mdc, "generic", mdcScope);
   }
 
   LineGobbler(final InputStream is,
               final Consumer<String> consumer,
               final ExecutorService executor,
               final Map<String, String> mdc,
-              final String caller) {
+              final String caller,
+              final MdcScope mdcScope) {
     this.is = IOs.newBufferedReader(is);
     this.consumer = consumer;
     this.executor = executor;
     this.mdc = mdc;
     this.caller = caller;
+    this.containerLogMDC = mdcScope;
   }
 
   @Override
@@ -62,7 +78,9 @@ public void voidCall() {
     try {
       String line;
       while ((line = is.readLine()) != null) {
-        consumer.accept(line);
+        try (containerLogMDC) {
+          consumer.accept(line);
+        }
       }
     } catch (final IOException i) {
       LOGGER.warn("{} gobbler IOException: {}. Typically happens when cancelling a job.", caller, i.getMessage());
diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
index 7194863c714c..190c1e9ebb19 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java
@@ -28,6 +28,8 @@
  */
 public class MdcScope implements AutoCloseable {
 
+  public final static MdcScope DEFAULT = new Builder().build();
+
   private final Map<String, String> originalContextMap;
 
   public MdcScope(final Map<String, String> keyValuesToAdd) {
@@ -38,22 +40,22 @@ public MdcScope(final Map<String, String> keyValuesToAdd) {
   }
 
   @Override
-  public void close() throws Exception {
+  public void close() {
     MDC.setContextMap(originalContextMap);
   }
 
-  public static class MdcScopeBuilder {
+  public static class Builder {
 
     private Optional<String> maybeLogPrefix = Optional.empty();
     private Optional<Color> maybePrefixColor = Optional.empty();
 
-    public MdcScopeBuilder setLogPrefix(final String logPrefix) {
+    public Builder setLogPrefix(final String logPrefix) {
       this.maybeLogPrefix = Optional.ofNullable(logPrefix);
 
       return this;
     }
 
-    public MdcScopeBuilder setPrefixColor(final Color color) {
+    public Builder setPrefixColor(final Color color) {
       this.maybePrefixColor = Optional.ofNullable(color);
 
       return this;
diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
index 4d985d5a6c6a..e52901e65e50 100644
--- a/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
+++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java
@@ -54,17 +54,14 @@ public void testMDCModified() {
 
           });
 
-    } catch (final Exception e) {
-      e.printStackTrace();
     }
   }
 
   @Test
   @DisplayName("The MDC context is properly restored")
   public void testMDCRestore() {
-    try (final MdcScope mdcScope = new MdcScope(modificationInMDC)) {} catch (final Exception e) {
-      e.printStackTrace();
-    }
+    try (final MdcScope mdcScope = new MdcScope(modificationInMDC)) {}
+
     final Map<String, String> mdcState = MDC.getCopyOfContextMap();
 
     Assertions.assertThat(mdcState).containsAllEntriesOf(originalMap);
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index 9bf76989307a..b6c8af3ecf7d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -9,6 +9,9 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper.Color;
+import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.Builder;
 import io.airbyte.config.WorkerDestinationConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -30,6 +33,10 @@
 public class DefaultAirbyteDestination implements AirbyteDestination {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteDestination.class);
+  private static final MdcScope CONTAINER_LOG_MDC = new Builder()
+      .setLogPrefix("container-log")
+      .setPrefixColor(Color.BLUE)
+      .build();
 
   private final IntegrationLauncher integrationLauncher;
   private final AirbyteStreamFactory streamFactory;
@@ -41,7 +48,7 @@ public class DefaultAirbyteDestination implements AirbyteDestination {
   private Iterator<AirbyteMessage> messageIterator = null;
 
   public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher) {
-    this(integrationLauncher, new DefaultAirbyteStreamFactory());
+    this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC));
 
   }
 
@@ -63,7 +70,7 @@ public void start(final WorkerDestinationConfig destinationConfig, final Path jo
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
         Jsons.serialize(destinationConfig.getCatalog()));
     // stdout logs are logged elsewhere since stdout also contains data
-    LineGobbler.gobble(destinationProcess.getErrorStream(), LOGGER::error, "airbyte-destination");
+    LineGobbler.gobble(destinationProcess.getErrorStream(), LOGGER::error, "airbyte-destination", CONTAINER_LOG_MDC);
 
     writer = new BufferedWriter(new OutputStreamWriter(destinationProcess.getOutputStream(), Charsets.UTF_8));
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index 2b17b4050fdc..f04ef16055e8 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -9,6 +9,9 @@
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper.Color;
+import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.Builder;
 import io.airbyte.config.WorkerSourceConfig;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.AirbyteMessage.Type;
@@ -35,6 +38,11 @@ public class DefaultAirbyteSource implements AirbyteSource {
   private static final Duration GRACEFUL_SHUTDOWN_DURATION = Duration.of(10, ChronoUnit.HOURS);
   private static final Duration FORCED_SHUTDOWN_DURATION = Duration.of(1, ChronoUnit.MINUTES);
 
+  private static final MdcScope CONTAINER_LOG_MDC = new Builder()
+      .setLogPrefix("container-log")
+      .setPrefixColor(Color.BLUE)
+      .build();
+
   private final IntegrationLauncher integrationLauncher;
   private final AirbyteStreamFactory streamFactory;
   private final HeartbeatMonitor heartbeatMonitor;
@@ -43,7 +51,7 @@ public class DefaultAirbyteSource implements AirbyteSource {
   private Iterator<AirbyteMessage> messageIterator = null;
 
   public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
-    this(integrationLauncher, new DefaultAirbyteStreamFactory(), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION));
+    this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION));
   }
 
   @VisibleForTesting
@@ -67,7 +75,7 @@ public void start(final WorkerSourceConfig sourceConfig, final Path jobRoot) thr
         sourceConfig.getState() == null ? null : WorkerConstants.INPUT_STATE_JSON_FILENAME,
         sourceConfig.getState() == null ? null : Jsons.serialize(sourceConfig.getState().getState()));
     // stdout logs are logged elsewhere since stdout also contains data
-    LineGobbler.gobble(sourceProcess.getErrorStream(), LOGGER::error, "airbyte-source");
+    LineGobbler.gobble(sourceProcess.getErrorStream(), LOGGER::error, "airbyte-source", CONTAINER_LOG_MDC);
 
     messageIterator = streamFactory.create(IOs.newBufferedReader(sourceProcess.getInputStream()))
         .peek(message -> heartbeatMonitor.beat())
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
index e12beb961a5e..e0c8c89e9da5 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactory.java
@@ -6,9 +6,8 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.logging.LoggingHelper.Color;
 import io.airbyte.commons.logging.MdcScope;
-import io.airbyte.commons.logging.MdcScope.MdcScopeBuilder;
+import io.airbyte.commons.logging.MdcScope.Builder;
 import io.airbyte.protocol.models.AirbyteLogMessage;
 import io.airbyte.protocol.models.AirbyteMessage;
 import java.io.BufferedReader;
@@ -30,21 +29,23 @@
 public class DefaultAirbyteStreamFactory implements AirbyteStreamFactory {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteStreamFactory.class);
-  private static final MdcScope containerLogMDC = new MdcScopeBuilder()
-      .setLogPrefix("container-log")
-      .setPrefixColor(Color.MAGENTA)
-      .build();
 
+  private final MdcScope containerLogMDC;
   private final AirbyteProtocolPredicate protocolValidator;
   private final Logger logger;
 
   public DefaultAirbyteStreamFactory() {
-    this(new AirbyteProtocolPredicate(), LOGGER);
+    this(new Builder().build());
   }
 
-  DefaultAirbyteStreamFactory(final AirbyteProtocolPredicate protocolPredicate, final Logger logger) {
+  public DefaultAirbyteStreamFactory(final MdcScope containerLogMDC) {
+    this(new AirbyteProtocolPredicate(), LOGGER, containerLogMDC);
+  }
+
+  DefaultAirbyteStreamFactory(final AirbyteProtocolPredicate protocolPredicate, final Logger logger, final MdcScope containerLogMDC) {
     protocolValidator = protocolPredicate;
     this.logger = logger;
+    this.containerLogMDC = containerLogMDC;
   }
 
   @Override
@@ -59,7 +60,7 @@ public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
             // want to make sure this info is available in the logs.
             try (containerLogMDC) {
               logger.info(line);
-            } catch (final Exception e) {}
+            }
           }
           return jsonLine.stream();
         })
@@ -84,7 +85,7 @@ public Stream<AirbyteMessage> create(final BufferedReader bufferedReader) {
           if (isLog) {
             try (containerLogMDC) {
               internalLog(airbyteMessage.getLog());
-            } catch (final Exception e) {}
+            }
           }
           return !isLog;
         });
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactoryTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactoryTest.java
index 74077fb11053..596f9e0f11f5 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactoryTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteStreamFactoryTest.java
@@ -14,6 +14,7 @@
 import static org.mockito.Mockito.when;
 
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.MdcScope.Builder;
 import io.airbyte.protocol.models.AirbyteLogMessage;
 import io.airbyte.protocol.models.AirbyteMessage;
 import java.io.BufferedReader;
@@ -120,7 +121,7 @@ public void testMissingNewLineBetweenValidRecords() {
   private Stream<AirbyteMessage> stringToMessageStream(final String inputString) {
     final InputStream inputStream = new ByteArrayInputStream(inputString.getBytes());
     final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
-    return new DefaultAirbyteStreamFactory(protocolPredicate, logger).create(bufferedReader);
+    return new DefaultAirbyteStreamFactory(protocolPredicate, logger, new Builder().build()).create(bufferedReader);
   }
 
 }

From be40626e44534135e29bb3924184e34bd66d4493 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Mon, 25 Oct 2021 17:08:36 -0700
Subject: [PATCH 23/32] Fix the check connection activity

---
 .../temporal/check/connection/CheckConnectionActivity.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivity.java
index 131372124ccc..e5178486610f 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivity.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivity.java
@@ -12,7 +12,7 @@
 import io.temporal.activity.ActivityMethod;
 
 @ActivityInterface
-interface CheckConnectionActivity {
+public interface CheckConnectionActivity {
 
   @ActivityMethod
   StandardCheckConnectionOutput run(JobRunConfig jobRunConfig,

From 62fa4859b4cbbcb735d9cc7f5b05ab2c8359502b Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Tue, 26 Oct 2021 10:04:00 -0700
Subject: [PATCH 24/32] Add tests

---
 .../commons/logging/LoggingHelper.java        |  5 ++-
 .../airbyte/DefaultAirbyteDestination.java    |  2 +-
 .../DefaultAirbyteDestinationTest.java        | 36 +++++++++++++++++++
 .../airbyte/DefaultAirbyteSourceTest.java     | 35 ++++++++++++++++++
 4 files changed, 76 insertions(+), 2 deletions(-)

diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
index 1975461edbe4..bf29cec7e590 100644
--- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
+++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java
@@ -4,6 +4,8 @@
 
 package io.airbyte.commons.logging;
 
+import com.google.common.annotations.VisibleForTesting;
+
 public class LoggingHelper {
 
   public enum Color {
@@ -31,7 +33,8 @@ public String getCode() {
 
   public static final String LOG_SOURCE_MDC_KEY = "log_source";
 
-  private static final String RESET = "\u001B[0m";
+  @VisibleForTesting
+  public static final String RESET = "\u001B[0m";
 
   public static String applyColor(final Color color, final String msg) {
     return color.getCode() + msg + RESET;
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index b6c8af3ecf7d..838318077f66 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -35,7 +35,7 @@ public class DefaultAirbyteDestination implements AirbyteDestination {
   private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteDestination.class);
   private static final MdcScope CONTAINER_LOG_MDC = new Builder()
       .setLogPrefix("container-log")
-      .setPrefixColor(Color.BLUE)
+      .setPrefixColor(Color.MAGENTA)
       .build();
 
   private final IntegrationLauncher integrationLauncher;
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
index bbf38be85d40..ef0fe3bcea28 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.workers.protocols.airbyte;
 
+import static io.airbyte.commons.logging.LoggingHelper.RESET;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
@@ -14,8 +15,11 @@
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.Lists;
+import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper.Color;
 import io.airbyte.config.WorkerDestinationConfig;
+import io.airbyte.config.helpers.LogClientSingleton;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.workers.TestConfigHelpers;
 import io.airbyte.workers.WorkerConstants;
@@ -31,6 +35,7 @@
 import java.nio.file.Path;
 import java.time.Duration;
 import java.util.List;
+import java.util.stream.Stream;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -120,6 +125,37 @@ public void testSuccessfulLifecycle() throws Exception {
     verify(process).exitValue();
   }
 
+  @Test
+  public void testTaggedLogs() throws Exception {
+    final Path jobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+    LogClientSingleton.setJobMdc(jobRoot);
+
+    final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory);
+    destination.start(DESTINATION_CONFIG, jobRoot);
+
+    final AirbyteMessage recordMessage = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "blue");
+    destination.accept(recordMessage);
+
+    final List<AirbyteMessage> messages = Lists.newArrayList();
+
+    messages.add(destination.attemptRead().get());
+    messages.add(destination.attemptRead().get());
+
+    when(process.isAlive()).thenReturn(false);
+
+    destination.notifyEndOfStream();
+
+    destination.close();
+
+    final Path logPath = jobRoot.resolve(LogClientSingleton.LOG_FILENAME);
+    final Stream<String> logs = IOs.readFile(logPath).lines();
+
+    logs.forEach(line -> {
+      org.assertj.core.api.Assertions.assertThat(line)
+          .startsWith(Color.MAGENTA.getCode() + "container-log" + RESET);
+    });
+  }
+
   @Test
   public void testCloseNotifiesLifecycle() throws Exception {
     final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher);
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
index 4f296984a504..b8ebbff455ab 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.workers.protocols.airbyte;
 
+import static io.airbyte.commons.logging.LoggingHelper.RESET;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -16,9 +17,12 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper.Color;
 import io.airbyte.config.State;
 import io.airbyte.config.WorkerSourceConfig;
+import io.airbyte.config.helpers.LogClientSingleton;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.CatalogHelpers;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
@@ -36,6 +40,7 @@
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -127,6 +132,36 @@ public void testSuccessfulLifecycle() throws Exception {
     verify(process).exitValue();
   }
 
+  @Test
+  public void testTaggedLogs() throws Exception {
+
+    final Path jobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+    LogClientSingleton.setJobMdc(jobRoot);
+
+    when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
+
+    final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor);
+    source.start(SOURCE_CONFIG, jobRoot);
+
+    final List<AirbyteMessage> messages = Lists.newArrayList();
+
+    messages.add(source.attemptRead().get());
+    messages.add(source.attemptRead().get());
+
+    when(process.isAlive()).thenReturn(false);
+    verify(heartbeatMonitor, times(2)).beat();
+
+    source.close();
+
+    final Path logPath = jobRoot.resolve(LogClientSingleton.LOG_FILENAME);
+    final Stream<String> logs = IOs.readFile(logPath).lines();
+
+    logs.forEach(line -> {
+      org.assertj.core.api.Assertions.assertThat(line)
+          .startsWith(Color.BLUE.getCode() + "container-log" + RESET);
+    });
+  }
+
   @Test
   public void testNonzeroExitCodeThrows() throws Exception {
     final AirbyteSource tap = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor);

From e1b1fd84aec4e2c736a8051f07328e3d09518008 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Tue, 26 Oct 2021 11:16:28 -0700
Subject: [PATCH 25/32] Test compilation

---
 .../airbyte/DefaultAirbyteSourceTest.java     | 51 +++++++++++--------
 1 file changed, 29 insertions(+), 22 deletions(-)

diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
index b8ebbff455ab..e98107c58596 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
@@ -102,6 +102,9 @@ public void setup() throws IOException, WorkerException {
   @SuppressWarnings({"OptionalGetWithoutIsPresent", "BusyWait"})
   @Test
   public void testSuccessfulLifecycle() throws Exception {
+    final Path jobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+    LogClientSingleton.setJobMdc(jobRoot);
+
     when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
 
     final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor);
@@ -130,28 +133,6 @@ public void testSuccessfulLifecycle() throws Exception {
     });
 
     verify(process).exitValue();
-  }
-
-  @Test
-  public void testTaggedLogs() throws Exception {
-
-    final Path jobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
-    LogClientSingleton.setJobMdc(jobRoot);
-
-    when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
-
-    final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor);
-    source.start(SOURCE_CONFIG, jobRoot);
-
-    final List<AirbyteMessage> messages = Lists.newArrayList();
-
-    messages.add(source.attemptRead().get());
-    messages.add(source.attemptRead().get());
-
-    when(process.isAlive()).thenReturn(false);
-    verify(heartbeatMonitor, times(2)).beat();
-
-    source.close();
 
     final Path logPath = jobRoot.resolve(LogClientSingleton.LOG_FILENAME);
     final Stream<String> logs = IOs.readFile(logPath).lines();
@@ -162,6 +143,32 @@ public void testTaggedLogs() throws Exception {
     });
   }
 
+  /*
+   * @Test public void testTaggedLogs() throws Exception {
+   *
+   * final Path jobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+   * LogClientSingleton.setJobMdc(jobRoot);
+   *
+   * when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
+   *
+   * final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory,
+   * heartbeatMonitor); source.start(SOURCE_CONFIG, jobRoot);
+   *
+   * final List<AirbyteMessage> messages = Lists.newArrayList();
+   *
+   * messages.add(source.attemptRead().get()); messages.add(source.attemptRead().get());
+   *
+   * when(process.isAlive()).thenReturn(false); verify(heartbeatMonitor, times(2)).beat();
+   *
+   * source.close();
+   *
+   * final Path logPath = jobRoot.resolve(LogClientSingleton.LOG_FILENAME); final Stream<String> logs
+   * = IOs.readFile(logPath).lines();
+   *
+   * logs.forEach(line -> { org.assertj.core.api.Assertions.assertThat(line)
+   * .startsWith(Color.BLUE.getCode() + "container-log" + RESET); }); }
+   */
+
   @Test
   public void testNonzeroExitCodeThrows() throws Exception {
     final AirbyteSource tap = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor);

From cfa5192b29965ba0eb840b088dce647524b44f07 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Tue, 26 Oct 2021 13:04:40 -0700
Subject: [PATCH 26/32] Change to logJobRoot name

---
 .../DefaultAirbyteDestinationTest.java        |  6 +-
 .../airbyte/DefaultAirbyteSourceTest.java     | 56 +++++++++----------
 2 files changed, 28 insertions(+), 34 deletions(-)

diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
index ef0fe3bcea28..271f820a54ee 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
@@ -78,7 +78,7 @@ public void setup() throws IOException, WorkerException {
         Jsons.serialize(DESTINATION_CONFIG.getDestinationConnectionConfiguration()),
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
         Jsons.serialize(DESTINATION_CONFIG.getCatalog())))
-            .thenReturn(process);
+        .thenReturn(process);
 
     when(process.isAlive()).thenReturn(true);
     when(process.getInputStream()).thenReturn(inputStream);
@@ -127,7 +127,7 @@ public void testSuccessfulLifecycle() throws Exception {
 
   @Test
   public void testTaggedLogs() throws Exception {
-    final Path jobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+    final Path logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
     LogClientSingleton.setJobMdc(jobRoot);
 
     final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory);
@@ -147,7 +147,7 @@ public void testTaggedLogs() throws Exception {
 
     destination.close();
 
-    final Path logPath = jobRoot.resolve(LogClientSingleton.LOG_FILENAME);
+    final Path logPath = logJobRoot.resolve(LogClientSingleton.LOG_FILENAME);
     final Stream<String> logs = IOs.readFile(logPath).lines();
 
     logs.forEach(line -> {
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
index e98107c58596..052cd2421bd8 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
@@ -102,9 +102,6 @@ public void setup() throws IOException, WorkerException {
   @SuppressWarnings({"OptionalGetWithoutIsPresent", "BusyWait"})
   @Test
   public void testSuccessfulLifecycle() throws Exception {
-    final Path jobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
-    LogClientSingleton.setJobMdc(jobRoot);
-
     when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
 
     final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor);
@@ -133,9 +130,32 @@ public void testSuccessfulLifecycle() throws Exception {
     });
 
     verify(process).exitValue();
+  }
+
+  @Test public void testTaggedLogs() throws Exception {
+
+    final Path logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+    LogClientSingleton.setJobMdc(logJobRoot);
+
+    when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
 
-    final Path logPath = jobRoot.resolve(LogClientSingleton.LOG_FILENAME);
-    final Stream<String> logs = IOs.readFile(logPath).lines();
+    final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory,
+        heartbeatMonitor);
+    source.start(SOURCE_CONFIG, jobRoot);
+
+    final List<AirbyteMessage> messages = Lists.newArrayList();
+
+    messages.add(source.attemptRead().get());
+    messages.add(source.attemptRead().get());
+
+    when(process.isAlive()).thenReturn(false);
+    verify(heartbeatMonitor, times(2)).beat();
+
+    source.close();
+
+    final Path logPath = logJobRoot.resolve(LogClientSingleton.LOG_FILENAME);
+    final Stream<String> logs
+        = IOs.readFile(logPath).lines();
 
     logs.forEach(line -> {
       org.assertj.core.api.Assertions.assertThat(line)
@@ -143,32 +163,6 @@ public void testSuccessfulLifecycle() throws Exception {
     });
   }
 
-  /*
-   * @Test public void testTaggedLogs() throws Exception {
-   *
-   * final Path jobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
-   * LogClientSingleton.setJobMdc(jobRoot);
-   *
-   * when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
-   *
-   * final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory,
-   * heartbeatMonitor); source.start(SOURCE_CONFIG, jobRoot);
-   *
-   * final List<AirbyteMessage> messages = Lists.newArrayList();
-   *
-   * messages.add(source.attemptRead().get()); messages.add(source.attemptRead().get());
-   *
-   * when(process.isAlive()).thenReturn(false); verify(heartbeatMonitor, times(2)).beat();
-   *
-   * source.close();
-   *
-   * final Path logPath = jobRoot.resolve(LogClientSingleton.LOG_FILENAME); final Stream<String> logs
-   * = IOs.readFile(logPath).lines();
-   *
-   * logs.forEach(line -> { org.assertj.core.api.Assertions.assertThat(line)
-   * .startsWith(Color.BLUE.getCode() + "container-log" + RESET); }); }
-   */
-
   @Test
   public void testNonzeroExitCodeThrows() throws Exception {
     final AirbyteSource tap = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor);

From 89e732de3f4cf4e806bba8ba90b8370c7c438d78 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Tue, 26 Oct 2021 17:24:13 -0700
Subject: [PATCH 27/32] Fix test setup

---
 .../DefaultAirbyteDestinationTest.java        | 26 +++++++++++---
 .../airbyte/DefaultAirbyteSourceTest.java     | 35 +++++++++++++++----
 2 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
index 271f820a54ee..1d028f7efc32 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
@@ -36,6 +36,7 @@
 import java.time.Duration;
 import java.util.List;
 import java.util.stream.Stream;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -54,6 +55,16 @@ class DefaultAirbyteDestinationTest {
       AirbyteMessageUtils.createStateMessage("checkpoint", "1"),
       AirbyteMessageUtils.createStateMessage("checkpoint", "2"));
 
+  private static Path logJobRoot;
+
+  static {
+    try {
+      logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+    } catch (final IOException e) {
+      e.printStackTrace();
+    }
+  }
+
   private Path jobRoot;
   private IntegrationLauncher integrationLauncher;
   private Process process;
@@ -78,7 +89,7 @@ public void setup() throws IOException, WorkerException {
         Jsons.serialize(DESTINATION_CONFIG.getDestinationConnectionConfiguration()),
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
         Jsons.serialize(DESTINATION_CONFIG.getCatalog())))
-        .thenReturn(process);
+            .thenReturn(process);
 
     when(process.isAlive()).thenReturn(true);
     when(process.getInputStream()).thenReturn(inputStream);
@@ -86,6 +97,16 @@ public void setup() throws IOException, WorkerException {
     streamFactory = noop -> MESSAGES.stream();
   }
 
+  @AfterEach
+  public void tearDown() throws IOException {
+    // The log file needs to be present and empty
+    final Path logFile = logJobRoot.resolve(LogClientSingleton.LOG_FILENAME);
+    if (Files.exists(logFile)) {
+      Files.delete(logFile);
+    }
+    Files.createFile(logFile);
+  }
+
   @SuppressWarnings("BusyWait")
   @Test
   public void testSuccessfulLifecycle() throws Exception {
@@ -127,9 +148,6 @@ public void testSuccessfulLifecycle() throws Exception {
 
   @Test
   public void testTaggedLogs() throws Exception {
-    final Path logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
-    LogClientSingleton.setJobMdc(jobRoot);
-
     final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory);
     destination.start(DESTINATION_CONFIG, jobRoot);
 
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
index 052cd2421bd8..b7b289e33b31 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
@@ -41,6 +41,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -70,6 +71,16 @@ class DefaultAirbyteSourceTest {
       AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "blue"),
       AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "yellow"));
 
+  private static Path logJobRoot;
+
+  static {
+    try {
+      logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+    } catch (final IOException e) {
+      e.printStackTrace();
+    }
+  }
+
   private Path jobRoot;
   private IntegrationLauncher integrationLauncher;
   private Process process;
@@ -97,11 +108,25 @@ public void setup() throws IOException, WorkerException {
     when(process.getErrorStream()).thenReturn(new ByteArrayInputStream("qwer".getBytes(StandardCharsets.UTF_8)));
 
     streamFactory = noop -> MESSAGES.stream();
+
+    LogClientSingleton.setJobMdc(logJobRoot);
+  }
+
+  @AfterEach
+  public void tearDown() throws IOException {
+    // The log file needs to be present and empty
+    final Path logFile = logJobRoot.resolve(LogClientSingleton.LOG_FILENAME);
+    if (Files.exists(logFile)) {
+      Files.delete(logFile);
+    }
+    Files.createFile(logFile);
   }
 
   @SuppressWarnings({"OptionalGetWithoutIsPresent", "BusyWait"})
   @Test
   public void testSuccessfulLifecycle() throws Exception {
+    when(process.getErrorStream()).thenReturn(new ByteArrayInputStream("qwer".getBytes(StandardCharsets.UTF_8)));
+
     when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
 
     final AirbyteSource source = new DefaultAirbyteSource(integrationLauncher, streamFactory, heartbeatMonitor);
@@ -132,10 +157,10 @@ public void testSuccessfulLifecycle() throws Exception {
     verify(process).exitValue();
   }
 
-  @Test public void testTaggedLogs() throws Exception {
+  @Test
+  public void testTaggedLogs() throws Exception {
 
-    final Path logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
-    LogClientSingleton.setJobMdc(logJobRoot);
+    when(process.getErrorStream()).thenReturn(new ByteArrayInputStream(("rewq").getBytes(StandardCharsets.UTF_8)));
 
     when(heartbeatMonitor.isBeating()).thenReturn(true).thenReturn(false);
 
@@ -149,13 +174,11 @@ public void testSuccessfulLifecycle() throws Exception {
     messages.add(source.attemptRead().get());
 
     when(process.isAlive()).thenReturn(false);
-    verify(heartbeatMonitor, times(2)).beat();
 
     source.close();
 
     final Path logPath = logJobRoot.resolve(LogClientSingleton.LOG_FILENAME);
-    final Stream<String> logs
-        = IOs.readFile(logPath).lines();
+    final Stream<String> logs = IOs.readFile(logPath).lines();
 
     logs.forEach(line -> {
       org.assertj.core.api.Assertions.assertThat(line)

From 430f3dd4bdf1e5ba1adef52773e13483c1069048 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Wed, 27 Oct 2021 13:46:57 -0700
Subject: [PATCH 28/32] Add tag to normalization and dbt

---
 .../workers/DbtTransformationRunner.java      | 37 ++++++---
 .../DefaultNormalizationRunner.java           | 81 +++++++++++--------
 .../airbyte/DefaultAirbyteDestination.java    |  2 +-
 .../airbyte/DefaultAirbyteSource.java         |  2 +-
 4 files changed, 73 insertions(+), 49 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index 6e09ef9bceb4..a1a32e745afa 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -8,6 +8,9 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.io.LineGobbler;
+import io.airbyte.commons.logging.LoggingHelper.Color;
+import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.Builder;
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.config.OperatorDbt;
 import io.airbyte.config.ResourceRequirements;
@@ -28,6 +31,10 @@ public class DbtTransformationRunner implements AutoCloseable {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DbtTransformationRunner.class);
   private static final String DBT_ENTRYPOINT_SH = "entrypoint.sh";
+  private static final MdcScope CONTAINER_LOG_MDC = new Builder()
+      .setLogPrefix("dbt-container-log")
+      .setPrefixColor(Color.CYAN)
+      .build();
 
   private final ProcessFactory processFactory;
   private final NormalizationRunner normalizationRunner;
@@ -48,7 +55,7 @@ public void start() throws Exception {
    * transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend
    * on the NormalizationRunner to configure the dbt project with the appropriate destination settings
    * and pull the custom git repository into the workspace.
-   *
+   * <p>
    * Once the workspace folder/files is setup to run, we invoke the custom transformation command as
    * provided by the user to execute whatever extra transformation has been implemented.
    */
@@ -59,10 +66,12 @@ public boolean run(final String jobId,
                      final ResourceRequirements resourceRequirements,
                      final OperatorDbt dbtConfig)
       throws Exception {
-    if (!normalizationRunner.configureDbt(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig)) {
-      return false;
+    try (CONTAINER_LOG_MDC) {
+      if (!normalizationRunner.configureDbt(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig)) {
+        return false;
+      }
+      return transform(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig);
     }
-    return transform(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig);
   }
 
   public boolean transform(final String jobId,
@@ -72,7 +81,7 @@ public boolean transform(final String jobId,
                            final ResourceRequirements resourceRequirements,
                            final OperatorDbt dbtConfig)
       throws Exception {
-    try {
+    try (CONTAINER_LOG_MDC) {
       final Map<String, String> files = ImmutableMap.of(
           DBT_ENTRYPOINT_SH, MoreResources.readResource("dbt_transformation_entrypoint.sh"),
           "sshtunneling.sh", MoreResources.readResource("sshtunneling.sh"));
@@ -104,16 +113,18 @@ public boolean transform(final String jobId,
 
   @Override
   public void close() throws Exception {
-    normalizationRunner.close();
+    try (CONTAINER_LOG_MDC) {
+      normalizationRunner.close();
 
-    if (process == null) {
-      return;
-    }
+      if (process == null) {
+        return;
+      }
 
-    LOGGER.debug("Closing dbt transformation process");
-    WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
-    if (process.isAlive() || process.exitValue() != 0) {
-      throw new WorkerException("Dbt transformation process wasn't successful");
+      LOGGER.debug("Closing dbt transformation process");
+      WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
+      if (process.isAlive() || process.exitValue() != 0) {
+        throw new WorkerException("Dbt transformation process wasn't successful");
+      }
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index 63f12fe2802d..571b1431a3a6 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -10,6 +10,9 @@
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.io.LineGobbler;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper.Color;
+import io.airbyte.commons.logging.MdcScope;
+import io.airbyte.commons.logging.MdcScope.Builder;
 import io.airbyte.config.OperatorDbt;
 import io.airbyte.config.ResourceRequirements;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
@@ -27,6 +30,10 @@
 public class DefaultNormalizationRunner implements NormalizationRunner {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DefaultNormalizationRunner.class);
+  private static final MdcScope CONTAINER_LOG_MDC = new Builder()
+      .setLogPrefix("normalization-container-log")
+      .setPrefixColor(Color.GREEN)
+      .build();
 
   private final DestinationType destinationType;
   private final ProcessFactory processFactory;
@@ -58,24 +65,26 @@ public boolean configureDbt(final String jobId,
                               final ResourceRequirements resourceRequirements,
                               final OperatorDbt dbtConfig)
       throws Exception {
-    final Map<String, String> files = ImmutableMap.of(
-        WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, Jsons.serialize(config));
-    final String gitRepoUrl = dbtConfig.getGitRepoUrl();
-    if (Strings.isNullOrEmpty(gitRepoUrl)) {
-      throw new WorkerException("Git Repo Url is required");
-    }
-    final String gitRepoBranch = dbtConfig.getGitRepoBranch();
-    if (Strings.isNullOrEmpty(gitRepoBranch)) {
-      return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
-          "--integration-type", destinationType.toString().toLowerCase(),
-          "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
-          "--git-repo", gitRepoUrl);
-    } else {
-      return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
-          "--integration-type", destinationType.toString().toLowerCase(),
-          "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
-          "--git-repo", gitRepoUrl,
-          "--git-branch", gitRepoBranch);
+    try (CONTAINER_LOG_MDC) {
+      final Map<String, String> files = ImmutableMap.of(
+          WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, Jsons.serialize(config));
+      final String gitRepoUrl = dbtConfig.getGitRepoUrl();
+      if (Strings.isNullOrEmpty(gitRepoUrl)) {
+        throw new WorkerException("Git Repo Url is required");
+      }
+      final String gitRepoBranch = dbtConfig.getGitRepoBranch();
+      if (Strings.isNullOrEmpty(gitRepoBranch)) {
+        return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
+            "--integration-type", destinationType.toString().toLowerCase(),
+            "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
+            "--git-repo", gitRepoUrl);
+      } else {
+        return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
+            "--integration-type", destinationType.toString().toLowerCase(),
+            "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
+            "--git-repo", gitRepoUrl,
+            "--git-branch", gitRepoBranch);
+      }
     }
   }
 
@@ -87,14 +96,16 @@ public boolean normalize(final String jobId,
                            final ConfiguredAirbyteCatalog catalog,
                            final ResourceRequirements resourceRequirements)
       throws Exception {
-    final Map<String, String> files = ImmutableMap.of(
-        WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, Jsons.serialize(config),
-        WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog));
-
-    return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "run",
-        "--integration-type", destinationType.toString().toLowerCase(),
-        "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
-        "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME);
+    try (CONTAINER_LOG_MDC) {
+      final Map<String, String> files = ImmutableMap.of(
+          WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, Jsons.serialize(config),
+          WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog));
+
+      return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "run",
+          "--integration-type", destinationType.toString().toLowerCase(),
+          "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
+          "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME);
+    }
   }
 
   private boolean runProcess(final String jobId,
@@ -104,7 +115,7 @@ private boolean runProcess(final String jobId,
                              final ResourceRequirements resourceRequirements,
                              final String... args)
       throws Exception {
-    try {
+    try (CONTAINER_LOG_MDC) {
       LOGGER.info("Running with normalization version: {}", normalizationImageName);
       process = processFactory.create(jobId, attempt, jobRoot, normalizationImageName, false, files, null, resourceRequirements,
           Map.of(KubeProcessFactory.JOB_TYPE, KubeProcessFactory.SYNC_JOB, KubeProcessFactory.SYNC_STEP, KubeProcessFactory.NORMALISE_STEP), args);
@@ -126,14 +137,16 @@ private boolean runProcess(final String jobId,
 
   @Override
   public void close() throws Exception {
-    if (process == null) {
-      return;
-    }
+    try (CONTAINER_LOG_MDC) {
+      if (process == null) {
+        return;
+      }
 
-    LOGGER.debug("Closing normalization process");
-    WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
-    if (process.isAlive() || process.exitValue() != 0) {
-      throw new WorkerException("Normalization process wasn't successful");
+      LOGGER.debug("Closing normalization process");
+      WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
+      if (process.isAlive() || process.exitValue() != 0) {
+        throw new WorkerException("Normalization process wasn't successful");
+      }
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index 838318077f66..a33466ee8efe 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -34,7 +34,7 @@ public class DefaultAirbyteDestination implements AirbyteDestination {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteDestination.class);
   private static final MdcScope CONTAINER_LOG_MDC = new Builder()
-      .setLogPrefix("container-log")
+      .setLogPrefix("destination-container-log")
       .setPrefixColor(Color.MAGENTA)
       .build();
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index f04ef16055e8..a9104256c8cd 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -39,7 +39,7 @@ public class DefaultAirbyteSource implements AirbyteSource {
   private static final Duration FORCED_SHUTDOWN_DURATION = Duration.of(1, ChronoUnit.MINUTES);
 
   private static final MdcScope CONTAINER_LOG_MDC = new Builder()
-      .setLogPrefix("container-log")
+      .setLogPrefix("source-container-log")
       .setPrefixColor(Color.BLUE)
       .build();
 

From d5abecde77cad0082bc6fd11e86754d77627c7ed Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Wed, 27 Oct 2021 14:22:25 -0700
Subject: [PATCH 29/32] Update test

---
 .../DefaultNormalizationRunnerTest.java       | 51 ++++++++++++++++++-
 .../DefaultAirbyteDestinationTest.java        |  4 +-
 .../airbyte/DefaultAirbyteSourceTest.java     | 11 ++--
 3 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
index 3c70af24a5fa..7ad05163f813 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
@@ -4,7 +4,9 @@
 
 package io.airbyte.workers.normalization;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static io.airbyte.commons.logging.LoggingHelper.RESET;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -12,7 +14,10 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
+import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.logging.LoggingHelper.Color;
+import io.airbyte.config.helpers.LogClientSingleton;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
 import io.airbyte.workers.WorkerConstants;
 import io.airbyte.workers.WorkerException;
@@ -25,6 +30,8 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -33,6 +40,17 @@ class DefaultNormalizationRunnerTest {
   private static final String JOB_ID = "0";
   private static final int JOB_ATTEMPT = 0;
 
+  private static Path logJobRoot;
+
+  static {
+    try {
+      logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+      LogClientSingleton.setJobMdc(logJobRoot);
+    } catch (final IOException e) {
+      e.printStackTrace();
+    }
+  }
+
   private Path jobRoot;
   private ProcessFactory processFactory;
   private Process process;
@@ -64,6 +82,16 @@ void setup() throws IOException, WorkerException {
     when(process.getErrorStream()).thenReturn(new ByteArrayInputStream("hello".getBytes()));
   }
 
+  @AfterEach
+  public void tearDown() throws IOException {
+    // The log file needs to be present and empty
+    final Path logFile = logJobRoot.resolve(LogClientSingleton.LOG_FILENAME);
+    if (Files.exists(logFile)) {
+      Files.delete(logFile);
+    }
+    Files.createFile(logFile);
+  }
+
   @Test
   void test() throws Exception {
     final NormalizationRunner runner =
@@ -74,6 +102,27 @@ void test() throws Exception {
     assertTrue(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, WorkerUtils.DEFAULT_RESOURCE_REQUIREMENTS));
   }
 
+  @Test
+  void testLog() throws Exception {
+
+    final NormalizationRunner runner =
+        new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME);
+
+    when(process.exitValue()).thenReturn(0);
+
+    assertTrue(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, WorkerUtils.DEFAULT_RESOURCE_REQUIREMENTS));
+
+    final Path logPath = logJobRoot.resolve(LogClientSingleton.LOG_FILENAME);
+    final Stream<String> logs = IOs.readFile(logPath).lines();
+
+    logs
+        .filter(line -> !line.contains("EnvConfigs(getEnvOrDefault)"))
+        .forEach(line -> {
+          org.assertj.core.api.Assertions.assertThat(line)
+              .startsWith(Color.BLUE.getCode() + "normalization-container-log" + RESET);
+        });
+  }
+
   @Test
   public void testClose() throws Exception {
     when(process.isAlive()).thenReturn(true).thenReturn(false);
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
index 1d028f7efc32..29df96a1f3f7 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
@@ -60,6 +60,7 @@ class DefaultAirbyteDestinationTest {
   static {
     try {
       logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+      LogClientSingleton.setJobMdc(logJobRoot);
     } catch (final IOException e) {
       e.printStackTrace();
     }
@@ -148,6 +149,7 @@ public void testSuccessfulLifecycle() throws Exception {
 
   @Test
   public void testTaggedLogs() throws Exception {
+
     final AirbyteDestination destination = new DefaultAirbyteDestination(integrationLauncher, streamFactory);
     destination.start(DESTINATION_CONFIG, jobRoot);
 
@@ -170,7 +172,7 @@ public void testTaggedLogs() throws Exception {
 
     logs.forEach(line -> {
       org.assertj.core.api.Assertions.assertThat(line)
-          .startsWith(Color.MAGENTA.getCode() + "container-log" + RESET);
+          .startsWith(Color.MAGENTA.getCode() + "destination-container-log" + RESET);
     });
   }
 
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
index b7b289e33b31..24c215a154bd 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
@@ -76,6 +76,7 @@ class DefaultAirbyteSourceTest {
   static {
     try {
       logJobRoot = Files.createTempDirectory(Path.of("/tmp"), "mdc_test");
+      LogClientSingleton.setJobMdc(logJobRoot);
     } catch (final IOException e) {
       e.printStackTrace();
     }
@@ -180,10 +181,12 @@ public void testTaggedLogs() throws Exception {
     final Path logPath = logJobRoot.resolve(LogClientSingleton.LOG_FILENAME);
     final Stream<String> logs = IOs.readFile(logPath).lines();
 
-    logs.forEach(line -> {
-      org.assertj.core.api.Assertions.assertThat(line)
-          .startsWith(Color.BLUE.getCode() + "container-log" + RESET);
-    });
+    logs
+        .filter(line -> !line.contains("EnvConfigs(getEnvOrDefault)"))
+        .forEach(line -> {
+          org.assertj.core.api.Assertions.assertThat(line)
+              .startsWith(Color.BLUE.getCode() + "source-container-log" + RESET);
+        });
   }
 
   @Test

From af40532633a2bacb5a3c7e817537d766e6dc1066 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Wed, 27 Oct 2021 14:53:06 -0700
Subject: [PATCH 30/32] PR comments

---
 .../workers/DbtTransformationRunner.java      | 47 +++++------
 .../DefaultNormalizationRunner.java           | 80 +++++++++----------
 .../airbyte/DefaultAirbyteDestination.java    |  2 +-
 .../airbyte/DefaultAirbyteSource.java         |  9 +--
 4 files changed, 63 insertions(+), 75 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index a1a32e745afa..70708f0c8964 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -32,7 +32,7 @@ public class DbtTransformationRunner implements AutoCloseable {
   private static final Logger LOGGER = LoggerFactory.getLogger(DbtTransformationRunner.class);
   private static final String DBT_ENTRYPOINT_SH = "entrypoint.sh";
   private static final MdcScope CONTAINER_LOG_MDC = new Builder()
-      .setLogPrefix("dbt-container-log")
+      .setLogPrefix("dbt")
       .setPrefixColor(Color.CYAN)
       .build();
 
@@ -50,14 +50,13 @@ public void start() throws Exception {
   }
 
   /**
-   * The docker image used by the DbtTransformationRunner is provided by the User, so we can't ensure
-   * to have the right python, dbt, dependencies etc software installed to successfully run our
-   * transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend
-   * on the NormalizationRunner to configure the dbt project with the appropriate destination settings
-   * and pull the custom git repository into the workspace.
+   * The docker image used by the DbtTransformationRunner is provided by the User, so we can't ensure to have the right python, dbt, dependencies etc
+   * software installed to successfully run our transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend on
+   * the NormalizationRunner to configure the dbt project with the appropriate destination settings and pull the custom git repository into the
+   * workspace.
    * <p>
-   * Once the workspace folder/files is setup to run, we invoke the custom transformation command as
-   * provided by the user to execute whatever extra transformation has been implemented.
+   * Once the workspace folder/files is setup to run, we invoke the custom transformation command as provided by the user to execute whatever extra
+   * transformation has been implemented.
    */
   public boolean run(final String jobId,
                      final int attempt,
@@ -66,12 +65,10 @@ public boolean run(final String jobId,
                      final ResourceRequirements resourceRequirements,
                      final OperatorDbt dbtConfig)
       throws Exception {
-    try (CONTAINER_LOG_MDC) {
-      if (!normalizationRunner.configureDbt(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig)) {
-        return false;
-      }
-      return transform(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig);
+    if (!normalizationRunner.configureDbt(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig)) {
+      return false;
     }
+    return transform(jobId, attempt, jobRoot, config, resourceRequirements, dbtConfig);
   }
 
   public boolean transform(final String jobId,
@@ -81,7 +78,7 @@ public boolean transform(final String jobId,
                            final ResourceRequirements resourceRequirements,
                            final OperatorDbt dbtConfig)
       throws Exception {
-    try (CONTAINER_LOG_MDC) {
+    try {
       final Map<String, String> files = ImmutableMap.of(
           DBT_ENTRYPOINT_SH, MoreResources.readResource("dbt_transformation_entrypoint.sh"),
           "sshtunneling.sh", MoreResources.readResource("sshtunneling.sh"));
@@ -96,8 +93,8 @@ public boolean transform(final String jobId,
               Map.of(KubeProcessFactory.JOB_TYPE, KubeProcessFactory.SYNC_JOB, KubeProcessFactory.SYNC_STEP, KubeProcessFactory.CUSTOM_STEP),
               dbtArguments);
 
-      LineGobbler.gobble(process.getInputStream(), LOGGER::info);
-      LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
+      LineGobbler.gobble(process.getInputStream(), LOGGER::info, CONTAINER_LOG_MDC);
+      LineGobbler.gobble(process.getErrorStream(), LOGGER::error, CONTAINER_LOG_MDC);
 
       WorkerUtils.wait(process);
 
@@ -113,18 +110,16 @@ public boolean transform(final String jobId,
 
   @Override
   public void close() throws Exception {
-    try (CONTAINER_LOG_MDC) {
-      normalizationRunner.close();
+    normalizationRunner.close();
 
-      if (process == null) {
-        return;
-      }
+    if (process == null) {
+      return;
+    }
 
-      LOGGER.debug("Closing dbt transformation process");
-      WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
-      if (process.isAlive() || process.exitValue() != 0) {
-        throw new WorkerException("Dbt transformation process wasn't successful");
-      }
+    LOGGER.debug("Closing dbt transformation process");
+    WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
+    if (process.isAlive() || process.exitValue() != 0) {
+      throw new WorkerException("Dbt transformation process wasn't successful");
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index 571b1431a3a6..39cbc52562dc 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -31,7 +31,7 @@ public class DefaultNormalizationRunner implements NormalizationRunner {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DefaultNormalizationRunner.class);
   private static final MdcScope CONTAINER_LOG_MDC = new Builder()
-      .setLogPrefix("normalization-container-log")
+      .setLogPrefix("normalization")
       .setPrefixColor(Color.GREEN)
       .build();
 
@@ -65,26 +65,24 @@ public boolean configureDbt(final String jobId,
                               final ResourceRequirements resourceRequirements,
                               final OperatorDbt dbtConfig)
       throws Exception {
-    try (CONTAINER_LOG_MDC) {
-      final Map<String, String> files = ImmutableMap.of(
-          WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, Jsons.serialize(config));
-      final String gitRepoUrl = dbtConfig.getGitRepoUrl();
-      if (Strings.isNullOrEmpty(gitRepoUrl)) {
-        throw new WorkerException("Git Repo Url is required");
-      }
-      final String gitRepoBranch = dbtConfig.getGitRepoBranch();
-      if (Strings.isNullOrEmpty(gitRepoBranch)) {
-        return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
-            "--integration-type", destinationType.toString().toLowerCase(),
-            "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
-            "--git-repo", gitRepoUrl);
-      } else {
-        return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
-            "--integration-type", destinationType.toString().toLowerCase(),
-            "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
-            "--git-repo", gitRepoUrl,
-            "--git-branch", gitRepoBranch);
-      }
+    final Map<String, String> files = ImmutableMap.of(
+        WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, Jsons.serialize(config));
+    final String gitRepoUrl = dbtConfig.getGitRepoUrl();
+    if (Strings.isNullOrEmpty(gitRepoUrl)) {
+      throw new WorkerException("Git Repo Url is required");
+    }
+    final String gitRepoBranch = dbtConfig.getGitRepoBranch();
+    if (Strings.isNullOrEmpty(gitRepoBranch)) {
+      return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
+          "--integration-type", destinationType.toString().toLowerCase(),
+          "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
+          "--git-repo", gitRepoUrl);
+    } else {
+      return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
+          "--integration-type", destinationType.toString().toLowerCase(),
+          "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
+          "--git-repo", gitRepoUrl,
+          "--git-branch", gitRepoBranch);
     }
   }
 
@@ -96,16 +94,14 @@ public boolean normalize(final String jobId,
                            final ConfiguredAirbyteCatalog catalog,
                            final ResourceRequirements resourceRequirements)
       throws Exception {
-    try (CONTAINER_LOG_MDC) {
-      final Map<String, String> files = ImmutableMap.of(
-          WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, Jsons.serialize(config),
-          WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog));
-
-      return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "run",
-          "--integration-type", destinationType.toString().toLowerCase(),
-          "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
-          "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME);
-    }
+    final Map<String, String> files = ImmutableMap.of(
+        WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, Jsons.serialize(config),
+        WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog));
+
+    return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "run",
+        "--integration-type", destinationType.toString().toLowerCase(),
+        "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
+        "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME);
   }
 
   private boolean runProcess(final String jobId,
@@ -115,13 +111,13 @@ private boolean runProcess(final String jobId,
                              final ResourceRequirements resourceRequirements,
                              final String... args)
       throws Exception {
-    try (CONTAINER_LOG_MDC) {
+    try {
       LOGGER.info("Running with normalization version: {}", normalizationImageName);
       process = processFactory.create(jobId, attempt, jobRoot, normalizationImageName, false, files, null, resourceRequirements,
           Map.of(KubeProcessFactory.JOB_TYPE, KubeProcessFactory.SYNC_JOB, KubeProcessFactory.SYNC_STEP, KubeProcessFactory.NORMALISE_STEP), args);
 
-      LineGobbler.gobble(process.getInputStream(), LOGGER::info);
-      LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
+      LineGobbler.gobble(process.getInputStream(), LOGGER::info, CONTAINER_LOG_MDC);
+      LineGobbler.gobble(process.getErrorStream(), LOGGER::error, CONTAINER_LOG_MDC);
 
       WorkerUtils.wait(process);
 
@@ -137,16 +133,14 @@ private boolean runProcess(final String jobId,
 
   @Override
   public void close() throws Exception {
-    try (CONTAINER_LOG_MDC) {
-      if (process == null) {
-        return;
-      }
+    if (process == null) {
+      return;
+    }
 
-      LOGGER.debug("Closing normalization process");
-      WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
-      if (process.isAlive() || process.exitValue() != 0) {
-        throw new WorkerException("Normalization process wasn't successful");
-      }
+    LOGGER.debug("Closing normalization process");
+    WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES);
+    if (process.isAlive() || process.exitValue() != 0) {
+      throw new WorkerException("Normalization process wasn't successful");
     }
   }
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
index a33466ee8efe..de4bacfcdc9b 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java
@@ -34,7 +34,7 @@ public class DefaultAirbyteDestination implements AirbyteDestination {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteDestination.class);
   private static final MdcScope CONTAINER_LOG_MDC = new Builder()
-      .setLogPrefix("destination-container-log")
+      .setLogPrefix("destination")
       .setPrefixColor(Color.MAGENTA)
       .build();
 
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index a9104256c8cd..06b53d701e2d 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -39,7 +39,7 @@ public class DefaultAirbyteSource implements AirbyteSource {
   private static final Duration FORCED_SHUTDOWN_DURATION = Duration.of(1, ChronoUnit.MINUTES);
 
   private static final MdcScope CONTAINER_LOG_MDC = new Builder()
-      .setLogPrefix("source-container-log")
+      .setLogPrefix("source")
       .setPrefixColor(Color.BLUE)
       .build();
 
@@ -54,10 +54,9 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
     this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION));
   }
 
-  @VisibleForTesting
-  DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
-                       final AirbyteStreamFactory streamFactory,
-                       final HeartbeatMonitor heartbeatMonitor) {
+  @VisibleForTesting DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
+                                          final AirbyteStreamFactory streamFactory,
+                                          final HeartbeatMonitor heartbeatMonitor) {
     this.integrationLauncher = integrationLauncher;
     this.streamFactory = streamFactory;
     this.heartbeatMonitor = heartbeatMonitor;

From 1f6f2c4b45d825c3eb091fc2ab722a0ef22de396 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Wed, 27 Oct 2021 14:57:44 -0700
Subject: [PATCH 31/32] Update test

---
 .../workers/normalization/DefaultNormalizationRunnerTest.java | 4 ++--
 .../protocols/airbyte/DefaultAirbyteDestinationTest.java      | 4 ++--
 .../workers/protocols/airbyte/DefaultAirbyteSourceTest.java   | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
index 7ad05163f813..a5c576738fdc 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
@@ -77,7 +77,7 @@ void setup() throws IOException, WorkerException {
         "--integration-type", "bigquery",
         "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
         "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME))
-            .thenReturn(process);
+        .thenReturn(process);
     when(process.getInputStream()).thenReturn(new ByteArrayInputStream("hello".getBytes()));
     when(process.getErrorStream()).thenReturn(new ByteArrayInputStream("hello".getBytes()));
   }
@@ -119,7 +119,7 @@ void testLog() throws Exception {
         .filter(line -> !line.contains("EnvConfigs(getEnvOrDefault)"))
         .forEach(line -> {
           org.assertj.core.api.Assertions.assertThat(line)
-              .startsWith(Color.BLUE.getCode() + "normalization-container-log" + RESET);
+              .startsWith(Color.GREEN.getCode() + "normalization" + RESET);
         });
   }
 
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
index 29df96a1f3f7..9f2be5a54cb1 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
@@ -90,7 +90,7 @@ public void setup() throws IOException, WorkerException {
         Jsons.serialize(DESTINATION_CONFIG.getDestinationConnectionConfiguration()),
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
         Jsons.serialize(DESTINATION_CONFIG.getCatalog())))
-            .thenReturn(process);
+        .thenReturn(process);
 
     when(process.isAlive()).thenReturn(true);
     when(process.getInputStream()).thenReturn(inputStream);
@@ -172,7 +172,7 @@ public void testTaggedLogs() throws Exception {
 
     logs.forEach(line -> {
       org.assertj.core.api.Assertions.assertThat(line)
-          .startsWith(Color.MAGENTA.getCode() + "destination-container-log" + RESET);
+          .startsWith(Color.MAGENTA.getCode() + "destination" + RESET);
     });
   }
 
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
index 24c215a154bd..412f4ec3c396 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java
@@ -185,7 +185,7 @@ public void testTaggedLogs() throws Exception {
         .filter(line -> !line.contains("EnvConfigs(getEnvOrDefault)"))
         .forEach(line -> {
           org.assertj.core.api.Assertions.assertThat(line)
-              .startsWith(Color.BLUE.getCode() + "source-container-log" + RESET);
+              .startsWith(Color.BLUE.getCode() + "source" + RESET);
         });
   }
 

From cc019c914c2c160093fa9ed117ce2bbc0a4f8513 Mon Sep 17 00:00:00 2001
From: Benoit Moriceau <benoit@airbyte.io>
Date: Wed, 27 Oct 2021 15:04:50 -0700
Subject: [PATCH 32/32] Format

---
 .../io/airbyte/workers/DbtTransformationRunner.java | 13 +++++++------
 .../protocols/airbyte/DefaultAirbyteSource.java     |  7 ++++---
 .../DefaultNormalizationRunnerTest.java             |  2 +-
 .../airbyte/DefaultAirbyteDestinationTest.java      |  2 +-
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
index 70708f0c8964..249d5eae4a8c 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/DbtTransformationRunner.java
@@ -50,13 +50,14 @@ public void start() throws Exception {
   }
 
   /**
-   * The docker image used by the DbtTransformationRunner is provided by the User, so we can't ensure to have the right python, dbt, dependencies etc
-   * software installed to successfully run our transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend on
-   * the NormalizationRunner to configure the dbt project with the appropriate destination settings and pull the custom git repository into the
-   * workspace.
+   * The docker image used by the DbtTransformationRunner is provided by the User, so we can't ensure
+   * to have the right python, dbt, dependencies etc software installed to successfully run our
+   * transform-config scripts (to translate Airbyte Catalogs into Dbt profiles file). Thus, we depend
+   * on the NormalizationRunner to configure the dbt project with the appropriate destination settings
+   * and pull the custom git repository into the workspace.
    * <p>
-   * Once the workspace folder/files is setup to run, we invoke the custom transformation command as provided by the user to execute whatever extra
-   * transformation has been implemented.
+   * Once the workspace folder/files is setup to run, we invoke the custom transformation command as
+   * provided by the user to execute whatever extra transformation has been implemented.
    */
   public boolean run(final String jobId,
                      final int attempt,
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
index 06b53d701e2d..fadf68e44ef0 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java
@@ -54,9 +54,10 @@ public DefaultAirbyteSource(final IntegrationLauncher integrationLauncher) {
     this(integrationLauncher, new DefaultAirbyteStreamFactory(CONTAINER_LOG_MDC), new HeartbeatMonitor(HEARTBEAT_FRESH_DURATION));
   }
 
-  @VisibleForTesting DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
-                                          final AirbyteStreamFactory streamFactory,
-                                          final HeartbeatMonitor heartbeatMonitor) {
+  @VisibleForTesting
+  DefaultAirbyteSource(final IntegrationLauncher integrationLauncher,
+                       final AirbyteStreamFactory streamFactory,
+                       final HeartbeatMonitor heartbeatMonitor) {
     this.integrationLauncher = integrationLauncher;
     this.streamFactory = streamFactory;
     this.heartbeatMonitor = heartbeatMonitor;
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
index a5c576738fdc..554f91ad90a8 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
@@ -77,7 +77,7 @@ void setup() throws IOException, WorkerException {
         "--integration-type", "bigquery",
         "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
         "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME))
-        .thenReturn(process);
+            .thenReturn(process);
     when(process.getInputStream()).thenReturn(new ByteArrayInputStream("hello".getBytes()));
     when(process.getErrorStream()).thenReturn(new ByteArrayInputStream("hello".getBytes()));
   }
diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
index 9f2be5a54cb1..52d0b958fa7e 100644
--- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
+++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestinationTest.java
@@ -90,7 +90,7 @@ public void setup() throws IOException, WorkerException {
         Jsons.serialize(DESTINATION_CONFIG.getDestinationConnectionConfiguration()),
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME,
         Jsons.serialize(DESTINATION_CONFIG.getCatalog())))
-        .thenReturn(process);
+            .thenReturn(process);
 
     when(process.isAlive()).thenReturn(true);
     when(process.getInputStream()).thenReturn(inputStream);