From f986687e89c5bb12cd2cac3c86168887cd1a9e3b Mon Sep 17 00:00:00 2001 From: Michel Tricot Date: Mon, 21 Sep 2020 22:21:41 -0700 Subject: [PATCH 1/2] remove invalid dir being created --- airbyte-commons/src/main/resources/log4j2.xml | 8 ++- .../commons/logging/Log4j2ConfigTest.java | 55 +++++++++++++------ .../io/airbyte/scheduler/JobSubmitter.java | 1 - .../airbyte/scheduler/JobSubmitterTest.java | 1 - 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-commons/src/main/resources/log4j2.xml index 9b924e7f575d..4e3178fb81e4 100644 --- a/airbyte-commons/src/main/resources/log4j2.xml +++ b/airbyte-commons/src/main/resources/log4j2.xml @@ -10,9 +10,13 @@ - + + + + + - + diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java index 34c1d05ad20a..cbdf442906e5 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java @@ -24,6 +24,7 @@ package io.airbyte.commons.logging; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import io.airbyte.commons.io.IOs; @@ -42,8 +43,6 @@ public class Log4j2ConfigTest { - private static final Logger LOGGER = LoggerFactory.getLogger(Log4j2ConfigTest.class); - private Path root; @BeforeEach @@ -53,51 +52,73 @@ void setUp() throws IOException { } @Test - void testWorkerDispatch() { + void testWorkerDispatch() throws InterruptedException { + final Logger logger = LoggerFactory.getLogger("testWorkerDispatch"); + final String filename = "logs.log"; - MDC.put("context", "worker"); - MDC.put("job_root", root.toString()); - MDC.put("job_log_filename", filename); - MDC.put("job_id", "1"); + ExecutorService executor = Executors.newFixedThreadPool(1); + executor.submit(() -> { + MDC.put("context", "worker"); + MDC.put("job_root", root.toString()); + MDC.put("job_log_filename", filename); + MDC.put("job_id", "1"); + logger.error("random message testWorkerDispatch"); + MDC.clear(); + }); - LOGGER.error("random message"); + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); - assertTrue(IOs.readFile(root, filename).contains("random message")); + assertTrue(IOs.readFile(root, filename).contains("random message testWorkerDispatch")); } @Test void testLogSeparateFiles() throws InterruptedException { + final Logger logger = LoggerFactory.getLogger("testLogSeparateFiles"); + final String filename = "logs.log"; final Path root1 = root.resolve("1"); final Path root2 = root.resolve("2"); - CountDownLatch latch = new CountDownLatch(2); ExecutorService executor = Executors.newFixedThreadPool(2); executor.submit(() -> { - MDC.put("context", "worker"); MDC.put("job_root", root1.toString()); MDC.put("job_log_filename", filename); MDC.put("job_id", "1"); - LOGGER.error("random message 1"); - latch.countDown(); + logger.error("random message 1"); }); executor.submit(() -> { - MDC.put("context", "worker"); MDC.put("job_root", root2.toString()); MDC.put("job_log_filename", filename); MDC.put("job_id", "2"); - LOGGER.error("random message 2"); - latch.countDown(); + logger.error("random message 2"); }); executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); - latch.await(); assertTrue(IOs.readFile(root1, filename).contains("random message 1")); assertTrue(IOs.readFile(root2, filename).contains("random message 2")); } + @Test + void testLogNoJobRoot() throws InterruptedException { + final Logger logger = LoggerFactory.getLogger("testWorkerDispatch"); + + final String filename = "logs.log"; + + ExecutorService executor = Executors.newFixedThreadPool(1); + executor.submit(() -> { + logger.error("random message testLogNoJobRoot"); + MDC.clear(); + }); + + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); + + assertFalse(Files.exists(root.resolve(filename))); + } + } diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobSubmitter.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobSubmitter.java index 957f01f40717..a4c05572dee8 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobSubmitter.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobSubmitter.java @@ -74,7 +74,6 @@ private void submitJob(Job job) { final Path logFilePath = workerRun.getJobRoot().resolve(WorkerConstants.LOG_FILENAME); persistence.updateLogPath(job.getId(), logFilePath); persistence.incrementAttempts(job.getId()); - MDC.put("context", "worker"); MDC.put("job_id", String.valueOf(job.getId())); MDC.put("job_root", logFilePath.getParent().toString()); MDC.put("job_log_filename", logFilePath.getFileName().toString()); diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/JobSubmitterTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/JobSubmitterTest.java index 55e74b7fb4bd..69a0ed609f3f 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/JobSubmitterTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/JobSubmitterTest.java @@ -168,7 +168,6 @@ void testMDC() throws Exception { assertEquals( ImmutableMap.of( - "context", "worker", "job_id", "1", "job_root", workerRun.getJobRoot().toString(), "job_log_filename", WorkerConstants.LOG_FILENAME), From 75d03c40826ee84ba199ac2009484519ca9e075e Mon Sep 17 00:00:00 2001 From: Michel Tricot Date: Mon, 21 Sep 2020 22:43:30 -0700 Subject: [PATCH 2/2] spoteless --- .../test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java index cbdf442906e5..bafb9faf6f50 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit;