From 62e9766f2ed42e161a8f6665978b1dcfdac94f98 Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Fri, 23 Jul 2021 03:42:04 +0100 Subject: [PATCH 01/14] Create cronjob value for scheduler --- src/main/resources/application.properties | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 067a1e6d..f8f1bfdb 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -45,4 +45,6 @@ spring.data.mongodb.port = 27017 #======================= # SPARQL endpoint #======================= -sparql.endpoint = http://localhost:3030/cwlviewer/ \ No newline at end of file +sparql.endpoint = http://localhost:3030/cwlviewer/ + +cronJob.clearTmpDir = "* * * * *" \ No newline at end of file From 9ea8783f77f262b022eda2414fc946d70c726bb8 Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Mon, 26 Jul 2021 10:59:39 +0100 Subject: [PATCH 02/14] Update tmpdir cron job cleaning settings --- src/main/resources/application.properties | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f1a0196d..4045d1bc 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -47,9 +47,7 @@ spring.data.mongodb.port = 27017 #======================= sparql.endpoint = http://localhost:3030/cwlviewer/ -<<<<<<< HEAD -cronJob.clearTmpDir = "* * * * *" -======= + #======================= # Scheduler settings @@ -60,6 +58,12 @@ cronJob.clearTmpDir = "* * * * *" # The expression below implies every hour at the 0th second and 0th minute i.e (01:00:00, 02:00::00,... etc) cron.deleteOldQueuedWorkflows = 0 0 * * * ? +# The expression below implies every day at the 0th second, 0th minute and 24th(0th) hour i.e ( time 00:00:00, every day) +cron.clearTmpDir = 0 0 0 * * ? + # Age limit for queued workflows in hours. queuedWorkflowAgeLimitHours = 24 ->>>>>>> main + +# Age limit for tmp directories in hours. +tmpdirAgeLimitHours = 12 + From eca1c9a1db7e6141b60c1a9cdb3f881b83d983d2 Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Mon, 26 Jul 2021 11:07:32 +0100 Subject: [PATCH 03/14] Create cron job to wipe tmp dir --- src/main/java/org/commonwl/view/Scheduler.java | 18 ++++++++++++++++++ src/main/resources/application.properties | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index b8a73c7c..9c98a992 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -24,6 +24,9 @@ public class Scheduler { @Value("${queuedWorkflowAgeLimitHours}") private Integer QUEUED_WORKFLOW_AGE_LIMIT_HOURS; + @Value("${tmpDirAgeLimitHours}") + private Integer TMP_DIR_AGE_LIMIT_HOURS; + @Autowired public Scheduler(QueuedWorkflowRepository queuedWorkflowRepository) { this.queuedWorkflowRepository = queuedWorkflowRepository; @@ -55,4 +58,19 @@ public void removeOldQueuedWorkflowEntries() { logger.info(queuedWorkflowRepository.deleteByTempRepresentation_RetrievedOnLessThanEqual(removeTime) + " Old queued workflows removed"); } + + + @Scheduled(cron = "${cron.clearTmpDir}") + public void clearTmpDir() { + Calendar calendar = Calendar.getInstance(); + Date now = new Date(); + calendar.setTime(now); + + // calculate time QUEUED_WORKFLOW_AGE_LIMIT_HOURS before now + calendar.add(Calendar.HOUR, -TMP_DIR_AGE_LIMIT_HOURS); + Date removeTime = calendar.getTime(); + + // access path to tmp dir + // wipe tmp dir and log info + } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 4045d1bc..0ff1a9d0 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -65,5 +65,5 @@ cron.clearTmpDir = 0 0 0 * * ? queuedWorkflowAgeLimitHours = 24 # Age limit for tmp directories in hours. -tmpdirAgeLimitHours = 12 +tmpDirAgeLimitHours = 12 From 9b0fc868d6308e003ba72b64a29b9c5356508fd1 Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Mon, 9 Aug 2021 07:08:49 +0100 Subject: [PATCH 04/14] Update /tmp wipe scheduled process --- .../java/org/commonwl/view/Scheduler.java | 37 ++++++++++++++----- .../org/commonwl/view/util/StreamGobbler.java | 2 +- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index 9c98a992..b2577285 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -1,6 +1,7 @@ package org.commonwl.view; +import org.commonwl.view.util.StreamGobbler; import org.commonwl.view.workflow.QueuedWorkflowRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +19,7 @@ @Component public class Scheduler { - private final Logger logger = LoggerFactory.getLogger(this.getClass()); + private final java.util.logging.Logger logger = LoggerFactory.getLogger(this.getClass()); private final QueuedWorkflowRepository queuedWorkflowRepository; @Value("${queuedWorkflowAgeLimitHours}") @@ -62,15 +63,33 @@ public void removeOldQueuedWorkflowEntries() { @Scheduled(cron = "${cron.clearTmpDir}") public void clearTmpDir() { - Calendar calendar = Calendar.getInstance(); - Date now = new Date(); - calendar.setTime(now); + // TODO: find source of tmp dir creation and delete from there - // calculate time QUEUED_WORKFLOW_AGE_LIMIT_HOURS before now - calendar.add(Calendar.HOUR, -TMP_DIR_AGE_LIMIT_HOURS); - Date removeTime = calendar.getTime(); - - // access path to tmp dir // wipe tmp dir and log info + try { + // Run command + String[] command = {"find", "/tmp", "-ctime", "+10", "-exec", "rm", "-rf", "{}", "+"}; + ProcessBuilder cwlToolProcess = new ProcessBuilder(command); + Process process = cwlToolProcess.start(); + + // Read output from the process using threads + StreamGobbler inputGobbler = new StreamGobbler(process.getInputStream()); + StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream()); + errorGobbler.start(); + inputGobbler.start(); + + // Wait for process to complete + int exitCode = process.waitFor(); + if (exitCode == 0) { + inputGobbler.join(); + logger.info(inputGobbler.getContent()); + } else { + errorGobbler.join(); + throw new Exception(errorGobbler.getContent()); + } + } catch (IOException|InterruptedException e) { + logger.error("Error running clear /tmp dir process", e); + throw new Exception("Error running /tmp dir clearing process"); + } } } diff --git a/src/main/java/org/commonwl/view/util/StreamGobbler.java b/src/main/java/org/commonwl/view/util/StreamGobbler.java index ac9f71c4..29eb462f 100644 --- a/src/main/java/org/commonwl/view/util/StreamGobbler.java +++ b/src/main/java/org/commonwl/view/util/StreamGobbler.java @@ -32,7 +32,7 @@ public class StreamGobbler extends Thread { private final String lineSeparator = System.getProperty("line.separator"); private InputStream is; - private String content = ""; + private String content = ""; // TODO: update to string builder public StreamGobbler(InputStream is) { this.is = is; From 8421be89ef7f54e619d125600e46e7b50eec9bdd Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Mon, 9 Aug 2021 07:16:56 +0100 Subject: [PATCH 05/14] Add logging to clearing process --- src/main/java/org/commonwl/view/Scheduler.java | 8 +++++--- src/main/resources/application.properties | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index b2577285..2cd5f51c 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -25,8 +25,8 @@ public class Scheduler { @Value("${queuedWorkflowAgeLimitHours}") private Integer QUEUED_WORKFLOW_AGE_LIMIT_HOURS; - @Value("${tmpDirAgeLimitHours}") - private Integer TMP_DIR_AGE_LIMIT_HOURS; + @Value("${tmpDirAgeLimitDays}") + private Integer TMP_DIR_AGE_LIMIT_DAYS; @Autowired public Scheduler(QueuedWorkflowRepository queuedWorkflowRepository) { @@ -68,8 +68,9 @@ public void clearTmpDir() { // wipe tmp dir and log info try { // Run command - String[] command = {"find", "/tmp", "-ctime", "+10", "-exec", "rm", "-rf", "{}", "+"}; + String[] command = {"find", "/tmp", "-ctime", "+" + TMP_DIR_AGE_LIMIT_DAYS, "-exec", "rm", "-rf", "{}", "+"}; ProcessBuilder cwlToolProcess = new ProcessBuilder(command); + logger.info("Clearing /tmp directory for content older than " + TMP_DIR_AGE_LIMIT_DAYS + " day" + (TMP_DIR_AGE_LIMIT_DAYS > 1 ? "s" : "") + "..."); Process process = cwlToolProcess.start(); // Read output from the process using threads @@ -83,6 +84,7 @@ public void clearTmpDir() { if (exitCode == 0) { inputGobbler.join(); logger.info(inputGobbler.getContent()); + logger.info("Successfully Cleared /tmp directory"); } else { errorGobbler.join(); throw new Exception(errorGobbler.getContent()); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 0ff1a9d0..73a5e437 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -64,6 +64,6 @@ cron.clearTmpDir = 0 0 0 * * ? # Age limit for queued workflows in hours. queuedWorkflowAgeLimitHours = 24 -# Age limit for tmp directories in hours. -tmpDirAgeLimitHours = 12 +# Age limit for tmp directories in days. +tmpDirAgeLimitDays = 1 From 431b62d9c77c406513b6cf7a0e6bbac5c631152a Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Mon, 9 Aug 2021 18:19:09 +0100 Subject: [PATCH 06/14] Fix scheduler logging issue --- src/main/java/org/commonwl/view/Scheduler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index 2cd5f51c..fba17f6e 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -19,7 +19,7 @@ @Component public class Scheduler { - private final java.util.logging.Logger logger = LoggerFactory.getLogger(this.getClass()); + private final Logger logger = LoggerFactory.getLogger(this.getClass()); private final QueuedWorkflowRepository queuedWorkflowRepository; @Value("${queuedWorkflowAgeLimitHours}") @@ -69,9 +69,9 @@ public void clearTmpDir() { try { // Run command String[] command = {"find", "/tmp", "-ctime", "+" + TMP_DIR_AGE_LIMIT_DAYS, "-exec", "rm", "-rf", "{}", "+"}; - ProcessBuilder cwlToolProcess = new ProcessBuilder(command); + ProcessBuilder clearProcess = new ProcessBuilder(command); logger.info("Clearing /tmp directory for content older than " + TMP_DIR_AGE_LIMIT_DAYS + " day" + (TMP_DIR_AGE_LIMIT_DAYS > 1 ? "s" : "") + "..."); - Process process = cwlToolProcess.start(); + Process process = clearProcess.start(); // Read output from the process using threads StreamGobbler inputGobbler = new StreamGobbler(process.getInputStream()); From 5469a452c938e8c60f3a9c3eab8f3e85a78d3d54 Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Mon, 9 Aug 2021 18:43:53 +0100 Subject: [PATCH 07/14] Fix exception module import error --- src/main/java/org/commonwl/view/Scheduler.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index fba17f6e..475a80bd 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -10,6 +10,9 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; + +import java.io.IOException; + import java.util.Calendar; import java.util.Date; @@ -87,11 +90,12 @@ public void clearTmpDir() { logger.info("Successfully Cleared /tmp directory"); } else { errorGobbler.join(); - throw new Exception(errorGobbler.getContent()); + logger.info("Could not clear /tmp directory"); + logger.error(errorGobbler.getContent()); + } } catch (IOException|InterruptedException e) { logger.error("Error running clear /tmp dir process", e); - throw new Exception("Error running /tmp dir clearing process"); } } } From 4f070ca7c5ac10a07a40cb2e87df3392960370d6 Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Mon, 9 Aug 2021 18:54:43 +0100 Subject: [PATCH 08/14] Update log levels for clearing process --- src/main/java/org/commonwl/view/Scheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index 475a80bd..aa7224d8 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -91,7 +91,7 @@ public void clearTmpDir() { } else { errorGobbler.join(); logger.info("Could not clear /tmp directory"); - logger.error(errorGobbler.getContent()); + logger.warn(errorGobbler.getContent()); } } catch (IOException|InterruptedException e) { From b88eb0b4733dd7113ef973aa9f83c6e60a853c7a Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Mon, 16 Aug 2021 09:40:11 +0100 Subject: [PATCH 09/14] Update log level in Scheduler --- src/main/java/org/commonwl/view/Scheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index aa7224d8..d8f709eb 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -90,7 +90,7 @@ public void clearTmpDir() { logger.info("Successfully Cleared /tmp directory"); } else { errorGobbler.join(); - logger.info("Could not clear /tmp directory"); + logger.warn("Could not clear /tmp directory"); logger.warn(errorGobbler.getContent()); } From 423a4287f1bd8de5e3609ceaf4cce83c4c29d23f Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Tue, 28 Sep 2021 17:02:18 +0100 Subject: [PATCH 10/14] Refactor code changes --- .../java/org/commonwl/view/Scheduler.java | 29 ++----- .../org/commonwl/view/util/FileUtils.java | 77 +++++++++++++++++++ 2 files changed, 82 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/commonwl/view/util/FileUtils.java diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index d8f709eb..fd27f4e3 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -2,6 +2,7 @@ import org.commonwl.view.util.StreamGobbler; +import org.commonwl.view.util.FileUtils; import org.commonwl.view.workflow.QueuedWorkflowRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,30 +71,10 @@ public void clearTmpDir() { // wipe tmp dir and log info try { - // Run command - String[] command = {"find", "/tmp", "-ctime", "+" + TMP_DIR_AGE_LIMIT_DAYS, "-exec", "rm", "-rf", "{}", "+"}; - ProcessBuilder clearProcess = new ProcessBuilder(command); - logger.info("Clearing /tmp directory for content older than " + TMP_DIR_AGE_LIMIT_DAYS + " day" + (TMP_DIR_AGE_LIMIT_DAYS > 1 ? "s" : "") + "..."); - Process process = clearProcess.start(); - - // Read output from the process using threads - StreamGobbler inputGobbler = new StreamGobbler(process.getInputStream()); - StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream()); - errorGobbler.start(); - inputGobbler.start(); - - // Wait for process to complete - int exitCode = process.waitFor(); - if (exitCode == 0) { - inputGobbler.join(); - logger.info(inputGobbler.getContent()); - logger.info("Successfully Cleared /tmp directory"); - } else { - errorGobbler.join(); - logger.warn("Could not clear /tmp directory"); - logger.warn(errorGobbler.getContent()); - - } + File file = new File("/tmp"); + FileUtils.deleteWithinDirectory(file, TMP_DIR_AGE_LIMIT_DAYS); + + //FileUtils.deleteWithinDirectoryCMD("/tmp", TMP_DIR_AGE_LIMIT_DAYS); } catch (IOException|InterruptedException e) { logger.error("Error running clear /tmp dir process", e); } diff --git a/src/main/java/org/commonwl/view/util/FileUtils.java b/src/main/java/org/commonwl/view/util/FileUtils.java new file mode 100644 index 00000000..50a85d77 --- /dev/null +++ b/src/main/java/org/commonwl/view/util/FileUtils.java @@ -0,0 +1,77 @@ +import java.io.IOException; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.commonwl.view.util.StreamGobbler; + +public abstract class FileUtils { + + private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); + private static long fileAge(File file) throws IOException { + + FileTime t = Files.getLastModifiedTime(file.toPath()); + Instant fileInstant = t.toInstant(); + Instant now = (Clock.systemUTC()).instant(); + Duration difference = Duration.between(fileInstant, now); + long days = difference.toDays(); + return days; + + } + + public static void deleteWithinDirectory(File file, long days) throws IOException{ + File[] files = file.listFiles(); + + if (files != null) { + for (File subfile : files) { + if (subfile.isDirectory()) { + deleteWithinDirectory(subfile); + } + + long daysOld = fileAge(subfile); + + if (daysOld >= days ) { + if (!subfile.isDirectory()) { + logger.info("deleting file " + subfile.getPath()); + subfile.delete(); + logger.info("deleted file " + subfile.getPath()); + } else { + File[] contents = subfile.listFiles(); + if (contents != null && contents.length == 0) { + logger.info("deteting Directory " + subfile.getPath()); + subfile.delete(); + logger.info("deleted Directory " + subfile.getPath()); + } + } + } + } + } + } + + public static void deleteWithinDirectoryCMD(Strig directoryPath, int days) throws IOException, InterruptedException { + + String[] command = {"find", directoryPath, "-ctime", "+" + days, "-exec", "rm", "-rf", "{}", "+"}; + ProcessBuilder clearProcess = new ProcessBuilder(command); + logger.info("Clearing /tmp directory for content older than " + days + " day" + (days > 1 ? "s" : "") + "..."); + Process process = clearProcess.start(); + + // Read output from the process using threads + StreamGobbler inputGobbler = new StreamGobbler(process.getInputStream()); + StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream()); + errorGobbler.start(); + inputGobbler.start(); + + // Wait for process to complete + int exitCode = process.waitFor(); + if (exitCode == 0) { + inputGobbler.join(); + logger.info(inputGobbler.getContent()); + logger.info("Successfully Cleared " + directoryPath + " directory"); + } else { + errorGobbler.join(); + logger.warn("Could not clear " + directoryPath + " directory"); + logger.warn(errorGobbler.getContent()); + + } + } + +} From 8c38030df243d80fd35293145c4c1f85fa7e54e9 Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Thu, 30 Sep 2021 10:40:14 +0100 Subject: [PATCH 11/14] Add missing imports --- src/main/java/org/commonwl/view/Scheduler.java | 4 +++- src/main/java/org/commonwl/view/util/FileUtils.java | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index fd27f4e3..05a4e4c6 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -11,6 +11,8 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import java.io.File; + import java.io.IOException; @@ -75,7 +77,7 @@ public void clearTmpDir() { FileUtils.deleteWithinDirectory(file, TMP_DIR_AGE_LIMIT_DAYS); //FileUtils.deleteWithinDirectoryCMD("/tmp", TMP_DIR_AGE_LIMIT_DAYS); - } catch (IOException|InterruptedException e) { + } catch (IOException e) { logger.error("Error running clear /tmp dir process", e); } } diff --git a/src/main/java/org/commonwl/view/util/FileUtils.java b/src/main/java/org/commonwl/view/util/FileUtils.java index 50a85d77..93dfec55 100644 --- a/src/main/java/org/commonwl/view/util/FileUtils.java +++ b/src/main/java/org/commonwl/view/util/FileUtils.java @@ -1,5 +1,11 @@ +import java.io.File; +import java.nio.file.Files; +import java.nio.file.attribute.FileTime; import java.io.IOException; -import java.io.IOException; +import java.time.Instant; +import java.time.Clock; +import java.time.Duration; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.commonwl.view.util.StreamGobbler; @@ -24,7 +30,7 @@ public static void deleteWithinDirectory(File file, long days) throws IOExceptio if (files != null) { for (File subfile : files) { if (subfile.isDirectory()) { - deleteWithinDirectory(subfile); + deleteWithinDirectory(subfile, days); } long daysOld = fileAge(subfile); @@ -47,7 +53,7 @@ public static void deleteWithinDirectory(File file, long days) throws IOExceptio } } - public static void deleteWithinDirectoryCMD(Strig directoryPath, int days) throws IOException, InterruptedException { + public static void deleteWithinDirectoryCMD(String directoryPath, int days) throws IOException, InterruptedException { String[] command = {"find", directoryPath, "-ctime", "+" + days, "-exec", "rm", "-rf", "{}", "+"}; ProcessBuilder clearProcess = new ProcessBuilder(command); From 96223fa3bf715533148c1f2b815673f5927d18bc Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Wed, 13 Oct 2021 06:17:41 +0100 Subject: [PATCH 12/14] Fix: class not found compile error --- src/main/java/org/commonwl/view/Scheduler.java | 5 ++--- .../java/org/commonwl/view/util/FileUtils.java | 18 +++++++++++++----- .../org/commonwl/view/util/StreamGobbler.java | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index 05a4e4c6..5d8d8012 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -69,13 +69,12 @@ public void removeOldQueuedWorkflowEntries() { @Scheduled(cron = "${cron.clearTmpDir}") public void clearTmpDir() { - // TODO: find source of tmp dir creation and delete from there // wipe tmp dir and log info try { File file = new File("/tmp"); - FileUtils.deleteWithinDirectory(file, TMP_DIR_AGE_LIMIT_DAYS); - + FileUtils utils = new FileUtils(logger); + utils.deleteWithinDirectory(file, TMP_DIR_AGE_LIMIT_DAYS); //FileUtils.deleteWithinDirectoryCMD("/tmp", TMP_DIR_AGE_LIMIT_DAYS); } catch (IOException e) { logger.error("Error running clear /tmp dir process", e); diff --git a/src/main/java/org/commonwl/view/util/FileUtils.java b/src/main/java/org/commonwl/view/util/FileUtils.java index 93dfec55..49cebe2a 100644 --- a/src/main/java/org/commonwl/view/util/FileUtils.java +++ b/src/main/java/org/commonwl/view/util/FileUtils.java @@ -1,3 +1,6 @@ +package org.commonwl.view.util; + + import java.io.File; import java.nio.file.Files; import java.nio.file.attribute.FileTime; @@ -10,10 +13,15 @@ import org.slf4j.LoggerFactory; import org.commonwl.view.util.StreamGobbler; -public abstract class FileUtils { +public class FileUtils { + + private final Logger logger; + + public FileUtils(Logger logger) { + this.logger = logger; + } - private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); - private static long fileAge(File file) throws IOException { + private long fileAge(File file) throws IOException { FileTime t = Files.getLastModifiedTime(file.toPath()); Instant fileInstant = t.toInstant(); @@ -24,7 +32,7 @@ private static long fileAge(File file) throws IOException { } - public static void deleteWithinDirectory(File file, long days) throws IOException{ + public void deleteWithinDirectory(File file, long days) throws IOException{ File[] files = file.listFiles(); if (files != null) { @@ -53,7 +61,7 @@ public static void deleteWithinDirectory(File file, long days) throws IOExceptio } } - public static void deleteWithinDirectoryCMD(String directoryPath, int days) throws IOException, InterruptedException { + public void deleteWithinDirectoryCMD(String directoryPath, int days) throws IOException, InterruptedException { String[] command = {"find", directoryPath, "-ctime", "+" + days, "-exec", "rm", "-rf", "{}", "+"}; ProcessBuilder clearProcess = new ProcessBuilder(command); diff --git a/src/main/java/org/commonwl/view/util/StreamGobbler.java b/src/main/java/org/commonwl/view/util/StreamGobbler.java index 29eb462f..031940cd 100644 --- a/src/main/java/org/commonwl/view/util/StreamGobbler.java +++ b/src/main/java/org/commonwl/view/util/StreamGobbler.java @@ -32,7 +32,7 @@ public class StreamGobbler extends Thread { private final String lineSeparator = System.getProperty("line.separator"); private InputStream is; - private String content = ""; // TODO: update to string builder + private String content = ""; // TODO: optimize - update to string builder public StreamGobbler(InputStream is) { this.is = is; From d8b3f2a11028128372564335274b80e34d9a290f Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Wed, 13 Oct 2021 06:58:04 +0100 Subject: [PATCH 13/14] Update log messages --- .../java/org/commonwl/view/Scheduler.java | 11 +---- .../org/commonwl/view/util/FileUtils.java | 46 ++++++++++++------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/commonwl/view/Scheduler.java b/src/main/java/org/commonwl/view/Scheduler.java index 5d8d8012..567b308b 100644 --- a/src/main/java/org/commonwl/view/Scheduler.java +++ b/src/main/java/org/commonwl/view/Scheduler.java @@ -69,15 +69,8 @@ public void removeOldQueuedWorkflowEntries() { @Scheduled(cron = "${cron.clearTmpDir}") public void clearTmpDir() { - // wipe tmp dir and log info - try { - File file = new File("/tmp"); - FileUtils utils = new FileUtils(logger); - utils.deleteWithinDirectory(file, TMP_DIR_AGE_LIMIT_DAYS); - //FileUtils.deleteWithinDirectoryCMD("/tmp", TMP_DIR_AGE_LIMIT_DAYS); - } catch (IOException e) { - logger.error("Error running clear /tmp dir process", e); - } + FileUtils utils = new FileUtils(logger); + utils.clearDirectory("/tmp", TMP_DIR_AGE_LIMIT_DAYS); } } diff --git a/src/main/java/org/commonwl/view/util/FileUtils.java b/src/main/java/org/commonwl/view/util/FileUtils.java index 49cebe2a..e610fd8a 100644 --- a/src/main/java/org/commonwl/view/util/FileUtils.java +++ b/src/main/java/org/commonwl/view/util/FileUtils.java @@ -21,18 +21,22 @@ public FileUtils(Logger logger) { this.logger = logger; } - private long fileAge(File file) throws IOException { + + public void clearDirectory(Stirng directoryPath, long days) { + try { + logger.info("Clearing "+ directoryPath + " directory for content older than " + days + " day" + (days > 1 ? "s" : "") + "..."); + File file = new File(directoryPath); + deleteWithinDirectory(file, days); + logger.info("Successfully cleared " + directoryPath + " directory"); + } catch(IOException e) { + logger.warn("Could not clear " + directoryPath + " directory"); + logger.error("Error running clear " + directoryPath + " dir process", e); + } - FileTime t = Files.getLastModifiedTime(file.toPath()); - Instant fileInstant = t.toInstant(); - Instant now = (Clock.systemUTC()).instant(); - Duration difference = Duration.between(fileInstant, now); - long days = difference.toDays(); - return days; - } - public void deleteWithinDirectory(File file, long days) throws IOException{ + + private void deleteWithinDirectory(File file, long days) throws IOException{ File[] files = file.listFiles(); if (files != null) { @@ -45,15 +49,15 @@ public void deleteWithinDirectory(File file, long days) throws IOException{ if (daysOld >= days ) { if (!subfile.isDirectory()) { - logger.info("deleting file " + subfile.getPath()); + logger.info("Deleting file " + subfile.getPath()); subfile.delete(); - logger.info("deleted file " + subfile.getPath()); + logger.info("Deleted file " + subfile.getPath()); } else { File[] contents = subfile.listFiles(); if (contents != null && contents.length == 0) { - logger.info("deteting Directory " + subfile.getPath()); + logger.info("Deleting directory " + subfile.getPath()); subfile.delete(); - logger.info("deleted Directory " + subfile.getPath()); + logger.info("Deleted directory " + subfile.getPath()); } } } @@ -61,11 +65,21 @@ public void deleteWithinDirectory(File file, long days) throws IOException{ } } - public void deleteWithinDirectoryCMD(String directoryPath, int days) throws IOException, InterruptedException { + private long fileAge(File file) throws IOException { + + FileTime t = Files.getLastModifiedTime(file.toPath()); + Instant fileInstant = t.toInstant(); + Instant now = (Clock.systemUTC()).instant(); + Duration difference = Duration.between(fileInstant, now); + long days = difference.toDays(); + return days; + + } + + private void deleteWithinDirectoryCMD(String directoryPath, int days) throws IOException, InterruptedException { String[] command = {"find", directoryPath, "-ctime", "+" + days, "-exec", "rm", "-rf", "{}", "+"}; ProcessBuilder clearProcess = new ProcessBuilder(command); - logger.info("Clearing /tmp directory for content older than " + days + " day" + (days > 1 ? "s" : "") + "..."); Process process = clearProcess.start(); // Read output from the process using threads @@ -79,7 +93,7 @@ public void deleteWithinDirectoryCMD(String directoryPath, int days) throws IOEx if (exitCode == 0) { inputGobbler.join(); logger.info(inputGobbler.getContent()); - logger.info("Successfully Cleared " + directoryPath + " directory"); + logger.info("Successfully cleared " + directoryPath + " directory"); } else { errorGobbler.join(); logger.warn("Could not clear " + directoryPath + " directory"); From 51c6415f49a46560ce5ce26273bb5d3ae798fac8 Mon Sep 17 00:00:00 2001 From: Osakpolor Obaseki Date: Wed, 13 Oct 2021 07:23:28 +0100 Subject: [PATCH 14/14] Fix: String type typo --- src/main/java/org/commonwl/view/util/FileUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/commonwl/view/util/FileUtils.java b/src/main/java/org/commonwl/view/util/FileUtils.java index e610fd8a..9ddf619a 100644 --- a/src/main/java/org/commonwl/view/util/FileUtils.java +++ b/src/main/java/org/commonwl/view/util/FileUtils.java @@ -22,7 +22,7 @@ public FileUtils(Logger logger) { } - public void clearDirectory(Stirng directoryPath, long days) { + public void clearDirectory(String directoryPath, long days) { try { logger.info("Clearing "+ directoryPath + " directory for content older than " + days + " day" + (days > 1 ? "s" : "") + "..."); File file = new File(directoryPath);