-
-
Notifications
You must be signed in to change notification settings - Fork 27
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Clear tmp dir #344
Clear tmp dir #344
Changes from 11 commits
62e9766
1ee0ccf
9ea8783
eca1c9a
9b0fc86
8421be8
431b62d
5469a45
4f070ca
b88eb0b
673d8c1
423a428
8c38030
d70e6a0
96223fa
d12c5fd
d8b3f2a
51c6415
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,7 @@ | ||
package org.commonwl.view; | ||
|
||
|
||
import org.commonwl.view.util.StreamGobbler; | ||
import org.commonwl.view.workflow.QueuedWorkflowRepository; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
|
@@ -9,6 +10,9 @@ | |
import org.springframework.scheduling.annotation.Scheduled; | ||
import org.springframework.stereotype.Component; | ||
|
||
|
||
import java.io.IOException; | ||
|
||
import java.util.Calendar; | ||
import java.util.Date; | ||
|
||
|
@@ -24,6 +28,9 @@ public class Scheduler { | |
@Value("${queuedWorkflowAgeLimitHours}") | ||
private Integer QUEUED_WORKFLOW_AGE_LIMIT_HOURS; | ||
|
||
@Value("${tmpDirAgeLimitDays}") | ||
private Integer TMP_DIR_AGE_LIMIT_DAYS; | ||
|
||
@Autowired | ||
public Scheduler(QueuedWorkflowRepository queuedWorkflowRepository) { | ||
this.queuedWorkflowRepository = queuedWorkflowRepository; | ||
|
@@ -55,4 +62,40 @@ public void removeOldQueuedWorkflowEntries() { | |
logger.info(queuedWorkflowRepository.deleteByTempRepresentation_RetrievedOnLessThanEqual(removeTime) | ||
+ " Old queued workflows removed"); | ||
} | ||
|
||
|
||
@Scheduled(cron = "${cron.clearTmpDir}") | ||
public void clearTmpDir() { | ||
// TODO: find source of tmp dir creation and delete from there | ||
|
||
// wipe tmp dir and log info | ||
try { | ||
// Run command | ||
String[] command = {"find", "/tmp", "-ctime", "+" + TMP_DIR_AGE_LIMIT_DAYS, "-exec", "rm", "-rf", "{}", "+"}; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As discussed, I suggest investigating a java-based solution instead of using a subprocess. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. +1 |
||
ProcessBuilder clearProcess = new ProcessBuilder(command); | ||
logger.info("Clearing /tmp directory for content older than " + TMP_DIR_AGE_LIMIT_DAYS + " day" + (TMP_DIR_AGE_LIMIT_DAYS > 1 ? "s" : "") + "..."); | ||
Process process = clearProcess.start(); | ||
|
||
// Read output from the process using threads | ||
StreamGobbler inputGobbler = new StreamGobbler(process.getInputStream()); | ||
StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream()); | ||
errorGobbler.start(); | ||
inputGobbler.start(); | ||
|
||
// Wait for process to complete | ||
int exitCode = process.waitFor(); | ||
if (exitCode == 0) { | ||
inputGobbler.join(); | ||
logger.info(inputGobbler.getContent()); | ||
logger.info("Successfully Cleared /tmp directory"); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. s/Cleared/cleared for consistency with other messages? 😬 Also, once the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Hi @kinow , Another idea was to delete the file the moment it is no longer in use, but an exact time would be difficult to determine Hence a solution that works would be to delete all files above a certain age. so I would be removing the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sounds good to me. The other comment about using a Java native solution is still valid I think. So instead of forking a new process we could try first using Java to iterate/walk the temp dir, filter for files/dirs older than the threshold, and finally delete or mark them for deletion (I assume we will try to delete it and in case it fails, call |
||
} else { | ||
errorGobbler.join(); | ||
logger.warn("Could not clear /tmp directory"); | ||
logger.warn(errorGobbler.getContent()); | ||
|
||
} | ||
} catch (IOException|InterruptedException e) { | ||
logger.error("Error running clear /tmp dir process", e); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -48,6 +48,7 @@ spring.data.mongodb.port = 27017 | |
sparql.endpoint = http://localhost:3030/cwlviewer/ | ||
|
||
|
||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit-picking, but unnecessary blank line 😬 |
||
#======================= | ||
# Scheduler settings | ||
#======================= | ||
|
@@ -57,5 +58,12 @@ sparql.endpoint = http://localhost:3030/cwlviewer/ | |
# The expression below implies every hour at the 0th second and 0th minute i.e (01:00:00, 02:00::00,... etc) | ||
cron.deleteOldQueuedWorkflows = 0 0 * * * ? | ||
|
||
# The expression below implies every day at the 0th second, 0th minute and 24th(0th) hour i.e ( time 00:00:00, every day) | ||
cron.clearTmpDir = 0 0 0 * * ? | ||
|
||
# Age limit for queued workflows in hours. | ||
queuedWorkflowAgeLimitHours = 24 | ||
queuedWorkflowAgeLimitHours = 24 | ||
|
||
# Age limit for tmp directories in days. | ||
tmpDirAgeLimitDays = 1 | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
👍 we can change the PR to draft until the
TODO
s are resolved.