diff --git a/MANPAGE.md b/MANPAGE.md index 0c00cf9..f2ccb73 100644 --- a/MANPAGE.md +++ b/MANPAGE.md @@ -25,21 +25,27 @@ If there is a failure when trying to kill a process, **earlyoom** sleeps for # OPTIONS -#### -m PERCENT -set available memory minimum to PERCENT of total (default 10 %) +#### -m PERCENT[,KILL_PERCENT] +set available memory minimum to PERCENT of total (default 10 %). +Send sigkill if at or below KILL_PERCENT (default 1/2 PERCENT), otherwise sigterm. -#### -s PERCENT +Use the same value for PERCENT and KILL_PERCENT if you always want to use sigkill. + +#### -s PERCENT[,KILL_PERCENT] set free swap minimum to PERCENT of total (default 10 %). +Send sigkill if at or below KILL_PERCENT (default 1/2 PERCENT), otherwise sigterm. You can use `-s 100` to have earlyoom effectively ignore swap usage: Processes are killed once available memory drops below the configured minimum, no matter how much swap is free. -#### -M SIZE -set available memory minimum to SIZE KiB +#### -M SIZE[,KILL_SIZE] +set available memory minimum to SIZE KiB. +Send sigkill if at or below KILL_SIZE (default 1/2 SIZE), otherwise sigterm. -#### -S SIZE -set free swap minimum to SIZE KiB +#### -S SIZE[,KILL_SIZE] +set free swap minimum to SIZE KiB. +Send sigkill if at or below KILL_SIZE (default 1/2 SIZE), otherwise sigterm. #### -k removed in earlyoom v1.2, ignored for compatibility diff --git a/README.md b/README.md index 68c3fda..519e15c 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,8 @@ Changelog * Gracefully handle the case of swap being added or removed after earlyoom was started ([issue 62](https://github.com/rfjakob/earlyoom/issues/62), [commit](https://github.com/rfjakob/earlyoom/commit/88e58903fec70b105aebba39cd584add5e1d1532)) + * Implement staged kill: first sigterm, then sigkill, with configurable limits + ([issue #67](https://github.com/rfjakob/earlyoom/issues/67)) * v1.1, 2018-07-07 * Fix possible shell code injection through GUI notifications ([commit](https://github.com/rfjakob/earlyoom/commit/ab79aa3895077676f50120f15e2bb22915446db9)) diff --git a/kill.c b/kill.c index 3b72a42..dfbe7c8 100644 --- a/kill.c +++ b/kill.c @@ -222,8 +222,8 @@ void userspace_kill(poll_loop_args_t args, int sig) // sig == 0 is used as a self-test during startup. Don't notifiy the user. if (sig != 0) { - fprintf(stderr, "Killing process: %s, pid: %d, badness: %d, VmRSS: %lu MiB\n", - victim_name, victim_pid, victim_badness, victim_vm_rss / 1024); + warn("Killing process '%s' with signal %d, pid: %d, badness: %d, VmRSS: %lu MiB\n", + victim_name, sig, victim_pid, victim_badness, victim_vm_rss / 1024); } int res = kill(victim_pid, sig); diff --git a/kill.h b/kill.h index 2079298..18a5642 100644 --- a/kill.h +++ b/kill.h @@ -9,8 +9,10 @@ typedef struct { DIR* procdir; /* if the available memory AND swap goes below these percentages, * we start killing processes */ - int mem_min_percent; - int swap_min_percent; + int mem_term_percent; + int mem_kill_percent; + int swap_term_percent; + int swap_kill_percent; /* ignore /proc/PID/oom_score_adj? */ bool ignore_oom_score_adj; /* notifcation command to launch when killing something. NULL = no-op. */ diff --git a/main.c b/main.c index 9fdf972..4c85b91 100644 --- a/main.c +++ b/main.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "kill.h" #include "meminfo.h" @@ -26,7 +27,7 @@ enum { }; static int set_oom_score_adj(int); -static void print_mem_stats(FILE* procdir, const meminfo_t m); +static void print_mem_stats(bool lowmem, const meminfo_t m); static void poll_loop(const poll_loop_args_t args); int enable_debug = 0; @@ -35,10 +36,13 @@ long page_size = 0; int main(int argc, char* argv[]) { poll_loop_args_t args = { + .mem_term_percent = 10, + .swap_term_percent = 10, + .mem_kill_percent = 5, + .swap_kill_percent = 5, .report_interval_ms = 1000, /* omitted fields are set to zero */ }; - long mem_min_kib = 0, swap_min_kib = 0; /* Same thing in KiB */ int set_my_priority = 0; char* prefer_cmds = NULL; char* avoid_cmds = NULL; @@ -71,38 +75,41 @@ int main(int argc, char* argv[]) { "help", no_argument, NULL, 'h' }, { 0, 0, NULL, 0 } /* end-of-array marker */ }; + bool have_m = 0, have_M = 0, have_s = 0, have_S = 0; while ((c = getopt_long(argc, argv, short_opt, long_opt, NULL)) != -1) { float report_interval_f = 0; + term_kill_tuple_t tuple; + switch (c) { case -1: /* no more arguments */ case 0: /* long option toggles */ break; case 'm': - args.mem_min_percent = strtol(optarg, NULL, 10); - // Using "-m 100" makes no sense - if (args.mem_min_percent <= 0 || args.mem_min_percent >= 100) { - fatal(15, "-m: invalid percentage '%s'\n", optarg); - } + // Use 99 as upper limit. Passing "-m 100" makes no sense. + tuple = parse_term_kill_tuple("-m", optarg, 99, 15); + args.mem_term_percent = tuple.term; + args.mem_kill_percent = tuple.kill; + have_m = 1; break; case 's': - args.swap_min_percent = strtol(optarg, NULL, 10); // Using "-s 100" is a valid way to ignore swap usage - if (args.swap_min_percent <= 0 || args.swap_min_percent > 100) { - fatal(16, "-s: invalid percentage: '%s'\n", optarg); - } + tuple = parse_term_kill_tuple("-s", optarg, 100, 16); + args.swap_term_percent = tuple.term; + args.swap_kill_percent = tuple.kill; + have_s = 1; break; case 'M': - mem_min_kib = strtol(optarg, NULL, 10); - if (mem_min_kib <= 0) { - fatal(15, "-M: invalid KiB value '%s'\n", optarg); - } + tuple = parse_term_kill_tuple("-M", optarg, m.MemTotalKiB * 100 / 99, 15); + args.mem_term_percent = 100 * tuple.term / m.MemTotalKiB; + args.mem_kill_percent = 100 * tuple.kill / m.MemTotalKiB; + have_M = 1; break; case 'S': - swap_min_kib = strtol(optarg, NULL, 10); - if (swap_min_kib <= 0) { - fatal(16, "-S: invalid KiB value: '%s'\n", optarg); - } + tuple = parse_term_kill_tuple("-S", optarg, m.SwapTotalKiB * 100 / 99, 16); + args.swap_term_percent = 100 * tuple.term / m.SwapTotalKiB; + args.swap_kill_percent = 100 * tuple.kill / m.SwapTotalKiB; + have_S = 1; break; case 'k': fprintf(stderr, "Option -k is ignored since earlyoom v1.2\n"); @@ -170,15 +177,12 @@ int main(int argc, char* argv[]) if (optind < argc) { fatal(13, "extra argument not understood: '%s'\n", argv[optind]); } - - if (args.mem_min_percent && mem_min_kib) { - fatal(2, "can't use -m with -M\n"); + if (have_m && have_M) { + fatal(2, "can't use both -m and -M\n"); } - - if (args.swap_min_percent && swap_min_kib) { - fatal(2, "can't use -s with -S\n"); + if (have_s && have_S) { + fatal(2, "can't use both -s and -S\n"); } - if (prefer_cmds) { args.prefer_regex = &_prefer_regex; if (regcomp(args.prefer_regex, prefer_cmds, REG_EXTENDED | REG_NOSUB) != 0) { @@ -186,7 +190,6 @@ int main(int argc, char* argv[]) } fprintf(stderr, "Prefering to kill process names that match regex '%s'\n", prefer_cmds); } - if (avoid_cmds) { args.avoid_regex = &_avoid_regex; if (regcomp(args.avoid_regex, avoid_cmds, REG_EXTENDED | REG_NOSUB) != 0) { @@ -194,33 +197,6 @@ int main(int argc, char* argv[]) } fprintf(stderr, "Avoiding to kill process names that match regex '%s'\n", avoid_cmds); } - - if (mem_min_kib) { - if (mem_min_kib >= m.MemTotalKiB) { - fatal(15, - "-M: the value you passed (%ld kiB) is at or above total memory (%ld kiB)\n", - mem_min_kib, m.MemTotalKiB); - } - args.mem_min_percent = 100 * mem_min_kib / m.MemTotalKiB; - } else { - if (!args.mem_min_percent) { - args.mem_min_percent = 10; - } - } - - if (swap_min_kib) { - if (swap_min_kib > m.SwapTotalKiB) { - fatal(16, - "-S: the value you passed (%ld kiB) is above total swap (%ld kiB)\n", - swap_min_kib, m.SwapTotalKiB); - } - args.swap_min_percent = 100 * swap_min_kib / m.SwapTotalKiB; - } else { - if (!args.swap_min_percent) { - args.swap_min_percent = 10; - } - } - if (set_my_priority) { bool fail = 0; if (setpriority(PRIO_PROCESS, 0, -20) != 0) { @@ -237,10 +213,10 @@ int main(int argc, char* argv[]) } // Print memory limits - fprintf(stderr, "mem total: %4d MiB, min: %2d %%\n", - m.MemTotalMiB, args.mem_min_percent); - fprintf(stderr, "swap total: %4d MiB, min: %2d %%\n", - m.SwapTotalMiB, args.swap_min_percent); + fprintf(stderr, "mem total: %4d MiB, sending sigterm at %2d %%, sigkill at %2d %%\n", + m.MemTotalMiB, args.mem_term_percent, args.mem_kill_percent); + fprintf(stderr, "swap total: %4d MiB, sending sigterm at %2d %%, sigkill at %2d %%\n", + m.SwapTotalMiB, args.swap_term_percent, args.swap_kill_percent); /* Dry-run oom kill to make sure stack grows to maximum size before * calling mlockall() @@ -259,10 +235,13 @@ int main(int argc, char* argv[]) * mem avail: 5259 MiB (67 %), swap free: 0 MiB (0 %)" * to the fd passed in out_fd. */ -static void print_mem_stats(FILE* out_fd, const meminfo_t m) +static void print_mem_stats(bool lowmem, const meminfo_t m) { - fprintf(out_fd, - "mem avail: %4d of %4d MiB (%2d %%), swap free: %4d of %4d MiB (%2d %%)\n", + int(*out_func)(const char* fmt, ...) = &printf; + if(lowmem) { + out_func=&warn; + } + out_func("mem avail: %4d of %4d MiB (%2d %%), swap free: %4d of %4d MiB (%2d %%)\n", m.MemAvailableMiB, m.MemTotalMiB, m.MemAvailablePercent, @@ -306,11 +285,11 @@ static int sleep_time_ms(const poll_loop_args_t* args, const meminfo_t* m) const int min_sleep = 100; const int max_sleep = 1000; - int mem_headroom_kib = (m->MemAvailablePercent - args->mem_min_percent) * 10 * m->MemTotalMiB; + int mem_headroom_kib = (m->MemAvailablePercent - args->mem_term_percent) * 10 * m->MemTotalMiB; if (mem_headroom_kib < 0) { mem_headroom_kib = 0; } - int swap_headroom_kib = (m->SwapFreePercent - args->swap_min_percent) * 10 * m->SwapTotalMiB; + int swap_headroom_kib = (m->SwapFreePercent - args->swap_term_percent) * 10 * m->SwapTotalMiB; if (swap_headroom_kib < 0) { swap_headroom_kib = 0; } @@ -334,18 +313,19 @@ static void poll_loop(const poll_loop_args_t args) while (1) { m = parse_meminfo(); - if (m.MemAvailablePercent <= args.mem_min_percent && m.SwapFreePercent <= args.swap_min_percent) { - fprintf(stderr, - "Low memory! mem avail: %d of %d MiB (%d) %% <= min %d %%, swap free: %d of %d MiB (%d %%) <= min %d %%\n", - m.MemAvailableMiB, - m.MemTotalMiB, - m.MemAvailablePercent, - args.mem_min_percent, - m.SwapFreeMiB, - m.SwapTotalMiB, - m.SwapFreePercent, - args.swap_min_percent); - userspace_kill(args, 9); + if (m.MemAvailablePercent <= args.mem_term_percent && m.SwapFreePercent <= args.swap_term_percent) { + int sig = 0; + if(m.MemAvailablePercent <= args.mem_kill_percent && m.SwapFreePercent <= args.swap_kill_percent) { + warn("Low memory! At or below sigkill limits (mem: %d %%, swap: %d %%)\n", + args.mem_kill_percent, args.swap_kill_percent); + sig = SIGKILL; + } else { + warn("Low Memory! At or below sigterm limits (mem: %d %%, swap: %d %%)\n", + args.mem_term_percent, args.swap_term_percent); + sig = SIGTERM; + } + print_mem_stats(1, m); + userspace_kill(args, sig); // With swap enabled, the kernel seems to need more than 100ms to free the memory // of the killed process. This means that earlyoom would immediately kill another // process. Sleep a little extra to give the kernel time to free the memory. @@ -356,7 +336,7 @@ static void poll_loop(const poll_loop_args_t args) report_countdown_ms -= cooldown_ms; } } else if (args.report_interval_ms && report_countdown_ms <= 0) { - print_mem_stats(stdout, m); + print_mem_stats(0, m); report_countdown_ms = args.report_interval_ms; } int sleep_ms = sleep_time_ms(&args, &m); diff --git a/msg.c b/msg.c index 030e687..3909f8d 100644 --- a/msg.c +++ b/msg.c @@ -5,6 +5,8 @@ #include #include +#include "msg.h" + // Print message to stderr and exit with "code". // Example: fatal(6, "could not compile regexp '%s'\n", regex_str); void fatal(int code, char* fmt, ...) @@ -25,8 +27,9 @@ void fatal(int code, char* fmt, ...) } // Print a yellow warning message to stderr. -void warn(char* fmt, ...) +int warn(const char* fmt, ...) { + int ret = 0; char* yellow = ""; char* reset = ""; if (isatty(fileno(stderr))) { @@ -37,6 +40,45 @@ void warn(char* fmt, ...) snprintf(fmt2, sizeof(fmt2), "%s%s%s", yellow, fmt, reset); va_list args; va_start(args, fmt); // yes fmt, NOT fmt2! - vfprintf(stderr, fmt2, args); + ret = vfprintf(stderr, fmt2, args); va_end(args); + return ret; +} + +term_kill_tuple_t parse_term_kill_tuple(char* opt, char* optarg, long upper_limit, int exitcode) +{ + term_kill_tuple_t tuple = { 0 }; + int n; + + n = sscanf(optarg, "%ld,%ld", &tuple.term, &tuple.kill); + if (n == 0) { + fatal(exitcode, "%s: could not parse '%s'\n", opt, optarg); + } + if (tuple.term == 0) { + fatal(exitcode, "%s: zero sigterm value in '%s'\n", opt, optarg); + } + if (tuple.term < 0) { + fatal(exitcode, "%s: negative sigterm value in '%s'\n", opt, optarg); + } + if (tuple.term > upper_limit) { + fatal(exitcode, "%s: sigterm value in '%s' exceeds limit %ld\n", + opt, optarg, upper_limit); + } + // User passed only "term" value + if (n == 1) { + tuple.kill = tuple.term / 2; + return tuple; + } + // User passed "term,kill" values + if (tuple.kill == 0) { + fatal(exitcode, "%s: zero sigkill value in '%s'\n", opt, optarg); + } + if (tuple.kill < 0) { + fatal(exitcode, "%s: negative sigkill value in '%s'\n", opt, optarg); + } + if (tuple.kill > tuple.term) { + fatal(exitcode, "%s: sigkill value exceeds sigterm value in '%s'\n", + opt, optarg); + } + return tuple; } diff --git a/msg.h b/msg.h index c57723b..6dc7f7e 100644 --- a/msg.h +++ b/msg.h @@ -3,6 +3,13 @@ #define MSG_H void fatal(int code, char* fmt, ...); -void warn(char* fmt, ...); +int warn(const char* fmt, ...); + +typedef struct { + long term; + long kill; +} term_kill_tuple_t; + +term_kill_tuple_t parse_term_kill_tuple(char* opt, char* optarg, long upper_limit, int exitcode); #endif diff --git a/tests/cli_test.go b/tests/cli_test.go index d540972..5bccb05 100644 --- a/tests/cli_test.go +++ b/tests/cli_test.go @@ -74,19 +74,19 @@ func TestCli(t *testing.T) { {args: []string{"-v"}, code: 0, stderrContains: "earlyoom v", stdoutEmpty: true}, {args: []string{"-d"}, code: -1, stdoutContains: "^ new victim (higher badness)"}, {args: []string{"-m", "1"}, code: -1, stderrContains: " 1 %", stdoutContains: memReport}, - {args: []string{"-m", "0"}, code: 15, stderrContains: "invalid percentage", stdoutEmpty: true}, - {args: []string{"-m", "-10"}, code: 15, stderrContains: "invalid percentage", stdoutEmpty: true}, + {args: []string{"-m", "0"}, code: 15, stderrContains: "fatal", stdoutEmpty: true}, + {args: []string{"-m", "-10"}, code: 15, stderrContains: "fatal", stdoutEmpty: true}, // Using "-m 100" makes no sense - {args: []string{"-m", "100"}, code: 15, stderrContains: "invalid percentage", stdoutEmpty: true}, + {args: []string{"-m", "100"}, code: 15, stderrContains: "fatal", stdoutEmpty: true}, {args: []string{"-s", "2"}, code: -1, stderrContains: " 2 %", stdoutContains: memReport}, // Using "-s 100" is a valid way to ignore swap usage {args: []string{"-s", "100"}, code: -1, stderrContains: " 100 %", stdoutContains: memReport}, - {args: []string{"-s", "101"}, code: 16, stderrContains: "invalid percentage", stdoutEmpty: true}, - {args: []string{"-s", "0"}, code: 16, stderrContains: "invalid percentage", stdoutEmpty: true}, - {args: []string{"-s", "-10"}, code: 16, stderrContains: "invalid percentage", stdoutEmpty: true}, + {args: []string{"-s", "101"}, code: 16, stderrContains: "fatal", stdoutEmpty: true}, + {args: []string{"-s", "0"}, code: 16, stderrContains: "fatal", stdoutEmpty: true}, + {args: []string{"-s", "-10"}, code: 16, stderrContains: "fatal", stdoutEmpty: true}, {args: []string{"-M", mem1percent}, code: -1, stderrContains: " 1 %", stdoutContains: memReport}, - {args: []string{"-M", "9999999999999999"}, code: 15, stderrContains: "at or above total memory", stdoutEmpty: true}, - {args: []string{"-S", "9999999999999999"}, code: 16, stderrContains: "above total swap", stdoutEmpty: true}, + {args: []string{"-M", "9999999999999999"}, code: 15, stderrContains: "fatal", stdoutEmpty: true}, + {args: []string{"-S", "9999999999999999"}, code: 16, stderrContains: "fatal", stdoutEmpty: true}, {args: []string{"-r", "0"}, code: -1, stderrContains: startupMsg, stdoutEmpty: true}, {args: []string{"-r", "0.1"}, code: -1, stderrContains: startupMsg, stdoutContains: memReport}, // Test --avoid and --prefer @@ -95,11 +95,20 @@ func TestCli(t *testing.T) { // Extra arguments should error out {args: []string{"xyz"}, code: 13, stderrContains: "extra argument not understood", stdoutEmpty: true}, {args: []string{"-i", "1"}, code: 13, stderrContains: "extra argument not understood", stdoutEmpty: true}, + // Tuples + {args: []string{"-m", "2,1"}, code: -1, stderrContains: "sending sigterm at 2 %, sigkill at 1 %", stdoutContains: memReport}, + {args: []string{"-m", "1,2"}, code: 15, stderrContains: "fatal", stdoutEmpty: true}, + {args: []string{"-m", "1,-1"}, code: 15, stderrContains: "fatal", stdoutEmpty: true}, + {args: []string{"-m", "1000,-1000"}, code: 15, stderrContains: "fatal", stdoutEmpty: true}, + {args: []string{"-s", "2,1"}, code: -1, stderrContains: "sending sigterm at 2 %, sigkill at 1 %", stdoutContains: memReport}, + {args: []string{"-s", "1,2"}, code: 16, stderrContains: "fatal", stdoutEmpty: true}, } if swapTotal > 0 { - // Test cannot work when there is no swap enabled - tc := cliTestCase{args: []string{"-S", swap2percent}, code: -1, stderrContains: " 2 %", stdoutContains: memReport} - testcases = append(testcases, tc) + // Tests that cannot work when there is no swap enabled + tc := []cliTestCase{ + cliTestCase{args: []string{"-S", swap2percent}, code: -1, stderrContains: " 2 %", stdoutContains: memReport}, + } + testcases = append(testcases, tc...) } for i, tc := range testcases {