From 5db21802804f11e71c1e360a7adf67e8cae0c9a2 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 21 Nov 2017 16:40:56 -0500 Subject: [PATCH 1/2] Extra way to check OOM of a command ran inside Docker Easy way to reproduce this error: 1. Import a project that compiles lxml (Read the Docs, for example) 2. The `pip` command will be killed and Docker won't return a specific status code, it will just return 1 3. In these cases, the `Killed` word will probably appear in the last 15 lines --- readthedocs/doc_builder/environments.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index d453c895448..df2e9529cc9 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -224,9 +224,12 @@ def run(self): self.exit_code = cmd_ret['ExitCode'] # Docker will exit with a special exit code to signify the command - # was killed due to memory usage, make the error code nicer. - if (self.exit_code == DOCKER_OOM_EXIT_CODE and - self.output == 'Killed\n'): + # was killed due to memory usage, make the error code + # nicer. Sometimes the kernel kills the command and Docker doesn't + # not use the specific exit code, so we check if the word `Killed` + # is in the last 15 lines of the command's output + killed_in_output = 'Killed' in '\n'.join(self.output.splitlines()[-15:]) + if self.exit_code == DOCKER_OOM_EXIT_CODE or killed_in_output: self.output = _('Command killed due to excessive memory ' 'consumption\n') except DockerAPIError: From 3e15665ef4b56cb01738b3207631187db79ac1e3 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 21 Nov 2017 18:15:34 -0500 Subject: [PATCH 2/2] Check status code and 'Killed' word --- readthedocs/doc_builder/environments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index df2e9529cc9..e6833e881f4 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -229,7 +229,7 @@ def run(self): # not use the specific exit code, so we check if the word `Killed` # is in the last 15 lines of the command's output killed_in_output = 'Killed' in '\n'.join(self.output.splitlines()[-15:]) - if self.exit_code == DOCKER_OOM_EXIT_CODE or killed_in_output: + if self.exit_code == DOCKER_OOM_EXIT_CODE or (self.exit_code == 1 and killed_in_output): self.output = _('Command killed due to excessive memory ' 'consumption\n') except DockerAPIError: