diff --git a/CHANGELOG.D/2800.feature b/CHANGELOG.D/2800.feature new file mode 100644 index 000000000..c6f7a2e38 --- /dev/null +++ b/CHANGELOG.D/2800.feature @@ -0,0 +1 @@ +Commands `neuro run`, `neuro logs`, `neuro attach` and `neuro exec` in non-quiet mode now prints details for cancelled and failed jobs. Also improved other indications of the job status. diff --git a/neuro-cli/src/neuro_cli/ael.py b/neuro-cli/src/neuro_cli/ael.py index bfdea17a4..48ae3acfb 100644 --- a/neuro-cli/src/neuro_cli/ael.py +++ b/neuro-cli/src/neuro_cli/ael.py @@ -21,6 +21,7 @@ from prompt_toolkit.keys import Keys from prompt_toolkit.output import Output, create_output from prompt_toolkit.shortcuts import PromptSession +from rich.markup import escape as rich_escape from neuro_sdk import ( JobDescription, @@ -37,7 +38,10 @@ log = logging.getLogger(__name__) -JOB_STARTED = "[dim]===== Job is running, press Ctrl-C to detach/kill =====[/dim]" +JOB_STARTED_NEURO_HAS_TTY = ( + "[green]√[/green] " + "[dim]===== Job is running, press Ctrl-C to detach/kill =====[/dim]" +) JOB_STARTED_NEURO_HAS_NO_TTY = ( "[dim]===== Job is running, press Ctrl-C to detach =====[/dim]" @@ -66,6 +70,7 @@ class InterruptAction(enum.Enum): class AttachHelper: attach_ready: bool log_printed: bool + job_started_msg: str write_sem: asyncio.Semaphore quiet: bool action: InterruptAction @@ -73,6 +78,7 @@ class AttachHelper: def __init__(self, *, quiet: bool) -> None: self.attach_ready = False self.log_printed = False + self.job_started_msg = "" self.write_sem = asyncio.Semaphore() self.quiet = quiet self.action = InterruptAction.NOTHING @@ -109,7 +115,10 @@ async def process_logs( if helper.attach_ready: return async with helper.write_sem: - helper.log_printed = True + if not helper.log_printed: + if not root.quiet: + root.print(helper.job_started_msg, markup=True) + helper.log_printed = True sys.stdout.write(txt) sys.stdout.flush() else: @@ -128,6 +137,10 @@ async def process_exec( finally: root.soft_reset_tty() + if not root.quiet: + status = await root.client.jobs.status(job) + print_job_result(root, status) + sys.exit(exit_code) @@ -278,16 +291,14 @@ async def _process_attach_single_try( root, job.id, logs, cluster_name=job.cluster_name ) - with JobStopProgress.create( - console=root.console, - quiet=root.quiet, - ) as progress: - if action == InterruptAction.KILL: + if action == InterruptAction.KILL: + with JobStopProgress.create(root.console, quiet=root.quiet) as progress: progress.kill(job) - sys.exit(128 + signal.SIGINT) - elif action == InterruptAction.DETACH: + sys.exit(128 + signal.SIGINT) + elif action == InterruptAction.DETACH: + with JobStopProgress.create(root.console, quiet=root.quiet) as progress: progress.detach(job) - sys.exit(0) + sys.exit(0) except ResourceNotFound: # Container already stopped, so we can ignore such error. pass @@ -315,28 +326,25 @@ async def _process_attach_single_try( # The class pins the current time in counstructor, # that's why we need to initialize # it AFTER the disconnection from attached session. - with JobStopProgress.create( - console=root.console, - quiet=root.quiet, - ) as progress: - while job.status == JobStatus.RUNNING: - await asyncio.sleep(0.2) - job = await root.client.jobs.status(job.id) + with JobStopProgress.create(root.console, quiet=root.quiet) as progress: + while not job.status.is_finished: if not progress.step(job): sys.exit(EX_IOERR) - if job.status == JobStatus.FAILED: - sys.exit(job.history.exit_code or EX_PLATFORMERROR) + await asyncio.sleep(0.2) + job = await root.client.jobs.status(job.id) + progress.end(job) + if job.status == JobStatus.FAILED: + sys.exit(job.history.exit_code or EX_PLATFORMERROR) + else: sys.exit(job.history.exit_code) async def _attach_tty( root: Root, job: str, logs: bool, *, cluster_name: Optional[str] ) -> InterruptAction: - if not root.quiet: - root.print(JOB_STARTED_TTY, markup=True) - loop = asyncio.get_event_loop() helper = AttachHelper(quiet=root.quiet) + helper.job_started_msg = JOB_STARTED_TTY stdout = create_output() h, w = stdout.get_size() @@ -357,6 +365,7 @@ async def _attach_tty( if status.status is not JobStatus.RUNNING: # Job is finished await logs_printer + print_job_result(root, status) if status.status == JobStatus.FAILED: sys.exit(status.history.exit_code or EX_PLATFORMERROR) else: @@ -484,18 +493,9 @@ async def _process_stdout_tty( else: txt = decoder.decode(chunk.data) async with helper.write_sem: - if not helper.quiet and not helper.attach_ready: - # Print header to stdout only, - # logs are printed to stdout and never to - # stderr (but logs printing is stopped by - # helper.attach_ready = True regardless - # what stream had receive text in attached mode. - if helper.log_printed: - s = ATTACH_STARTED_AFTER_LOGS - if root.tty: - s = "[green]√[/green] " + s - root.print(s, markup=True) - helper.attach_ready = True + if not helper.attach_ready: + await _print_header(root, helper) + helper.attach_ready = True stdout.write_raw(txt) stdout.flush() @@ -503,14 +503,12 @@ async def _process_stdout_tty( async def _attach_non_tty( root: Root, job: str, logs: bool, *, cluster_name: Optional[str] ) -> InterruptAction: - if not root.quiet: - s = JOB_STARTED_NEURO_HAS_NO_TTY - if root.tty: - s = "[green]√[/green] " + JOB_STARTED - root.print(s, markup=True) - loop = asyncio.get_event_loop() helper = AttachHelper(quiet=root.quiet) + if root.tty: + helper.job_started_msg = JOB_STARTED_NEURO_HAS_TTY + else: + helper.job_started_msg = JOB_STARTED_NEURO_HAS_NO_TTY if logs: logs_printer = loop.create_task( @@ -527,6 +525,7 @@ async def _attach_non_tty( if status.history.exit_code is not None: # Wait for logs printing finish before exit await logs_printer + print_job_result(root, status) sys.exit(status.history.exit_code) input_task = None @@ -580,18 +579,9 @@ async def _process_stdout_non_tty( async def _write(fileno: int, txt: str) -> None: f = streams[fileno] async with helper.write_sem: - if not helper.quiet and not helper.attach_ready: - # Print header to stdout only, - # logs are printed to stdout and never to - # stderr (but logs printing is stopped by - # helper.attach_ready = True regardless - # what stream had receive text in attached mode. - if helper.log_printed: - s = ATTACH_STARTED_AFTER_LOGS - if root.tty: - s = "[green]√[/green] " + s - root.print(s, markup=True) - helper.attach_ready = True + if not helper.attach_ready: + await _print_header(root, helper) + helper.attach_ready = True f.write(txt) f.flush() @@ -608,6 +598,23 @@ async def _write(fileno: int, txt: str) -> None: await _write(chunk.fileno, txt) +async def _print_header(root: Root, helper: AttachHelper) -> None: + if not helper.quiet and not helper.attach_ready: + # Print header to stdout only, + # logs are printed to stdout and never to + # stderr (but logs printing is stopped by + # helper.attach_ready = True regardless + # what stream had receive text in attached mode. + if helper.log_printed: + s = ATTACH_STARTED_AFTER_LOGS + if root.tty: + s = "[green]√[/green] " + s + root.print(s, markup=True) + else: + if not root.quiet: + root.print(helper.job_started_msg, markup=True) + + def _create_interruption_dialog() -> PromptSession[InterruptAction]: bindings = KeyBindings() @@ -701,3 +708,25 @@ async def _cancel_attach_output(root: Root, output_task: "asyncio.Task[Any]") -> if ex and isinstance(ex, StdStreamError): return await root.cancel_with_logging(output_task) + + +def print_job_result(root: Root, job: JobDescription) -> None: + if job.status == JobStatus.SUCCEEDED and root.verbosity > 0: + msg = f"Job [b]{job.id}[/b] finished successfully" + if root.tty: + msg = "[green]√[/green] " + msg + root.print(msg, markup=True) + if job.status == JobStatus.CANCELLED and root.verbosity >= 0: + msg = f"Job [b]{job.id}[/b] was cancelled" + if root.tty: + msg = "[green]√[/green] " + msg + if job.history.reason: + msg += f" ({rich_escape(job.history.reason)})" + root.print(msg, markup=True) + if job.status == JobStatus.FAILED and root.verbosity >= 0: + msg = f"Job [b]{job.id}[/b] failed" + if root.tty: + msg = "[red]×[/red] " + msg + if job.history.reason: + msg += f" ({rich_escape(job.history.reason)})" + root.print(msg, markup=True) diff --git a/neuro-cli/src/neuro_cli/formatters/jobs.py b/neuro-cli/src/neuro_cli/formatters/jobs.py index b98973662..8e9c97a53 100644 --- a/neuro-cli/src/neuro_cli/formatters/jobs.py +++ b/neuro-cli/src/neuro_cli/formatters/jobs.py @@ -542,7 +542,7 @@ def _get_status_reason_message(self, job: JobDescription) -> str: return "" def _get_status_description_message(self, job: JobDescription) -> str: - description = job.history.description or "" + description = job.history.description if description: return f"({description})" return "" @@ -599,13 +599,13 @@ def begin(self, job: JobDescription) -> None: def step(self, job: JobDescription) -> None: new_time = self.time_factory() dt = new_time - self._time - if job.status == JobStatus.PENDING: + if job.status.is_pending: msg = Text("-", "yellow") - elif job.status == JobStatus.FAILED: - msg = Text("×", "red") - else: - # RUNNING or SUCCEDED + elif job.status in (JobStatus.RUNNING, JobStatus.SUCCEEDED): msg = Text("√", "green") + else: + # FAILED or CANCELLED or UNKNOWN + msg = Text("×", "red") msg = Text.assemble(msg, " Status: ", fmt_status(job.status)) reason = self._get_status_reason_message(job) @@ -635,7 +635,7 @@ def end(self, job: JobDescription) -> None: self._prev = empty self._live_render.set_renderable(empty) - if job.status != JobStatus.FAILED: + if not job.status.is_finished: http_url = job.http_url if http_url: out.append(f"{yes()} [b]Http URL[/b]: {rich_escape(str(http_url))}") @@ -739,6 +739,9 @@ def step(self, job: JobDescription) -> bool: def tick(self, job: JobDescription) -> None: pass + def end(self, job: JobDescription) -> None: + pass + def timeout(self, job: JobDescription) -> None: pass @@ -790,18 +793,29 @@ def kill(self, job: JobDescription) -> None: ] ) + def end(self, job: JobDescription) -> None: + if job.status == JobStatus.SUCCEEDED: + msg = yes() + f" Job [b]{job.id}[/b] finished successfully" + elif job.status == JobStatus.CANCELLED: + msg = yes() + f" Job [b]{job.id}[/b] was cancelled" + if job.history.reason: + msg += f" ({rich_escape(job.history.reason)})" + else: + msg = no() + f" Job [b]{job.id}[/b] failed" + if job.history.reason: + msg += f" ({rich_escape(job.history.reason)})" + + self._live_render.set_renderable(Text.from_markup(msg)) + with self._console: + self._console.print(Control()) + def tick(self, job: JobDescription) -> None: new_time = self.time_factory() dt = new_time - self._time - - if job.status == JobStatus.RUNNING: - msg = ( - "[yellow]-[/yellow]" - + f" Wait for stop {next(self._spinner)} [{dt:.1f} sec]" - ) - else: - msg = yes() + f" Job [b]{job.id}[/b] stopped" - + msg = ( + "[yellow]-[/yellow]" + + f" Wait for stop {next(self._spinner)} [{dt:.1f} sec]" + ) self._live_render.set_renderable(Text.from_markup(msg)) with self._console: self._console.print(Control()) @@ -853,7 +867,7 @@ class StreamJobStopProgress(JobStopProgress): def __init__(self, console: Console) -> None: super().__init__() self._console = console - self._console.print("Wait for stopping") + self._first = True def detach(self, job: JobDescription) -> None: pass @@ -861,8 +875,22 @@ def detach(self, job: JobDescription) -> None: def kill(self, job: JobDescription) -> None: self._console.print("Job was killed") + def end(self, job: JobDescription) -> None: + if job.status == JobStatus.CANCELLED: + msg = "Job was cancelled" + if job.history.reason: + msg += f" ({job.history.reason})" + self._console.print(msg) + if job.status == JobStatus.FAILED: + msg = "Job failed" + if job.history.reason: + msg += f" ({job.history.reason})" + self._console.print(msg) + def tick(self, job: JobDescription) -> None: - pass + if self._first: + self._console.print("Wait for stopping") + self._first = False def timeout(self, job: JobDescription) -> None: self._console.print("") diff --git a/neuro-cli/src/neuro_cli/job.py b/neuro-cli/src/neuro_cli/job.py index 9d396be0d..f611bb384 100644 --- a/neuro-cli/src/neuro_cli/job.py +++ b/neuro-cli/src/neuro_cli/job.py @@ -38,7 +38,7 @@ from neuro_cli.parse_utils import parse_sort_keys from neuro_cli.utils import parse_org_name, resolve_disk -from .ael import process_attach, process_exec, process_logs +from .ael import print_job_result, process_attach, process_exec, process_logs from .click_types import ( CLUSTER, JOB, @@ -234,6 +234,9 @@ async def logs(root: Root, since: str, job: str, timestamps: bool) -> None: since=_parse_date(since), timestamps=timestamps, ) + if not root.quiet: + status = await root.client.jobs.status(id) + print_job_result(root, status) @command() @@ -258,11 +261,14 @@ async def attach(root: Root, job: str, port_forward: List[Tuple[int, int]]) -> N status=JobStatus.items(), ) status = await root.client.jobs.status(id) - progress = JobStartProgress.create(console=root.console, quiet=root.quiet) - while status.status.is_pending: - await asyncio.sleep(0.2) - status = await root.client.jobs.status(id) - progress.step(status) + if status.status.is_pending: + with JobStartProgress.create(root.console, quiet=root.quiet) as progress: + progress.step(status) + while status.status.is_pending: + await asyncio.sleep(0.2) + status = await root.client.jobs.status(id) + progress.step(status) + tty = status.container.tty _check_tty(root, tty) @@ -1291,7 +1297,7 @@ async def _force_disk_id(disk_uri: URL) -> URL: await root.client.users.share(user, permission) with JobStartProgress.create(console=root.console, quiet=root.quiet) as progress: progress.begin(job) - while wait_start and job.status == JobStatus.PENDING: + while wait_start and job.status.is_pending: await asyncio.sleep(0.2) job = await root.client.jobs.status(job.id) progress.step(job) diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[cancelled]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[cancelled]_0.ref new file mode 100644 index 000000000..e69de29bb diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[failed]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[failed]_0.ref new file mode 100644 index 000000000..e69de29bb diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[pending]_0.ref similarity index 100% rename from neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end_0.ref rename to neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[pending]_0.ref diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[running]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[running]_0.ref new file mode 100644 index 000000000..b1e56a811 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[running]_0.ref @@ -0,0 +1 @@ +√ Http URL: http://local.host.test/ diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[succeeded]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[succeeded]_0.ref new file mode 100644 index 000000000..e69de29bb diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[suspended]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[suspended]_0.ref new file mode 100644 index 000000000..b1e56a811 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_end[suspended]_0.ref @@ -0,0 +1 @@ +√ Http URL: http://local.host.test/ diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[cancelled]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[cancelled]_0.ref new file mode 100644 index 000000000..dca7ca542 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[cancelled]_0.ref @@ -0,0 +1,2 @@ +- Status: pending Pulling - Status: pending Pulling d [2.0 sec] - Status: pending Pulling +- Status: pending Pulling d [2.0 sec] × Status: cancelled reason diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[failed]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[failed]_0.ref new file mode 100644 index 000000000..f3fe60533 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[failed]_0.ref @@ -0,0 +1,2 @@ +- Status: pending Pulling - Status: pending Pulling d [2.0 sec] - Status: pending Pulling +- Status: pending Pulling d [2.0 sec] × Status: failed reason diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[pending]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[pending]_0.ref new file mode 100644 index 000000000..9d7f9b10b --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[pending]_0.ref @@ -0,0 +1,2 @@ +- Status: pending Pulling - Status: pending Pulling d [2.0 sec] - Status: pending Pulling +- Status: pending Pulling d [2.0 sec] - Status: pending reason diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[running]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[running]_0.ref new file mode 100644 index 000000000..ff4e274c7 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[running]_0.ref @@ -0,0 +1,2 @@ +- Status: pending Pulling - Status: pending Pulling d [2.0 sec] - Status: pending Pulling +- Status: pending Pulling d [2.0 sec] √ Status: running reason diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[succeeded]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[succeeded]_0.ref new file mode 100644 index 000000000..96e8a949e --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[succeeded]_0.ref @@ -0,0 +1,2 @@ +- Status: pending Pulling - Status: pending Pulling d [2.0 sec] - Status: pending Pulling +- Status: pending Pulling d [2.0 sec] √ Status: succeeded reason diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[suspended]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[suspended]_0.ref new file mode 100644 index 000000000..bf26f9593 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step[suspended]_0.ref @@ -0,0 +1,2 @@ +- Status: pending Pulling - Status: pending Pulling d [2.0 sec] - Status: pending Pulling +- Status: pending Pulling d [2.0 sec] - Status: suspended reason diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step_0.ref deleted file mode 100644 index 1a92c9a25..000000000 --- a/neuro-cli/tests/unit/formatters/ascii/TestJobStartProgress.test_tty_step_0.ref +++ /dev/null @@ -1,2 +0,0 @@ -- Status: pending Pulling - Status: pending Pulling ◢ [2.0 sec] - Status: pending Pulling -- Status: pending Pulling ◢ [2.0 sec] √ Status: running reason diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_detach_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_detach_0.ref new file mode 100644 index 000000000..e69de29bb diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_end[cancelled]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_end[cancelled]_0.ref new file mode 100644 index 000000000..015057273 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_end[cancelled]_0.ref @@ -0,0 +1 @@ +Job was cancelled (reason) diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_end[failed]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_end[failed]_0.ref new file mode 100644 index 000000000..d9707c0df --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_end[failed]_0.ref @@ -0,0 +1 @@ +Job failed (reason) diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_end[succeeded]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_end[succeeded]_0.ref new file mode 100644 index 000000000..e69de29bb diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_kill_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_kill_0.ref new file mode 100644 index 000000000..9566b3906 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_kill_0.ref @@ -0,0 +1 @@ +Job was killed diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_step_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_step_0.ref new file mode 100644 index 000000000..b381ae9a1 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_no_tty_step_0.ref @@ -0,0 +1 @@ +Wait for stopping diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_quiet_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_quiet_0.ref new file mode 100644 index 000000000..e69de29bb diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_quiet_1.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_quiet_1.ref new file mode 100644 index 000000000..e69de29bb diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_quiet_2.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_quiet_2.ref new file mode 100644 index 000000000..e69de29bb diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_detach_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_detach_0.ref new file mode 100644 index 000000000..450523a22 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_detach_0.ref @@ -0,0 +1,9 @@ +× Terminal was detached but job is still running + Re-attach to job: + neuro attach test-job + Check job status: + neuro status test-job + Kill job: + neuro kill test-job + Fetch job logs: + neuro logs test-job diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[cancelled]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[cancelled]_0.ref new file mode 100644 index 000000000..e2e5017f4 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[cancelled]_0.ref @@ -0,0 +1 @@ +√ Job test-job was cancelled (reason) diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[failed]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[failed]_0.ref new file mode 100644 index 000000000..027fdea09 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[failed]_0.ref @@ -0,0 +1 @@ +× Job test-job failed (reason) diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[succeeded]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[succeeded]_0.ref new file mode 100644 index 000000000..ad246113c --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_end[succeeded]_0.ref @@ -0,0 +1 @@ +√ Job test-job finished successfully diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_kill_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_kill_0.ref new file mode 100644 index 000000000..549a03af7 --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_kill_0.ref @@ -0,0 +1,5 @@ +× Job was killed + Get job status: + neuro status test-job + Fetch job logs: + neuro logs test-job diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[pending]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[pending]_0.ref new file mode 100644 index 000000000..52d7a81ae --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[pending]_0.ref @@ -0,0 +1 @@ +- Wait for stop d [2.0 sec] - Wait for stop q [4.0 sec] diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[running]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[running]_0.ref new file mode 100644 index 000000000..52d7a81ae --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[running]_0.ref @@ -0,0 +1 @@ +- Wait for stop d [2.0 sec] - Wait for stop q [4.0 sec] diff --git a/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[suspended]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[suspended]_0.ref new file mode 100644 index 000000000..52d7a81ae --- /dev/null +++ b/neuro-cli/tests/unit/formatters/ascii/TestJobStopProgress.test_tty_step[suspended]_0.ref @@ -0,0 +1 @@ +- Wait for stop d [2.0 sec] - Wait for stop q [4.0 sec] diff --git a/neuro-cli/tests/unit/formatters/test_jobs_formatters.py b/neuro-cli/tests/unit/formatters/test_jobs_formatters.py index 7cbb97b07..1ca5a3618 100644 --- a/neuro-cli/tests/unit/formatters/test_jobs_formatters.py +++ b/neuro-cli/tests/unit/formatters/test_jobs_formatters.py @@ -1,6 +1,5 @@ import io import itertools -import sys from dataclasses import replace from datetime import datetime, timedelta, timezone from decimal import Decimal @@ -28,9 +27,11 @@ Volume, ) +import neuro_cli.formatters.jobs from neuro_cli.formatters.jobs import ( JobStartProgress, JobStatusFormatter, + JobStopProgress, JobTelemetryFormatter, LifeSpanUpdateFormatter, SimpleJobsFormatter, @@ -145,57 +146,57 @@ def job_descr() -> JobDescription: ) -class TestJobStartProgress: - def make_job( - self, - status: JobStatus, - reason: str, - *, - name: Optional[str] = None, - life_span: Optional[float] = None, - description: str = "ErrorDesc", - total_price_credits: Decimal = Decimal("150"), - price_credits_per_hour: Decimal = Decimal("15"), - ) -> JobDescription: - return JobDescription( - name=name, +def make_job( + status: JobStatus, + reason: str, + *, + name: Optional[str] = None, + life_span: Optional[float] = None, + description: str = "ErrorDesc", + total_price_credits: Decimal = Decimal("150"), + price_credits_per_hour: Decimal = Decimal("15"), +) -> JobDescription: + return JobDescription( + name=name, + status=status, + owner="test-user", + cluster_name="default", + id="test-job", + uri=URL("job://default/test-user/test-job"), + description="test job description", + http_url=URL("http://local.host.test/"), + history=JobStatusHistory( status=status, - owner="test-user", - cluster_name="default", - id="test-job", - uri=URL("job://default/test-user/test-job"), - description="test job description", - http_url=URL("http://local.host.test/"), - history=JobStatusHistory( - status=status, - reason=reason, - description=description, - created_at=isoparse("2018-09-25T12:28:21.298672+00:00"), - started_at=isoparse("2018-09-25T12:28:59.759433+00:00"), - finished_at=isoparse("2018-09-25T12:28:59.759433+00:00"), - ), - container=Container( - command="test-command", - image=RemoteImage.new_external_image(name="test-image"), - resources=Resources( - 16, - 0.1, - 4, - "nvidia-tesla-p4", - True, - tpu_type="v2-8", - tpu_software_version="1.14", - ), + reason=reason, + description=description, + created_at=isoparse("2018-09-25T12:28:21.298672+00:00"), + started_at=isoparse("2018-09-25T12:28:59.759433+00:00"), + finished_at=isoparse("2018-09-25T12:28:59.759433+00:00"), + ), + container=Container( + command="test-command", + image=RemoteImage.new_external_image(name="test-image"), + resources=Resources( + 16, + 0.1, + 4, + "nvidia-tesla-p4", + True, + tpu_type="v2-8", + tpu_software_version="1.14", ), - scheduler_enabled=False, - pass_config=True, - life_span=life_span, - total_price_credits=total_price_credits, - price_credits_per_hour=price_credits_per_hour, - ) + ), + scheduler_enabled=False, + pass_config=True, + life_span=life_span, + total_price_credits=total_price_credits, + price_credits_per_hour=price_credits_per_hour, + ) + +class TestJobStartProgress: def test_quiet(self, rich_cmp: Any, new_console: _NewConsole) -> None: - job = self.make_job(JobStatus.PENDING, "") + job = make_job(JobStatus.PENDING, "") console = new_console(tty=True, color=True) with JobStartProgress.create(console, quiet=True) as progress: progress.begin(job) @@ -208,7 +209,7 @@ def test_quiet(self, rich_cmp: Any, new_console: _NewConsole) -> None: def test_no_tty_begin(self, rich_cmp: Any, new_console: _NewConsole) -> None: console = new_console(tty=False, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.begin(self.make_job(JobStatus.PENDING, "")) + progress.begin(make_job(JobStatus.PENDING, "")) rich_cmp(console) def test_no_tty_begin_with_name( @@ -216,55 +217,63 @@ def test_no_tty_begin_with_name( ) -> None: console = new_console(tty=False, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.begin(self.make_job(JobStatus.PENDING, "", name="job-name")) + progress.begin(make_job(JobStatus.PENDING, "", name="job-name")) rich_cmp(console) def test_no_tty_step(self, rich_cmp: Any, new_console: _NewConsole) -> None: console = new_console(tty=False, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.step(self.make_job(JobStatus.PENDING, "")) - progress.step(self.make_job(JobStatus.PENDING, "")) - progress.step(self.make_job(JobStatus.RUNNING, "reason")) + progress.step(make_job(JobStatus.PENDING, "")) + progress.step(make_job(JobStatus.PENDING, "")) + progress.step(make_job(JobStatus.RUNNING, "reason")) rich_cmp(console) def test_no_tty_end(self, rich_cmp: Any, new_console: _NewConsole) -> None: console = new_console(tty=False, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.end(self.make_job(JobStatus.RUNNING, "")) + progress.end(make_job(JobStatus.RUNNING, "")) rich_cmp(console) def test_tty_begin(self, rich_cmp: Any, new_console: _NewConsole) -> None: console = new_console(tty=True, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.begin(self.make_job(JobStatus.PENDING, "")) + progress.begin(make_job(JobStatus.PENDING, "")) rich_cmp(console) def test_tty_begin_with_name(self, rich_cmp: Any, new_console: _NewConsole) -> None: console = new_console(tty=True, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.begin(self.make_job(JobStatus.PENDING, "", name="job-name")) + progress.begin(make_job(JobStatus.PENDING, "", name="job-name")) rich_cmp(console) - @pytest.mark.skipif( - sys.platform == "win32", reason="On Windows spinner uses another characters set" - ) + @pytest.mark.parametrize("status", JobStatus.items()) def test_tty_step( - self, rich_cmp: Any, new_console: _NewConsole, monkeypatch: Any + self, + rich_cmp: Any, + new_console: _NewConsole, + monkeypatch: Any, + status: JobStatus, ) -> None: monkeypatch.setattr( JobStartProgress, "time_factory", itertools.count(10).__next__ ) + monkeypatch.setattr( + neuro_cli.formatters.jobs, "SPINNER", itertools.cycle("dqpb") + ) console = new_console(tty=True, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.step(self.make_job(JobStatus.PENDING, "Pulling", description="")) - progress.step(self.make_job(JobStatus.PENDING, "Pulling", description="")) - progress.step(self.make_job(JobStatus.RUNNING, "reason", description="")) + progress.step(make_job(JobStatus.PENDING, "Pulling", description="")) + progress.step(make_job(JobStatus.PENDING, "Pulling", description="")) + progress.step(make_job(status, "reason", description="")) rich_cmp(console) - def test_tty_end(self, rich_cmp: Any, new_console: _NewConsole) -> None: + @pytest.mark.parametrize("status", JobStatus.items()) + def test_tty_end( + self, rich_cmp: Any, new_console: _NewConsole, status: JobStatus + ) -> None: console = new_console(tty=True, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.end(self.make_job(JobStatus.RUNNING, "")) + progress.end(make_job(status, "reason")) rich_cmp(console) def test_tty_end_with_life_span( @@ -272,7 +281,91 @@ def test_tty_end_with_life_span( ) -> None: console = new_console(tty=True, color=True) with JobStartProgress.create(console, quiet=False) as progress: - progress.end(self.make_job(JobStatus.RUNNING, "", life_span=24 * 3600)) + progress.end(make_job(JobStatus.RUNNING, "", life_span=24 * 3600)) + + +class TestJobStopProgress: + def test_quiet(self, rich_cmp: Any, new_console: _NewConsole) -> None: + job = make_job(JobStatus.RUNNING, "") + console = new_console(tty=True, color=True) + with JobStopProgress.create(console, quiet=True) as progress: + progress.step(job) + rich_cmp(console, index=0) + progress.step(job) + rich_cmp(console, index=1) + progress.end(make_job(JobStatus.FAILED, "OOMKilled")) + rich_cmp(console, index=2) + + def test_no_tty_step(self, rich_cmp: Any, new_console: _NewConsole) -> None: + console = new_console(tty=False, color=True) + with JobStopProgress.create(console, quiet=False) as progress: + progress.step(make_job(JobStatus.RUNNING, "")) + progress.step(make_job(JobStatus.RUNNING, "")) + progress.step(make_job(JobStatus.RUNNING, "")) + rich_cmp(console) + + @pytest.mark.parametrize("status", JobStatus.finished_items()) + def test_no_tty_end( + self, rich_cmp: Any, new_console: _NewConsole, status: JobStatus + ) -> None: + console = new_console(tty=False, color=True) + with JobStopProgress.create(console, quiet=False) as progress: + progress.end(make_job(status, "reason")) + rich_cmp(console) + + @pytest.mark.parametrize("status", JobStatus.active_items()) + def test_tty_step( + self, + rich_cmp: Any, + new_console: _NewConsole, + monkeypatch: Any, + status: JobStatus, + ) -> None: + monkeypatch.setattr( + JobStopProgress, "time_factory", itertools.count(10).__next__ + ) + monkeypatch.setattr( + neuro_cli.formatters.jobs, "SPINNER", itertools.cycle("dqpb") + ) + console = new_console(tty=True, color=True) + with JobStopProgress.create(console, quiet=False) as progress: + progress.step(make_job(JobStatus.RUNNING, "", description="")) + progress.step(make_job(status, "reason", description="")) + rich_cmp(console) + progress.step(make_job(JobStatus.RUNNING, "", description="")) + + @pytest.mark.parametrize("status", JobStatus.finished_items()) + def test_tty_end( + self, rich_cmp: Any, new_console: _NewConsole, status: JobStatus + ) -> None: + console = new_console(tty=True, color=True) + with JobStopProgress.create(console, quiet=False) as progress: + progress.end(make_job(status, "reason")) + rich_cmp(console) + + def test_no_tty_detach(self, rich_cmp: Any, new_console: _NewConsole) -> None: + console = new_console(tty=False, color=True) + with JobStopProgress.create(console, quiet=False) as progress: + progress.detach(make_job(JobStatus.RUNNING, "")) + rich_cmp(console) + + def test_tty_detach(self, rich_cmp: Any, new_console: _NewConsole) -> None: + console = new_console(tty=True, color=True) + with JobStopProgress.create(console, quiet=False) as progress: + progress.detach(make_job(JobStatus.RUNNING, "")) + rich_cmp(console) + + def test_no_tty_kill(self, rich_cmp: Any, new_console: _NewConsole) -> None: + console = new_console(tty=False, color=True) + with JobStopProgress.create(console, quiet=False) as progress: + progress.kill(make_job(JobStatus.RUNNING, "")) + rich_cmp(console) + + def test_tty_kill(self, rich_cmp: Any, new_console: _NewConsole) -> None: + console = new_console(tty=True, color=True) + with JobStopProgress.create(console, quiet=False) as progress: + progress.kill(make_job(JobStatus.RUNNING, "")) + rich_cmp(console) class TestJobOutputFormatter: