From e97afec40e8a102fa1a54b8165231d9882c489e4 Mon Sep 17 00:00:00 2001 From: Samuel Anderson <119458760+AWS-Samuel@users.noreply.github.com> Date: Tue, 14 May 2024 15:08:19 +0000 Subject: [PATCH] feat: add ability to configure timeouts in submission UI Signed-off-by: Samuel Anderson <119458760+AWS-Samuel@users.noreply.github.com> --- .../expected_job_bundle/template.yaml | 3 + .../expected_job_bundle/template.yaml | 3 + .../expected_job_bundle/template.yaml | 3 + .../expected_job_bundle/template.yaml | 3 + .../ocio/expected_job_bundle/template.yaml | 3 + src/deadline/nuke_submitter/data_classes.py | 5 + .../deadline_submitter_for_nuke.py | 44 +++++ .../ui/components/scene_settings_tab.py | 152 +++++++++++++++++- 8 files changed, 211 insertions(+), 5 deletions(-) diff --git a/job_bundle_output_tests/cwd-path/expected_job_bundle/template.yaml b/job_bundle_output_tests/cwd-path/expected_job_bundle/template.yaml index 502fd4b..af47942 100644 --- a/job_bundle_output_tests/cwd-path/expected_job_bundle/template.yaml +++ b/job_bundle_output_tests/cwd-path/expected_job_bundle/template.yaml @@ -96,6 +96,7 @@ steps: - file://{{ Env.File.initData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 86400 onExit: command: NukeAdaptor args: @@ -105,6 +106,7 @@ steps: - '{{ Session.WorkingDirectory }}/connection.json' cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 3600 script: embeddedFiles: - name: runData @@ -123,3 +125,4 @@ steps: - file://{{ Task.File.runData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 518400 \ No newline at end of file diff --git a/job_bundle_output_tests/group-read/expected_job_bundle/template.yaml b/job_bundle_output_tests/group-read/expected_job_bundle/template.yaml index 722e5ab..4da29da 100644 --- a/job_bundle_output_tests/group-read/expected_job_bundle/template.yaml +++ b/job_bundle_output_tests/group-read/expected_job_bundle/template.yaml @@ -95,6 +95,7 @@ steps: - file://{{ Env.File.initData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 86400 onExit: command: NukeAdaptor args: @@ -104,6 +105,7 @@ steps: - '{{ Session.WorkingDirectory }}/connection.json' cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 3600 script: embeddedFiles: - name: runData @@ -122,3 +124,4 @@ steps: - file://{{ Task.File.runData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 518400 diff --git a/job_bundle_output_tests/multi-load-save/expected_job_bundle/template.yaml b/job_bundle_output_tests/multi-load-save/expected_job_bundle/template.yaml index 62f2472..ea685e1 100644 --- a/job_bundle_output_tests/multi-load-save/expected_job_bundle/template.yaml +++ b/job_bundle_output_tests/multi-load-save/expected_job_bundle/template.yaml @@ -98,6 +98,7 @@ steps: - file://{{ Env.File.initData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 86400 onExit: command: NukeAdaptor args: @@ -107,6 +108,7 @@ steps: - '{{ Session.WorkingDirectory }}/connection.json' cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 3600 script: embeddedFiles: - name: runData @@ -125,3 +127,4 @@ steps: - file://{{ Task.File.runData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 518400 diff --git a/job_bundle_output_tests/noise-saver/expected_job_bundle/template.yaml b/job_bundle_output_tests/noise-saver/expected_job_bundle/template.yaml index d37d7a4..807e48a 100644 --- a/job_bundle_output_tests/noise-saver/expected_job_bundle/template.yaml +++ b/job_bundle_output_tests/noise-saver/expected_job_bundle/template.yaml @@ -96,6 +96,7 @@ steps: - file://{{ Env.File.initData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 86400 onExit: command: NukeAdaptor args: @@ -105,6 +106,7 @@ steps: - '{{ Session.WorkingDirectory }}/connection.json' cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 3600 script: embeddedFiles: - name: runData @@ -123,3 +125,4 @@ steps: - file://{{ Task.File.runData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 518400 diff --git a/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml b/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml index 122697f..809e93d 100644 --- a/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml +++ b/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml @@ -96,6 +96,7 @@ steps: - file://{{ Env.File.initData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 86400 onExit: command: NukeAdaptor args: @@ -105,6 +106,7 @@ steps: - '{{ Session.WorkingDirectory }}/connection.json' cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 3600 script: embeddedFiles: - name: runData @@ -123,3 +125,4 @@ steps: - file://{{ Task.File.runData }} cancelation: mode: NOTIFY_THEN_TERMINATE + timeout: 518400 diff --git a/src/deadline/nuke_submitter/data_classes.py b/src/deadline/nuke_submitter/data_classes.py index e9daa5d..ead2440 100644 --- a/src/deadline/nuke_submitter/data_classes.py +++ b/src/deadline/nuke_submitter/data_classes.py @@ -31,6 +31,11 @@ class RenderSubmitterUISettings: # pylint: disable=too-many-instance-attributes input_directories: list[str] = field(default_factory=list, metadata={"sticky": True}) output_directories: list[str] = field(default_factory=list, metadata={"sticky": True}) + timeouts_enabled: bool = field(default=True, metadata={"sticky": True}) + on_run_timeout_seconds: int = field(default=518400, metadata={"sticky": True}) # 6 days + on_enter_timeout_seconds: int = field(default=86400, metadata={"sticky": True}) # 1 day + on_exit_timeout_seconds: int = field(default=3600, metadata={"sticky": True}) # 1 hour + # developer options include_adaptor_wheels: bool = field(default=False, metadata={"sticky": True}) diff --git a/src/deadline/nuke_submitter/deadline_submitter_for_nuke.py b/src/deadline/nuke_submitter/deadline_submitter_for_nuke.py index 73e526f..ed60bfb 100644 --- a/src/deadline/nuke_submitter/deadline_submitter_for_nuke.py +++ b/src/deadline/nuke_submitter/deadline_submitter_for_nuke.py @@ -51,6 +51,33 @@ def _get_write_node(settings: RenderSubmitterUISettings) -> tuple[Node, str]: return write_node, settings.write_node_selection +def _set_timeouts(template: dict[str, Any], settings: RenderSubmitterUISettings) -> None: + """ + Timeouts are an OpenJD field applicable to actions but for specification 2023-09, timeouts must + be hard-coded in the job template. There are three types of actions: OnRun, onEnter, and onExit. + This function does an in-place modification of timeout values for each action in the template. + """ + + def _handle_environment(environment: dict): + if "script" in environment: + actions = environment["script"]["actions"] + actions["onEnter"]["timeout"] = settings.on_enter_timeout_seconds + if "onExit" in actions: + actions["onExit"]["timeout"] = settings.on_exit_timeout_seconds + + def _handle_step(step: dict): + for environment in step.get("stepEnvironments", []): + _handle_environment(environment) + + step["script"]["actions"]["onRun"]["timeout"] = settings.on_run_timeout_seconds + + for environment in template.get("jobEnvironments", []): + _handle_environment(environment) + + for step in template.get("steps", []): + _handle_step(step) + + def _get_job_template(settings: RenderSubmitterUISettings) -> dict[str, Any]: # Load the default Nuke job template, and then fill in scene-specific # values it needs. @@ -62,6 +89,9 @@ def _get_job_template(settings: RenderSubmitterUISettings) -> dict[str, Any]: if settings.description: job_template["description"] = settings.description + # Set the timeouts for each action: + _set_timeouts(job_template, settings) + # Get a map of the parameter definitions for easier lookup parameter_def_map = {param["name"]: param for param in job_template["parameterDefinitions"]} @@ -271,6 +301,20 @@ def on_create_job_bundle_callback( if result == QMessageBox.Yes: nuke.scriptSave() + if settings.timeouts_enabled: + message = "The following timeout value(s) must be greater than 0: \n" + zero_timeouts = [] + if not settings.on_run_timeout_seconds: + zero_timeouts.append("Render Timeout") + if not settings.on_enter_timeout_seconds: + zero_timeouts.append("Setup Timeout") + if not settings.on_exit_timeout_seconds: + zero_timeouts.append("Teardown Timeout") + if zero_timeouts: + message += ", ".join(zero_timeouts) + message += "\n\nPlease configure these value(s) in the 'Job-Specific Settings' tab." + raise DeadlineOperationError(message) + job_bundle_path = Path(job_bundle_dir) job_template = _get_job_template(settings) diff --git a/src/deadline/nuke_submitter/ui/components/scene_settings_tab.py b/src/deadline/nuke_submitter/ui/components/scene_settings_tab.py index a5a428b..5e98cf6 100644 --- a/src/deadline/nuke_submitter/ui/components/scene_settings_tab.py +++ b/src/deadline/nuke_submitter/ui/components/scene_settings_tab.py @@ -10,11 +10,14 @@ QCheckBox, QComboBox, QGridLayout, + QGroupBox, QLabel, QLineEdit, + QMessageBox, QSizePolicy, QSpacerItem, QWidget, + QSpinBox, ) from ...assets import find_all_write_nodes @@ -48,31 +51,147 @@ def _build_ui(self): self.write_node_box.addItem(write_node.fullName(), write_node.fullName()) lyt.addWidget(QLabel("Write Nodes"), 0, 0) - lyt.addWidget(self.write_node_box, 0, 1) + lyt.addWidget(self.write_node_box, 0, 1, 1, -1) self.views_box = QComboBox(self) self.views_box.addItem("All Views", "") for view in sorted(nuke.views()): self.views_box.addItem(view, view) lyt.addWidget(QLabel("Views"), 1, 0) - lyt.addWidget(self.views_box, 1, 1) + lyt.addWidget(self.views_box, 1, 1, 1, -1) self.frame_override_chck = QCheckBox("Override Frame Range", self) self.frame_override_txt = QLineEdit(self) lyt.addWidget(self.frame_override_chck, 2, 0) - lyt.addWidget(self.frame_override_txt, 2, 1) + lyt.addWidget(self.frame_override_txt, 2, 1, 1, -1) self.frame_override_chck.stateChanged.connect(self.activate_frame_override_changed) self.proxy_mode_check = QCheckBox("Use Proxy Mode", self) lyt.addWidget(self.proxy_mode_check, 3, 0) + self.timeout_checkbox = QCheckBox("Use Timeouts", self) + self.timeout_checkbox.setChecked(True) + self.timeout_checkbox.clicked.connect(self.activate_timeout_changed) + self.timeout_checkbox.setToolTip( + "Set a maximum duration for actions from this job. See AWS Deadline Cloud documentation to learn more" + ) + lyt.addWidget(self.timeout_checkbox, 4, 0) + self.timeouts_subtext = QLabel("Set a maximum duration for actions from this job") + self.timeouts_subtext.setStyleSheet("font-style: italic") + lyt.addWidget(self.timeouts_subtext, 4, 1, 1, -1) + + self.timeouts_box = QGroupBox() + timeouts_lyt = QGridLayout(self.timeouts_box) + lyt.addWidget(self.timeouts_box, 5, 0, 1, -1) + + def create_timeout_row(label, tooltip, row): + qlabel = QLabel(label) + qlabel.setToolTip(tooltip) + timeouts_lyt.addWidget(qlabel, row, 0) + + days_box = QSpinBox(self, minimum=0, maximum=365) + days_box.setSuffix(" days") + timeouts_lyt.addWidget(days_box, row, 1) + + hours_box = QSpinBox(self, minimum=0, maximum=23) + hours_box.setSuffix(" hours") + timeouts_lyt.addWidget(hours_box, row, 2) + + minutes_box = QSpinBox(self, minimum=0, maximum=59) + minutes_box.setSuffix(" minutes") + timeouts_lyt.addWidget(minutes_box, row, 3) + + return qlabel, days_box, hours_box, minutes_box + + def hookup_zero_callback(timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox]): + def indicate_is_valid_callback(value: int): + self.indicate_if_valid(timeout_boxes) + + for timeout_box in timeout_boxes[1:]: + timeout_box.valueChanged.connect(indicate_is_valid_callback) + + self.on_run_timeouts = create_timeout_row( + label="Render Task Timeout", + tooltip="Maximum duration of each action which performs a render. Default is 6 days.", + row=0, + ) + hookup_zero_callback(self.on_run_timeouts) + + self.on_enter_timeouts = create_timeout_row( + label="Setup Timeout", + tooltip="Maximum duration of each action which sets up the job for rendering, such as scene load. Default is 1 day.", + row=1, + ) + hookup_zero_callback(self.on_enter_timeouts) + + self.on_exit_timeouts = create_timeout_row( + label="Teardown Timeout", + tooltip="Maximum duration of action which tears down the setup required for rendering. Default is 1 hour.", + row=2, + ) + hookup_zero_callback(self.on_exit_timeouts) + if self.developer_options: self.include_adaptor_wheels = QCheckBox( "Developer Option: Include Adaptor Wheels", self ) - lyt.addWidget(self.include_adaptor_wheels, 4, 0) + lyt.addWidget(self.include_adaptor_wheels, 6, 0, 1, 2) - lyt.addItem(QSpacerItem(0, 0, QSizePolicy.Minimum, QSizePolicy.Expanding), 10, 0) + lyt.addItem(QSpacerItem(0, 0, QSizePolicy.Minimum, QSizePolicy.Expanding), 7, 0) + + def indicate_if_valid(self, timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox]): + if ( + self._calculate_timeout_seconds(timeout_boxes) == 0 + and self.timeout_checkbox.isChecked() + ): + timeout_boxes[0].setStyleSheet("color: red") + else: + timeout_boxes[0].setStyleSheet("") + + # If the spin box has a value of 1, we should not make the suffix plural. + for box in timeout_boxes[1:4]: + if box.value() == 1: + box.setSuffix(box.suffix().strip("s")) + elif not box.suffix().endswith("s"): + box.setSuffix(box.suffix() + "s") + + def activate_timeout_changed(self, _=None, warn=True): + state = self.timeout_checkbox.checkState() + if state == Qt.Unchecked and warn: + result = QMessageBox.warning( + self, + "Warning", + "Removing timeouts in your submission can result in a task that runs indefinitely. Are you sure you want to remove timeouts?", + QMessageBox.Yes | QMessageBox.No, + QMessageBox.No, + ) + if result == QMessageBox.No: + self.timeout_checkbox.setChecked(True) + for timeout_boxes in (self.on_run_timeouts, self.on_enter_timeouts, self.on_exit_timeouts): + for timeout_box in timeout_boxes: + timeout_box.setEnabled(state == Qt.Checked) + self.indicate_if_valid(timeout_boxes) + + def _calculate_timeout_seconds( + self, timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox] + ): + return ( + timeout_boxes[1].value() * 86400 + + timeout_boxes[2].value() * 3600 + + timeout_boxes[3].value() * 60 + ) + + @property + def on_run_timeout_seconds(self): + return self._calculate_timeout_seconds(self.on_run_timeouts) + + @property + def on_enter_timeout_seconds(self): + return self._calculate_timeout_seconds(self.on_enter_timeouts) + + @property + def on_exit_timeout_seconds(self): + return self._calculate_timeout_seconds(self.on_exit_timeouts) def refresh_ui(self, settings: RenderSubmitterUISettings): self.frame_override_chck.setChecked(settings.override_frame_range) @@ -89,9 +208,27 @@ def refresh_ui(self, settings: RenderSubmitterUISettings): self.proxy_mode_check.setChecked(settings.is_proxy_mode) + self.timeout_checkbox.setChecked(settings.timeouts_enabled) + + def _set_timeout( + timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox], timeout_seconds: int + ): + days = timeout_seconds // 86400 + hours = (timeout_seconds % 86400) // 3600 + minutes = (timeout_seconds % 3600) // 60 + timeout_boxes[1].setValue(days) + timeout_boxes[2].setValue(hours) + timeout_boxes[3].setValue(minutes) + + _set_timeout(self.on_run_timeouts, settings.on_run_timeout_seconds) + _set_timeout(self.on_enter_timeouts, settings.on_enter_timeout_seconds) + _set_timeout(self.on_exit_timeouts, settings.on_exit_timeout_seconds) + if self.developer_options: self.include_adaptor_wheels.setChecked(settings.include_adaptor_wheels) + self.activate_timeout_changed(warn=False) # don't warn when loading from sticky settings + def update_settings(self, settings: RenderSubmitterUISettings): """ Update a scene settings object with the latest values. @@ -103,6 +240,11 @@ def update_settings(self, settings: RenderSubmitterUISettings): settings.view_selection = self.views_box.currentData() settings.is_proxy_mode = self.proxy_mode_check.isChecked() + settings.timeouts_enabled = self.timeout_checkbox.isChecked() + settings.on_run_timeout_seconds = self.on_run_timeout_seconds + settings.on_enter_timeout_seconds = self.on_enter_timeout_seconds + settings.on_exit_timeout_seconds = self.on_exit_timeout_seconds + if self.developer_options: settings.include_adaptor_wheels = self.include_adaptor_wheels.isChecked() else: