feat: add ability to configure timeouts in submission UI (#143)

Signed-off-by: Samuel Anderson <119458760+AWS-Samuel@users.noreply.github.com>

job_bundle_output_tests/cwd-path/expected_job_bundle/template.yaml

@@ -96,6 +96,7 @@ steps:
           - file://{{ Env.File.initData }}
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 86400
         onExit:
           command: NukeAdaptor
           args:
@@ -105,6 +106,7 @@ steps:
           - '{{ Session.WorkingDirectory }}/connection.json'
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 3600
   script:
     embeddedFiles:
     - name: runData
@@ -123,3 +125,4 @@ steps:
         - file://{{ Task.File.runData }}
         cancelation:
           mode: NOTIFY_THEN_TERMINATE
+        timeout: 518400

job_bundle_output_tests/group-read/expected_job_bundle/template.yaml

@@ -95,6 +95,7 @@ steps:
           - file://{{ Env.File.initData }}
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 86400
         onExit:
           command: NukeAdaptor
           args:
@@ -104,6 +105,7 @@ steps:
           - '{{ Session.WorkingDirectory }}/connection.json'
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 3600
   script:
     embeddedFiles:
     - name: runData
@@ -122,3 +124,4 @@ steps:
         - file://{{ Task.File.runData }}
         cancelation:
           mode: NOTIFY_THEN_TERMINATE
+        timeout: 518400

job_bundle_output_tests/multi-load-save/expected_job_bundle/template.yaml

@@ -98,6 +98,7 @@ steps:
           - file://{{ Env.File.initData }}
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 86400
         onExit:
           command: NukeAdaptor
           args:
@@ -107,6 +108,7 @@ steps:
           - '{{ Session.WorkingDirectory }}/connection.json'
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 3600
   script:
     embeddedFiles:
     - name: runData
@@ -125,3 +127,4 @@ steps:
         - file://{{ Task.File.runData }}
         cancelation:
           mode: NOTIFY_THEN_TERMINATE
+        timeout: 518400

job_bundle_output_tests/noise-saver/expected_job_bundle/template.yaml

@@ -96,6 +96,7 @@ steps:
           - file://{{ Env.File.initData }}
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 86400
         onExit:
           command: NukeAdaptor
           args:
@@ -105,6 +106,7 @@ steps:
           - '{{ Session.WorkingDirectory }}/connection.json'
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 3600
   script:
     embeddedFiles:
     - name: runData
@@ -123,3 +125,4 @@ steps:
         - file://{{ Task.File.runData }}
         cancelation:
           mode: NOTIFY_THEN_TERMINATE
+        timeout: 518400

job_bundle_output_tests/ocio/expected_job_bundle/template.yaml

@@ -96,6 +96,7 @@ steps:
           - file://{{ Env.File.initData }}
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 86400
         onExit:
           command: NukeAdaptor
           args:
@@ -105,6 +106,7 @@ steps:
           - '{{ Session.WorkingDirectory }}/connection.json'
           cancelation:
             mode: NOTIFY_THEN_TERMINATE
+          timeout: 3600
   script:
     embeddedFiles:
     - name: runData
@@ -123,3 +125,4 @@ steps:
         - file://{{ Task.File.runData }}
         cancelation:
           mode: NOTIFY_THEN_TERMINATE
+        timeout: 518400

src/deadline/nuke_submitter/data_classes.py

@@ -31,6 +31,11 @@ class RenderSubmitterUISettings:  # pylint: disable=too-many-instance-attributes
     input_directories: list[str] = field(default_factory=list, metadata={"sticky": True})
     output_directories: list[str] = field(default_factory=list, metadata={"sticky": True})
 
+    timeouts_enabled: bool = field(default=True, metadata={"sticky": True})
+    on_run_timeout_seconds: int = field(default=518400, metadata={"sticky": True})  # 6 days
+    on_enter_timeout_seconds: int = field(default=86400, metadata={"sticky": True})  # 1 day
+    on_exit_timeout_seconds: int = field(default=3600, metadata={"sticky": True})  # 1 hour
+
     # developer options
     include_adaptor_wheels: bool = field(default=False, metadata={"sticky": True})
 

src/deadline/nuke_submitter/deadline_submitter_for_nuke.py

@@ -51,6 +51,33 @@ def _get_write_node(settings: RenderSubmitterUISettings) -> tuple[Node, str]:
     return write_node, settings.write_node_selection
 
 
+def _set_timeouts(template: dict[str, Any], settings: RenderSubmitterUISettings) -> None:
+    """
+    Timeouts are an OpenJD field applicable to actions but for specification 2023-09, timeouts must
+    be hard-coded in the job template. There are three types of actions: OnRun, onEnter, and onExit.
+    This function does an in-place modification of timeout values for each action in the template.
+    """
+
+    def _handle_environment(environment: dict):
+        if "script" in environment:
+            actions = environment["script"]["actions"]
+            actions["onEnter"]["timeout"] = settings.on_enter_timeout_seconds
+            if "onExit" in actions:
+                actions["onExit"]["timeout"] = settings.on_exit_timeout_seconds
+
+    def _handle_step(step: dict):
+        for environment in step.get("stepEnvironments", []):
+            _handle_environment(environment)
+
+        step["script"]["actions"]["onRun"]["timeout"] = settings.on_run_timeout_seconds
+
+    for environment in template.get("jobEnvironments", []):
+        _handle_environment(environment)
+
+    for step in template.get("steps", []):
+        _handle_step(step)
+
+
 def _get_job_template(settings: RenderSubmitterUISettings) -> dict[str, Any]:
     # Load the default Nuke job template, and then fill in scene-specific
     # values it needs.
@@ -62,6 +89,9 @@ def _get_job_template(settings: RenderSubmitterUISettings) -> dict[str, Any]:
     if settings.description:
         job_template["description"] = settings.description
 
+    # Set the timeouts for each action:
+    _set_timeouts(job_template, settings)
+
     # Get a map of the parameter definitions for easier lookup
     parameter_def_map = {param["name"]: param for param in job_template["parameterDefinitions"]}
 
@@ -271,6 +301,20 @@ def on_create_job_bundle_callback(
             if result == QMessageBox.Yes:
                 nuke.scriptSave()
 
+        if settings.timeouts_enabled:
+            message = "The following timeout value(s) must be greater than 0: \n"
+            zero_timeouts = []
+            if not settings.on_run_timeout_seconds:
+                zero_timeouts.append("Render Timeout")
+            if not settings.on_enter_timeout_seconds:
+                zero_timeouts.append("Setup Timeout")
+            if not settings.on_exit_timeout_seconds:
+                zero_timeouts.append("Teardown Timeout")
+            if zero_timeouts:
+                message += ", ".join(zero_timeouts)
+                message += "\n\nPlease configure these value(s) in the 'Job-Specific Settings' tab."
+                raise DeadlineOperationError(message)
+
         job_bundle_path = Path(job_bundle_dir)
         job_template = _get_job_template(settings)
 

src/deadline/nuke_submitter/ui/components/scene_settings_tab.py

@@ -10,11 +10,14 @@
     QCheckBox,
     QComboBox,
     QGridLayout,
+    QGroupBox,
     QLabel,
     QLineEdit,
+    QMessageBox,
     QSizePolicy,
     QSpacerItem,
     QWidget,
+    QSpinBox,
 )
 
 from ...assets import find_all_write_nodes
@@ -48,31 +51,147 @@ def _build_ui(self):
             self.write_node_box.addItem(write_node.fullName(), write_node.fullName())
 
         lyt.addWidget(QLabel("Write Nodes"), 0, 0)
-        lyt.addWidget(self.write_node_box, 0, 1)
+        lyt.addWidget(self.write_node_box, 0, 1, 1, -1)
 
         self.views_box = QComboBox(self)
         self.views_box.addItem("All Views", "")
         for view in sorted(nuke.views()):
             self.views_box.addItem(view, view)
         lyt.addWidget(QLabel("Views"), 1, 0)
-        lyt.addWidget(self.views_box, 1, 1)
+        lyt.addWidget(self.views_box, 1, 1, 1, -1)
 
         self.frame_override_chck = QCheckBox("Override Frame Range", self)
         self.frame_override_txt = QLineEdit(self)
         lyt.addWidget(self.frame_override_chck, 2, 0)
-        lyt.addWidget(self.frame_override_txt, 2, 1)
+        lyt.addWidget(self.frame_override_txt, 2, 1, 1, -1)
         self.frame_override_chck.stateChanged.connect(self.activate_frame_override_changed)
 
         self.proxy_mode_check = QCheckBox("Use Proxy Mode", self)
         lyt.addWidget(self.proxy_mode_check, 3, 0)
 
+        self.timeout_checkbox = QCheckBox("Use Timeouts", self)
+        self.timeout_checkbox.setChecked(True)
+        self.timeout_checkbox.clicked.connect(self.activate_timeout_changed)
+        self.timeout_checkbox.setToolTip(
+            "Set a maximum duration for actions from this job. See AWS Deadline Cloud documentation to learn more"
+        )
+        lyt.addWidget(self.timeout_checkbox, 4, 0)
+        self.timeouts_subtext = QLabel("Set a maximum duration for actions from this job")
+        self.timeouts_subtext.setStyleSheet("font-style: italic")
+        lyt.addWidget(self.timeouts_subtext, 4, 1, 1, -1)
+
+        self.timeouts_box = QGroupBox()
+        timeouts_lyt = QGridLayout(self.timeouts_box)
+        lyt.addWidget(self.timeouts_box, 5, 0, 1, -1)
+
+        def create_timeout_row(label, tooltip, row):
+            qlabel = QLabel(label)
+            qlabel.setToolTip(tooltip)
+            timeouts_lyt.addWidget(qlabel, row, 0)
+
+            days_box = QSpinBox(self, minimum=0, maximum=365)
+            days_box.setSuffix(" days")
+            timeouts_lyt.addWidget(days_box, row, 1)
+
+            hours_box = QSpinBox(self, minimum=0, maximum=23)
+            hours_box.setSuffix(" hours")
+            timeouts_lyt.addWidget(hours_box, row, 2)
+
+            minutes_box = QSpinBox(self, minimum=0, maximum=59)
+            minutes_box.setSuffix(" minutes")
+            timeouts_lyt.addWidget(minutes_box, row, 3)
+
+            return qlabel, days_box, hours_box, minutes_box
+
+        def hookup_zero_callback(timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox]):
+            def indicate_is_valid_callback(value: int):
+                self.indicate_if_valid(timeout_boxes)
+
+            for timeout_box in timeout_boxes[1:]:
+                timeout_box.valueChanged.connect(indicate_is_valid_callback)
+
+        self.on_run_timeouts = create_timeout_row(
+            label="Render Task Timeout",
+            tooltip="Maximum duration of each action which performs a render. Default is 6 days.",
+            row=0,
+        )
+        hookup_zero_callback(self.on_run_timeouts)
+
+        self.on_enter_timeouts = create_timeout_row(
+            label="Setup Timeout",
+            tooltip="Maximum duration of each action which sets up the job for rendering, such as scene load. Default is 1 day.",
+            row=1,
+        )
+        hookup_zero_callback(self.on_enter_timeouts)
+
+        self.on_exit_timeouts = create_timeout_row(
+            label="Teardown Timeout",
+            tooltip="Maximum duration of action which tears down the setup required for rendering. Default is 1 hour.",
+            row=2,
+        )
+        hookup_zero_callback(self.on_exit_timeouts)
+
         if self.developer_options:
             self.include_adaptor_wheels = QCheckBox(
                 "Developer Option: Include Adaptor Wheels", self
             )
-            lyt.addWidget(self.include_adaptor_wheels, 4, 0)
+            lyt.addWidget(self.include_adaptor_wheels, 6, 0, 1, 2)
 
-        lyt.addItem(QSpacerItem(0, 0, QSizePolicy.Minimum, QSizePolicy.Expanding), 10, 0)
+        lyt.addItem(QSpacerItem(0, 0, QSizePolicy.Minimum, QSizePolicy.Expanding), 7, 0)
+
+    def indicate_if_valid(self, timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox]):
+        if (
+            self._calculate_timeout_seconds(timeout_boxes) == 0
+            and self.timeout_checkbox.isChecked()
+        ):
+            timeout_boxes[0].setStyleSheet("color: red")
+        else:
+            timeout_boxes[0].setStyleSheet("")
+
+        # If the spin box has a value of 1, we should not make the suffix plural.
+        for box in timeout_boxes[1:4]:
+            if box.value() == 1:
+                box.setSuffix(box.suffix().strip("s"))
+            elif not box.suffix().endswith("s"):
+                box.setSuffix(box.suffix() + "s")
+
+    def activate_timeout_changed(self, _=None, warn=True):
+        state = self.timeout_checkbox.checkState()
+        if state == Qt.Unchecked and warn:
+            result = QMessageBox.warning(
+                self,
+                "Warning",
+                "Removing timeouts in your submission can result in a task that runs indefinitely. Are you sure you want to remove timeouts?",
+                QMessageBox.Yes | QMessageBox.No,
+                QMessageBox.No,
+            )
+            if result == QMessageBox.No:
+                self.timeout_checkbox.setChecked(True)
+        for timeout_boxes in (self.on_run_timeouts, self.on_enter_timeouts, self.on_exit_timeouts):
+            for timeout_box in timeout_boxes:
+                timeout_box.setEnabled(state == Qt.Checked)
+            self.indicate_if_valid(timeout_boxes)
+
+    def _calculate_timeout_seconds(
+        self, timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox]
+    ):
+        return (
+            timeout_boxes[1].value() * 86400
+            + timeout_boxes[2].value() * 3600
+            + timeout_boxes[3].value() * 60
+        )
+
+    @property
+    def on_run_timeout_seconds(self):
+        return self._calculate_timeout_seconds(self.on_run_timeouts)
+
+    @property
+    def on_enter_timeout_seconds(self):
+        return self._calculate_timeout_seconds(self.on_enter_timeouts)
+
+    @property
+    def on_exit_timeout_seconds(self):
+        return self._calculate_timeout_seconds(self.on_exit_timeouts)
 
     def refresh_ui(self, settings: RenderSubmitterUISettings):
         self.frame_override_chck.setChecked(settings.override_frame_range)
@@ -89,9 +208,27 @@ def refresh_ui(self, settings: RenderSubmitterUISettings):
 
         self.proxy_mode_check.setChecked(settings.is_proxy_mode)
 
+        self.timeout_checkbox.setChecked(settings.timeouts_enabled)
+
+        def _set_timeout(
+            timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox], timeout_seconds: int
+        ):
+            days = timeout_seconds // 86400
+            hours = (timeout_seconds % 86400) // 3600
+            minutes = (timeout_seconds % 3600) // 60
+            timeout_boxes[1].setValue(days)
+            timeout_boxes[2].setValue(hours)
+            timeout_boxes[3].setValue(minutes)
+
+        _set_timeout(self.on_run_timeouts, settings.on_run_timeout_seconds)
+        _set_timeout(self.on_enter_timeouts, settings.on_enter_timeout_seconds)
+        _set_timeout(self.on_exit_timeouts, settings.on_exit_timeout_seconds)
+
         if self.developer_options:
             self.include_adaptor_wheels.setChecked(settings.include_adaptor_wheels)
 
+        self.activate_timeout_changed(warn=False)  # don't warn when loading from sticky settings
+
     def update_settings(self, settings: RenderSubmitterUISettings):
         """
         Update a scene settings object with the latest values.
@@ -103,6 +240,11 @@ def update_settings(self, settings: RenderSubmitterUISettings):
         settings.view_selection = self.views_box.currentData()
         settings.is_proxy_mode = self.proxy_mode_check.isChecked()
 
+        settings.timeouts_enabled = self.timeout_checkbox.isChecked()
+        settings.on_run_timeout_seconds = self.on_run_timeout_seconds
+        settings.on_enter_timeout_seconds = self.on_enter_timeout_seconds
+        settings.on_exit_timeout_seconds = self.on_exit_timeout_seconds
+
         if self.developer_options:
             settings.include_adaptor_wheels = self.include_adaptor_wheels.isChecked()
         else: