From d612a1744750c6af5280ef482ca8fce977ee64d7 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Fri, 8 Sep 2023 23:24:25 +0000 Subject: [PATCH 01/12] Add custom OCIO config to job attachments when enabled Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- src/deadline/nuke_submitter/assets.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/deadline/nuke_submitter/assets.py b/src/deadline/nuke_submitter/assets.py index 5c6bbb2..57a1a35 100644 --- a/src/deadline/nuke_submitter/assets.py +++ b/src/deadline/nuke_submitter/assets.py @@ -9,6 +9,7 @@ from sys import platform import nuke +import PyOpenColorIO as OCIO from deadline.client.job_bundle.submission import AssetReferences from deadline.client.exceptions import DeadlineOperationError @@ -31,6 +32,22 @@ def get_project_path() -> str: project_path = os.getcwd() return project_path +def is_custom_ocio_config_enabled() -> bool: + """True if the script is using a custom OCIO config""" + return nuke.root().knob("colorManagement").value() == "OCIO" and nuke.root().knob("OCIO_config").value() == "custom" + +def get_custom_ocio_config_path() -> str: + """This is the path to the custom OCIO config used by the script""" + return nuke.root().knob("customOCIOConfigPath").getEvaluatedValue() + +def get_custom_ocio_config_luts_dir(ocio_config_file) -> str: + """Returns the directory containing the LUTs for the provided OCIO config""" + ocio_config = OCIO.Config.CreateFromFile(ocio_config_file) + + # At least for all of the AMPAS OCIO configs this is always a relative path to the "luts" directory + search_path = ocio_config.getSearchPath() + + return join(dirname(ocio_config_file), search_path) def get_scene_asset_references() -> AssetReferences: """Traverses all nodes to determine both input and output asset references""" @@ -77,6 +94,14 @@ def get_scene_asset_references() -> AssetReferences: for filename in get_node_filenames(node): asset_references.output_directories.add(dirname(filename)) + # if using a custom OCIO config, add the config file and associated LUT directory + if is_custom_ocio_config_enabled(): + ocio_config_path = get_custom_ocio_config_path() + ocio_config_luts_dir = get_custom_ocio_config_luts_dir(ocio_config_path) + + asset_references.input_filenames.add(ocio_config_path) + asset_references.input_directories.add(ocio_config_luts_dir) + return asset_references From 916e62d1b5deb745283f4d516cdab01b708fc688 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Mon, 11 Sep 2023 18:31:16 +0000 Subject: [PATCH 02/12] fix formatting Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- src/deadline/nuke_submitter/assets.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/deadline/nuke_submitter/assets.py b/src/deadline/nuke_submitter/assets.py index 57a1a35..f4e95bc 100644 --- a/src/deadline/nuke_submitter/assets.py +++ b/src/deadline/nuke_submitter/assets.py @@ -32,14 +32,20 @@ def get_project_path() -> str: project_path = os.getcwd() return project_path + def is_custom_ocio_config_enabled() -> bool: """True if the script is using a custom OCIO config""" - return nuke.root().knob("colorManagement").value() == "OCIO" and nuke.root().knob("OCIO_config").value() == "custom" + return ( + nuke.root().knob("colorManagement").value() == "OCIO" + and nuke.root().knob("OCIO_config").value() == "custom" + ) + def get_custom_ocio_config_path() -> str: """This is the path to the custom OCIO config used by the script""" return nuke.root().knob("customOCIOConfigPath").getEvaluatedValue() + def get_custom_ocio_config_luts_dir(ocio_config_file) -> str: """Returns the directory containing the LUTs for the provided OCIO config""" ocio_config = OCIO.Config.CreateFromFile(ocio_config_file) @@ -49,6 +55,7 @@ def get_custom_ocio_config_luts_dir(ocio_config_file) -> str: return join(dirname(ocio_config_file), search_path) + def get_scene_asset_references() -> AssetReferences: """Traverses all nodes to determine both input and output asset references""" nuke.tprint("Walking scene graph to auto-detect input/output asset references...") From eb7bd0a29a3320cbbaabebe5d4f8135d58544970 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Mon, 11 Sep 2023 18:31:59 +0000 Subject: [PATCH 03/12] Mock PyOpenColorIO for tests to pass Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- pyproject.toml | 2 +- test/deadline_submitter_for_nuke/unit/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 45bc332..b351170 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ explicit_package_bases = true mypy_path = "src" [[tool.mypy.overrides]] -module = ["nuke.*","PySide2.*"] +module = ["nuke.*","PySide2.*","PyOpenColorIO.*"] ignore_missing_imports = true [tool.ruff] diff --git a/test/deadline_submitter_for_nuke/unit/__init__.py b/test/deadline_submitter_for_nuke/unit/__init__.py index 731a2c4..ddf29b1 100644 --- a/test/deadline_submitter_for_nuke/unit/__init__.py +++ b/test/deadline_submitter_for_nuke/unit/__init__.py @@ -12,6 +12,7 @@ "PySide2.QtCore", "PySide2.QtGui", "PySide2.QtWidgets", + "PyOpenColorIO", ] for module in mock_modules: From 19d5d749ffc1fe81fa5b77232cbe241a931a1123 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Mon, 11 Sep 2023 20:59:34 +0000 Subject: [PATCH 04/12] Update test_get_scene_asset_references Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- src/deadline/nuke_submitter/assets.py | 2 +- .../unit/mock_stubs.py | 13 ++++++++ .../unit/test_assets.py | 32 ++++++++++++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 test/deadline_submitter_for_nuke/unit/mock_stubs.py diff --git a/src/deadline/nuke_submitter/assets.py b/src/deadline/nuke_submitter/assets.py index f4e95bc..b4c637d 100644 --- a/src/deadline/nuke_submitter/assets.py +++ b/src/deadline/nuke_submitter/assets.py @@ -46,7 +46,7 @@ def get_custom_ocio_config_path() -> str: return nuke.root().knob("customOCIOConfigPath").getEvaluatedValue() -def get_custom_ocio_config_luts_dir(ocio_config_file) -> str: +def get_custom_ocio_config_luts_dir(ocio_config_file: str) -> str: """Returns the directory containing the LUTs for the provided OCIO config""" ocio_config = OCIO.Config.CreateFromFile(ocio_config_file) diff --git a/test/deadline_submitter_for_nuke/unit/mock_stubs.py b/test/deadline_submitter_for_nuke/unit/mock_stubs.py new file mode 100644 index 0000000..d007d3f --- /dev/null +++ b/test/deadline_submitter_for_nuke/unit/mock_stubs.py @@ -0,0 +1,13 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + +class MockOCIOConfig: + """Mock class which emulates an OCIO Config""" + + __name__ = "Config" + + def __init__(self, search_path: str): + self._search_path = search_path + + def getSearchPath(self) -> str: + return self._search_path diff --git a/test/deadline_submitter_for_nuke/unit/test_assets.py b/test/deadline_submitter_for_nuke/unit/test_assets.py index 9b977ea..0825fd2 100644 --- a/test/deadline_submitter_for_nuke/unit/test_assets.py +++ b/test/deadline_submitter_for_nuke/unit/test_assets.py @@ -16,6 +16,8 @@ get_scene_asset_references, ) +from .mock_stubs import MockOCIOConfig + def _activated_reading_write_node_knobs(knob_name: str): """Side effect function to allow knob() to return different values @@ -36,8 +38,19 @@ def _activated_reading_write_node_knobs(knob_name: str): "deadline.nuke_submitter.assets.get_node_filenames", return_value=["/one/asset.png", "/two/asset.png"], ) +@patch("deadline.nuke_submitter.assets.is_custom_ocio_config_enabled", return_value=False) +@patch( + "deadline.nuke_submitter.assets.get_custom_ocio_config_path", + return_value="/this/ocio_configs/config.ocio", +) +@patch("PyOpenColorIO.Config.CreateFromFile", return_value=MockOCIOConfig(search_path="luts")) def test_get_scene_asset_references( - mock_get_node_filenames: Mock, mock_get_nuke_script_file: Mock, mock_path_isfile: Mock + mock_ocio_config_create_from_file: Mock, + mock_get_custom_ocio_config_path: Mock, + mock_is_custom_ocio_config_enabled: Mock, + mock_get_node_filenames: Mock, + mock_get_nuke_script_file: Mock, + mock_path_isfile: Mock, ): # GIVEN expected_assets = mock_get_node_filenames.return_value @@ -75,6 +88,23 @@ def test_get_scene_asset_references( assert expected_script_file in results.input_filenames assert all(asset in results.input_filenames for asset in expected_assets) + # GIVEN + expected_ocio_config_path = mock_get_custom_ocio_config_path.return_value + expected_ocio_config_luts_dir = os.path.join( + os.path.dirname(expected_ocio_config_path), + mock_ocio_config_create_from_file.return_value.getSearchPath(), + ) + nuke.allNodes.return_value = [] + mock_is_custom_ocio_config_enabled.return_value = True + + # WHEN + results = get_scene_asset_references() + + # THEN + assert expected_script_file in results.input_filenames + assert expected_ocio_config_path in results.input_filenames + assert expected_ocio_config_luts_dir in results.input_directories + @patch("os.path.isfile", return_value=False) @patch("deadline.nuke_submitter.assets.get_nuke_script_file", return_value="/this/scriptfile.nk") From b312ac3b53b5935fe69267aec6d6907016e6dc74 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:50:52 +0000 Subject: [PATCH 05/12] Normalize nuke version in job bundle output tests Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- .../job_bundle_output_test_runner.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py index aac3c06..be51ad3 100644 --- a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py +++ b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py @@ -25,6 +25,7 @@ import shutil import filecmp import difflib +import yaml from typing import Any from pathlib import Path from datetime import datetime, timezone @@ -39,7 +40,7 @@ from deadline.client.ui import gui_error_handler from deadline.client.ui.dialogs import submit_job_to_deadline_dialog from deadline.client.exceptions import DeadlineOperationError -from .deadline_submitter_for_nuke import show_nuke_render_submitter_noargs +from .deadline_submitter_for_nuke import show_nuke_render_submitter_noargs, get_nuke_version # The following functions expose a DCC interface to the job bundle output test logic. @@ -111,6 +112,22 @@ def _show_deadline_cloud_submitter(mainwin: Any): return show_nuke_render_submitter_noargs() +def _get_expected_nuke_version(expected_job_bundle_dir: str) -> str: + """Returns the nuke version specified in provided job bundle""" + if os.path.exists(expected_job_bundle_dir): + expected_parameter_values_file = os.path.join( + expected_job_bundle_dir, "parameter_values.yaml" + ) + if os.path.isfile(expected_parameter_values_file): + with open(expected_parameter_values_file, encoding="utf8") as f: + expected_parameter_values = yaml.safe_load(f)["parameterValues"] + for parameter_value in expected_parameter_values: + if parameter_value["name"] == "NukeVersion": + return parameter_value["value"] + + return None + + # The following functions implement the test logic. @@ -217,7 +234,15 @@ def _run_job_bundle_output_test(test_dir: str, dcc_scene_file: str, report_fh, m # Close the DCC scene file _close_dcc_scene_file() - # Process every file in the job bundle to replace the temp dir with a standardized path + expected_job_bundle_dir = os.path.join(test_dir, "expected_job_bundle") + + test_nuke_version = get_nuke_version() + expected_nuke_version = _get_expected_nuke_version(expected_job_bundle_dir) + + # Process every file in the job bundle to replace the temp dir with a standardized path. + # Also replace the nuke version in the test job bundle with the nuke version in the + # expected job bundle (if there is one). This allows job bundle tests to be run from + # different nuke versions than they were created with. for filename in os.listdir(temp_job_bundle_dir): full_filename = os.path.join(temp_job_bundle_dir, filename) with open(full_filename, encoding="utf8") as f: @@ -229,12 +254,13 @@ def _run_job_bundle_output_test(test_dir: str, dcc_scene_file: str, report_fh, m contents = contents.replace(tempdir, "/normalized/job/bundle/dir") contents = contents.replace(tempdir.replace("\\", "/"), "/normalized/job/bundle/dir") contents = contents.replace(os.getcwd(), "/normalized/cwd") + if expected_nuke_version != test_nuke_version: + contents = contents.replace(test_nuke_version, expected_nuke_version) with open(full_filename, "w", encoding="utf8") as f: f.write(contents) # If there's an expected job bundle to compare with, do the comparison, # otherwise copy the one we created to be that expected job bundle. - expected_job_bundle_dir = os.path.join(test_dir, "expected_job_bundle") if os.path.exists(expected_job_bundle_dir): test_job_bundle_dir = os.path.join(test_dir, "test_job_bundle") if os.path.exists(test_job_bundle_dir): From 070a96dd136f68f72b09dfcbc6826cb967ef5059 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:10:53 +0000 Subject: [PATCH 06/12] Fix linter errors Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- requirements-testing.txt | 3 ++- src/deadline/nuke_submitter/job_bundle_output_test_runner.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 4a7176e..0bd8f2b 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -6,4 +6,5 @@ pytest-xdist ~= 3.3 twine ~= 4.0 mypy ~= 1.5 black ~= 23.7 -ruff ~= 0.0.287 \ No newline at end of file +ruff ~= 0.0.287 +types-PyYAML ~= 6.0.12 \ No newline at end of file diff --git a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py index be51ad3..8d3ee9b 100644 --- a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py +++ b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py @@ -125,7 +125,7 @@ def _get_expected_nuke_version(expected_job_bundle_dir: str) -> str: if parameter_value["name"] == "NukeVersion": return parameter_value["value"] - return None + return "" # The following functions implement the test logic. From 4fc85c94aa6e4d55e8d78e96af9e5daaf1181d88 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Wed, 13 Sep 2023 18:57:54 +0000 Subject: [PATCH 07/12] Support multiple OCIO config search paths Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- src/deadline/nuke_submitter/assets.py | 20 +++++++++++-------- .../unit/mock_stubs.py | 10 ++++++---- .../unit/test_assets.py | 4 ++-- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/deadline/nuke_submitter/assets.py b/src/deadline/nuke_submitter/assets.py index b4c637d..f9f6a19 100644 --- a/src/deadline/nuke_submitter/assets.py +++ b/src/deadline/nuke_submitter/assets.py @@ -7,6 +7,7 @@ from collections.abc import Generator from os.path import commonpath, dirname, join, normpath, samefile from sys import platform +from typing import List import nuke import PyOpenColorIO as OCIO @@ -46,14 +47,15 @@ def get_custom_ocio_config_path() -> str: return nuke.root().knob("customOCIOConfigPath").getEvaluatedValue() -def get_custom_ocio_config_luts_dir(ocio_config_file: str) -> str: - """Returns the directory containing the LUTs for the provided OCIO config""" +def get_custom_ocio_config_search_paths(ocio_config_file: str) -> List[str]: + """Returns the directories containing the LUTs for the provided OCIO config""" ocio_config = OCIO.Config.CreateFromFile(ocio_config_file) - # At least for all of the AMPAS OCIO configs this is always a relative path to the "luts" directory - search_path = ocio_config.getSearchPath() + # A config can have multiple search paths and they can be relative or absolute. + # At least for all of the AMPAS OCIO configs this is always a single relative path to the "luts" directory + search_paths = ocio_config.getSearchPaths() - return join(dirname(ocio_config_file), search_path) + return [join(dirname(ocio_config_file), search_path) for search_path in search_paths] def get_scene_asset_references() -> AssetReferences: @@ -101,13 +103,15 @@ def get_scene_asset_references() -> AssetReferences: for filename in get_node_filenames(node): asset_references.output_directories.add(dirname(filename)) - # if using a custom OCIO config, add the config file and associated LUT directory + # if using a custom OCIO config, add the config file and associated search directories if is_custom_ocio_config_enabled(): ocio_config_path = get_custom_ocio_config_path() - ocio_config_luts_dir = get_custom_ocio_config_luts_dir(ocio_config_path) + ocio_config_search_paths = get_custom_ocio_config_search_paths(ocio_config_path) asset_references.input_filenames.add(ocio_config_path) - asset_references.input_directories.add(ocio_config_luts_dir) + + for search_path in ocio_config_search_paths: + asset_references.input_directories.add(search_path) return asset_references diff --git a/test/deadline_submitter_for_nuke/unit/mock_stubs.py b/test/deadline_submitter_for_nuke/unit/mock_stubs.py index d007d3f..6233b2e 100644 --- a/test/deadline_submitter_for_nuke/unit/mock_stubs.py +++ b/test/deadline_submitter_for_nuke/unit/mock_stubs.py @@ -1,13 +1,15 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +from typing import List + class MockOCIOConfig: """Mock class which emulates an OCIO Config""" __name__ = "Config" - def __init__(self, search_path: str): - self._search_path = search_path + def __init__(self, search_paths: List[str]): + self._search_paths = search_paths - def getSearchPath(self) -> str: - return self._search_path + def getSearchPaths(self) -> List[str]: + return self._search_paths diff --git a/test/deadline_submitter_for_nuke/unit/test_assets.py b/test/deadline_submitter_for_nuke/unit/test_assets.py index 0825fd2..d300cd3 100644 --- a/test/deadline_submitter_for_nuke/unit/test_assets.py +++ b/test/deadline_submitter_for_nuke/unit/test_assets.py @@ -43,7 +43,7 @@ def _activated_reading_write_node_knobs(knob_name: str): "deadline.nuke_submitter.assets.get_custom_ocio_config_path", return_value="/this/ocio_configs/config.ocio", ) -@patch("PyOpenColorIO.Config.CreateFromFile", return_value=MockOCIOConfig(search_path="luts")) +@patch("PyOpenColorIO.Config.CreateFromFile", return_value=MockOCIOConfig(search_paths=["luts"])) def test_get_scene_asset_references( mock_ocio_config_create_from_file: Mock, mock_get_custom_ocio_config_path: Mock, @@ -92,7 +92,7 @@ def test_get_scene_asset_references( expected_ocio_config_path = mock_get_custom_ocio_config_path.return_value expected_ocio_config_luts_dir = os.path.join( os.path.dirname(expected_ocio_config_path), - mock_ocio_config_create_from_file.return_value.getSearchPath(), + mock_ocio_config_create_from_file.return_value.getSearchPaths()[0], ) nuke.allNodes.return_value = [] mock_is_custom_ocio_config_enabled.return_value = True From edc2fbe0c213da45bc0f367b0e36cf98a8a3803a Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Thu, 14 Sep 2023 00:52:11 +0000 Subject: [PATCH 08/12] Fix nuke version replacement in job bundle test runner Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- src/deadline/nuke_submitter/job_bundle_output_test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py index 8d3ee9b..ef8a4c5 100644 --- a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py +++ b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py @@ -254,7 +254,7 @@ def _run_job_bundle_output_test(test_dir: str, dcc_scene_file: str, report_fh, m contents = contents.replace(tempdir, "/normalized/job/bundle/dir") contents = contents.replace(tempdir.replace("\\", "/"), "/normalized/job/bundle/dir") contents = contents.replace(os.getcwd(), "/normalized/cwd") - if expected_nuke_version != test_nuke_version: + if expected_nuke_version and expected_nuke_version != test_nuke_version: contents = contents.replace(test_nuke_version, expected_nuke_version) with open(full_filename, "w", encoding="utf8") as f: f.write(contents) From 0a29ddce33183902d3da4c7fa2a610faa0f1be9d Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Thu, 14 Sep 2023 00:52:28 +0000 Subject: [PATCH 09/12] ocio config job bundle test Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- .../expected_job_bundle/asset_references.yaml | 11 + .../expected_job_bundle/parameter_values.yaml | 19 ++ .../ocio/expected_job_bundle/template.yaml | 227 ++++++++++++++++++ .../ocio/scene/config.ocio | 25 ++ job_bundle_output_tests/ocio/scene/ocio.nk | 75 ++++++ 5 files changed, 357 insertions(+) create mode 100644 job_bundle_output_tests/ocio/expected_job_bundle/asset_references.yaml create mode 100644 job_bundle_output_tests/ocio/expected_job_bundle/parameter_values.yaml create mode 100644 job_bundle_output_tests/ocio/expected_job_bundle/template.yaml create mode 100644 job_bundle_output_tests/ocio/scene/config.ocio create mode 100644 job_bundle_output_tests/ocio/scene/ocio.nk diff --git a/job_bundle_output_tests/ocio/expected_job_bundle/asset_references.yaml b/job_bundle_output_tests/ocio/expected_job_bundle/asset_references.yaml new file mode 100644 index 0000000..ca11f4d --- /dev/null +++ b/job_bundle_output_tests/ocio/expected_job_bundle/asset_references.yaml @@ -0,0 +1,11 @@ +assetReferences: + inputs: + directories: + - /normalized/job/bundle/dir/luts + - /tmp/luts + filenames: + - /normalized/job/bundle/dir/config.ocio + - /normalized/job/bundle/dir/ocio.nk + outputs: + directories: + - output diff --git a/job_bundle_output_tests/ocio/expected_job_bundle/parameter_values.yaml b/job_bundle_output_tests/ocio/expected_job_bundle/parameter_values.yaml new file mode 100644 index 0000000..7fae662 --- /dev/null +++ b/job_bundle_output_tests/ocio/expected_job_bundle/parameter_values.yaml @@ -0,0 +1,19 @@ +parameterValues: +- name: deadline:priority + value: 50 +- name: deadline:targetTaskRunStatus + value: READY +- name: deadline:maxFailedTasksCount + value: 100 +- name: deadline:maxRetriesPerTask + value: 5 +- name: Frames + value: 1-100 +- name: NukeScriptFile + value: /normalized/job/bundle/dir/ocio.nk +- name: ProxyMode + value: 'false' +- name: NukeVersion + value: 14.0v5 +- name: RezPackages + value: nuke-13 deadline_cloud_for_nuke diff --git a/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml b/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml new file mode 100644 index 0000000..91c6ddc --- /dev/null +++ b/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml @@ -0,0 +1,227 @@ +specificationVersion: jobtemplate-2023-09 +name: ocio.nk +parameterDefinitions: +- name: NukeScriptFile + type: PATH + objectType: FILE + dataFlow: IN + userInterface: + control: CHOOSE_INPUT_FILE + label: Nuke Script File + fileFilters: + - label: Nuke Script Files + patterns: + - '*.nk' + - label: All Files + patterns: + - '*' + description: The Nuke script file to render. +- name: Frames + type: STRING + description: The frames to render. E.g. 1-3,8,11-15 + minLength: 1 +- name: WriteNode + type: STRING + userInterface: + control: DROPDOWN_LIST + label: Write Node + description: Which write node to render ('All Write Nodes' for all of them) + default: All Write Nodes + allowedValues: + - All Write Nodes + - Write1 +- name: View + type: STRING + userInterface: + control: DROPDOWN_LIST + description: Which view to render ('All Views' for all of them) + default: All Views + allowedValues: + - All Views + - main +- name: ProxyMode + type: STRING + userInterface: + control: CHECK_BOX + label: Proxy Mode + description: Render in Proxy Mode. + default: 'false' + allowedValues: + - 'true' + - 'false' +- name: ContinueOnError + type: STRING + userInterface: + control: CHECK_BOX + label: Continue On Error + description: Continue processing when errors occur. + default: 'false' + allowedValues: + - 'true' + - 'false' +- name: NukeVersion + type: STRING + userInterface: + control: LINE_EDIT + label: Nuke Version + description: The version of Nuke. +- name: RezPackages + type: STRING + userInterface: + control: LINE_EDIT + label: Rez Packages + description: A space-separated list of Rez packages to install + default: nuke-13 deadline_cloud_for_nuke +jobEnvironments: +- name: Rez + description: Initializes and destroys the Rez environment for the job. + script: + actions: + onEnter: + command: '{{ Env.File.Enter }}' + onExit: + command: '{{ Env.File.Exit }}' + embeddedFiles: + - name: Enter + filename: rez-enter.sh + type: TEXT + runnable: true + data: | + #!/bin/env bash + + set -euo pipefail + + if [ ! -z "{{Param.RezPackages}}" ]; then + echo "Rez Package List:" + echo " {{Param.RezPackages}}" + + # Create the environment + /usr/local/bin/deadline-rez init \ + -d "{{Session.WorkingDirectory}}" \ + {{Param.RezPackages}} + + # Capture the environment's vars + {{Env.File.InitialVars}} + . /usr/local/bin/deadline-rez activate \ + -d "{{Session.WorkingDirectory}}" + {{Env.File.CaptureVars}} + else + echo "No Rez Packages, skipping environment creation." + fi + - name: Exit + filename: rez-exit.sh + type: TEXT + runnable: true + data: | + #!/bin/env bash + + set -euo pipefail + + if [ ! -z "{{Param.RezPackages}}" ]; then + echo "Rez Package List:" + echo " {{Param.RezPackages}}" + + /usr/local/bin/deadline-rez destroy \ + -d "{{ Session.WorkingDirectory }}" + else + echo "No Rez Packages, skipping environment teardown." + fi + - name: InitialVars + filename: initial-vars.sh + type: TEXT + runnable: true + data: | + #!/usr/bin/env python3 + import os, json + envfile = "{{Session.WorkingDirectory}}/.envInitial" + with open(envfile, "w", encoding="utf8") as f: + json.dump(dict(os.environ), f) + - name: CaptureVars + filename: capture-vars.sh + type: TEXT + runnable: true + data: | + #!/usr/bin/env python3 + import os, json, sys + envfile = "{{Session.WorkingDirectory}}/.envInitial" + if os.path.isfile(envfile): + with open(envfile, "r", encoding="utf8") as f: + before = json.load(f) + else: + print("No initial environment found, must run Env.File.CaptureVars script first") + sys.exit(1) + after = dict(os.environ) + + put = {k: v for k, v in after.items() if v != before.get(k)} + delete = {k for k in before if k not in after} + + for k, v in put.items(): + print(f"updating {k}={v}") + print(f"openjd_env: {k}={v}") + for k in delete: + print(f"openjd_unset_env: {k}") +steps: +- name: Render + parameterSpace: + taskParameterDefinitions: + - name: Frame + type: INT + range: '{{Param.Frames}}' + stepEnvironments: + - name: Nuke + description: Runs Nuke in the background with a script file loaded. + script: + embeddedFiles: + - name: initData + filename: init-data.yaml + type: TEXT + data: | + continue_on_error: {{Param.ContinueOnError}} + proxy: {{Param.ProxyMode}} + script_file: '{{Param.NukeScriptFile}}' + version: '{{Param.NukeVersion}}' + write_nodes: + - '{{Param.WriteNode}}' + views: + - '{{Param.View}}' + actions: + onEnter: + command: NukeAdaptor + args: + - daemon + - start + - --path-mapping-rules + - file://{{Session.PathMappingRulesFile}} + - --connection-file + - '{{Session.WorkingDirectory}}/connection.json' + - --init-data + - file://{{ Env.File.initData }} + cancelation: + mode: NOTIFY_THEN_TERMINATE + onExit: + command: NukeAdaptor + args: + - daemon + - stop + - --connection-file + - '{{ Session.WorkingDirectory }}/connection.json' + cancelation: + mode: NOTIFY_THEN_TERMINATE + script: + embeddedFiles: + - name: runData + filename: run-data.yaml + type: TEXT + data: 'frame: {{Task.Param.Frame}}' + actions: + onRun: + command: NukeAdaptor + args: + - daemon + - run + - --connection-file + - '{{Session.WorkingDirectory}}/connection.json' + - --run-data + - file://{{ Task.File.runData }} + cancelation: + mode: NOTIFY_THEN_TERMINATE diff --git a/job_bundle_output_tests/ocio/scene/config.ocio b/job_bundle_output_tests/ocio/scene/config.ocio new file mode 100644 index 0000000..54cf9b6 --- /dev/null +++ b/job_bundle_output_tests/ocio/scene/config.ocio @@ -0,0 +1,25 @@ +ocio_profile_version: 2.1 + +environment: + {} +search_path: + - luts + - /tmp/luts +strictparsing: true +luma: [0.2126, 0.7152, 0.0722] +name: deadline-cloud-for-nuke-test-config + +roles: + {} + +file_rules: + - ! {name: Default, colorspace: default} + +displays: + {} + +active_displays: [] +active_views: [] + +colorspaces: + [] \ No newline at end of file diff --git a/job_bundle_output_tests/ocio/scene/ocio.nk b/job_bundle_output_tests/ocio/scene/ocio.nk new file mode 100644 index 0000000..8d55835 --- /dev/null +++ b/job_bundle_output_tests/ocio/scene/ocio.nk @@ -0,0 +1,75 @@ +#! /opt/nuke/Nuke14.0v5/libnuke-14.0.5.so -nx +#write_info Write1 file:"output/output_%04d.png" format:"2048 1556 1" chans:":rgba.red:rgba.green:rgba.blue:" framerange:"1 100" fps:"0" colorspace:"matte_paint" datatype:"8 bit" transfer:"unknown" views:"main" colorManagement:"OCIO" +version 14.0 v5 +define_window_layout_xml { + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +} +Root { + inputs 0 + name /home/rocky/src/deadline-cloud-for-nuke/job_bundle_output_tests/ocio/scene/ocio.nk + format "2048 1556 0 0 2048 1556 1 2K_Super_35(full-ap)" + proxy_type scale + proxy_format "1024 778 0 0 1024 778 1 1K_Super_35(full-ap)" + colorManagement OCIO + OCIO_config custom + customOCIOConfigPath /home/rocky/src/deadline-cloud-for-nuke/job_bundle_output_tests/ocio/scene/config.ocio + defaultViewerLUT "OCIO LUTs" + workingSpaceLUT scene_linear + monitorLut 0 + monitorOutLUT "sRGB (default)" + int8Lut matte_paint + int16Lut texture_paint + logLut compositing_log + floatLut scene_linear +} +ColorBars { + inputs 0 + name ColorBars1 + xpos -53 + ypos -134 +} +set Nca0bef0 [stack 0] +Write { + file output/output_####.png + colorspace matte_paint + file_type png + checkHashOnRead false + name Write1 + xpos 11 + ypos -17 +} +push $Nca0bef0 +Viewer { + frame_range 1-100 + name Viewer1 + xpos -139 + ypos 52 +} From e8987691f451de90147a921bbf0eb7c1db79bb29 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Fri, 15 Sep 2023 22:10:25 +0000 Subject: [PATCH 10/12] Update test_assets for multiple ocio config search paths Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- .../deadline_submitter_for_nuke/unit/test_assets.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/deadline_submitter_for_nuke/unit/test_assets.py b/test/deadline_submitter_for_nuke/unit/test_assets.py index d300cd3..db86636 100644 --- a/test/deadline_submitter_for_nuke/unit/test_assets.py +++ b/test/deadline_submitter_for_nuke/unit/test_assets.py @@ -90,10 +90,10 @@ def test_get_scene_asset_references( # GIVEN expected_ocio_config_path = mock_get_custom_ocio_config_path.return_value - expected_ocio_config_luts_dir = os.path.join( - os.path.dirname(expected_ocio_config_path), - mock_ocio_config_create_from_file.return_value.getSearchPaths()[0], - ) + expected_ocio_config_search_paths = [ + os.path.join(os.path.dirname(expected_ocio_config_path), search_path) + for search_path in mock_ocio_config_create_from_file.return_value.getSearchPaths() + ] nuke.allNodes.return_value = [] mock_is_custom_ocio_config_enabled.return_value = True @@ -103,7 +103,10 @@ def test_get_scene_asset_references( # THEN assert expected_script_file in results.input_filenames assert expected_ocio_config_path in results.input_filenames - assert expected_ocio_config_luts_dir in results.input_directories + assert all( + search_path in expected_ocio_config_search_paths + for search_path in results.input_directories + ) @patch("os.path.isfile", return_value=False) From ce6d3937c034502f2f36964d11a7b2ad05d19cb8 Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:30:46 +0000 Subject: [PATCH 11/12] Fix rebase Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- .../expected_job_bundle/asset_references.yaml | 3 ++- .../expected_job_bundle/parameter_values.yaml | 20 ++++++-------- .../ocio/expected_job_bundle/template.yaml | 14 ---------- requirements-testing.txt | 3 +-- .../job_bundle_output_test_runner.py | 27 ++----------------- 5 files changed, 13 insertions(+), 54 deletions(-) diff --git a/job_bundle_output_tests/ocio/expected_job_bundle/asset_references.yaml b/job_bundle_output_tests/ocio/expected_job_bundle/asset_references.yaml index ca11f4d..0bc20a1 100644 --- a/job_bundle_output_tests/ocio/expected_job_bundle/asset_references.yaml +++ b/job_bundle_output_tests/ocio/expected_job_bundle/asset_references.yaml @@ -8,4 +8,5 @@ assetReferences: - /normalized/job/bundle/dir/ocio.nk outputs: directories: - - output + - /normalized/cwd/output + referencedPaths: [] diff --git a/job_bundle_output_tests/ocio/expected_job_bundle/parameter_values.yaml b/job_bundle_output_tests/ocio/expected_job_bundle/parameter_values.yaml index 7fae662..a2aa969 100644 --- a/job_bundle_output_tests/ocio/expected_job_bundle/parameter_values.yaml +++ b/job_bundle_output_tests/ocio/expected_job_bundle/parameter_values.yaml @@ -1,19 +1,15 @@ parameterValues: -- name: deadline:priority - value: 50 -- name: deadline:targetTaskRunStatus - value: READY -- name: deadline:maxFailedTasksCount - value: 100 -- name: deadline:maxRetriesPerTask - value: 5 - name: Frames value: 1-100 - name: NukeScriptFile value: /normalized/job/bundle/dir/ocio.nk - name: ProxyMode value: 'false' -- name: NukeVersion - value: 14.0v5 -- name: RezPackages - value: nuke-13 deadline_cloud_for_nuke +- name: deadline:targetTaskRunStatus + value: READY +- name: deadline:maxFailedTasksCount + value: 20 +- name: deadline:maxRetriesPerTask + value: 5 +- name: deadline:priority + value: 50 diff --git a/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml b/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml index 91c6ddc..d85a237 100644 --- a/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml +++ b/job_bundle_output_tests/ocio/expected_job_bundle/template.yaml @@ -59,19 +59,6 @@ parameterDefinitions: allowedValues: - 'true' - 'false' -- name: NukeVersion - type: STRING - userInterface: - control: LINE_EDIT - label: Nuke Version - description: The version of Nuke. -- name: RezPackages - type: STRING - userInterface: - control: LINE_EDIT - label: Rez Packages - description: A space-separated list of Rez packages to install - default: nuke-13 deadline_cloud_for_nuke jobEnvironments: - name: Rez description: Initializes and destroys the Rez environment for the job. @@ -179,7 +166,6 @@ steps: continue_on_error: {{Param.ContinueOnError}} proxy: {{Param.ProxyMode}} script_file: '{{Param.NukeScriptFile}}' - version: '{{Param.NukeVersion}}' write_nodes: - '{{Param.WriteNode}}' views: diff --git a/requirements-testing.txt b/requirements-testing.txt index 0bd8f2b..4a7176e 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -6,5 +6,4 @@ pytest-xdist ~= 3.3 twine ~= 4.0 mypy ~= 1.5 black ~= 23.7 -ruff ~= 0.0.287 -types-PyYAML ~= 6.0.12 \ No newline at end of file +ruff ~= 0.0.287 \ No newline at end of file diff --git a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py index ef8a4c5..bbb910c 100644 --- a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py +++ b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py @@ -25,7 +25,6 @@ import shutil import filecmp import difflib -import yaml from typing import Any from pathlib import Path from datetime import datetime, timezone @@ -40,7 +39,7 @@ from deadline.client.ui import gui_error_handler from deadline.client.ui.dialogs import submit_job_to_deadline_dialog from deadline.client.exceptions import DeadlineOperationError -from .deadline_submitter_for_nuke import show_nuke_render_submitter_noargs, get_nuke_version +from .deadline_submitter_for_nuke import show_nuke_render_submitter_noargs # The following functions expose a DCC interface to the job bundle output test logic. @@ -112,22 +111,6 @@ def _show_deadline_cloud_submitter(mainwin: Any): return show_nuke_render_submitter_noargs() -def _get_expected_nuke_version(expected_job_bundle_dir: str) -> str: - """Returns the nuke version specified in provided job bundle""" - if os.path.exists(expected_job_bundle_dir): - expected_parameter_values_file = os.path.join( - expected_job_bundle_dir, "parameter_values.yaml" - ) - if os.path.isfile(expected_parameter_values_file): - with open(expected_parameter_values_file, encoding="utf8") as f: - expected_parameter_values = yaml.safe_load(f)["parameterValues"] - for parameter_value in expected_parameter_values: - if parameter_value["name"] == "NukeVersion": - return parameter_value["value"] - - return "" - - # The following functions implement the test logic. @@ -234,11 +217,6 @@ def _run_job_bundle_output_test(test_dir: str, dcc_scene_file: str, report_fh, m # Close the DCC scene file _close_dcc_scene_file() - expected_job_bundle_dir = os.path.join(test_dir, "expected_job_bundle") - - test_nuke_version = get_nuke_version() - expected_nuke_version = _get_expected_nuke_version(expected_job_bundle_dir) - # Process every file in the job bundle to replace the temp dir with a standardized path. # Also replace the nuke version in the test job bundle with the nuke version in the # expected job bundle (if there is one). This allows job bundle tests to be run from @@ -254,13 +232,12 @@ def _run_job_bundle_output_test(test_dir: str, dcc_scene_file: str, report_fh, m contents = contents.replace(tempdir, "/normalized/job/bundle/dir") contents = contents.replace(tempdir.replace("\\", "/"), "/normalized/job/bundle/dir") contents = contents.replace(os.getcwd(), "/normalized/cwd") - if expected_nuke_version and expected_nuke_version != test_nuke_version: - contents = contents.replace(test_nuke_version, expected_nuke_version) with open(full_filename, "w", encoding="utf8") as f: f.write(contents) # If there's an expected job bundle to compare with, do the comparison, # otherwise copy the one we created to be that expected job bundle. + expected_job_bundle_dir = os.path.join(test_dir, "expected_job_bundle") if os.path.exists(expected_job_bundle_dir): test_job_bundle_dir = os.path.join(test_dir, "test_job_bundle") if os.path.exists(test_job_bundle_dir): From 513d402fc487403dd960f9619759d60679d8a69b Mon Sep 17 00:00:00 2001 From: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:39:52 +0000 Subject: [PATCH 12/12] Cleanup Signed-off-by: Alex Gerveshi <15710060+agerveshi@users.noreply.github.com> --- src/deadline/nuke_submitter/assets.py | 3 +-- .../nuke_submitter/job_bundle_output_test_runner.py | 5 +---- test/deadline_submitter_for_nuke/unit/mock_stubs.py | 6 +++--- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/deadline/nuke_submitter/assets.py b/src/deadline/nuke_submitter/assets.py index f9f6a19..7f1aa03 100644 --- a/src/deadline/nuke_submitter/assets.py +++ b/src/deadline/nuke_submitter/assets.py @@ -7,7 +7,6 @@ from collections.abc import Generator from os.path import commonpath, dirname, join, normpath, samefile from sys import platform -from typing import List import nuke import PyOpenColorIO as OCIO @@ -47,7 +46,7 @@ def get_custom_ocio_config_path() -> str: return nuke.root().knob("customOCIOConfigPath").getEvaluatedValue() -def get_custom_ocio_config_search_paths(ocio_config_file: str) -> List[str]: +def get_custom_ocio_config_search_paths(ocio_config_file: str) -> list[str]: """Returns the directories containing the LUTs for the provided OCIO config""" ocio_config = OCIO.Config.CreateFromFile(ocio_config_file) diff --git a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py index bbb910c..aac3c06 100644 --- a/src/deadline/nuke_submitter/job_bundle_output_test_runner.py +++ b/src/deadline/nuke_submitter/job_bundle_output_test_runner.py @@ -217,10 +217,7 @@ def _run_job_bundle_output_test(test_dir: str, dcc_scene_file: str, report_fh, m # Close the DCC scene file _close_dcc_scene_file() - # Process every file in the job bundle to replace the temp dir with a standardized path. - # Also replace the nuke version in the test job bundle with the nuke version in the - # expected job bundle (if there is one). This allows job bundle tests to be run from - # different nuke versions than they were created with. + # Process every file in the job bundle to replace the temp dir with a standardized path for filename in os.listdir(temp_job_bundle_dir): full_filename = os.path.join(temp_job_bundle_dir, filename) with open(full_filename, encoding="utf8") as f: diff --git a/test/deadline_submitter_for_nuke/unit/mock_stubs.py b/test/deadline_submitter_for_nuke/unit/mock_stubs.py index 6233b2e..7d71bd2 100644 --- a/test/deadline_submitter_for_nuke/unit/mock_stubs.py +++ b/test/deadline_submitter_for_nuke/unit/mock_stubs.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -from typing import List +from __future__ import annotations class MockOCIOConfig: @@ -8,8 +8,8 @@ class MockOCIOConfig: __name__ = "Config" - def __init__(self, search_paths: List[str]): + def __init__(self, search_paths: list[str]): self._search_paths = search_paths - def getSearchPaths(self) -> List[str]: + def getSearchPaths(self) -> list[str]: return self._search_paths