Moved validation logic into build action

aws_lambda_builders/workflows/python_pip/actions.py

@@ -2,8 +2,12 @@
 Action to resolve Python dependencies using PIP
 """
 
+from typing import Optional, Tuple
+
 from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
 from aws_lambda_builders.architecture import X86_64
+from aws_lambda_builders.binary_path import BinaryPath
+from aws_lambda_builders.exceptions import MisMatchRuntimeError, RuntimeValidatorError
 from aws_lambda_builders.workflows.python_pip.utils import OSUtils
 
 from .exceptions import MissingPipError
@@ -27,20 +31,21 @@ def __init__(
         self.binaries = binaries
         self.architecture = architecture
 
+        self._os_utils = OSUtils()
+
     def execute(self):
-        os_utils = OSUtils()
-        python_path = self.binaries[self.LANGUAGE].binary_path
         try:
-            pip = SubprocessPip(osutils=os_utils, python_exe=python_path)
+            pip, python_with_pip = self._find_runtime_with_pip()
+            python_path = self.binaries[self.LANGUAGE].binary_path = python_with_pip
         except MissingPipError as ex:
             raise ActionFailedError(str(ex))
         pip_runner = PipRunner(python_exe=python_path, pip=pip)
         dependency_builder = DependencyBuilder(
-            osutils=os_utils, pip_runner=pip_runner, runtime=self.runtime, architecture=self.architecture
+            osutils=self._os_utils, pip_runner=pip_runner, runtime=self.runtime, architecture=self.architecture
         )
 
         package_builder = PythonPipDependencyBuilder(
-            osutils=os_utils, runtime=self.runtime, dependency_builder=dependency_builder
+            osutils=self._os_utils, runtime=self.runtime, dependency_builder=dependency_builder
         )
         try:
             target_artifact_dir = self.artifacts_dir
@@ -55,3 +60,29 @@ def execute(self):
             )
         except PackagerError as ex:
             raise ActionFailedError(str(ex))
+
+    def _find_runtime_with_pip(self) -> Tuple[str, str]:
+        """
+        foo bar
+        """
+        binary_object: Optional[BinaryPath] = self.binaries.get(self.LANGUAGE)
+
+        if not binary_object:
+            raise ActionFailedError("Failed to fetch Python binaries from the PATH.")
+
+        for python_path in binary_object.resolver.exec_paths:
+            try:
+                valid_python_path = binary_object.validator.validate(python_path)
+
+                if valid_python_path:
+                    pip = SubprocessPip(osutils=self._os_utils, python_exe=valid_python_path)
+
+                    return (pip, valid_python_path)
+            except (MissingPipError, MisMatchRuntimeError, RuntimeValidatorError):
+                # runtime and mismatch exceptions should have been caught
+                # during the init phase
+
+                # we can ignore these and let the action fail at the end
+                pass
+
+        raise ActionFailedError("Failed to find a Python runtime containing pip on the PATH.")

aws_lambda_builders/workflows/python_pip/validator.py

@@ -8,9 +8,8 @@
 
 from aws_lambda_builders.exceptions import MisMatchRuntimeError
 from aws_lambda_builders.validator import RuntimeValidator
-from aws_lambda_builders.workflows.python_pip.compat import pip_import_string
-from aws_lambda_builders.workflows.python_pip.exceptions import MissingPipError
-from aws_lambda_builders.workflows.python_pip.utils import OSUtils
+
+from .utils import OSUtils
 
 LOG = logging.getLogger(__name__)
 
@@ -21,7 +20,7 @@ def __init__(self, runtime, architecture):
         self.language = "python"
         self._valid_runtime_path = None
 
-    def validate(self, runtime_path: str) -> str:
+    def validate(self, runtime_path):
         """
         Checks if the language supplied matches the required lambda runtime
 
@@ -38,8 +37,7 @@ def validate(self, runtime_path: str) -> str:
         Raises
         ------
         MisMatchRuntimeError
-            Raise runtime is not support or runtime does not support architecture or
-            if the Python runtime does not contain `pip`.
+            Raise runtime is not support or runtime does not support architecture.
         """
 
         runtime_path = super(PythonRuntimeValidator, self).validate(runtime_path)
@@ -50,24 +48,11 @@ def validate(self, runtime_path: str) -> str:
             cmd, cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=OSUtils().original_environ()
         )
         p.communicate()
-
         if p.returncode != 0:
             raise MisMatchRuntimeError(language=self.language, required_runtime=self.runtime, runtime_path=runtime_path)
-
-        try:
-            # call method to import `pip`
-            # ignoring method return values since we only want to check
-            # if `pip` is imported successfully
-            pip_import_string(runtime_path)
-        except MissingPipError as ex:
-            LOG.debug(f"Invalid Python runtime {runtime_path}, runtime does not contain pip")
-
-            raise MisMatchRuntimeError(
-                language=self.language, required_runtime=self.runtime, runtime_path=runtime_path
-            ) from ex
-
-        self._valid_runtime_path = runtime_path
-        return self._valid_runtime_path
+        else:
+            self._valid_runtime_path = runtime_path
+            return self._valid_runtime_path
 
     def _validate_python_cmd(self, runtime_path):
         major, minor = self.runtime.replace(self.language, "").split(".")

tests/unit/workflows/python_pip/test_actions.py

@@ -1,7 +1,7 @@
 import sys
 
 from unittest import TestCase
-from unittest.mock import patch, Mock, ANY
+from unittest.mock import MagicMock, patch, Mock, ANY
 
 from aws_lambda_builders.actions import ActionFailedError
 from aws_lambda_builders.architecture import ARM64, X86_64
@@ -15,8 +15,10 @@
 class TestPythonPipBuildAction(TestCase):
     @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
     @patch("aws_lambda_builders.workflows.python_pip.actions.DependencyBuilder")
-    def test_action_must_call_builder(self, DependencyBuilderMock, PythonPipDependencyBuilderMock):
-        builder_instance = PythonPipDependencyBuilderMock.return_value
+    @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
+    def test_action_must_call_builder(self, find_runtime_mock, dependency_builder_mock, pip_dependency_builder_mock):
+        builder_instance = pip_dependency_builder_mock.return_value
+        find_runtime_mock.return_value = (Mock(), Mock())
 
         action = PythonPipBuildAction(
             "artifacts",
@@ -28,16 +30,20 @@ def test_action_must_call_builder(self, DependencyBuilderMock, PythonPipDependen
         )
         action.execute()
 
-        DependencyBuilderMock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=X86_64)
+        dependency_builder_mock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=X86_64)
 
         builder_instance.build_dependencies.assert_called_with(
             artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest"
         )
 
     @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
     @patch("aws_lambda_builders.workflows.python_pip.actions.DependencyBuilder")
-    def test_action_must_call_builder_with_architecture(self, DependencyBuilderMock, PythonPipDependencyBuilderMock):
-        builder_instance = PythonPipDependencyBuilderMock.return_value
+    @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
+    def test_action_must_call_builder_with_architecture(
+        self, find_runtime_mock, dependency_builder_mock, pip_dependency_builder_mock
+    ):
+        builder_instance = pip_dependency_builder_mock.return_value
+        find_runtime_mock.return_value = (Mock(), Mock())
 
         action = PythonPipBuildAction(
             "artifacts",
@@ -50,16 +56,18 @@ def test_action_must_call_builder_with_architecture(self, DependencyBuilderMock,
         )
         action.execute()
 
-        DependencyBuilderMock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=ARM64)
+        dependency_builder_mock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=ARM64)
 
         builder_instance.build_dependencies.assert_called_with(
             artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest"
         )
 
     @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
-    def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock):
-        builder_instance = PythonPipDependencyBuilderMock.return_value
+    @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
+    def test_must_raise_exception_on_failure(self, find_runtime_mock, pip_dependency_builder_mock):
+        builder_instance = pip_dependency_builder_mock.return_value
         builder_instance.build_dependencies.side_effect = PackagerError()
+        find_runtime_mock.return_value = (Mock(), Mock())
 
         action = PythonPipBuildAction(
             "artifacts",
@@ -73,25 +81,29 @@ def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock):
         with self.assertRaises(ActionFailedError):
             action.execute()
 
-    @patch("aws_lambda_builders.workflows.python_pip.actions.SubprocessPip")
-    def test_must_raise_exception_on_pip_failure(self, PythonSubProcessPipMock):
-        PythonSubProcessPipMock.side_effect = MissingPipError(python_path="mockpath")
+    # @patch("aws_lambda_builders.workflows.python_pip.actions.SubprocessPip")
+    # @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
+    # def test_must_raise_exception_on_pip_failure(self, find_runtime_mock, pip_sub_process_mock):
+    #     pip_sub_process_mock.side_effect = MissingPipError(python_path="mockpath")
+    #     find_runtime_mock.return_value = (Mock(), Mock())
 
-        action = PythonPipBuildAction(
-            "artifacts",
-            "scratch_dir",
-            "manifest",
-            "runtime",
-            None,
-            {"python": BinaryPath(resolver=Mock(), validator=Mock(), binary="python", binary_path=sys.executable)},
-        )
+    #     action = PythonPipBuildAction(
+    #         "artifacts",
+    #         "scratch_dir",
+    #         "manifest",
+    #         "runtime",
+    #         None,
+    #         {"python": BinaryPath(resolver=Mock(), validator=Mock(), binary="python", binary_path=sys.executable)},
+    #     )
 
-        with self.assertRaises(ActionFailedError):
-            action.execute()
+    #     with self.assertRaises(ActionFailedError):
+    #         action.execute()
 
     @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
-    def test_action_must_call_builder_with_dependencies_dir(self, PythonPipDependencyBuilderMock):
-        builder_instance = PythonPipDependencyBuilderMock.return_value
+    @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
+    def test_action_must_call_builder_with_dependencies_dir(self, find_runtime_mock, pip_dependency_builder_mock):
+        builder_instance = pip_dependency_builder_mock.return_value
+        find_runtime_mock.return_value = (Mock(), Mock())
 
         action = PythonPipBuildAction(
             "artifacts",
@@ -106,3 +118,30 @@ def test_action_must_call_builder_with_dependencies_dir(self, PythonPipDependenc
         builder_instance.build_dependencies.assert_called_with(
             artifacts_dir_path="dependencies_dir", scratch_dir_path="scratch_dir", requirements_path="manifest"
         )
+
+    def test_find_runtime_missing_binary_object(self):
+        mock_binaries = {}
+
+        with self.assertRaises(ActionFailedError) as ex:
+            PythonPipBuildAction(Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries)._find_runtime_with_pip()
+
+            self.assertEqual(str(ex.exception), "Failed to fetch Python binaries from the PATH.")
+
+    def test_find_runtime_empty_exec_paths(self):
+        mock_resolver = Mock()
+        mock_resolver.resolver = Mock()
+        mock_resolver.resolver.exec_paths = {}
+
+        mock_binaries = Mock()
+        mock_binaries.get = Mock(return_value=mock_resolver)
+
+        with self.assertRaises(ActionFailedError) as ex:
+            PythonPipBuildAction(Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries)._find_runtime_with_pip()
+
+            self.assertEqual(str(ex.exception), "Failed to fetch Python binaries from the PATH.")
+
+    def test_find_runtime_found_pip(self):
+        pass
+
+    def test_find_runtime_no_matches(self):
+        pass

tests/unit/workflows/python_pip/test_validator.py

@@ -3,7 +3,6 @@
 from parameterized import parameterized
 
 from aws_lambda_builders.exceptions import MisMatchRuntimeError
-from aws_lambda_builders.workflows.python_pip.exceptions import MissingPipError
 from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator
 from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError
 
@@ -20,52 +19,36 @@ class TestPythonRuntimeValidator(TestCase):
     def setUp(self):
         self.validator = PythonRuntimeValidator(runtime="python3.7", architecture="x86_64")
 
-    @mock.patch("aws_lambda_builders.workflows.python_pip.validator.pip_import_string")
-    def test_runtime_validate_unsupported_language_fail_open(self, pip_mock):
+    def test_runtime_validate_unsupported_language_fail_open(self):
         validator = PythonRuntimeValidator(runtime="python2.6", architecture="arm64")
         with self.assertRaises(UnsupportedRuntimeError):
             validator.validate(runtime_path="/usr/bin/python2.6")
 
-    @mock.patch("aws_lambda_builders.workflows.python_pip.validator.pip_import_string")
-    def test_runtime_validate_supported_version_runtime(self, pip_mock):
+    def test_runtime_validate_supported_version_runtime(self):
         with mock.patch("subprocess.Popen") as mock_subprocess:
             mock_subprocess.return_value = MockSubProcess(0)
             self.validator.validate(runtime_path="/usr/bin/python3.7")
             self.assertTrue(mock_subprocess.call_count, 1)
 
-    @mock.patch("aws_lambda_builders.workflows.python_pip.validator.pip_import_string")
-    def test_runtime_validate_mismatch_version_runtime(self, pip_mock):
+    def test_runtime_validate_mismatch_version_runtime(self):
         with mock.patch("subprocess.Popen") as mock_subprocess:
             mock_subprocess.return_value = MockSubProcess(1)
             with self.assertRaises(MisMatchRuntimeError):
                 self.validator.validate(runtime_path="/usr/bin/python3.9")
                 self.assertTrue(mock_subprocess.call_count, 1)
 
-    @mock.patch("aws_lambda_builders.workflows.python_pip.validator.pip_import_string")
-    def test_python_command(self, pip_mock):
+    def test_python_command(self):
         cmd = self.validator._validate_python_cmd(runtime_path="/usr/bin/python3.7")
         version_strings = ["sys.version_info.major == 3", "sys.version_info.minor == 7"]
         for version_string in version_strings:
             self.assertTrue(all([part for part in cmd if version_string in part]))
 
-    @mock.patch("aws_lambda_builders.workflows.python_pip.validator.pip_import_string")
-    @mock.patch("aws_lambda_builders.workflows.python_pip.validator.subprocess.Popen")
-    def test_runtime_exists_missing_pip(self, popen_mock, pip_mock):
-        popen_mock.return_code = 0
-        pip_mock.side_effect = [MissingPipError(python_path="message")]
-
-        with self.assertRaises(MisMatchRuntimeError) as ex:
-            self.validator.validate(runtime_path="/usr/bin/python3.7")
-
-        self.assertEqual(self.validator._valid_runtime_path, None)
-
     @parameterized.expand(
         [
             ("python3.7", "arm64"),
         ]
     )
-    @mock.patch("aws_lambda_builders.workflows.python_pip.validator.pip_import_string")
-    def test_runtime_validate_with_incompatible_architecture(self, runtime, architecture, pip_mock):
+    def test_runtime_validate_with_incompatible_architecture(self, runtime, architecture):
         validator = PythonRuntimeValidator(runtime=runtime, architecture=architecture)
         with self.assertRaises(UnsupportedArchitectureError):
             validator.validate(runtime_path="/usr/bin/python")