Skip to content

Commit

Permalink
fix: Validate if pip is installed in the runtime path (#549)
Browse files Browse the repository at this point in the history 
* Validate if pip is installed in the current env

* make format

* Fixed and added unit test

* Changed relative import to absolute

* Added type hinting to method

* Moved validation logic into build action

* Add doc strings and remaining tests
  • Loading branch information
@lucashuy
lucashuy authored Sep 28, 2023
1 parent a01d058 commit 704f7ec
Show file tree
Hide file tree
Showing 2 changed files with 167 additions and 39 deletions.
77 changes: 65 additions & 12 deletions aws_lambda_builders/workflows/python_pip/actions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,24 @@
Action to resolve Python dependencies using PIP
"""

import logging
from typing import Optional, Tuple

from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
from aws_lambda_builders.architecture import X86_64
from aws_lambda_builders.binary_path import BinaryPath
from aws_lambda_builders.exceptions import MisMatchRuntimeError, RuntimeValidatorError
from aws_lambda_builders.workflows.python_pip.exceptions import MissingPipError
from aws_lambda_builders.workflows.python_pip.packager import (
DependencyBuilder,
PackagerError,
PipRunner,
PythonPipDependencyBuilder,
SubprocessPip,
)
from aws_lambda_builders.workflows.python_pip.utils import OSUtils

from .exceptions import MissingPipError
from .packager import DependencyBuilder, PackagerError, PipRunner, PythonPipDependencyBuilder, SubprocessPip
LOG = logging.getLogger(__name__)


class PythonPipBuildAction(BaseAction):
Expand All @@ -27,20 +39,21 @@ def __init__(
self.binaries = binaries
self.architecture = architecture

def execute(self):
os_utils = OSUtils()
python_path = self.binaries[self.LANGUAGE].binary_path
try:
pip = SubprocessPip(osutils=os_utils, python_exe=python_path)
except MissingPipError as ex:
raise ActionFailedError(str(ex))
pip_runner = PipRunner(python_exe=python_path, pip=pip)
self._os_utils = OSUtils()

def execute(self) -> None:
"""
Executes the build action for Python `pip` workflows.
"""
pip, python_with_pip = self._find_runtime_with_pip()
pip_runner = PipRunner(python_exe=python_with_pip, pip=pip)

dependency_builder = DependencyBuilder(
osutils=os_utils, pip_runner=pip_runner, runtime=self.runtime, architecture=self.architecture
osutils=self._os_utils, pip_runner=pip_runner, runtime=self.runtime, architecture=self.architecture
)

package_builder = PythonPipDependencyBuilder(
osutils=os_utils, runtime=self.runtime, dependency_builder=dependency_builder
osutils=self._os_utils, runtime=self.runtime, dependency_builder=dependency_builder
)
try:
target_artifact_dir = self.artifacts_dir
Expand All @@ -55,3 +68,43 @@ def execute(self):
)
except PackagerError as ex:
raise ActionFailedError(str(ex))

def _find_runtime_with_pip(self) -> Tuple[SubprocessPip, str]:
"""
Finds a Python runtime that also contains `pip`.
Returns
-------
Tuple[SubprocessPip, str]
Returns a tuple of the SubprocessPip object created from
a valid Python runtime and the runtime path itself
Raises
------
ActionFailedError
Raised if the method is not able to find a valid runtime
that has the correct Python and pip installed
"""
binary_object: Optional[BinaryPath] = self.binaries.get(self.LANGUAGE)

if not binary_object:
raise ActionFailedError("Failed to fetch Python binaries from the PATH.")

for python_path in binary_object.resolver.exec_paths:
try:
valid_python_path = binary_object.validator.validate(python_path)

if valid_python_path:
pip = SubprocessPip(osutils=self._os_utils, python_exe=valid_python_path)

return (pip, valid_python_path)
except (MisMatchRuntimeError, RuntimeValidatorError):
# runtime and mismatch exceptions should have been caught
# during the init phase

# we can ignore these and let the action fail at the end
LOG.debug(f"Python runtime path '{valid_python_path}' does not match the workflow")
except MissingPipError:
LOG.debug(f"Python runtime path '{valid_python_path}' does not contain pip")

raise ActionFailedError("Failed to find a Python runtime containing pip on the PATH.")
129 changes: 102 additions & 27 deletions tests/unit/workflows/python_pip/test_actions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,16 @@

from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction
from aws_lambda_builders.workflows.python_pip.exceptions import MissingPipError
from aws_lambda_builders.workflows.python_pip.packager import PackagerError
from aws_lambda_builders.workflows.python_pip.packager import PackagerError, SubprocessPip


class TestPythonPipBuildAction(TestCase):
@patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
@patch("aws_lambda_builders.workflows.python_pip.actions.DependencyBuilder")
def test_action_must_call_builder(self, DependencyBuilderMock, PythonPipDependencyBuilderMock):
builder_instance = PythonPipDependencyBuilderMock.return_value
@patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
def test_action_must_call_builder(self, find_runtime_mock, dependency_builder_mock, pip_dependency_builder_mock):
builder_instance = pip_dependency_builder_mock.return_value
find_runtime_mock.return_value = (Mock(), Mock())

action = PythonPipBuildAction(
"artifacts",
Expand All @@ -28,16 +30,20 @@ def test_action_must_call_builder(self, DependencyBuilderMock, PythonPipDependen
)
action.execute()

DependencyBuilderMock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=X86_64)
dependency_builder_mock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=X86_64)

builder_instance.build_dependencies.assert_called_with(
artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest"
)

@patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
@patch("aws_lambda_builders.workflows.python_pip.actions.DependencyBuilder")
def test_action_must_call_builder_with_architecture(self, DependencyBuilderMock, PythonPipDependencyBuilderMock):
builder_instance = PythonPipDependencyBuilderMock.return_value
@patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
def test_action_must_call_builder_with_architecture(
self, find_runtime_mock, dependency_builder_mock, pip_dependency_builder_mock
):
builder_instance = pip_dependency_builder_mock.return_value
find_runtime_mock.return_value = (Mock(), Mock())

action = PythonPipBuildAction(
"artifacts",
Expand All @@ -50,32 +56,18 @@ def test_action_must_call_builder_with_architecture(self, DependencyBuilderMock,
)
action.execute()

DependencyBuilderMock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=ARM64)
dependency_builder_mock.assert_called_with(osutils=ANY, pip_runner=ANY, runtime="runtime", architecture=ARM64)

builder_instance.build_dependencies.assert_called_with(
artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest"
)

@patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock):
builder_instance = PythonPipDependencyBuilderMock.return_value
@patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
def test_must_raise_exception_on_failure(self, find_runtime_mock, pip_dependency_builder_mock):
builder_instance = pip_dependency_builder_mock.return_value
builder_instance.build_dependencies.side_effect = PackagerError()

action = PythonPipBuildAction(
"artifacts",
"scratch_dir",
"manifest",
"runtime",
None,
{"python": BinaryPath(resolver=Mock(), validator=Mock(), binary="python", binary_path=sys.executable)},
)

with self.assertRaises(ActionFailedError):
action.execute()

@patch("aws_lambda_builders.workflows.python_pip.actions.SubprocessPip")
def test_must_raise_exception_on_pip_failure(self, PythonSubProcessPipMock):
PythonSubProcessPipMock.side_effect = MissingPipError(python_path="mockpath")
find_runtime_mock.return_value = (Mock(), Mock())

action = PythonPipBuildAction(
"artifacts",
Expand All @@ -90,8 +82,10 @@ def test_must_raise_exception_on_pip_failure(self, PythonSubProcessPipMock):
action.execute()

@patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
def test_action_must_call_builder_with_dependencies_dir(self, PythonPipDependencyBuilderMock):
builder_instance = PythonPipDependencyBuilderMock.return_value
@patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipBuildAction._find_runtime_with_pip")
def test_action_must_call_builder_with_dependencies_dir(self, find_runtime_mock, pip_dependency_builder_mock):
builder_instance = pip_dependency_builder_mock.return_value
find_runtime_mock.return_value = (Mock(), Mock())

action = PythonPipBuildAction(
"artifacts",
Expand All @@ -106,3 +100,84 @@ def test_action_must_call_builder_with_dependencies_dir(self, PythonPipDependenc
builder_instance.build_dependencies.assert_called_with(
artifacts_dir_path="dependencies_dir", scratch_dir_path="scratch_dir", requirements_path="manifest"
)

def test_find_runtime_missing_binary_object(self):
mock_binaries = {}

with self.assertRaises(ActionFailedError) as ex:
PythonPipBuildAction(Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries)._find_runtime_with_pip()

self.assertEqual(str(ex.exception), "Failed to fetch Python binaries from the PATH.")

def test_find_runtime_empty_exec_paths(self):
mock_resolver = Mock()
mock_resolver.resolver = Mock()
mock_resolver.resolver.exec_paths = []

mock_binaries = Mock()
mock_binaries.get = Mock(return_value=mock_resolver)

with self.assertRaises(ActionFailedError) as ex:
PythonPipBuildAction(Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries)._find_runtime_with_pip()

self.assertEqual(str(ex.exception), "Failed to fetch Python binaries from the PATH.")

@patch("aws_lambda_builders.workflows.python_pip.actions.SubprocessPip")
def test_find_runtime_found_pip(self, pip_subprocess_mock):
expected_pip = Mock()
pip_subprocess_mock.return_value = expected_pip

expected_python_path = "my_python_path"

mock_binary_path = Mock()
mock_binary_path.resolver = Mock()
mock_binary_path.resolver.exec_paths = [expected_python_path]
mock_binary_path.validator = Mock()
mock_binary_path.validator.validate.return_value = expected_python_path

mock_binaries = Mock()
mock_binaries.get = Mock(return_value=mock_binary_path)

pip, runtime_path = PythonPipBuildAction(
Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries
)._find_runtime_with_pip()

self.assertEqual(pip, expected_pip)
self.assertEqual(runtime_path, expected_python_path)

@patch("aws_lambda_builders.workflows.python_pip.actions.SubprocessPip")
def test_find_runtime_no_pip_matches(self, pip_subprocess_mock):
python_path = "my_python_path"

pip_subprocess_mock.side_effect = [MissingPipError(python_path="message")]

mock_binary_path = Mock()
mock_binary_path.resolver = Mock()
mock_binary_path.resolver.exec_paths = [python_path]
mock_binary_path.validator = Mock()
mock_binary_path.validator.validate.return_value = python_path

mock_binaries = Mock()
mock_binaries.get = Mock(return_value=mock_binary_path)

with self.assertRaises(ActionFailedError) as ex:
PythonPipBuildAction(Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries)._find_runtime_with_pip()

self.assertEqual(str(ex.exception), "Failed to find a Python runtime containing pip on the PATH.")

def test_find_runtime_no_python_matches(self):
python_path = "my_python_path"

mock_binary_path = Mock()
mock_binary_path.resolver = Mock()
mock_binary_path.resolver.exec_paths = [python_path]
mock_binary_path.validator = Mock()
mock_binary_path.validator.validate.return_value = None

mock_binaries = Mock()
mock_binaries.get = Mock(return_value=mock_binary_path)

with self.assertRaises(ActionFailedError) as ex:
PythonPipBuildAction(Mock(), Mock(), Mock(), Mock(), Mock(), mock_binaries)._find_runtime_with_pip()

self.assertEqual(str(ex.exception), "Failed to find a Python runtime containing pip on the PATH.")

0 comments on commit 704f7ec

Please sign in to comment.