From 21c0396da08b258e558238e6386673c964ac4e92 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 18 Sep 2024 22:20:15 +1200 Subject: [PATCH 01/31] Offer pip install ops[testing] and if available expose Scenario in ops.testing. --- ops/testing.py | 146 ++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 3 + test/test_testing.py | 39 ++++++++++++ tox.ini | 2 + 4 files changed, 187 insertions(+), 3 deletions(-) diff --git a/ops/testing.py b/ops/testing.py index e93ba7bcb..514f324bf 100644 --- a/ops/testing.py +++ b/ops/testing.py @@ -32,6 +32,10 @@ on testing charms, see `Charm SDK | Testing `_. """ +import typing as _typing +from importlib.metadata import PackageNotFoundError as _PackageNotFoundError +from importlib.metadata import version as _get_package_version + from ._private.harness import ( ActionFailed, ActionOutput, @@ -56,8 +60,144 @@ storage, ) -# The Harness testing framework. -_ = ActionFailed +# If the 'ops.testing' optional extra is installed, make those +# names available in this namespace. +try: + _version = _get_package_version('ops-scenario') +except _PackageNotFoundError: + pass +else: + if _version and int(_version.split('.', 1)[0]) >= 7: + from scenario import ( + ActionFailed as _ScenarioActionFailed, + ) + from scenario import ( + ActiveStatus, + Address, + AnyJson, + BindAddress, + BlockedStatus, + CheckInfo, + CloudCredential, + CloudSpec, + Container, + Context, + DeferredEvent, + ErrorStatus, + Exec, + ICMPPort, + JujuLogLine, + MaintenanceStatus, + Manager, + Model, + Mount, + Network, + Notice, + PeerRelation, + Port, + RawDataBagContents, + RawSecretRevisionContents, + Relation, + RelationBase, + Resource, + Secret, + State, + StateValidationError, + Storage, + StoredState, + SubordinateRelation, + TCPPort, + UDPPort, + UnitID, + UnknownStatus, + WaitingStatus, + ) + + # The Scenario unit testing framework. + _ = ActiveStatus + _ = Address + _ = AnyJson + _ = BindAddress + _ = BlockedStatus + _ = CheckInfo + _ = CloudCredential + _ = CloudSpec + _ = Container + _ = Context + _ = DeferredEvent + _ = ErrorStatus + _ = Exec + _ = ICMPPort + _ = JujuLogLine + _ = MaintenanceStatus + _ = Manager + _ = Model + _ = Mount + _ = Network + _ = Notice + _ = PeerRelation + _ = Port + _ = RawDataBagContents + _ = RawSecretRevisionContents + _ = Relation + _ = RelationBase + _ = Resource + _ = Secret + _ = State + _ = StateValidationError + _ = Storage + _ = StoredState + _ = SubordinateRelation + _ = TCPPort + _ = UDPPort + _ = UnitID + _ = UnknownStatus + _ = WaitingStatus + + # Handle the name clash between Harness's and Scenario's ActionFailed. + class _MergedActionFailed(ActionFailed, _ScenarioActionFailed): + """Raised when :code:`event.fail()` is called during an action handler.""" + + message: str + """Optional details of the failure, as provided by :meth:`ops.ActionEvent.fail`.""" + + output: ActionOutput + """Any logs and results set by the Charm. + + When using Context.run, both logs and results will be empty - these + can be found in Context.action_logs and Context.action_results. + """ + + state: _typing.Optional[State] + """The Juju state after the action has been run. + + When using Harness.run_action, this will be None. + """ + + def __init__( + self, + message: str, + output: _typing.Optional[ActionOutput] = None, + *, + state: _typing.Optional[State] = None, + ): + self.message = message + self.output = ActionOutput([], {}) if output is None else output + self.state = state + + ActionFailed = _MergedActionFailed + + # Also monkeypatch this merged one in so that isinstance checks work. + import ops._private.harness as _harness + + _harness.ActionFailed = _MergedActionFailed + import scenario.context as _context + + _context.ActionFailed = _MergedActionFailed + + +# The Harness unit testing framework. +_ = ActionFailed # If Scenario has been installed, then this will be the merged ActionFailed. _ = ActionOutput _ = AppUnitOrName _ = CharmType @@ -71,7 +211,7 @@ # Names exposed for backwards compatibility _ = CharmBase _ = CharmMeta -_ = Container +_ = Container # If Scenario has been installed, then this will be scenario.Container. _ = ExecProcess _ = RelationNotFoundError _ = RelationRole diff --git a/pyproject.toml b/pyproject.toml index 4bfa2d1db..ad29c607a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ docs = [ "sphinxcontrib-jquery", "sphinxext-opengraph" ] +testing = [ + "ops-scenario>=7.0.5,<8", +] [project.urls] "Homepage" = "https://juju.is/docs/sdk" diff --git a/test/test_testing.py b/test/test_testing.py index 965ebc894..24290a02b 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -7025,3 +7025,42 @@ def test_get_cloud_spec_without_set_error(self, request: pytest.FixtureRequest): harness.begin() with pytest.raises(ops.ModelError): harness.model.get_cloud_spec() + + +@pytest.mark.skipif( + not hasattr(ops.testing, 'Context'), reason='requires optional ops[testing] install' +) +def test_scenario_available(): + ctx = ops.testing.Context(ops.CharmBase, meta={'name': 'foo'}) + state = ctx.run(ctx.on.start(), ops.testing.State()) + assert isinstance(state.unit_status, ops.testing.UnknownStatus) + + +@pytest.mark.skipif( + not hasattr(ops.testing, 'Context'), reason='requires optional ops[testing] install' +) +def test_merged_actionfailed(): + class MyCharm(ops.CharmBase): + def __init__(self, framework): + super().__init__(framework) + framework.observe(self.on.go_action, self._on_go) + + def _on_go(self, event): + event.log('\U0001f680') + event.set_results({'interim': '\U0001f97a'}) + event.fail('\U0001f61e') + + harness = ops.testing.Harness(MyCharm, actions="""go:\n description: go""") + harness.begin() + with pytest.raises(ops.testing.ActionFailed) as excinfo: + harness.run_action('go') + assert excinfo.value.message == '\U0001f61e' + assert excinfo.value.output.logs == ['\U0001f680'] + assert excinfo.value.output.results == {'interim': '\U0001f97a'} + + ctx = ops.testing.Context(MyCharm, meta={'name': 'foo'}, actions={'go': {}}) + with pytest.raises(ops.testing.ActionFailed) as excinfo: + ctx.run(ctx.on.action('go'), ops.testing.State()) + assert excinfo.value.message == '\U0001f61e' + assert ctx.action_logs == ['\U0001f680'] + assert ctx.action_results == {'interim': '\U0001f97a'} diff --git a/tox.ini b/tox.ini index cf097a12f..ca45096c6 100644 --- a/tox.ini +++ b/tox.ini @@ -80,6 +80,7 @@ deps = pyright==1.1.377 pytest~=7.2 typing_extensions~=4.2 + ops-scenario>=7.0.5,<8.0 commands = pyright {posargs} @@ -94,6 +95,7 @@ deps = pytest~=7.2 pytest-xdist~=3.6 typing_extensions~=4.2 + ops-scenario>=7.0.5,<8.0 commands = pytest -n auto --ignore={[vars]tst_path}smoke -v --tb native {posargs} From cd0ecbf4b8779550ce28dca31fb8533ea73ea28b Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Thu, 19 Sep 2024 11:54:59 +1200 Subject: [PATCH 02/31] Use an explicit reference to help Sphinx out. --- ops/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ops/model.py b/ops/model.py index d03c4b584..f4fbb4083 100644 --- a/ops/model.py +++ b/ops/model.py @@ -1982,7 +1982,7 @@ class UnknownStatus(StatusBase): charm has not called status-set yet. This status is read-only; trying to set unit or application status to - ``UnknownStatus`` will raise :class:`InvalidStatusError`. + ``UnknownStatus`` will raise :class:`~ops.InvalidStatusError`. """ name = 'unknown' @@ -2002,7 +2002,7 @@ class ErrorStatus(StatusBase): human intervention in order to operate correctly). This status is read-only; trying to set unit or application status to - ``ErrorStatus`` will raise :class:`InvalidStatusError`. + ``ErrorStatus`` will raise :class:`~ops.InvalidStatusError`. """ name = 'error' From 4eedb72dbf98df39f3219f9170ebef71283dc761 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Thu, 19 Sep 2024 11:55:39 +1200 Subject: [PATCH 03/31] Create a new class rather than inheriting from both the originals. --- ops/testing.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/ops/testing.py b/ops/testing.py index 514f324bf..1f3735e55 100644 --- a/ops/testing.py +++ b/ops/testing.py @@ -37,7 +37,6 @@ from importlib.metadata import version as _get_package_version from ._private.harness import ( - ActionFailed, ActionOutput, AppUnitOrName, CharmBase, @@ -65,12 +64,9 @@ try: _version = _get_package_version('ops-scenario') except _PackageNotFoundError: - pass + from ops._private.harness import ActionFailed # type: ignore else: if _version and int(_version.split('.', 1)[0]) >= 7: - from scenario import ( - ActionFailed as _ScenarioActionFailed, - ) from scenario import ( ActiveStatus, Address, @@ -102,7 +98,6 @@ Resource, Secret, State, - StateValidationError, Storage, StoredState, SubordinateRelation, @@ -111,6 +106,7 @@ UnitID, UnknownStatus, WaitingStatus, + errors, ) # The Scenario unit testing framework. @@ -144,7 +140,6 @@ _ = Resource _ = Secret _ = State - _ = StateValidationError _ = Storage _ = StoredState _ = SubordinateRelation @@ -153,9 +148,10 @@ _ = UnitID _ = UnknownStatus _ = WaitingStatus + _ = errors # Handle the name clash between Harness's and Scenario's ActionFailed. - class _MergedActionFailed(ActionFailed, _ScenarioActionFailed): + class ActionFailed(Exception): # noqa: N818 """Raised when :code:`event.fail()` is called during an action handler.""" message: str @@ -185,16 +181,15 @@ def __init__( self.output = ActionOutput([], {}) if output is None else output self.state = state - ActionFailed = _MergedActionFailed - - # Also monkeypatch this merged one in so that isinstance checks work. + # Monkeypatch this merged one in so that isinstance checks work. import ops._private.harness as _harness - _harness.ActionFailed = _MergedActionFailed + _harness.ActionFailed = ActionFailed import scenario.context as _context - _context.ActionFailed = _MergedActionFailed - + _context.ActionFailed = ActionFailed + else: + from ops._private.harness import ActionFailed # type: ignore # The Harness unit testing framework. _ = ActionFailed # If Scenario has been installed, then this will be the merged ActionFailed. From b61469b29f4489767588547960f55f639828bc6c Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Thu, 19 Sep 2024 11:56:02 +1200 Subject: [PATCH 04/31] Add types in the test class. --- test/test_testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_testing.py b/test/test_testing.py index 24290a02b..b3fc67800 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -7041,11 +7041,11 @@ def test_scenario_available(): ) def test_merged_actionfailed(): class MyCharm(ops.CharmBase): - def __init__(self, framework): + def __init__(self, framework: ops.Framework): super().__init__(framework) framework.observe(self.on.go_action, self._on_go) - def _on_go(self, event): + def _on_go(self, event: ops.ActionEvent): event.log('\U0001f680') event.set_results({'interim': '\U0001f97a'}) event.fail('\U0001f61e') From 7ee4fe5c5991f33eb9bafaca50d79c7a3ca77b34 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Thu, 19 Sep 2024 23:48:38 +1200 Subject: [PATCH 05/31] Use explicit references to ops so that copying the docstrings works. --- ops/charm.py | 73 ++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/ops/charm.py b/ops/charm.py index edef8db32..deac3715c 100644 --- a/ops/charm.py +++ b/ops/charm.py @@ -118,10 +118,11 @@ class ActionEvent(EventBase): invokes a Juju Action. Callbacks bound to these events may be used for responding to the administrator's Juju Action request. - To read the parameters for the action, see the instance variable :attr:`params`. - To respond with the result of the action, call :meth:`set_results`. To add - progress messages that are visible as the action is progressing use - :meth:`log`. + To read the parameters for the action, see the instance variable + :attr:`~ops.ActionEvent.params`. + To respond with the result of the action, call + :meth:`~ops.ActionEvent.set_results`. To add progress messages that are + visible as the action is progressing use :meth:`~ops.ActionEvent.log`. """ id: str = '' @@ -240,7 +241,7 @@ class StartEvent(HookEvent): """Event triggered immediately after first configuration change. This event is triggered immediately after the first - :class:`ConfigChangedEvent`. Callback methods bound to the event should be + :class:`~ops.ConfigChangedEvent`. Callback methods bound to the event should be used to ensure that the charm's software is in a running state. Note that the charm's software should be configured so as to persist in this state through reboots without further intervention on Juju's part. @@ -298,8 +299,8 @@ class ConfigChangedEvent(HookEvent): specifically has changed in the config). - Right after the unit starts up for the first time. This event notifies the charm of its initial configuration. - Typically, this event will fire between an :class:`install ` - and a :class:`start ` during the startup sequence + Typically, this event will fire between an :class:`~ops.InstallEvent` + and a :class:~`ops.StartEvent` during the startup sequence (when a unit is first deployed), but in general it will fire whenever the unit is (re)started, for example after pod churn on Kubernetes, on unit rescheduling, on unit upgrade or refresh, and so on. @@ -323,7 +324,7 @@ class UpdateStatusEvent(HookEvent): to this event should determine the "health" of the application and set the status appropriately. - The interval between :class:`update-status ` events can + The interval between :class:`~ops.UpdateStatusEvent` events can be configured model-wide, e.g. ``juju model-config update-status-hook-interval=1m``. """ @@ -355,7 +356,7 @@ class PreSeriesUpgradeEvent(HookEvent): It can be assumed that only after all units on a machine have executed the callback method associated with this event, the administrator will initiate steps to actually upgrade the series. After the upgrade has been completed, - the :class:`PostSeriesUpgradeEvent` will fire. + the :class:`~ops.PostSeriesUpgradeEvent` will fire. .. jujuremoved:: 4.0 """ @@ -412,7 +413,7 @@ class LeaderSettingsChangedEvent(HookEvent): .. deprecated:: 2.4.0 This event has been deprecated in favor of using a Peer relation, and having the leader set a value in the Application data bag for - that peer relation. (See :class:`RelationChangedEvent`.) + that peer relation. (See :class:`~ops.RelationChangedEvent`.) """ @@ -550,7 +551,7 @@ class RelationCreatedEvent(RelationEvent): This is triggered when a new integration with another app is added in Juju. This can occur before units for those applications have started. All existing - relations will trigger `RelationCreatedEvent` before :class:`StartEvent` is + relations will trigger `RelationCreatedEvent` before :class:`~ops.StartEvent` is emitted. """ @@ -582,7 +583,7 @@ class RelationChangedEvent(RelationEvent): to see the new information, where ``event`` is the event object passed to the callback method bound to this event. - This event always fires once, after :class:`RelationJoinedEvent`, and + This event always fires once, after :class:`~ops.RelationJoinedEvent`, and will subsequently fire whenever that remote unit changes its data for the relation. Callback methods bound to this event should be the only ones that rely on remote relation data. They should not error if the data @@ -597,7 +598,7 @@ class RelationChangedEvent(RelationEvent): class RelationDepartedEvent(RelationEvent): """Event triggered when a unit leaves a relation. - This is the inverse of the :class:`RelationJoinedEvent`, representing when a + This is the inverse of the :class:`~ops.RelationJoinedEvent`, representing when a unit is leaving the relation (the unit is being removed, the app is being removed, the relation is being removed). For remaining units, this event is emitted once for each departing unit. For departing units, this event is @@ -610,7 +611,7 @@ class RelationDepartedEvent(RelationEvent): unit has already shut down. Once all callback methods bound to this event have been run for such a - relation, the unit agent will fire the :class:`RelationBrokenEvent`. + relation, the unit agent will fire the :class:`~ops.RelationBrokenEvent`. """ unit: model.Unit # pyright: ignore[reportIncompatibleVariableOverride] @@ -670,7 +671,7 @@ class RelationBrokenEvent(RelationEvent): The event indicates that the current relation is no longer valid, and that the charm's software must be configured as though the relation had never existed. It will only be called after every callback method bound to - :class:`RelationDepartedEvent` has been run. If a callback method + :class:`~ops.RelationDepartedEvent` has been run. If a callback method bound to this event is being executed, it is guaranteed that no remote units are currently known locally. """ @@ -741,7 +742,7 @@ class StorageAttachedEvent(StorageEvent): Callback methods bound to this event allow the charm to run code when storage has been added. Such methods will be run before the - :class:`InstallEvent` fires, so that the installation routine may + :class:`~ops.InstallEvent` fires, so that the installation routine may use the storage. The name prefix of this hook will depend on the storage key defined in the ``metadata.yaml`` file. """ @@ -755,7 +756,7 @@ class StorageDetachingEvent(StorageEvent): Callback methods bound to this event allow the charm to run code before storage is removed. Such methods will be run before storage - is detached, and always before the :class:`StopEvent` fires, thereby + is detached, and always before the :class:`~ops.StopEvent` fires, thereby allowing the charm to gracefully release resources before they are removed and before the unit terminates. The name prefix of the hook will depend on the storage key defined in the ``metadata.yaml`` @@ -905,7 +906,7 @@ class PebbleCheckFailedEvent(PebbleCheckEvent): """Event triggered when a Pebble check exceeds the configured failure threshold. Note that the check may have started passing by the time this event is - emitted (which will mean that a :class:`PebbleCheckRecoveredEvent` will be + emitted (which will mean that a :class:`~ops.PebbleCheckRecoveredEvent` will be emitted next). If the handler is executing code that should only be done if the check is currently failing, check the current status with ``event.info.status == ops.pebble.CheckStatus.DOWN``. @@ -1185,59 +1186,59 @@ class CharmEvents(ObjectEvents): # each event class's docstring. Please keep in sync. install = EventSource(InstallEvent) - """Triggered when a charm is installed (see :class:`InstallEvent`).""" + """Triggered when a charm is installed (see :class:`~ops.InstallEvent`).""" start = EventSource(StartEvent) - """Triggered immediately after first configuration change (see :class:`StartEvent`).""" + """Triggered immediately after first configuration change (see :class:`~ops.StartEvent`).""" stop = EventSource(StopEvent) - """Triggered when a charm is shut down (see :class:`StopEvent`).""" + """Triggered when a charm is shut down (see :class:`~ops.StopEvent`).""" remove = EventSource(RemoveEvent) - """Triggered when a unit is about to be terminated (see :class:`RemoveEvent`).""" + """Triggered when a unit is about to be terminated (see :class:`~ops.RemoveEvent`).""" update_status = EventSource(UpdateStatusEvent) """Triggered periodically by a status update request from Juju (see - :class:`UpdateStatusEvent`). + :class:`~ops.UpdateStatusEvent`). """ config_changed = EventSource(ConfigChangedEvent) - """Triggered when a configuration change occurs (see :class:`ConfigChangedEvent`).""" + """Triggered when a configuration change occurs (see :class:`~ops.ConfigChangedEvent`).""" upgrade_charm = EventSource(UpgradeCharmEvent) - """Triggered by request to upgrade the charm (see :class:`UpgradeCharmEvent`).""" + """Triggered by request to upgrade the charm (see :class:`~ops.UpgradeCharmEvent`).""" pre_series_upgrade = EventSource(PreSeriesUpgradeEvent) - """Triggered to prepare a unit for series upgrade (see :class:`PreSeriesUpgradeEvent`). + """Triggered to prepare a unit for series upgrade (see :class:`~ops.PreSeriesUpgradeEvent`). .. jujuremoved:: 4.0 """ post_series_upgrade = EventSource(PostSeriesUpgradeEvent) - """Triggered after a series upgrade (see :class:`PostSeriesUpgradeEvent`). + """Triggered after a series upgrade (see :class:`~ops.PostSeriesUpgradeEvent`). .. jujuremoved:: 4.0 """ leader_elected = EventSource(LeaderElectedEvent) - """Triggered when a new leader has been elected (see :class:`LeaderElectedEvent`).""" + """Triggered when a new leader has been elected (see :class:`~ops.LeaderElectedEvent`).""" leader_settings_changed = EventSource(LeaderSettingsChangedEvent) """Triggered when leader changes any settings (see - :class:`LeaderSettingsChangedEvent`). + :class:`~ops.LeaderSettingsChangedEvent`). .. deprecated:: 2.4.0 """ collect_metrics = EventSource(CollectMetricsEvent) - """Triggered by Juju to collect metrics (see :class:`CollectMetricsEvent`). + """Triggered by Juju to collect metrics (see :class:`~ops.CollectMetricsEvent`). .. jujuremoved:: 4.0 """ secret_changed = EventSource(SecretChangedEvent) """Triggered by Juju on the observer when the secret owner changes its contents (see - :class:`SecretChangedEvent`). + :class:`~ops.SecretChangedEvent`). .. jujuadded:: 3.0 Charm secrets added in Juju 3.0, user secrets added in Juju 3.3 @@ -1245,33 +1246,33 @@ class CharmEvents(ObjectEvents): secret_expired = EventSource(SecretExpiredEvent) """Triggered by Juju on the owner when a secret's expiration time elapses (see - :class:`SecretExpiredEvent`). + :class:`~ops.SecretExpiredEvent`). .. jujuadded:: 3.0 """ secret_rotate = EventSource(SecretRotateEvent) """Triggered by Juju on the owner when the secret's rotation policy elapses (see - :class:`SecretRotateEvent`). + :class:`~ops.SecretRotateEvent`). .. jujuadded:: 3.0 """ secret_remove = EventSource(SecretRemoveEvent) """Triggered by Juju on the owner when a secret revision can be removed (see - :class:`SecretRemoveEvent`). + :class:`~ops.SecretRemoveEvent`). .. jujuadded:: 3.0 """ collect_app_status = EventSource(CollectStatusEvent) """Triggered on the leader at the end of every hook to collect app statuses for evaluation - (see :class:`CollectStatusEvent`). + (see :class:`~ops.CollectStatusEvent`). """ collect_unit_status = EventSource(CollectStatusEvent) """Triggered at the end of every hook to collect unit statuses for evaluation - (see :class:`CollectStatusEvent`). + (see :class:`~ops.CollectStatusEvent`). """ From 40323f23be2255eb80b8007c066ea4f701375962 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Thu, 19 Sep 2024 23:49:05 +1200 Subject: [PATCH 06/31] CharmEvents should be exposed in ops.testing (maybe also in scenario top level?). --- ops/testing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ops/testing.py b/ops/testing.py index 1f3735e55..46c6f1384 100644 --- a/ops/testing.py +++ b/ops/testing.py @@ -108,6 +108,7 @@ WaitingStatus, errors, ) + from scenario.context import CharmEvents # The Scenario unit testing framework. _ = ActiveStatus @@ -115,6 +116,7 @@ _ = AnyJson _ = BindAddress _ = BlockedStatus + _ = CharmEvents _ = CheckInfo _ = CloudCredential _ = CloudSpec From 55f0910d4026876859c0e76cdd6e1538899c83d6 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Thu, 19 Sep 2024 23:50:44 +1200 Subject: [PATCH 07/31] Add Scenario and Harness explicitly rather than via automodule, to work around our trickery. Drop modindex (only has ops and ops.pebble, seems pointless) and search (it's on the left-hand side all the time, no point in a blank page) as drive-bys. --- docs/index.rst | 143 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index c27e2bbe5..b99e5b913 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,7 +64,148 @@ ops.pebble module ops.testing module ================== +Frameworks for unit testing charms in a simulated Juju environment. + +Two frameworks are available: + +* State-transition testing, which tests the charm's state transitions in response + to events. This is the recommended approach. Install ops with the `testing` + extra to use this framework; for example: `pip install ops[testing]` +* Harness, which provides an API similar to the Juju CLI. This is a legacy + framework, and has issues, particularly with resetting the charm state between + Juju events. + +.. note:: + Unit testing is only one aspect of a comprehensive testing strategy. For more + on testing charms, see `Charm SDK | Testing `_. + +State-transition testing +------------------------ + +Write tests that declaratively define the Juju state all at once, define the +Juju context against which to test the charm, and fire a single event on the +charm to execute its logic. The tests can then assert that the Juju state has +changed as expected. + +These tests are 'state-transition' tests, a way to test isolated units of charm +functionality (how the state changes in reaction to events). They are not +necessarily tests of individual methods or functions (but might be, depending on +the charm's event observers); they are testing the 'contract' of the charm: given +a certain state, when a certain event happens, the charm should transition to a +certain (likely different) state. They do not test against a real Juju +controller and model, and focus on a single Juju unit, unlike integration tests. +For simplicity, we refer to them as 'unit' tests in the charm context. + +Writing these tests should nudge you into thinking of a charm as a black-box +input->output function. The input is the union of an `Event` (why am I, charm, +being executed), a `State` (am I leader? what is my integration data? what is my +config?...) and the charm's execution `Context` (what integrations can I have? +what containers can I have?...). The output is another `State`: the state after +the charm has had a chance to interact with the mocked Juju model and affect the +state. + +.. image:: https://raw.githubusercontent.com/canonical/ops-scenario/main/resources/state-transition-model.png + :alt: Transition diagram, with the input state and event on the left, the context including the charm in the centre, and the state out on the right + +Writing unit tests for a charm, then, means verifying that: + +- the charm does not raise uncaught exceptions while handling the event +- the output state (as compared with the input state) is as expected. + +A test consists of three broad steps: + +- **Arrange**: + - declare the context + - declare the input state +- **Act**: + - select an event to fire + - run the context (i.e. obtain the output state, given the input state and the event) +- **Assert**: + - verify that the output state (as compared with the input state) is how you expect it to be + - verify that the charm has seen a certain sequence of statuses, events, and `juju-log` calls + - optionally, you can use a context manager to get a hold of the charm instance and run + assertions on APIs and state internal to it. + +The most basic scenario is one in which all is defaulted and barely any data is +available. The charm has no config, no integrations, no leadership, and its +status is `unknown`. With that, we can write the simplest possible test: + +.. code-block:: python + + def test_base(): + ctx = Context(MyCharm) + out = ctx.run(ctx.on.start(), State()) + assert out.unit_status == UnknownStatus() + +.. autoclass:: ops.testing.ActionFailed +.. autoclass:: ops.testing.ActiveStatus +.. autoclass:: ops.testing.Address +.. autoclass:: ops.testing.BindAddress +.. autoclass:: ops.testing.BlockedStatus +.. autoclass:: ops.testing.CharmEvents +.. autoclass:: ops.testing.CheckInfo +.. autoclass:: ops.testing.CloudCredential +.. autoclass:: ops.testing.CloudSpec +.. autoclass:: ops.testing.Container +.. autoclass:: ops.testing.Context + :special-members: __call__ +.. autoclass:: ops.testing.DeferredEvent +.. autoclass:: ops.testing.ErrorStatus +.. autoclass:: ops.testing.Exec +.. autoclass:: ops.testing.ICMPPort +.. autoclass:: ops.testing.JujuLogLine +.. autoclass:: ops.testing.MaintenanceStatus +.. autoclass:: ops.testing.Manager +.. autoclass:: ops.testing.Model +.. autoclass:: ops.testing.Mount +.. autoclass:: ops.testing.Network +.. autoclass:: ops.testing.Notice +.. autoclass:: ops.testing.PeerRelation +.. autoclass:: ops.testing.Port +.. autoclass:: ops.testing.RawDataBagContents +.. autoclass:: ops.testing.RawSecretRevisionContents +.. autoclass:: ops.testing.Relation +.. autoclass:: ops.testing.RelationBase +.. autoclass:: ops.testing.Resource +.. autoclass:: ops.testing.Secret +.. autoclass:: ops.testing.State +.. autoclass:: ops.testing.Storage +.. autoclass:: ops.testing.StoredState +.. autoclass:: ops.testing.SubordinateRelation +.. autoclass:: ops.testing.TCPPort +.. autoclass:: ops.testing.UDPPort +.. autoclass:: ops.testing.UnknownStatus +.. autoclass:: ops.testing.WaitingStatus +.. autoclass:: ops.testing.errors.ContextSetupError +.. autoclass:: ops.testing.errors.AlreadyEmittedError +.. autoclass:: ops.testing.errors.ScenarioRuntimeError +.. autoclass:: ops.testing.errors.UncaughtCharmError +.. autoclass:: ops.testing.errors.InconsistentScenarioError +.. autoclass:: ops.testing.errors.StateValidationError +.. autoclass:: ops.testing.errors.MetadataNotFoundError +.. autoclass:: ops.testing.errors.ActionMissingFromContextError +.. autoclass:: ops.testing.errors.NoObserverError +.. autoclass:: ops.testing.errors.BadOwnerPath + +Harness +------- + +The Harness framework includes: + +- :class:`ops.testing.Harness`, a class to set up the simulated environment, + that provides: + + - :meth:`~ops.testing.Harness.add_relation` method, to declare a relation + (integration) with another app. + - :meth:`~ops.testing.Harness.begin` and :meth:`~ops.testing.Harness.cleanup` + methods to start and end the testing lifecycle. + - :meth:`~ops.testing.Harness.evaluate_status` method, which aggregates the + status of the charm after test interactions. + - :attr:`~ops.testing.Harness.model` attribute, which exposes e.g. the + :attr:`~ops.Model.unit` attribute for detailed assertions on the unit's state. + .. autoclass:: ops.testing.ActionFailed + :noindex: .. autoclass:: ops.testing.ActionOutput .. autoclass:: ops.testing.ExecArgs .. autoclass:: ops.testing.ExecResult @@ -75,5 +216,3 @@ Indices ======= * :ref:`genindex` -* :ref:`modindex` -* :ref:`search` From e7870cd8df11a2f9562ff7517b4f8896d2ee8ddf Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 20 Sep 2024 10:15:37 +1200 Subject: [PATCH 08/31] Adjust the monkeypatching to also include runtime. --- ops/testing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ops/testing.py b/ops/testing.py index 46c6f1384..db1a1a4ae 100644 --- a/ops/testing.py +++ b/ops/testing.py @@ -188,8 +188,10 @@ def __init__( _harness.ActionFailed = ActionFailed import scenario.context as _context + import scenario.runtime as _runtime - _context.ActionFailed = ActionFailed + _context.ActionFailed = ActionFailed # type: ignore[reportPrivateImportUsage] + _runtime.ActionFailed = ActionFailed # type: ignore[reportPrivateImportUsage] else: from ops._private.harness import ActionFailed # type: ignore From 41a985450ddd3097fe9964b180922e335c125e47 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 20 Sep 2024 10:21:39 +1200 Subject: [PATCH 09/31] Fix markup. Note that Harness is included by default. --- docs/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b99e5b913..1266bc684 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -69,11 +69,11 @@ Frameworks for unit testing charms in a simulated Juju environment. Two frameworks are available: * State-transition testing, which tests the charm's state transitions in response - to events. This is the recommended approach. Install ops with the `testing` - extra to use this framework; for example: `pip install ops[testing]` + to events. This is the recommended approach. Install ops with the ``testing`` + extra to use this framework; for example: ``pip install ops[testing]`` * Harness, which provides an API similar to the Juju CLI. This is a legacy framework, and has issues, particularly with resetting the charm state between - Juju events. + Juju events. This is currently included with a base ``ops`` install. .. note:: Unit testing is only one aspect of a comprehensive testing strategy. For more From 2b60277afe1aeef9fec28e11ac272b6bc554ec33 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 20 Sep 2024 12:15:20 +1200 Subject: [PATCH 10/31] Merge in the Scenario customisations. --- docs/custom_conf.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/custom_conf.py b/docs/custom_conf.py index f567b91ec..64fdeff3c 100644 --- a/docs/custom_conf.py +++ b/docs/custom_conf.py @@ -18,6 +18,8 @@ from docutils import nodes +import inspect +import sphinx.ext.autodoc from sphinx import addnodes from sphinx.util.docutils import SphinxDirective @@ -44,7 +46,6 @@ def _compute_navigation_tree(context): # Pull in fix from https://github.com/sphinx-doc/sphinx/pull/11222/files to fix # "invalid signature for autoattribute ('ops.pebble::ServiceDict.backoff-delay')" import re # noqa: E402 -import sphinx.ext.autodoc # noqa: E402 sphinx.ext.autodoc.py_ext_sig_re = re.compile( r"""^ ([\w.]+::)? # explicit module name @@ -216,7 +217,9 @@ def _compute_navigation_tree(context): # pyspelling, sphinx, sphinx-autobuild, sphinx-copybutton, sphinx-design, # sphinx-notfound-page, sphinx-reredirects, sphinx-tabs, sphinxcontrib-jquery, # sphinxext-opengraph -custom_required_modules = [] +custom_required_modules = [ + 'ops-scenario>=7.0.5,<8', +] # Add files or directories that should be excluded from processing. custom_excludes = [ @@ -315,6 +318,8 @@ def _compute_navigation_tree(context): # Please keep this list sorted alphabetically. ('py:class', '_ChangeDict'), ('py:class', '_CheckInfoDict'), + ('py:class', '_EntityStatus'), + ('py:class', '_Event'), ('py:class', '_FileInfoDict'), ('py:class', '_NoticeDict'), ('py:class', '_ProgressDict'), @@ -326,6 +331,8 @@ def _compute_navigation_tree(context): ('py:class', '_TextOrBinaryIO'), ('py:class', '_WarningDict'), ('py:class', '_Writeable'), + ('py:class', 'AnyJson'), + ('py:class', 'CharmType'), ('py:obj', 'ops._private.harness.CharmType'), ('py:class', 'ops._private.harness.CharmType'), ('py:class', 'ops.charm._ContainerBaseDict'), @@ -345,9 +352,34 @@ def _compute_navigation_tree(context): ('py:class', 'ops.testing._ConfigOption'), ('py:class', 'ops.testing.CharmType'), ('py:obj', 'ops.testing.CharmType'), + ('py:class', 'scenario.state._EntityStatus'), + ('py:class', 'scenario.state._Event'), + ('py:class', 'scenario.state._max_posargs.._MaxPositionalArgs'), ] +# Monkeypatch Sphinx to look for __init__ rather than __new__ for the subclasses +# of _MaxPositionalArgs. +_real_get_signature = sphinx.ext.autodoc.ClassDocumenter._get_signature + + +def _custom_get_signature(self): + if any(p.__name__ == '_MaxPositionalArgs' for p in self.object.__mro__): + signature = inspect.signature(self.object) + parameters = [] + for position, param in enumerate(signature.parameters.values()): + if position >= self.object._max_positional_args: + parameters.append(param.replace(kind=inspect.Parameter.KEYWORD_ONLY)) + else: + parameters.append(param) + signature = signature.replace(parameters=parameters) + return None, None, signature + return _real_get_signature(self) + + +sphinx.ext.autodoc.ClassDocumenter._get_signature = _custom_get_signature + + # This is very strongly based on # https://github.com/sphinx-doc/sphinx/blob/03b9134ee00e98df4f8b5f6d22f345cdafe31870/sphinx/domains/changeset.py#L46 # Unfortunately, the built-in class is not easily subclassable without also From c647130c72a8f2de499962b053412fc38d35119f Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 20 Sep 2024 12:16:03 +1200 Subject: [PATCH 11/31] Re-pin with ops-scenario added. --- docs/requirements.txt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 340a58046..0c4e6f1b1 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --extra=docs --output-file=docs/requirements.txt pyproject.toml @@ -73,6 +73,10 @@ mdurl==0.1.2 # via markdown-it-py myst-parser==4.0.0 # via ops (pyproject.toml) +ops==2.16.1 + # via ops-scenario +ops-scenario==7.0.5 + # via ops (pyproject.toml) packaging==24.1 # via sphinx pygments==2.18.0 @@ -85,7 +89,9 @@ pyspelling==2.10 pyyaml==6.0.2 # via # myst-parser + # ops # ops (pyproject.toml) + # ops-scenario # pyspelling requests==2.32.3 # via @@ -160,6 +166,8 @@ wcmatch==9.0 webencodings==0.5.1 # via html5lib websocket-client==1.8.0 - # via ops (pyproject.toml) + # via + # ops + # ops (pyproject.toml) websockets==12.0 # via sphinx-autobuild From 5f21f9472c95919cb42d60b2282a49ced29a27f6 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Fri, 20 Sep 2024 12:16:20 +1200 Subject: [PATCH 12/31] Add ops-scenario to the docs dependencies. --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ad29c607a..56004d415 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,8 @@ docs = [ "sphinx-notfound-page", "sphinx-tabs", "sphinxcontrib-jquery", - "sphinxext-opengraph" + "sphinxext-opengraph", + "ops-scenario>=7.0.5,<8", ] testing = [ "ops-scenario>=7.0.5,<8", From d6c7d2d93e51dfc54f652c2aae3bdf579a0fcf02 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 23 Sep 2024 17:19:34 +1200 Subject: [PATCH 13/31] Use __all__ and move ActionFailed to Harness. --- ops/_private/harness.py | 38 ++++++-- ops/testing.py | 191 ++++++++++++++++++---------------------- test/test_testing.py | 30 ------- 3 files changed, 117 insertions(+), 142 deletions(-) diff --git a/ops/_private/harness.py b/ops/_private/harness.py index 09334406e..c9df5667a 100644 --- a/ops/_private/harness.py +++ b/ops/_private/harness.py @@ -64,6 +64,17 @@ from ops.model import Container, RelationNotFoundError, StatusName, _NetworkDict from ops.pebble import ExecProcess +if typing.TYPE_CHECKING: + try: + from ops.testing import State # type: ignore + except ImportError: + # This is used in the ActionFailed type annotations: it will never be + # used in this case, because it's only relevant when ops.testing has + # the State class, so we just define it as a dummy class. + class State: + pass + + ReadableBuffer = Union[bytes, str, StringIO, BytesIO, BinaryIO] _StringOrPath = Union[str, pathlib.PurePosixPath, pathlib.Path] _FileKwargs = TypedDict( @@ -161,18 +172,35 @@ class ActionOutput: """The action's results, as set or updated by :meth:`ops.ActionEvent.set_results`.""" -class ActionFailed(Exception): # noqa - """Raised when :code:`event.fail()` is called during a :meth:`Harness.run_action` call.""" +class ActionFailed(Exception): # noqa: N818 (name doesn't end with "Error") + """Raised when :code:`event.fail()` is called during an action handler.""" message: str """Optional details of the failure, as provided by :meth:`ops.ActionEvent.fail`.""" output: ActionOutput - """Any logs and results set by the Charm.""" + """Any logs and results set by the Charm. + + When using Context.run, both logs and results will be empty - these + can be found in Context.action_logs and Context.action_results. + """ + + state: typing.Optional['State'] + """The Juju state after the action has been run. - def __init__(self, message: str, output: ActionOutput): + When using Harness.run_action, this will be None. + """ + + def __init__( + self, + message: str, + output: typing.Optional[ActionOutput] = None, + *, + state: typing.Optional['State'] = None, + ): self.message = message - self.output = output + self.output = ActionOutput([], {}) if output is None else output + self.state = state def __str__(self): if self.message: diff --git a/ops/testing.py b/ops/testing.py index db1a1a4ae..be243e962 100644 --- a/ops/testing.py +++ b/ops/testing.py @@ -32,11 +32,12 @@ on testing charms, see `Charm SDK | Testing `_. """ -import typing as _typing -from importlib.metadata import PackageNotFoundError as _PackageNotFoundError -from importlib.metadata import version as _get_package_version +# ruff: noqa: F401 (unused import) + +import importlib.metadata from ._private.harness import ( + ActionFailed, ActionOutput, AppUnitOrName, CharmBase, @@ -59,12 +60,26 @@ storage, ) +# The Harness unit testing framework. +__all__ = [ + 'ActionFailed', + 'ActionOutput', + 'AppUnitOrName', + 'CharmType', + 'ExecArgs', + 'ExecHandler', + 'ExecResult', + 'Harness', + 'ReadableBuffer', + 'YAMLStringOrFile', +] + # If the 'ops.testing' optional extra is installed, make those # names available in this namespace. try: - _version = _get_package_version('ops-scenario') -except _PackageNotFoundError: - from ops._private.harness import ActionFailed # type: ignore + _version = importlib.metadata.version('ops-scenario') +except importlib.metadata.PackageNotFoundError: + pass else: if _version and int(_version.split('.', 1)[0]) >= 7: from scenario import ( @@ -108,114 +123,76 @@ WaitingStatus, errors, ) + + # This can be imported in the group above after Scenario exposes it at the top level. + # https://github.com/canonical/ops-scenario/pull/200 from scenario.context import CharmEvents # The Scenario unit testing framework. - _ = ActiveStatus - _ = Address - _ = AnyJson - _ = BindAddress - _ = BlockedStatus - _ = CharmEvents - _ = CheckInfo - _ = CloudCredential - _ = CloudSpec - _ = Container - _ = Context - _ = DeferredEvent - _ = ErrorStatus - _ = Exec - _ = ICMPPort - _ = JujuLogLine - _ = MaintenanceStatus - _ = Manager - _ = Model - _ = Mount - _ = Network - _ = Notice - _ = PeerRelation - _ = Port - _ = RawDataBagContents - _ = RawSecretRevisionContents - _ = Relation - _ = RelationBase - _ = Resource - _ = Secret - _ = State - _ = Storage - _ = StoredState - _ = SubordinateRelation - _ = TCPPort - _ = UDPPort - _ = UnitID - _ = UnknownStatus - _ = WaitingStatus - _ = errors - - # Handle the name clash between Harness's and Scenario's ActionFailed. - class ActionFailed(Exception): # noqa: N818 - """Raised when :code:`event.fail()` is called during an action handler.""" - - message: str - """Optional details of the failure, as provided by :meth:`ops.ActionEvent.fail`.""" - - output: ActionOutput - """Any logs and results set by the Charm. - - When using Context.run, both logs and results will be empty - these - can be found in Context.action_logs and Context.action_results. - """ - - state: _typing.Optional[State] - """The Juju state after the action has been run. - - When using Harness.run_action, this will be None. - """ - - def __init__( - self, - message: str, - output: _typing.Optional[ActionOutput] = None, - *, - state: _typing.Optional[State] = None, - ): - self.message = message - self.output = ActionOutput([], {}) if output is None else output - self.state = state - - # Monkeypatch this merged one in so that isinstance checks work. - import ops._private.harness as _harness - - _harness.ActionFailed = ActionFailed + __all__.extend([ + 'ActiveStatus', + 'Address', + 'AnyJson', + 'BindAddress', + 'BlockedStatus', + 'CharmEvents', + 'CheckInfo', + 'CloudCredential', + 'CloudSpec', + 'Container', + 'Context', + 'DeferredEvent', + 'ErrorStatus', + 'Exec', + 'ICMPPort', + 'JujuLogLine', + 'MaintenanceStatus', + 'Manager', + 'Model', + 'Mount', + 'Network', + 'Notice', + 'PeerRelation', + 'Port', + 'RawDataBagContents', + 'RawSecretRevisionContents', + 'Relation', + 'RelationBase', + 'Resource', + 'Secret', + 'State', + 'Storage', + 'StoredState', + 'SubordinateRelation', + 'TCPPort', + 'UDPPort', + 'UnitID', + 'UnknownStatus', + 'WaitingStatus', + 'errors', + ]) + + # Until Scenario uses the ops._private.harness.ActionFailed, we need to + # monkeypatch it in, so that the ops.testing.ActionFailed object is the + # one that we expect, even if people are mixing Harness and Scenario. + # https://github.com/canonical/ops-scenario/issues/201 import scenario.context as _context import scenario.runtime as _runtime _context.ActionFailed = ActionFailed # type: ignore[reportPrivateImportUsage] _runtime.ActionFailed = ActionFailed # type: ignore[reportPrivateImportUsage] - else: - from ops._private.harness import ActionFailed # type: ignore - -# The Harness unit testing framework. -_ = ActionFailed # If Scenario has been installed, then this will be the merged ActionFailed. -_ = ActionOutput -_ = AppUnitOrName -_ = CharmType -_ = ExecArgs -_ = ExecHandler -_ = ExecResult -_ = Harness -_ = ReadableBuffer -_ = YAMLStringOrFile # Names exposed for backwards compatibility -_ = CharmBase -_ = CharmMeta -_ = Container # If Scenario has been installed, then this will be scenario.Container. -_ = ExecProcess -_ = RelationNotFoundError -_ = RelationRole -_ = charm -_ = framework -_ = model -_ = pebble -_ = storage +__all__.extend([ + 'CharmBase', + 'CharmMeta', + 'Container', # If Scenario has been installed, then this will be scenario.Container. + 'ExecProcess', + 'RelationNotFoundError', + 'RelationRole', + 'charm', + 'framework', + 'model', + 'pebble', + 'storage', +]) diff --git a/test/test_testing.py b/test/test_testing.py index b3fc67800..832142730 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -7034,33 +7034,3 @@ def test_scenario_available(): ctx = ops.testing.Context(ops.CharmBase, meta={'name': 'foo'}) state = ctx.run(ctx.on.start(), ops.testing.State()) assert isinstance(state.unit_status, ops.testing.UnknownStatus) - - -@pytest.mark.skipif( - not hasattr(ops.testing, 'Context'), reason='requires optional ops[testing] install' -) -def test_merged_actionfailed(): - class MyCharm(ops.CharmBase): - def __init__(self, framework: ops.Framework): - super().__init__(framework) - framework.observe(self.on.go_action, self._on_go) - - def _on_go(self, event: ops.ActionEvent): - event.log('\U0001f680') - event.set_results({'interim': '\U0001f97a'}) - event.fail('\U0001f61e') - - harness = ops.testing.Harness(MyCharm, actions="""go:\n description: go""") - harness.begin() - with pytest.raises(ops.testing.ActionFailed) as excinfo: - harness.run_action('go') - assert excinfo.value.message == '\U0001f61e' - assert excinfo.value.output.logs == ['\U0001f680'] - assert excinfo.value.output.results == {'interim': '\U0001f97a'} - - ctx = ops.testing.Context(MyCharm, meta={'name': 'foo'}, actions={'go': {}}) - with pytest.raises(ops.testing.ActionFailed) as excinfo: - ctx.run(ctx.on.action('go'), ops.testing.State()) - assert excinfo.value.message == '\U0001f61e' - assert ctx.action_logs == ['\U0001f680'] - assert ctx.action_results == {'interim': '\U0001f97a'} From 8905747fe9015208f41dbcb171dde2cc837b1003 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 23 Sep 2024 17:56:22 +1200 Subject: [PATCH 14/31] Add a test to ensure that we don't forget to document ops.testing classes. --- test/test_infra.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/test_infra.py b/test/test_infra.py index 445bbce2d..6bda3a1e9 100644 --- a/test/test_infra.py +++ b/test/test_infra.py @@ -19,6 +19,8 @@ import pytest +import ops.testing + @pytest.mark.parametrize( 'mod_name', @@ -45,3 +47,35 @@ def test_import(mod_name: str, tmp_path: pathlib.Path): proc = subprocess.run([sys.executable, testfile], env=environ) assert proc.returncode == 0 + + +def test_ops_testing_doc(): + """Ensure that ops.testing's documentation includes all the expected names.""" + prefix = '.. autoclass:: ops.testing.' + # We don't document the type aliases. + expected_names = set( + name + for name in ops.testing.__all__ + if name != 'errors' + and name not in ops.testing._compatibility_names + and getattr(ops.testing, name).__class__.__module__ != 'typing' + ) + expected_names.update( + f'errors.{name}' for name in dir(ops.testing.errors) if not name.startswith('_') + ) + # ops.testing.UnitID is `int` - we don't document it, but it's hard to fit + # into the above logic, so we just exclude it here. + expected_names.discard('UnitID') + # ops.testing.Container is a documented class when Scenario is installed, + # but exported for compatibility when not, so we do want to have it present + # even though the above compatibility_names logic would exclude it. + expected_names.add('Container') + + with open('docs/index.rst') as testing_doc: + found_names = { + line.split(prefix, 1)[1].strip() + for line in testing_doc + if line.strip().startswith(prefix) + } + + assert expected_names == found_names From 98f0e302b7a58746b803897da6918417db8cf563 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 23 Sep 2024 18:00:03 +1200 Subject: [PATCH 15/31] Minor tweaks, mostly based on review. --- docs/index.rst | 52 +++++++++++++++++++++++++++++--------------------- ops/testing.py | 5 +++-- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 1266bc684..e2dd56c78 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -71,9 +71,10 @@ Two frameworks are available: * State-transition testing, which tests the charm's state transitions in response to events. This is the recommended approach. Install ops with the ``testing`` extra to use this framework; for example: ``pip install ops[testing]`` -* Harness, which provides an API similar to the Juju CLI. This is a legacy +* Harness, which provides an API similar to the Juju CLI. This is a deprecated framework, and has issues, particularly with resetting the charm state between - Juju events. This is currently included with a base ``ops`` install. + Juju events. It will be moved out of the base ``ops`` install in a future + release. .. note:: Unit testing is only one aspect of a comprehensive testing strategy. For more @@ -82,11 +83,25 @@ Two frameworks are available: State-transition testing ------------------------ -Write tests that declaratively define the Juju state all at once, define the -Juju context against which to test the charm, and fire a single event on the +State-transition tests expect you to define the Juju state all at once, define +the Juju context against which to test the charm, and fire a single event on the charm to execute its logic. The tests can then assert that the Juju state has changed as expected. +A very simple test, where the charm has no config, no integrations, the unit +is the leader, and has a `start` handler that sets the status to active might +look like this: + +.. code-block:: python + + from ops import testing + + def test_base(): + ctx = testing.Context(MyCharm) + state = testing.State(leader=True) + out = ctx.run(ctx.on.start(), state) + assert out.unit_status == testing.ActiveStatus() + These tests are 'state-transition' tests, a way to test isolated units of charm functionality (how the state changes in reaction to events). They are not necessarily tests of individual methods or functions (but might be, depending on @@ -109,8 +124,8 @@ state. Writing unit tests for a charm, then, means verifying that: +- the output state (as compared with the input state) is as expected - the charm does not raise uncaught exceptions while handling the event -- the output state (as compared with the input state) is as expected. A test consists of three broad steps: @@ -118,24 +133,18 @@ A test consists of three broad steps: - declare the context - declare the input state - **Act**: - - select an event to fire - - run the context (i.e. obtain the output state, given the input state and the event) + - run an event (ie. obtain the output state, given the input state and the event) - **Assert**: - verify that the output state (as compared with the input state) is how you expect it to be - verify that the charm has seen a certain sequence of statuses, events, and `juju-log` calls - - optionally, you can use a context manager to get a hold of the charm instance and run - assertions on APIs and state internal to it. - -The most basic scenario is one in which all is defaulted and barely any data is -available. The charm has no config, no integrations, no leadership, and its -status is `unknown`. With that, we can write the simplest possible test: -.. code-block:: python - - def test_base(): - ctx = Context(MyCharm) - out = ctx.run(ctx.on.start(), State()) - assert out.unit_status == UnknownStatus() +.. + _The list here is manually maintained, because the `automodule` directive + expects to document names defined in the module, and not imported ones, and + we're doing the opposite of that - and we also want to use the 'ops.testing' + namespace, not expose the 'ops._private.harness' and 'scenario' ones. + Ideally, someone will figure out a nicer way to do this that doesn't require + keeping this list in sync (see test/test_infra.py for a check that we are ok). .. autoclass:: ops.testing.ActionFailed .. autoclass:: ops.testing.ActiveStatus @@ -162,8 +171,6 @@ status is `unknown`. With that, we can write the simplest possible test: .. autoclass:: ops.testing.Notice .. autoclass:: ops.testing.PeerRelation .. autoclass:: ops.testing.Port -.. autoclass:: ops.testing.RawDataBagContents -.. autoclass:: ops.testing.RawSecretRevisionContents .. autoclass:: ops.testing.Relation .. autoclass:: ops.testing.RelationBase .. autoclass:: ops.testing.Resource @@ -190,7 +197,8 @@ status is `unknown`. With that, we can write the simplest possible test: Harness ------- -The Harness framework includes: +The Harness framework is deprecated and will be moved out of the base install in +a future ops release. The Harness framework includes: - :class:`ops.testing.Harness`, a class to set up the simulated environment, that provides: diff --git a/ops/testing.py b/ops/testing.py index be243e962..9a98787b5 100644 --- a/ops/testing.py +++ b/ops/testing.py @@ -183,7 +183,7 @@ _runtime.ActionFailed = ActionFailed # type: ignore[reportPrivateImportUsage] # Names exposed for backwards compatibility -__all__.extend([ +_compatibility_names = [ 'CharmBase', 'CharmMeta', 'Container', # If Scenario has been installed, then this will be scenario.Container. @@ -195,4 +195,5 @@ 'model', 'pebble', 'storage', -]) +] +__all__.extend(_compatibility_names) From 3decc03baff766bd20965ac99435859ab60752f9 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 23 Sep 2024 18:10:31 +1200 Subject: [PATCH 16/31] Update docs/index.rst Co-authored-by: Ben Hoyt --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index e2dd56c78..0b13c43fe 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -135,7 +135,7 @@ A test consists of three broad steps: - **Act**: - run an event (ie. obtain the output state, given the input state and the event) - **Assert**: - - verify that the output state (as compared with the input state) is how you expect it to be + - verify that the output state is what you expect it to be - verify that the charm has seen a certain sequence of statuses, events, and `juju-log` calls .. From 3a6c58f3ad10667484c962e5e534831fa8bd296a Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 23 Sep 2024 18:11:42 +1200 Subject: [PATCH 17/31] Update docs/index.rst Co-authored-by: Ben Hoyt --- docs/index.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 0b13c43fe..98711cd91 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -102,14 +102,14 @@ look like this: out = ctx.run(ctx.on.start(), state) assert out.unit_status == testing.ActiveStatus() -These tests are 'state-transition' tests, a way to test isolated units of charm -functionality (how the state changes in reaction to events). They are not -necessarily tests of individual methods or functions (but might be, depending on -the charm's event observers); they are testing the 'contract' of the charm: given -a certain state, when a certain event happens, the charm should transition to a -certain (likely different) state. They do not test against a real Juju -controller and model, and focus on a single Juju unit, unlike integration tests. -For simplicity, we refer to them as 'unit' tests in the charm context. +These 'state-transition' tests give charm authors a way to test +how the state changes in reaction to events. They are not +necessarily tests of individual methods or functions; +they are testing the 'contract' of the charm: given +a certain state, when a certain event happens, the charm should transition to +another state. Unlike integration tests, they do not test using a real Juju +controller and model, and focus on a single Juju unit. +For simplicity, we refer to them as 'unit' tests. Writing these tests should nudge you into thinking of a charm as a black-box input->output function. The input is the union of an `Event` (why am I, charm, From b92d4e3f83db8e37455ce95f445215a4f5cdff98 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 23 Sep 2024 18:12:18 +1200 Subject: [PATCH 18/31] Update docs/index.rst Co-authored-by: Ben Hoyt --- docs/index.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 98711cd91..37d33932e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -112,12 +112,15 @@ controller and model, and focus on a single Juju unit. For simplicity, we refer to them as 'unit' tests. Writing these tests should nudge you into thinking of a charm as a black-box -input->output function. The input is the union of an `Event` (why am I, charm, -being executed), a `State` (am I leader? what is my integration data? what is my -config?...) and the charm's execution `Context` (what integrations can I have? -what containers can I have?...). The output is another `State`: the state after -the charm has had a chance to interact with the mocked Juju model and affect the -state. +'input to output' function. The inputs are: + +- Event: why am I, the charm, being executed +- State: am I the leader? what is my integration data? what is my config? +- Context: what integrations can I have? what containers can I have? + +The output is another `State`: the state after +the charm has interacted with the mocked Juju model. +The output state is the same type of data structure as the input state. .. image:: https://raw.githubusercontent.com/canonical/ops-scenario/main/resources/state-transition-model.png :alt: Transition diagram, with the input state and event on the left, the context including the charm in the centre, and the state out on the right From cbf52b30dd2dce8ef1844454bb9a0b975fd55820 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 23 Sep 2024 18:12:49 +1200 Subject: [PATCH 19/31] Tweak, per review. --- docs/index.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 37d33932e..8a859e37f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -71,10 +71,10 @@ Two frameworks are available: * State-transition testing, which tests the charm's state transitions in response to events. This is the recommended approach. Install ops with the ``testing`` extra to use this framework; for example: ``pip install ops[testing]`` -* Harness, which provides an API similar to the Juju CLI. This is a deprecated - framework, and has issues, particularly with resetting the charm state between - Juju events. It will be moved out of the base ``ops`` install in a future - release. +* Harness, which provides an API reminiscent of the Juju CLI. This is a + deprecated framework, and has issues, particularly with resetting the charm + state between Juju events. It will be moved out of the base ``ops`` install in + a future release. .. note:: Unit testing is only one aspect of a comprehensive testing strategy. For more From 67b5a7aa2d8190c0eb7a95a104a9d95fb2b4cee0 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 23 Sep 2024 18:59:44 +1200 Subject: [PATCH 20/31] WiP docs on separate pages. --- docs/index.rst | 180 +++++---------------------------------------- ops/__init__.py | 2 +- test/test_infra.py | 3 + 3 files changed, 24 insertions(+), 161 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8a859e37f..39ef53382 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,3 @@ - ops library API reference ========================= @@ -6,33 +5,34 @@ The `ops` library is a Python framework for writing and testing Juju charms. See more: `Charm SDK documentation `_ -The library provides: +The library (`available on PyPI`_) provides: - :ref:`ops_main_entry_point`, used to initialise and run your charm; - :ref:`ops_module`, the API to respond to Juju events and manage the application; - :ref:`ops_pebble_module`, the Pebble client, a low-level API for Kubernetes containers; -- :ref:`ops_testing_module`, the framework for unit testing charms in a simulated environment; +- :ref:`ops_testing_module` frameworks for unit testing charms in a simulated environment; You can structure your charm however you like, but with the `ops` library, you get a framework that promotes consistency and readability by following best practices. It also helps you organise your code better by separating different -aspects of the charm, such as managing the application’s state, handling +aspects of the charm, such as managing the application's state, handling integrations with other services, and making the charm easier to test. +.. _available on PyPI: https://pypi.org/project/ops/ .. toctree:: :maxdepth: 2 :caption: Contents: - .. _ops_module: -ops module -========== +ops +=== .. automodule:: ops :exclude-members: main +.. _ops_main_entry_point: .. _ops_main_entry_point: @@ -43,185 +43,45 @@ The main entry point to initialise and run your charm. .. autofunction:: ops.main - legacy main module ------------------ .. automodule:: ops.main :noindex: - .. _ops_pebble_module: -ops.pebble module -================= +ops.pebble +========== .. automodule:: ops.pebble - .. _ops_testing_module: -ops.testing module -================== +.. _ops_testing_module: -Frameworks for unit testing charms in a simulated Juju environment. +Testing +======= -Two frameworks are available: +Two frameworks for unit testing charms in a simulated Juju environment are +available: -* State-transition testing, which tests the charm's state transitions in response +* `State-transition testing`_, which tests the charm's state transitions in response to events. This is the recommended approach. Install ops with the ``testing`` extra to use this framework; for example: ``pip install ops[testing]`` -* Harness, which provides an API reminiscent of the Juju CLI. This is a +* `Harness`_, which provides an API reminiscent of the Juju CLI. This is a deprecated framework, and has issues, particularly with resetting the charm state between Juju events. It will be moved out of the base ``ops`` install in a future release. +.. _State-transition testing: +.. _Harness: + + .. note:: Unit testing is only one aspect of a comprehensive testing strategy. For more on testing charms, see `Charm SDK | Testing `_. -State-transition testing ------------------------- - -State-transition tests expect you to define the Juju state all at once, define -the Juju context against which to test the charm, and fire a single event on the -charm to execute its logic. The tests can then assert that the Juju state has -changed as expected. - -A very simple test, where the charm has no config, no integrations, the unit -is the leader, and has a `start` handler that sets the status to active might -look like this: - -.. code-block:: python - - from ops import testing - - def test_base(): - ctx = testing.Context(MyCharm) - state = testing.State(leader=True) - out = ctx.run(ctx.on.start(), state) - assert out.unit_status == testing.ActiveStatus() - -These 'state-transition' tests give charm authors a way to test -how the state changes in reaction to events. They are not -necessarily tests of individual methods or functions; -they are testing the 'contract' of the charm: given -a certain state, when a certain event happens, the charm should transition to -another state. Unlike integration tests, they do not test using a real Juju -controller and model, and focus on a single Juju unit. -For simplicity, we refer to them as 'unit' tests. - -Writing these tests should nudge you into thinking of a charm as a black-box -'input to output' function. The inputs are: - -- Event: why am I, the charm, being executed -- State: am I the leader? what is my integration data? what is my config? -- Context: what integrations can I have? what containers can I have? - -The output is another `State`: the state after -the charm has interacted with the mocked Juju model. -The output state is the same type of data structure as the input state. - -.. image:: https://raw.githubusercontent.com/canonical/ops-scenario/main/resources/state-transition-model.png - :alt: Transition diagram, with the input state and event on the left, the context including the charm in the centre, and the state out on the right - -Writing unit tests for a charm, then, means verifying that: - -- the output state (as compared with the input state) is as expected -- the charm does not raise uncaught exceptions while handling the event - -A test consists of three broad steps: - -- **Arrange**: - - declare the context - - declare the input state -- **Act**: - - run an event (ie. obtain the output state, given the input state and the event) -- **Assert**: - - verify that the output state is what you expect it to be - - verify that the charm has seen a certain sequence of statuses, events, and `juju-log` calls - -.. - _The list here is manually maintained, because the `automodule` directive - expects to document names defined in the module, and not imported ones, and - we're doing the opposite of that - and we also want to use the 'ops.testing' - namespace, not expose the 'ops._private.harness' and 'scenario' ones. - Ideally, someone will figure out a nicer way to do this that doesn't require - keeping this list in sync (see test/test_infra.py for a check that we are ok). - -.. autoclass:: ops.testing.ActionFailed -.. autoclass:: ops.testing.ActiveStatus -.. autoclass:: ops.testing.Address -.. autoclass:: ops.testing.BindAddress -.. autoclass:: ops.testing.BlockedStatus -.. autoclass:: ops.testing.CharmEvents -.. autoclass:: ops.testing.CheckInfo -.. autoclass:: ops.testing.CloudCredential -.. autoclass:: ops.testing.CloudSpec -.. autoclass:: ops.testing.Container -.. autoclass:: ops.testing.Context - :special-members: __call__ -.. autoclass:: ops.testing.DeferredEvent -.. autoclass:: ops.testing.ErrorStatus -.. autoclass:: ops.testing.Exec -.. autoclass:: ops.testing.ICMPPort -.. autoclass:: ops.testing.JujuLogLine -.. autoclass:: ops.testing.MaintenanceStatus -.. autoclass:: ops.testing.Manager -.. autoclass:: ops.testing.Model -.. autoclass:: ops.testing.Mount -.. autoclass:: ops.testing.Network -.. autoclass:: ops.testing.Notice -.. autoclass:: ops.testing.PeerRelation -.. autoclass:: ops.testing.Port -.. autoclass:: ops.testing.Relation -.. autoclass:: ops.testing.RelationBase -.. autoclass:: ops.testing.Resource -.. autoclass:: ops.testing.Secret -.. autoclass:: ops.testing.State -.. autoclass:: ops.testing.Storage -.. autoclass:: ops.testing.StoredState -.. autoclass:: ops.testing.SubordinateRelation -.. autoclass:: ops.testing.TCPPort -.. autoclass:: ops.testing.UDPPort -.. autoclass:: ops.testing.UnknownStatus -.. autoclass:: ops.testing.WaitingStatus -.. autoclass:: ops.testing.errors.ContextSetupError -.. autoclass:: ops.testing.errors.AlreadyEmittedError -.. autoclass:: ops.testing.errors.ScenarioRuntimeError -.. autoclass:: ops.testing.errors.UncaughtCharmError -.. autoclass:: ops.testing.errors.InconsistentScenarioError -.. autoclass:: ops.testing.errors.StateValidationError -.. autoclass:: ops.testing.errors.MetadataNotFoundError -.. autoclass:: ops.testing.errors.ActionMissingFromContextError -.. autoclass:: ops.testing.errors.NoObserverError -.. autoclass:: ops.testing.errors.BadOwnerPath - -Harness -------- - -The Harness framework is deprecated and will be moved out of the base install in -a future ops release. The Harness framework includes: - -- :class:`ops.testing.Harness`, a class to set up the simulated environment, - that provides: - - - :meth:`~ops.testing.Harness.add_relation` method, to declare a relation - (integration) with another app. - - :meth:`~ops.testing.Harness.begin` and :meth:`~ops.testing.Harness.cleanup` - methods to start and end the testing lifecycle. - - :meth:`~ops.testing.Harness.evaluate_status` method, which aggregates the - status of the charm after test interactions. - - :attr:`~ops.testing.Harness.model` attribute, which exposes e.g. the - :attr:`~ops.Model.unit` attribute for detailed assertions on the unit's state. - -.. autoclass:: ops.testing.ActionFailed - :noindex: -.. autoclass:: ops.testing.ActionOutput -.. autoclass:: ops.testing.ExecArgs -.. autoclass:: ops.testing.ExecResult -.. autoclass:: ops.testing.Harness - Indices ======= diff --git a/ops/__init__.py b/ops/__init__.py index 0e34ef169..53a3b9822 100644 --- a/ops/__init__.py +++ b/ops/__init__.py @@ -14,7 +14,7 @@ """The API to respond to Juju events and manage the application. -This module provides code freatures to your charm, including: +This module provides code features to your charm, including: - :class:`~ops.CharmBase`, the base class for charms and :class:`~ops.Object`, the base class for charm libraries. diff --git a/test/test_infra.py b/test/test_infra.py index 6bda3a1e9..24afb8976 100644 --- a/test/test_infra.py +++ b/test/test_infra.py @@ -49,6 +49,9 @@ def test_import(mod_name: str, tmp_path: pathlib.Path): assert proc.returncode == 0 +@pytest.mark.skipif( + not hasattr(ops.testing, 'Context'), reason='requires optional ops[testing] install' +) def test_ops_testing_doc(): """Ensure that ops.testing's documentation includes all the expected names.""" prefix = '.. autoclass:: ops.testing.' From 1f5801e88c3d52f60afab120dbe087a3438d738f Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 11:17:02 +1200 Subject: [PATCH 21/31] Split the testing classes into separate pages. --- docs/custom_conf.py | 19 +---- docs/harness.rst | 31 ++++++++ docs/index.rst | 25 ++++--- docs/state-transition-testing.rst | 120 ++++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 30 deletions(-) create mode 100644 docs/harness.rst create mode 100644 docs/state-transition-testing.rst diff --git a/docs/custom_conf.py b/docs/custom_conf.py index 64fdeff3c..fb14823a1 100644 --- a/docs/custom_conf.py +++ b/docs/custom_conf.py @@ -23,29 +23,12 @@ from sphinx import addnodes from sphinx.util.docutils import SphinxDirective -import furo -import furo.navigation sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) -# Furo patch to get local TOC to show in sidebar (as sphinx-rtd-theme did) -# See https://github.com/pradyunsg/furo/blob/490527b2aef00b1198770c3389a1979911ee1fcb/src/furo/__init__.py#L115-L128 - -_old_compute_navigation_tree = furo._compute_navigation_tree - - -def _compute_navigation_tree(context): - tree_html = _old_compute_navigation_tree(context) - if not tree_html and context.get('toc'): - tree_html = furo.navigation.get_navigation_tree(context['toc']) - return tree_html - - -furo._compute_navigation_tree = _compute_navigation_tree - # Pull in fix from https://github.com/sphinx-doc/sphinx/pull/11222/files to fix # "invalid signature for autoattribute ('ops.pebble::ServiceDict.backoff-delay')" -import re # noqa: E402 +import re sphinx.ext.autodoc.py_ext_sig_re = re.compile( r"""^ ([\w.]+::)? # explicit module name diff --git a/docs/harness.rst b/docs/harness.rst new file mode 100644 index 000000000..62afc9687 --- /dev/null +++ b/docs/harness.rst @@ -0,0 +1,31 @@ +:orphan: + +.. _harness: + +Harness Unit Test Framework +=========================== + +.. deprecated:: 2.17 + The Harness framework is deprecated and will be moved out of the base + install in a future ops release. + +The Harness framework includes: + +- :class:`ops.testing.Harness`, a class to set up the simulated environment, + that provides: + + - :meth:`~ops.testing.Harness.add_relation` method, to declare a relation + (integration) with another app. + - :meth:`~ops.testing.Harness.begin` and :meth:`~ops.testing.Harness.cleanup` + methods to start and end the testing lifecycle. + - :meth:`~ops.testing.Harness.evaluate_status` method, which aggregates the + status of the charm after test interactions. + - :attr:`~ops.testing.Harness.model` attribute, which exposes e.g. the + :attr:`~ops.Model.unit` attribute for detailed assertions on the unit's state. + +.. autoclass:: ops.testing.ActionFailed + :noindex: +.. autoclass:: ops.testing.ActionOutput +.. autoclass:: ops.testing.ExecArgs +.. autoclass:: ops.testing.ExecResult +.. autoclass:: ops.testing.Harness diff --git a/docs/index.rst b/docs/index.rst index 39ef53382..003906a80 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,5 @@ -ops library API reference -========================= +API reference +============= The `ops` library is a Python framework for writing and testing Juju charms. @@ -21,13 +21,17 @@ integrations with other services, and making the charm easier to test. .. _available on PyPI: https://pypi.org/project/ops/ .. toctree:: + :hidden: :maxdepth: 2 - :caption: Contents: + + self + state-transition-testing + harness .. _ops_module: ops -=== +--- .. automodule:: ops :exclude-members: main @@ -37,7 +41,7 @@ ops .. _ops_main_entry_point: ops.main entry point -==================== +-------------------- The main entry point to initialise and run your charm. @@ -52,7 +56,7 @@ legacy main module .. _ops_pebble_module: ops.pebble -========== +---------- .. automodule:: ops.pebble @@ -61,22 +65,19 @@ ops.pebble .. _ops_testing_module: Testing -======= +------- Two frameworks for unit testing charms in a simulated Juju environment are available: -* `State-transition testing`_, which tests the charm's state transitions in response +* :doc:`State-transition testing `, which tests the charm's state transitions in response to events. This is the recommended approach. Install ops with the ``testing`` extra to use this framework; for example: ``pip install ops[testing]`` -* `Harness`_, which provides an API reminiscent of the Juju CLI. This is a +* :doc:`Harness `, which provides an API reminiscent of the Juju CLI. This is a deprecated framework, and has issues, particularly with resetting the charm state between Juju events. It will be moved out of the base ``ops`` install in a future release. -.. _State-transition testing: -.. _Harness: - .. note:: Unit testing is only one aspect of a comprehensive testing strategy. For more diff --git a/docs/state-transition-testing.rst b/docs/state-transition-testing.rst new file mode 100644 index 000000000..6464664e6 --- /dev/null +++ b/docs/state-transition-testing.rst @@ -0,0 +1,120 @@ +:orphan: + +.. _state-transition-tests: + +Unit Test Framework +=================== + +State-transition tests expect you to define the Juju state all at once, define +the Juju context against which to test the charm, and fire a single event on the +charm to execute its logic. The tests can then assert that the Juju state has +changed as expected. + +A very simple test, where the charm has no config, no integrations, the unit +is the leader, and has a `start` handler that sets the status to active might +look like this: + +.. code-block:: python + + from ops import testing + + def test_base(): + ctx = testing.Context(MyCharm) + state = testing.State(leader=True) + out = ctx.run(ctx.on.start(), state) + assert out.unit_status == testing.ActiveStatus() + +These 'state-transition' tests give charm authors a way to test +how the state changes in reaction to events. They are not +necessarily tests of individual methods or functions; +they are testing the 'contract' of the charm: given +a certain state, when a certain event happens, the charm should transition to +another state. Unlike integration tests, they do not test using a real Juju +controller and model, and focus on a single Juju unit. +For simplicity, we refer to them as 'unit' tests. + +Writing these tests should nudge you into thinking of a charm as a black-box +'input to output' function. The inputs are: + +- Event: why am I, the charm, being executed +- State: am I the leader? what is my integration data? what is my config? +- Context: what integrations can I have? what containers can I have? + +The output is another `State`: the state after +the charm has interacted with the mocked Juju model. +The output state is the same type of data structure as the input state. + +.. image:: https://raw.githubusercontent.com/canonical/ops-scenario/main/resources/state-transition-model.png + :alt: Transition diagram, with the input state and event on the left, the context including the charm in the centre, and the state out on the right + +Writing unit tests for a charm, then, means verifying that: + +- the output state (as compared with the input state) is as expected +- the charm does not raise uncaught exceptions while handling the event + +A test consists of three broad steps: + +- **Arrange**: + - declare the context + - declare the input state +- **Act**: + - run an event (ie. obtain the output state, given the input state and the event) +- **Assert**: + - verify that the output state is what you expect it to be + - verify that the charm has seen a certain sequence of statuses, events, and `juju-log` calls + +.. + _The list here is manually maintained, because the `automodule` directive + expects to document names defined in the module, and not imported ones, and + we're doing the opposite of that - and we also want to use the 'ops.testing' + namespace, not expose the 'ops._private.harness' and 'scenario' ones. + Ideally, someone will figure out a nicer way to do this that doesn't require + keeping this list in sync (see test/test_infra.py for a check that we are ok). + +.. autoclass:: ops.testing.ActionFailed +.. autoclass:: ops.testing.ActiveStatus +.. autoclass:: ops.testing.Address +.. autoclass:: ops.testing.BindAddress +.. autoclass:: ops.testing.BlockedStatus +.. autoclass:: ops.testing.CharmEvents +.. autoclass:: ops.testing.CheckInfo +.. autoclass:: ops.testing.CloudCredential +.. autoclass:: ops.testing.CloudSpec +.. autoclass:: ops.testing.Container +.. autoclass:: ops.testing.Context + :special-members: __call__ +.. autoclass:: ops.testing.DeferredEvent +.. autoclass:: ops.testing.ErrorStatus +.. autoclass:: ops.testing.Exec +.. autoclass:: ops.testing.ICMPPort +.. autoclass:: ops.testing.JujuLogLine +.. autoclass:: ops.testing.MaintenanceStatus +.. autoclass:: ops.testing.Manager +.. autoclass:: ops.testing.Model +.. autoclass:: ops.testing.Mount +.. autoclass:: ops.testing.Network +.. autoclass:: ops.testing.Notice +.. autoclass:: ops.testing.PeerRelation +.. autoclass:: ops.testing.Port +.. autoclass:: ops.testing.Relation +.. autoclass:: ops.testing.RelationBase +.. autoclass:: ops.testing.Resource +.. autoclass:: ops.testing.Secret +.. autoclass:: ops.testing.State +.. autoclass:: ops.testing.Storage +.. autoclass:: ops.testing.StoredState +.. autoclass:: ops.testing.SubordinateRelation +.. autoclass:: ops.testing.TCPPort +.. autoclass:: ops.testing.UDPPort +.. autoclass:: ops.testing.UnknownStatus +.. autoclass:: ops.testing.WaitingStatus +.. autoclass:: ops.testing.errors.ContextSetupError +.. autoclass:: ops.testing.errors.AlreadyEmittedError +.. autoclass:: ops.testing.errors.ScenarioRuntimeError +.. autoclass:: ops.testing.errors.UncaughtCharmError +.. autoclass:: ops.testing.errors.InconsistentScenarioError +.. autoclass:: ops.testing.errors.StateValidationError +.. autoclass:: ops.testing.errors.MetadataNotFoundError +.. autoclass:: ops.testing.errors.ActionMissingFromContextError +.. autoclass:: ops.testing.errors.NoObserverError +.. autoclass:: ops.testing.errors.BadOwnerPath From 151ec9fcf2d66b6e9977b93be4aadaef46a8df0f Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 11:35:19 +1200 Subject: [PATCH 22/31] Fix merge. --- docs/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 003906a80..ae1fb62dc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,8 +38,6 @@ ops .. _ops_main_entry_point: -.. _ops_main_entry_point: - ops.main entry point -------------------- From 04bac0b2e986ec67ffaf337b019b10cc09cea6ce Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 11:55:53 +1200 Subject: [PATCH 23/31] Update test for doc split. --- ops/testing.py | 3 ++- test/test_infra.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ops/testing.py b/ops/testing.py index 9a98787b5..fc916ef55 100644 --- a/ops/testing.py +++ b/ops/testing.py @@ -33,6 +33,7 @@ """ # ruff: noqa: F401 (unused import) +# pyright: reportUnusedImport=false import importlib.metadata @@ -196,4 +197,4 @@ 'pebble', 'storage', ] -__all__.extend(_compatibility_names) +__all__.extend(_compatibility_names) # type: ignore[reportUnsupportedDunderAll] diff --git a/test/test_infra.py b/test/test_infra.py index 24afb8976..47eaec39a 100644 --- a/test/test_infra.py +++ b/test/test_infra.py @@ -16,6 +16,7 @@ import pathlib import subprocess import sys +import typing import pytest @@ -74,11 +75,13 @@ def test_ops_testing_doc(): # even though the above compatibility_names logic would exclude it. expected_names.add('Container') - with open('docs/index.rst') as testing_doc: - found_names = { - line.split(prefix, 1)[1].strip() - for line in testing_doc - if line.strip().startswith(prefix) - } + found_names: typing.Set[str] = set() + for test_doc in ('docs/harness.rst', 'docs/state-transition-testing.rst'): + with open(test_doc) as testing_doc: + found_names.update({ + line.split(prefix, 1)[1].strip() + for line in testing_doc + if line.strip().startswith(prefix) + }) assert expected_names == found_names From 08c95f86cf200b7e2752b48b14c3a55990166b16 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 12:50:07 +1200 Subject: [PATCH 24/31] Fix rebase. --- docs/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ae1fb62dc..c53923974 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,8 +60,6 @@ ops.pebble .. _ops_testing_module: -.. _ops_testing_module: - Testing ------- From cf61118676d64dc08d86f70041ede86c4e49c274 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 14:19:21 +1200 Subject: [PATCH 25/31] Allow 'pip install ops[harness], which is the same as pip install ops for now, to ease transition. --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 56004d415..867022fb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ docs = [ testing = [ "ops-scenario>=7.0.5,<8", ] +# Empty for now, because Harness is bundled with the base install, but allow +# specifying the extra to ease transition later. +harness = [] [project.urls] "Homepage" = "https://juju.is/docs/sdk" From 8b1d1eeed39e2fafa2ea5314d0a819507bfb79f9 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 14:19:39 +1200 Subject: [PATCH 26/31] Typo from previous PR. --- ops/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/__init__.py b/ops/__init__.py index 53a3b9822..b4c05f3f2 100644 --- a/ops/__init__.py +++ b/ops/__init__.py @@ -14,7 +14,7 @@ """The API to respond to Juju events and manage the application. -This module provides code features to your charm, including: +This module provides core features to your charm, including: - :class:`~ops.CharmBase`, the base class for charms and :class:`~ops.Object`, the base class for charm libraries. From d0ce232f839b24ce17c7830640e4a1653bf8fb41 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 14:19:57 +1200 Subject: [PATCH 27/31] Add a PendingDeprecationWarning for Harness. --- ops/_private/harness.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ops/_private/harness.py b/ops/_private/harness.py index c9df5667a..59bb9aa69 100644 --- a/ops/_private/harness.py +++ b/ops/_private/harness.py @@ -314,6 +314,13 @@ def __init__( juju_debug_at=self._juju_context.debug_at, ) + warnings.warn( + 'Harness is deprecated; we recommend using state transition testing ' + "(previously known as 'Scenario') instead", + PendingDeprecationWarning, + stacklevel=2, + ) + def _event_context(self, event_name: str): """Configures the Harness to behave as if an event hook were running. From 24686d3663457b589c21a0adabb14e9d46a62e61 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 14:20:25 +1200 Subject: [PATCH 28/31] Tweaks per review. --- docs/harness.rst | 11 +++++++---- docs/index.rst | 31 ++++++++++--------------------- docs/state-transition-testing.rst | 24 +++++++++++++++--------- 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/docs/harness.rst b/docs/harness.rst index 62afc9687..5503790fe 100644 --- a/docs/harness.rst +++ b/docs/harness.rst @@ -1,9 +1,7 @@ -:orphan: - .. _harness: -Harness Unit Test Framework -=========================== +Harness (legacy unit testing) +============================= .. deprecated:: 2.17 The Harness framework is deprecated and will be moved out of the base @@ -23,6 +21,11 @@ The Harness framework includes: - :attr:`~ops.testing.Harness.model` attribute, which exposes e.g. the :attr:`~ops.Model.unit` attribute for detailed assertions on the unit's state. +.. note:: + Unit testing is only one aspect of a comprehensive testing strategy. For more + on testing charms, see `Charm SDK | Testing `_. + + .. autoclass:: ops.testing.ActionFailed :noindex: .. autoclass:: ops.testing.ActionOutput diff --git a/docs/index.rst b/docs/index.rst index c53923974..06555a7b8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,10 +7,18 @@ The `ops` library is a Python framework for writing and testing Juju charms. The library (`available on PyPI`_) provides: -- :ref:`ops_main_entry_point`, used to initialise and run your charm; - :ref:`ops_module`, the API to respond to Juju events and manage the application; +- :ref:`ops_main_entry_point`, used to initialise and run your charm; - :ref:`ops_pebble_module`, the Pebble client, a low-level API for Kubernetes containers; -- :ref:`ops_testing_module` frameworks for unit testing charms in a simulated environment; +- the APIs for unit testing charms in a simulated environment: + + - :doc:`State-transition testing `. This is the + recommended approach. + - :doc:`Harness `. This is a deprecated framework, and has issues, + particularly with resetting the charm state between Juju events. It will be + moved out of the base ``ops`` install in an ops release in the future. Charm + authors that don't want to upgrade will still be able to use it with + ``pip install ops[harness]``. You can structure your charm however you like, but with the `ops` library, you get a framework that promotes consistency and readability by following best @@ -60,25 +68,6 @@ ops.pebble .. _ops_testing_module: -Testing -------- - -Two frameworks for unit testing charms in a simulated Juju environment are -available: - -* :doc:`State-transition testing `, which tests the charm's state transitions in response - to events. This is the recommended approach. Install ops with the ``testing`` - extra to use this framework; for example: ``pip install ops[testing]`` -* :doc:`Harness `, which provides an API reminiscent of the Juju CLI. This is a - deprecated framework, and has issues, particularly with resetting the charm - state between Juju events. It will be moved out of the base ``ops`` install in - a future release. - - -.. note:: - Unit testing is only one aspect of a comprehensive testing strategy. For more - on testing charms, see `Charm SDK | Testing `_. - Indices ======= diff --git a/docs/state-transition-testing.rst b/docs/state-transition-testing.rst index 6464664e6..156848023 100644 --- a/docs/state-transition-testing.rst +++ b/docs/state-transition-testing.rst @@ -1,14 +1,15 @@ -:orphan: - .. _state-transition-tests: -Unit Test Framework -=================== +Unit testing (was: Scenario) +============================ + +Install ops with the ``testing`` extra to use this API; for example: +``pip install ops[testing]`` -State-transition tests expect you to define the Juju state all at once, define -the Juju context against which to test the charm, and fire a single event on the -charm to execute its logic. The tests can then assert that the Juju state has -changed as expected. +State-transition tests, previously known as 'Scenario', expect you to define the +Juju state all at once, define the Juju context against which to test the charm, +and fire a single event on the charm to execute its logic. The tests can then +assert that the Juju state has changed as expected. A very simple test, where the charm has no config, no integrations, the unit is the leader, and has a `start` handler that sets the status to active might @@ -16,7 +17,7 @@ look like this: .. code-block:: python - from ops import testing + from ops import testing def test_base(): ctx = testing.Context(MyCharm) @@ -63,6 +64,11 @@ A test consists of three broad steps: - verify that the output state is what you expect it to be - verify that the charm has seen a certain sequence of statuses, events, and `juju-log` calls +.. note:: + Unit testing is only one aspect of a comprehensive testing strategy. For more + on testing charms, see `Charm SDK | Testing `_. + + .. _The list here is manually maintained, because the `automodule` directive expects to document names defined in the module, and not imported ones, and From bac91d7824c7a8a5f459622cdbe6e4af2c751416 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 14:22:19 +1200 Subject: [PATCH 29/31] Update ops/_private/harness.py Co-authored-by: Dima Tisnek --- ops/_private/harness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/_private/harness.py b/ops/_private/harness.py index 59bb9aa69..1bc9f51e3 100644 --- a/ops/_private/harness.py +++ b/ops/_private/harness.py @@ -199,7 +199,7 @@ def __init__( state: typing.Optional['State'] = None, ): self.message = message - self.output = ActionOutput([], {}) if output is None else output + self.output = output or ActionOutput([], {}) self.state = state def __str__(self): From 2472e7c2addca35d36ab81829362d3b97b352a4c Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 14:45:13 +1200 Subject: [PATCH 30/31] Update docs/index.rst Co-authored-by: Ben Hoyt --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 06555a7b8..a16662f69 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,7 +13,7 @@ The library (`available on PyPI`_) provides: - the APIs for unit testing charms in a simulated environment: - :doc:`State-transition testing `. This is the - recommended approach. + recommended approach (it was previously known as 'Scenario'). - :doc:`Harness `. This is a deprecated framework, and has issues, particularly with resetting the charm state between Juju events. It will be moved out of the base ``ops`` install in an ops release in the future. Charm From bc05df84743d17a341eaef4e46be6a09b457b86c Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 24 Sep 2024 14:54:30 +1200 Subject: [PATCH 31/31] Move some index content to the harness page. --- docs/harness.rst | 9 +++++++-- docs/index.rst | 5 +---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/harness.rst b/docs/harness.rst index 5503790fe..a03cded49 100644 --- a/docs/harness.rst +++ b/docs/harness.rst @@ -5,9 +5,10 @@ Harness (legacy unit testing) .. deprecated:: 2.17 The Harness framework is deprecated and will be moved out of the base - install in a future ops release. + install in a future ops release. Charm authors that don't want to upgrade + will still be able to use it with ``pip install ops[harness]``. -The Harness framework includes: +The Harness API includes: - :class:`ops.testing.Harness`, a class to set up the simulated environment, that provides: @@ -21,6 +22,10 @@ The Harness framework includes: - :attr:`~ops.testing.Harness.model` attribute, which exposes e.g. the :attr:`~ops.Model.unit` attribute for detailed assertions on the unit's state. +.. warning:: The Harness API has flaws with resetting the charm state between + Juju events. Care must be taken when emitting multiple events with the same + Harness object. + .. note:: Unit testing is only one aspect of a comprehensive testing strategy. For more on testing charms, see `Charm SDK | Testing `_. diff --git a/docs/index.rst b/docs/index.rst index a16662f69..24f0a7f4b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,10 +15,7 @@ The library (`available on PyPI`_) provides: - :doc:`State-transition testing `. This is the recommended approach (it was previously known as 'Scenario'). - :doc:`Harness `. This is a deprecated framework, and has issues, - particularly with resetting the charm state between Juju events. It will be - moved out of the base ``ops`` install in an ops release in the future. Charm - authors that don't want to upgrade will still be able to use it with - ``pip install ops[harness]``. + particularly with resetting the charm state between Juju events. You can structure your charm however you like, but with the `ops` library, you get a framework that promotes consistency and readability by following best