Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrating Message Infrastructure into Zambeze #77

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
bce7896
First step in integration
JoshuaSBrown Nov 15, 2022
48329d4
Merge branch 'devel' into JoshuaSBrown-integrate-message-framework-in…
JoshuaSBrown Nov 15, 2022
1ff35e5
Apply formatting
JoshuaSBrown Nov 15, 2022
3785e0c
Specify multipledispatch version
JoshuaSBrown Nov 15, 2022
7e5fa78
Small fixes to typing
JoshuaSBrown Nov 15, 2022
4a6a278
Use standard supported overload decorator
JoshuaSBrown Nov 15, 2022
749c109
Fix double overload
JoshuaSBrown Nov 15, 2022
aed47fe
Remove + from string concatenation
JoshuaSBrown Nov 15, 2022
8828a34
Simplifying design by using dataclass
JoshuaSBrown Nov 16, 2022
22cdb14
Improve message framework
JoshuaSBrown Nov 21, 2022
ca8239b
Progress to integrating message framework into zambeze
JoshuaSBrown Nov 28, 2022
6f92593
Add missing file
JoshuaSBrown Nov 28, 2022
bb173af
Address merge conflicts
JoshuaSBrown Nov 29, 2022
f14b8c9
A few small type fixes
JoshuaSBrown Nov 29, 2022
2f8efdb
Fix defaults in dataclass
JoshuaSBrown Nov 29, 2022
0dd4735
Fix plugins
JoshuaSBrown Nov 29, 2022
e90f69b
Address overloading
JoshuaSBrown Nov 29, 2022
905cfc3
Address some of the type hinting issues
JoshuaSBrown Nov 29, 2022
9b6c42b
Attempt to fix rsync template
JoshuaSBrown Nov 29, 2022
8d9d72f
Attempt to fix message validation step
JoshuaSBrown Nov 29, 2022
7c96664
Get tests working
JoshuaSBrown Nov 29, 2022
8c48977
Fix plugins gitlab-runner test hopefully
JoshuaSBrown Nov 29, 2022
a6c8258
Debugging CI and plugins
JoshuaSBrown Nov 30, 2022
00f3d27
Major rework to address design flaw, and enforce separation of concerns
JoshuaSBrown Nov 30, 2022
cd45b51
Some small fixes to imports
JoshuaSBrown Dec 8, 2022
5392e4e
Fixed unit tests
JoshuaSBrown Dec 13, 2022
402e305
Uncomment and fix unit tests
JoshuaSBrown Dec 13, 2022
8791c3c
Make messaging format consistent with design document
JoshuaSBrown Dec 14, 2022
53bdee8
Add name to logger if logger not provided
JoshuaSBrown Dec 14, 2022
4b43f33
Apply formatting
JoshuaSBrown Dec 14, 2022
bee5e19
Fix gitlab runner tests
JoshuaSBrown Dec 14, 2022
fbf47b6
Apply formatting
JoshuaSBrown Dec 14, 2022
c6ae19b
Clear up formatting and unused imports
JoshuaSBrown Dec 15, 2022
b106a13
Address line length
JoshuaSBrown Dec 15, 2022
7e49f28
Fix some type checks
JoshuaSBrown Dec 15, 2022
28c2b88
Fix conflicting formatting
JoshuaSBrown Dec 15, 2022
2feb00c
Final formatting issue
JoshuaSBrown Dec 15, 2022
f3dc6fa
Made steps into integrating
JoshuaSBrown Dec 15, 2022
3db223d
remove pyre checks
JoshuaSBrown Dec 15, 2022
a1d5a47
Add checks for submission_time
JoshuaSBrown Dec 15, 2022
233d424
Working unit tests for shell plugin
JoshuaSBrown Dec 22, 2022
aa3742b
clean up stray print statements, formatting
JoshuaSBrown Dec 22, 2022
b413d36
Merge pull request #81 from ORNL/JoshuaSBrown-add-missing-shell-plugi…
JoshuaSBrown Dec 22, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions notes.txt

This file was deleted.

8 changes: 5 additions & 3 deletions tests/test_message_activity.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# Local imports
from zambeze.orchestration.message.message_activity import MessageActivity
from zambeze.orchestration.message.activity_message.message_activity import (
MessageActivity,
)

from zambeze.orchestration.zambeze_types import MessageType

Expand All @@ -15,15 +17,15 @@ def test_message_activity_type():

def test_message_activity_message():
msg_activity = MessageActivity({"body": "testing"})
assert msg_activity.message["body"] == "testing"
assert msg_activity.data["body"] == "testing"


def test_message_activity_immutability():
"""Demonstrate immutability of message once created"""
msg_activity = MessageActivity({"body": "testing"})
failed = False
try:
msg_activity.message["new_key"] == "should fail"
msg_activity.data["new_key"] == "should fail"
except Exception:
failed = True
pass
Expand Down
112 changes: 80 additions & 32 deletions tests/test_message_activity_validator.py
Original file line number Diff line number Diff line change
@@ -1,61 +1,109 @@
import pytest
import time
import uuid

from zambeze.orchestration.message.message_activity_validator import (
from zambeze.orchestration.message.activity_message.message_activity_validator import (
MessageActivityValidator,
)

# fmt: off
from zambeze.orchestration.message.activity_message.\
message_activity_template_generator import createActivityTemplate
# fmt: on
from zambeze.orchestration.zambeze_types import ActivityType

###############################################################################
# Testing Action: Control
###############################################################################


@pytest.mark.unit
def test_message_activity_validator1():
"""This check should fail missing required fields"""
def test_message_activity_validator():
"""This check should fail because a dict is passed to a validator instead
of an immutable message type."""
activity_message = {
"message_id": "",
"submission_time": "",
"type": "",
"activity_id": "",
"message_id": str(uuid.uuid4()),
"submission_time": str(int(time.time())),
"type": "ACTIVITY",
"activity_id": str(uuid.uuid4()),
"campaign_id": str(uuid.uuid4()),
"credential": {},
"body": {},
}
validator = MessageActivityValidator()
result = validator.check(activity_message)
assert result[0] is False


@pytest.mark.unit
def test_message_activity_validator2():
"""This test should be true all required fields are included"""
activity_message = {
"message_id": "",
"type": "",
"activity_id": "",
"agent_id": "",
"campaign_id": "",
"credential": {},
"submission_time": "",
"body": {},
}
def test_message_activity_validator_required_shell():
"""This test should be true all required fields are included

The following attributes should exist
message_id: "",
type: "ACTIVITY",
activity_id: "",
agent_id: "",
campaign_id: "",
credential: {},
submission_time: "",
body: {},
"""
validator = MessageActivityValidator()
activity_message = createActivityTemplate(ActivityType.SHELL)
activity_message.message_id = str(uuid.uuid4())
activity_message.activity_id = str(uuid.uuid4())
activity_message.agent_id = str(uuid.uuid4())
activity_message.campaign_id = str(uuid.uuid4())
activity_message.credential = {}
activity_message.submission_time = str(int(time.time()))
assert activity_message.body.type == "SHELL"

result = validator.check(activity_message)
assert result[0] is True


@pytest.mark.unit
def test_message_activity_validator3():
"""This test should be true all required fields are included as well as all
def test_message_activity_validator_required_plugin():
"""This test should be true all required fields are included

The following attributes should exist
message_id: "",
type: "ACTIVITY",
activity_id: "",
agent_id: "",
campaign_id: "",
credential: {},
submission_time: "",
body: {},
"""
validator = MessageActivityValidator()
activity_message = createActivityTemplate(ActivityType.PLUGIN)
activity_message.message_id = str(uuid.uuid4())
activity_message.activity_id = str(uuid.uuid4())
activity_message.agent_id = str(uuid.uuid4())
activity_message.campaign_id = str(uuid.uuid4())
activity_message.credential = {}
activity_message.submission_time = str(int(time.time()))
assert activity_message.body.type == "PLUGIN"
result = validator.check(activity_message)
assert result[0] is True


@pytest.mark.unit
def test_message_activity_validator_required_and_optional_shell():
"""This test should be true all required fields are defined as well as all
optional fields"""
activity_message = {
"message_id": "",
"type": "",
"activity_id": "",
"agent_id": "",
"campaign_id": "",
"credential": {},
"submission_time": "",
"body": {},
"needs": [],
}
validator = MessageActivityValidator()
activity_message = createActivityTemplate(ActivityType.SHELL)
activity_message.message_id = str(uuid.uuid4())
activity_message.activity_id = str(uuid.uuid4())
activity_message.agent_id = str(uuid.uuid4())
activity_message.campaign_id = str(uuid.uuid4())
activity_message.credential = {}
activity_message.submission_time = str(int(time.time()))
assert activity_message.body.type == "SHELL"
activity_message.needs = []
result = validator.check(activity_message)
assert result[0] is True

Expand Down
147 changes: 140 additions & 7 deletions tests/test_message_factory.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# Local imports
from zambeze.orchestration.message.message_factory import MessageFactory

from zambeze.orchestration.zambeze_types import MessageType
from zambeze.orchestration.zambeze_types import MessageType, ActivityType
from zambeze.orchestration.plugins import Plugins

# Standard imports
import pytest
import time
import uuid


@pytest.mark.unit
Expand Down Expand Up @@ -38,11 +39,11 @@ def test_factory_fail():
"""At this point the template can be used to fill in the items required
to exceute the message after it is sent"""
print(status_tuple)
status_tuple[1]["message_id"] = 1
status_tuple[1]["submission_time"] = time.time()
status_tuple[1].message_id = 1
status_tuple[1].submission_time = str(int(time.time()))

"""This field should force failure"""
status_tuple[1]["unsupported_field"] = "should make fail"
status_tuple[1].unsupported_field = "should make fail"

"""Should generate an immutable Status message, it will also verify that
the correct fields have been included."""
Expand All @@ -56,7 +57,100 @@ def test_factory_fail():


@pytest.mark.unit
def test_factory_success():
def test_factory_activity_template_plugin_globus():
"""This test should be true all required fields are defined as well as all
optional fields"""
plugins = Plugins()
msg_factory = MessageFactory(plugins)
activity_template = msg_factory.createTemplate(
MessageType.ACTIVITY,
ActivityType.PLUGIN,
args={"plugin": "globus", "action": "transfer"},
)[1]
print(activity_template)
no_fail = True
try:
activity_template.message_id = str(uuid.uuid4())
activity_template.activity_id = str(uuid.uuid4())
activity_template.agent_id = str(uuid.uuid4())
activity_template.campaign_id = str(uuid.uuid4())
activity_template.credential = {}
activity_template.submission_time = str(int(time.time()))
assert activity_template.body.parameters.transfer.type == "synchronous"
assert activity_template.body.type == "PLUGIN"
assert activity_template.body.plugin == "globus"
activity_template.body.parameters.transfer.items[0].source = ""
activity_template.body.parameters.transfer.items[0].destination = ""
activity_template.needs = []
except Exception as e:
print(e)
no_fail = False
assert no_fail


@pytest.mark.unit
def test_factory_activity_template_plugin_rsync():
"""This test should be true all required fields are defined as well as all
optional fields"""
plugins = Plugins()
msg_factory = MessageFactory(plugins)
activity_template = msg_factory.createTemplate(
MessageType.ACTIVITY,
ActivityType.PLUGIN,
{"plugin": "rsync", "action": "transfer"},
)[1]
print(activity_template)
no_fail = True
try:
activity_template.message_id = str(uuid.uuid4())
activity_template.activity_id = str(uuid.uuid4())
activity_template.agent_id = str(uuid.uuid4())
activity_template.campaign_id = str(uuid.uuid4())
activity_template.credential = {}
activity_template.submission_time = str(int(time.time()))
assert activity_template.body.parameters.transfer.type == "synchronous"
assert activity_template.body.type == "PLUGIN"
assert activity_template.body.plugin == "rsync"
activity_template.body.parameters.transfer.items[0].source.path = ""
activity_template.body.parameters.transfer.items[0].source.user = ""
activity_template.body.parameters.transfer.items[0].destination.path = ""
activity_template.body.parameters.transfer.items[0].destination.user = ""
activity_template.needs = []
except Exception as e:
print(e)
no_fail = False
assert no_fail


@pytest.mark.unit
def test_factory_activity_template_shell():
"""This test should be true all required fields are defined as well as all
optional fields"""
plugins = Plugins()
msg_factory = MessageFactory(plugins)
activity_template = msg_factory.createTemplate(
MessageType.ACTIVITY, ActivityType.SHELL, args={"shell": "bash"}
)[1]
print(activity_template)
no_fail = True
try:
activity_template.message_id = str(uuid.uuid4())
activity_template.activity_id = str(uuid.uuid4())
activity_template.agent_id = str(uuid.uuid4())
activity_template.campaign_id = str(uuid.uuid4())
activity_template.credential = {}
activity_template.submission_time = str(int(time.time()))
assert activity_template.body.type == "SHELL"
assert activity_template.body.shell == "bash"

except Exception as e:
print(e)
no_fail = False
assert no_fail


@pytest.mark.unit
def test_factory_success_status():
plugins = Plugins()
msg_factory = MessageFactory(plugins)

Expand All @@ -83,11 +177,50 @@ def test_factory_success():
status_tuple = msg_factory.createTemplate(MessageType.STATUS)
"""At this point the template can be used to fill in the items required
to exceute the message after it is sent"""
status_tuple[1]["message_id"] = 1
status_tuple[1]["submission_time"] = time.time()
status_tuple[1].message_id = str(uuid.uuid4())
status_tuple[1].submission_time = str(int(time.time()))
status_tuple[1].activity_id = str(uuid.uuid4())
status_tuple[1].target_id = str(uuid.uuid4())
status_tuple[1].campaign_id = str(uuid.uuid4())
status_tuple[1].agent_id = str(uuid.uuid4())
status_tuple[1].body = {}

"""Should generate an immutable Status message, it will also verify that
the correct fields have been included."""
status_msg = msg_factory.create(status_tuple)

assert status_msg.type == MessageType.STATUS


@pytest.mark.unit
def test_factory_create_plugin_rsync():
"""This test should be true all required fields are defined as well as all
optional fields"""
plugins = Plugins()
msg_factory = MessageFactory(plugins)
activity_tuple = msg_factory.createTemplate(
MessageType.ACTIVITY,
ActivityType.PLUGIN,
{"plugin": "rsync", "action": "transfer"},
)
print(activity_tuple)

activity_tuple[1].message_id = str(uuid.uuid4())
activity_tuple[1].activity_id = str(uuid.uuid4())
activity_tuple[1].agent_id = str(uuid.uuid4())
activity_tuple[1].campaign_id = str(uuid.uuid4())
activity_tuple[1].credential = {}
activity_tuple[1].submission_time = str(int(time.time()))
activity_tuple[1].body.parameters.transfer.items[0].source.path = ""
activity_tuple[1].body.parameters.transfer.items[0].source.user = ""
activity_tuple[1].body.parameters.transfer.items[0].source.ip = "127.0.0.1"
activity_tuple[1].body.parameters.transfer.items[0].destination.path = ""
activity_tuple[1].body.parameters.transfer.items[0].destination.user = ""
activity_tuple[1].body.parameters.transfer.items[0].destination.ip = "127.0.0.1"
activity_tuple[1].needs = []

"""Should generate an immutable Activity message, it will also verify that
the correct fields have been included."""
status_msg = msg_factory.create(activity_tuple)

assert status_msg.type == MessageType.ACTIVITY
6 changes: 3 additions & 3 deletions tests/test_message_status.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Local imports
from zambeze.orchestration.message.message_status import MessageStatus
from zambeze.orchestration.message.status_message.message_status import MessageStatus

from zambeze.orchestration.zambeze_types import MessageType

Expand All @@ -15,15 +15,15 @@ def test_message_status_type():

def test_message_status_message():
msg_status = MessageStatus({"body": "testing"})
assert msg_status.message["body"] == "testing"
assert msg_status.data["body"] == "testing"


def test_message_status_immutability():
"""Demonstrate immutability of message once created"""
msg_status = MessageStatus({"body": "testing"})
failed = False
try:
msg_status.message["new_key"] == "should fail"
msg_status.data["new_key"] == "should fail"
except Exception:
failed = True
pass
Expand Down
Loading