Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PLINT-542] Add machines metrics #19285

Open
wants to merge 7 commits into
base: sarah/add-octopus-integration
Choose a base branch
from
Open

[PLINT-542] Add machines metrics #19285

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d662619
Add support for machine metrics
sarah-witt Dec 18, 2024
9e4b6b5
add role tags
sarah-witt Dec 18, 2024
caae890
change to startswith
sarah-witt Dec 19, 2024
78a5fb0
Resolve conflicts
sarah-witt Dec 20, 2024
cf47801
validate metadata
sarah-witt Dec 20, 2024
9660a83
Merge branch 'sarah/add-octopus-integration' into sarah/add-machines-…
sarah-witt Dec 26, 2024
69d8d13
Address comments
sarah-witt Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions octopus_deploy/datadog_checks/octopus_deploy/check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ def _process_spaces(self):
self._process_project_groups(
space_id, space_name, space_config.get("project_groups") if space_config else None
)
self._collect_machine_metrics(space_id)
if self.collect_events:
self._collect_new_events(space_id, space_name)

Expand Down Expand Up @@ -341,6 +342,29 @@ def _collect_server_nodes_metrics(self):
self.gauge("server_node.in_maintenance_mode", maintenance_mode, tags=self._base_tags + server_tags)
self.gauge("server_node.max_concurrent_tasks", max_tasks, tags=self._base_tags + server_tags)

def _collect_machine_metrics(self, space_id):
self.log.debug("Collecting server node metrics.")
url = f"api/{space_id}/machines"
response_json = self._process_paginated_endpoint(url)
machines = response_json.get('Items', [])

for machine in machines:
machine_id = machine.get("Id")
machine_name = machine.get("Name")
machine_slug = machine.get("Slug")
roles = machine.get("Roles", [])
health_status = machine.get("HealthStatus", None)
jose-manuel-almaza marked this conversation as resolved.
Show resolved Hide resolved
is_healthy = health_status.startswith("Healthy")
machine_tags = [
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you think about adding OperatingSystem as a machine_os tag?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added!

f"machine_id:{machine_id}",
f"machine_name:{machine_name}",
f"machine_slug:{machine_slug}",
f"health_status:{health_status}",
]
machine_tags += roles
jose-manuel-almaza marked this conversation as resolved.
Show resolved Hide resolved
self.gauge("machine.count", 1, tags=self._base_tags + machine_tags)
self.gauge("machine.is_healthy", is_healthy, tags=self._base_tags + machine_tags)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the purpose of also reporting it as a metric if we already report it as a tag?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's easier to monitor on, and the tag gives more specific/detailed information


def _collect_deployment_logs(self, space_id, task_id, tags):
url = f"api/{space_id}/tasks/{task_id}/details"
activity_logs = self._process_endpoint(url).get('ActivityLogs', [])
Expand Down
2 changes: 2 additions & 0 deletions octopus_deploy/metadata.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ octopus_deploy.deployment.completed_time,gauge,,second,,Duration of deployment.,
octopus_deploy.deployment.count,gauge,,,,Number of deployments monitored.,-1,octopus_deploy,octopus_deploy deploy count,,
octopus_deploy.deployment.executing_time,gauge,,second,,How long the deployment has been executing.,-1,octopus_deploy,octopus_deploy deploy dur,,
octopus_deploy.deployment.queued_time,gauge,,second,,Time deployment was in queue.,-1,octopus_deploy,octopus_deploy deploy queue,,
octopus_deploy.machine.count,gauge,,,,Number of machines discovered.,-1,octopus_deploy,octopus_deploy machine count,,
octopus_deploy.machine.is_healthy,gauge,,,,Whether or not the machine is healthy.,-1,octopus_deploy,octopus_deploy machine health,,
octopus_deploy.project.count,gauge,,,,Number of projects discovered.,-1,octopus_deploy,octopus_deploy projects count,,
octopus_deploy.project_group.count,gauge,,,,Number of project groups discovered.,-1,octopus_deploy,octopus_deploy project group count,,
octopus_deploy.server_node.count,gauge,,,,Number of Octopus server nodes discovered.,-1,octopus_deploy,octopus_deploy server count,,
Expand Down
2 changes: 2 additions & 0 deletions octopus_deploy/tests/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
"octopus_deploy.server_node.count",
"octopus_deploy.server_node.in_maintenance_mode",
"octopus_deploy.server_node.max_concurrent_tasks",
"octopus_deploy.machine.count",
"octopus_deploy.machine.is_healthy",
]

ALL_METRICS = ["octopus_deploy.deployment.completed_time"] + E2E_METRICS
Expand Down
10 changes: 9 additions & 1 deletion octopus_deploy/tests/docker/Caddyfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,15 @@
rewrite * /GET/api/octopusservernodes/skip=0/take=30/response.json
file_server
}
@get_tasks_1844 {
@get_machines {
method GET
path /api/Spaces-1/machines
}
route @get_machines {
rewrite * /GET/api/Spaces-1/machines/skip=0/take=30/response.json
file_server
}
@get_tasks_1844 {
method GET
path /api/Spaces-1/tasks/ServerTasks-1844/details
}
Expand Down
68 changes: 68 additions & 0 deletions octopus_deploy/tests/fixtures/GET/api/Spaces-1/machines/skip=0/take=2/response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
{
"ItemType": "Machine",
"TotalResults": 1,
"ItemsPerPage": 2,
"NumberOfPages": 1,
"LastPageNumber": 0,
"Items": [
{
"Id": "Machines-1",
"EnvironmentIds": [
"Environments-1"
],
"Roles": [
"test-tag"
],
"TenantedDeploymentParticipation": "Untenanted",
"TenantIds": [],
"TenantTags": [],
"SpaceId": "Spaces-1",
"Name": "test-machine",
"Thumbprint": "test-footprint",
"Uri": "hostname-test:10933/",
"IsDisabled": false,
"MachinePolicyId": "MachinePolicies-1",
"HealthStatus": "Healthy",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're only testing HealthStatus with Healthy value

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added multiple health status values!

"HasLatestCalamari": false,
"StatusSummary": "Octopus was able to successfully establish a connection with this machine on Wednesday",
"IsInProcess": false,
"Endpoint": {
"CommunicationStyle": "TentaclePassive",
"ProxyId": null,
"Thumbprint": "test-footprint",
"Uri": "hostname-test:10933/",
"TentacleVersionDetails": {
"UpgradeLocked": false,
"Version": "8.2.2517",
"UpgradeSuggested": false,
"UpgradeRequired": false,
"UpgradeAvailable": false
},
"CertificateSignatureAlgorithm": "sha256RSA",
"Id": null,
"LastModifiedOn": null,
"LastModifiedBy": null,
"Links": {}
},
"OperatingSystem": "Ubuntu 24.04.1 LTS",
"ShellName": "Bash",
"ShellVersion": "5.2.21(1)-release",
"Architecture": "x86_64",
"Slug": "test-machine",
"SkipInitialHealthCheck": false,
"Links": {
"Self": "/api/Spaces-1/machines/Machines-1",
"Connection": "/api/Spaces-1/machines/Machines-1/connection",
"TasksTemplate": "/api/Spaces-1/machines/Machines-1/tasks{?skip,take,type}",
"SinglyScopedVariableDetails": "/api/Spaces-1/machines/Machines-1/singlyScopedVariableDetails"
}
}
],
"Links": {
"Self": "/api/Spaces-1/machines?skip=0&take=30",
"Template": "/api/Spaces-1/machines{?skip,take,name,ids,partialName,roles,isDisabled,healthStatuses,commStyles,tenantIds,tenantTags,environmentIds,shellNames,deploymentTargetTypes}",
"Page.All": "/api/Spaces-1/machines?skip=0&take=2147483647",
"Page.Current": "/api/Spaces-1/machines?skip=0&take=30",
"Page.Last": "/api/Spaces-1/machines?skip=0&take=30"
}
}
68 changes: 68 additions & 0 deletions octopus_deploy/tests/fixtures/GET/api/Spaces-1/machines/skip=0/take=30/response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
{
"ItemType": "Machine",
"TotalResults": 1,
"ItemsPerPage": 30,
"NumberOfPages": 1,
"LastPageNumber": 0,
"Items": [
{
"Id": "Machines-1",
"EnvironmentIds": [
"Environments-1"
],
"Roles": [
"test-tag"
],
"TenantedDeploymentParticipation": "Untenanted",
"TenantIds": [],
"TenantTags": [],
"SpaceId": "Spaces-1",
"Name": "test-machine",
"Thumbprint": "test-footprint",
"Uri": "hostname-test:10933/",
"IsDisabled": false,
"MachinePolicyId": "MachinePolicies-1",
"HealthStatus": "Healthy",
"HasLatestCalamari": false,
"StatusSummary": "Octopus was able to successfully establish a connection with this machine on Wednesday",
"IsInProcess": false,
"Endpoint": {
"CommunicationStyle": "TentaclePassive",
"ProxyId": null,
"Thumbprint": "test-footprint",
"Uri": "hostname-test:10933/",
"TentacleVersionDetails": {
"UpgradeLocked": false,
"Version": "8.2.2517",
"UpgradeSuggested": false,
"UpgradeRequired": false,
"UpgradeAvailable": false
},
"CertificateSignatureAlgorithm": "sha256RSA",
"Id": null,
"LastModifiedOn": null,
"LastModifiedBy": null,
"Links": {}
},
"OperatingSystem": "Ubuntu 24.04.1 LTS",
"ShellName": "Bash",
"ShellVersion": "5.2.21(1)-release",
"Architecture": "x86_64",
"Slug": "test-machine",
"SkipInitialHealthCheck": false,
"Links": {
"Self": "/api/Spaces-1/machines/Machines-1",
"Connection": "/api/Spaces-1/machines/Machines-1/connection",
"TasksTemplate": "/api/Spaces-1/machines/Machines-1/tasks{?skip,take,type}",
"SinglyScopedVariableDetails": "/api/Spaces-1/machines/Machines-1/singlyScopedVariableDetails"
}
}
],
"Links": {
"Self": "/api/Spaces-1/machines?skip=0&take=30",
"Template": "/api/Spaces-1/machines{?skip,take,name,ids,partialName,roles,isDisabled,healthStatuses,commStyles,tenantIds,tenantTags,environmentIds,shellNames,deploymentTargetTypes}",
"Page.All": "/api/Spaces-1/machines?skip=0&take=2147483647",
"Page.Current": "/api/Spaces-1/machines?skip=0&take=30",
"Page.Last": "/api/Spaces-1/machines?skip=0&take=30"
}
}
108 changes: 108 additions & 0 deletions octopus_deploy/tests/test_unit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1272,3 +1272,111 @@ def test_paginated_limit_tasks(
skip_take_args += [(list(args), skip, take)]

assert skip_take_args == expected_skip_take_args


@pytest.mark.usefixtures('mock_http_get')
@mock.patch("datadog_checks.octopus_deploy.check.get_current_datetime")
def test_machines_metrics(
get_current_datetime,
dd_run_check,
aggregator,
):
instance = {'octopus_endpoint': 'http://localhost:80'}

check = OctopusDeployCheck('octopus_deploy', {}, [instance])

get_current_datetime.return_value = MOCKED_TIME1
dd_run_check(check)
aggregator.assert_metric(
"octopus_deploy.machine.count",
1,
tags=[
"machine_id:Machines-1",
"machine_name:test-machine",
"machine_slug:test-machine",
"health_status:Healthy",
"test-tag",
],
)
aggregator.assert_metric(
"octopus_deploy.machine.is_healthy",
1,
tags=[
"machine_id:Machines-1",
"machine_name:test-machine",
"machine_slug:test-machine",
"health_status:Healthy",
"test-tag",
],
)


@pytest.mark.parametrize(
('paginated_limit, expected_skip_take_args'),
[
pytest.param(
30,
[
(['http://localhost:80/api/Spaces-1/machines'], 0, 30),
],
id='high limit',
),
pytest.param(
2,
[
(['http://localhost:80/api/Spaces-1/machines'], 0, 2),
],
id='low limit',
),
],
)
@pytest.mark.usefixtures('mock_http_get')
@mock.patch("datadog_checks.octopus_deploy.check.get_current_datetime")
def test_machines_pagination(
get_current_datetime,
dd_run_check,
aggregator,
expected_skip_take_args,
mock_http_get,
paginated_limit,
):
instance = {'octopus_endpoint': 'http://localhost:80'}
instance['paginated_limit'] = paginated_limit

check = OctopusDeployCheck('octopus_deploy', {}, [instance])

get_current_datetime.return_value = MOCKED_TIME1
dd_run_check(check)

skip_take_args = []
for call in mock_http_get.call_args_list:
args, kwargs = call
take = kwargs.get('params', {}).get('take')
skip = kwargs.get('params', {}).get('skip')
if 'http://localhost:80/api/Spaces-1/machines' == args[0]:
skip_take_args += [(list(args), skip, take)]

assert skip_take_args == expected_skip_take_args

aggregator.assert_metric(
"octopus_deploy.machine.count",
1,
tags=[
"machine_id:Machines-1",
"machine_name:test-machine",
"machine_slug:test-machine",
"health_status:Healthy",
"test-tag",
],
)
aggregator.assert_metric(
"octopus_deploy.machine.is_healthy",
1,
tags=[
"machine_id:Machines-1",
"machine_name:test-machine",
"machine_slug:test-machine",
"health_status:Healthy",
"test-tag",
],
)
Loading