From 9bf01896011dc8544d999808ddaf450a4dbba9c3 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 1 May 2024 11:00:13 +0100 Subject: [PATCH 1/6] Add playbook for deploying radosgw usage exporter --- .../ansible/deploy-radosgw-usage-exporter.yml | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml diff --git a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml new file mode 100644 index 000000000..b6c99d1d6 --- /dev/null +++ b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml @@ -0,0 +1,88 @@ +--- +- name: Deploy RADOS gateway usage exporter + hosts: monitoring + gather_facts: false + vars: + venv: "{{ virtualenv_path }}/openstack" + tasks: + - name: Check ec2 credential for admin + command: > + {{ venv }}/bin/openstack + ec2 credentials list --user admin + --format json + environment: "{{ openstack_auth_env }}" + delegate_to: localhost + register: credential_check + + - name: Create ec2 credential if there's none + command: > + {{ venv }}/bin/openstack + ec2 credentials create --user admin + --format json + environment: "{{ openstack_auth_env }}" + delegate_to: localhost + when: "{{ credential_check.stdout == [] }}" + + - name: Query ec2 credential for admin + command: > + {{ venv }}/bin/openstack + ec2 credentials list --user admin + --format json + environment: "{{ openstack_auth_env }}" + delegate_to: localhost + register: credential + + - name: Get object storage endpoint + command: > + {{ venv }}/bin/openstack + endpoint list --service object-store --interface internal + --format json + environment: "{{ openstack_auth_env }}" + delegate_to: localhost + register: endpoint + + - name: Ensure radosgw_usage_exporter container is running + community.docker.docker_container: + name: radosgw_usage_exporter + image: ghcr.io/stackhpc/radosgw_usage_exporter:0.1.0 + network_mode: host + env: + RADOSGW_SERVER: "{{ radosgw_server }}" + ADMIN_ENTRY: admin + ACCESS_KEY: "{{ ec2.Access }}" + SECRET_KEY: "{{ ec2.Secret }}" + vars: + ec2: "{{ credential.stdout | from_json | first }}" + host: "{{ endpoint.stdout | from_json | first }}" + radosgw_server: "{{ host.URL | regex_replace('(https?://)([0-9.]+):([0-9]+)/.*', '\\1\\2:\\3') }}" + become: true + + - name: Get target ip addresses + set_fact: + prometheus_targets: "{{ prometheus_targets | default([]) +[( internal_net_name | net_ip( item ) + ':9242' )] }}" + loop: "{{ groups['monitoring'] }}" + + - name: Ensure radosgw_usage_exporter is targeted by prometheus-server + blockinfile: + path: /etc/kolla/prometheus-server/prometheus.yml + block: | + - honor_labels: true + job_name: ceph_radosgw_usage_exporter + scrape_interval: 15s + static_configs: + - targets: + {% for item in prometheus_targets %} + - {{ item }} + {% endfor %} + metric_relabel_configs: + - source_labels: [owner,user] + target_label: tenant_id + separator: "" + regex: (.+) + replacement: ${1} + become: true + + - name: Restart prometheus-server + ansible.builtin.command: + cmd: systemctl restart kolla-prometheus_server-container + become: true From 6a67f7f2355f02328bb8f99f50de0d6f8613e352 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 1 May 2024 11:32:51 +0100 Subject: [PATCH 2/6] Fix regex to include FQDN --- etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml index b6c99d1d6..74ded1e3c 100644 --- a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml +++ b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml @@ -54,7 +54,7 @@ vars: ec2: "{{ credential.stdout | from_json | first }}" host: "{{ endpoint.stdout | from_json | first }}" - radosgw_server: "{{ host.URL | regex_replace('(https?://)([0-9.]+):([0-9]+)/.*', '\\1\\2:\\3') }}" + radosgw_server: "{{ host.URL | regex_replace('(https?://)([^:/]+):([0-9]+)/.*', '\\1\\2:\\3') }}" # Drop Swift api address become: true - name: Get target ip addresses From 8b09451f5e5acc36eeb933e9f9c98e63c89f3e66 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Fri, 3 May 2024 16:05:56 +0100 Subject: [PATCH 3/6] Improve playbook with run_once --- .../ansible/deploy-radosgw-usage-exporter.yml | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml index 74ded1e3c..47ca9e735 100644 --- a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml +++ b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml @@ -13,6 +13,7 @@ environment: "{{ openstack_auth_env }}" delegate_to: localhost register: credential_check + run_once: true - name: Create ec2 credential if there's none command: > @@ -21,6 +22,7 @@ --format json environment: "{{ openstack_auth_env }}" delegate_to: localhost + run_once: true when: "{{ credential_check.stdout == [] }}" - name: Query ec2 credential for admin @@ -31,6 +33,7 @@ environment: "{{ openstack_auth_env }}" delegate_to: localhost register: credential + run_once: true - name: Get object storage endpoint command: > @@ -40,6 +43,7 @@ environment: "{{ openstack_auth_env }}" delegate_to: localhost register: endpoint + run_once: true - name: Ensure radosgw_usage_exporter container is running community.docker.docker_container: @@ -57,17 +61,21 @@ radosgw_server: "{{ host.URL | regex_replace('(https?://)([^:/]+):([0-9]+)/.*', '\\1\\2:\\3') }}" # Drop Swift api address become: true - - name: Get target ip addresses + - name: Get Prometheus target ip addresses of RADOS gateway usage exporters set_fact: prometheus_targets: "{{ prometheus_targets | default([]) +[( internal_net_name | net_ip( item ) + ':9242' )] }}" loop: "{{ groups['monitoring'] }}" + run_once: true - - name: Ensure radosgw_usage_exporter is targeted by prometheus-server - blockinfile: - path: /etc/kolla/prometheus-server/prometheus.yml - block: | - - honor_labels: true - job_name: ceph_radosgw_usage_exporter + - name: Print config to append on prometheus.yml + debug: + msg: | + Add Prometheus target for RADOS gateway usage exporter below at prometheus.yml then reconfigure prometheus_server. + You can find a template to prometheus.yml at ``ansible/roles/prometheus/templates/prometheus.yml.j2`` under kolla-ansible directory. + Make sure to use {% raw %} and {% endraw %} tags appropriately on prometheus.yml if you're overriding it for the first time. + + - job_name: ceph_radosgw_usage_exporter + honor_labels: true scrape_interval: 15s static_configs: - targets: @@ -80,9 +88,5 @@ separator: "" regex: (.+) replacement: ${1} - become: true + run_once: true - - name: Restart prometheus-server - ansible.builtin.command: - cmd: systemctl restart kolla-prometheus_server-container - become: true From cb4a84386b91aa440675b226cbd843e1deefa6f3 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 22 May 2024 09:32:10 +0100 Subject: [PATCH 4/6] Add general condition variable to enable rgw exporter --- .../ansible/deploy-radosgw-usage-exporter.yml | 38 ++++--------------- etc/kayobe/stackhpc-monitoring.yml | 5 +++ 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml index 47ca9e735..8a5a1d2f3 100644 --- a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml +++ b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml @@ -14,6 +14,7 @@ delegate_to: localhost register: credential_check run_once: true + when: stackhpc_enable_radosgw_usage_exporter - name: Create ec2 credential if there's none command: > @@ -23,7 +24,9 @@ environment: "{{ openstack_auth_env }}" delegate_to: localhost run_once: true - when: "{{ credential_check.stdout == [] }}" + when: + - "{{ credential_check.stdout == [] }}" + - stackhpc_enable_radosgw_usage_exporter - name: Query ec2 credential for admin command: > @@ -34,6 +37,7 @@ delegate_to: localhost register: credential run_once: true + when: stackhpc_enable_radosgw_usage_exporter - name: Get object storage endpoint command: > @@ -44,6 +48,7 @@ delegate_to: localhost register: endpoint run_once: true + when: stackhpc_enable_radosgw_usage_exporter - name: Ensure radosgw_usage_exporter container is running community.docker.docker_container: @@ -60,33 +65,4 @@ host: "{{ endpoint.stdout | from_json | first }}" radosgw_server: "{{ host.URL | regex_replace('(https?://)([^:/]+):([0-9]+)/.*', '\\1\\2:\\3') }}" # Drop Swift api address become: true - - - name: Get Prometheus target ip addresses of RADOS gateway usage exporters - set_fact: - prometheus_targets: "{{ prometheus_targets | default([]) +[( internal_net_name | net_ip( item ) + ':9242' )] }}" - loop: "{{ groups['monitoring'] }}" - run_once: true - - - name: Print config to append on prometheus.yml - debug: - msg: | - Add Prometheus target for RADOS gateway usage exporter below at prometheus.yml then reconfigure prometheus_server. - You can find a template to prometheus.yml at ``ansible/roles/prometheus/templates/prometheus.yml.j2`` under kolla-ansible directory. - Make sure to use {% raw %} and {% endraw %} tags appropriately on prometheus.yml if you're overriding it for the first time. - - - job_name: ceph_radosgw_usage_exporter - honor_labels: true - scrape_interval: 15s - static_configs: - - targets: - {% for item in prometheus_targets %} - - {{ item }} - {% endfor %} - metric_relabel_configs: - - source_labels: [owner,user] - target_label: tenant_id - separator: "" - regex: (.+) - replacement: ${1} - run_once: true - + when: stackhpc_enable_radosgw_usage_exporter diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml index e8e0bb91f..7a49bdaca 100644 --- a/etc/kayobe/stackhpc-monitoring.yml +++ b/etc/kayobe/stackhpc-monitoring.yml @@ -20,6 +20,11 @@ alertmanager_warn_network_bond_single_link: true # targets being templated during deployment. stackhpc_enable_os_capacity: true +# Whether the RADOS gateway usage exporter is enabled. +# Enabling this will result in templating radosge_usage_exporter endpoint as +# Prometheus scrape targets during deployment. +stackhpc_enable_radosgw_usage_exporter: false + # Whether TLS certificate verification is enabled for the OpenStack Capacity # exporter during Keystone authentication. stackhpc_os_capacity_openstack_verify: true From 2813fc3bbfe143dc885518fca28a8b811ff7b7d1 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 22 May 2024 09:38:23 +0100 Subject: [PATCH 5/6] Add prometheus target template for rgw exporter --- .../prometheus.yml.d/80-radosgw-exporter.yml | 21 +++++++++++++++++++ etc/kayobe/stackhpc-monitoring.yml | 1 + 2 files changed, 22 insertions(+) create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml new file mode 100644 index 000000000..304736a80 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml @@ -0,0 +1,21 @@ +# yamllint disable-file +--- +{% if stackhpc_enable_radosgw_usage_exporter | bool %} +{% raw %} +scrape_configs: + - job_name: ceph_radosgw_usage_exporter + honor_labels: true + scrape_interval: 15s + metric_relabel_configs: + - replacement: ${1} + source_labels: [owner,user] + target_label: tenant_id + separator: "" + regex: (.+) + static_configs: + - targets: + {% for host in groups['monitoring'] %} + - "{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{% endraw %}{{ stackhpc_radosgw_usage_exporter_port }}{% raw %}" + {% endfor %} +{% endraw %} +{% endif %} diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml index 7a49bdaca..18a625ca6 100644 --- a/etc/kayobe/stackhpc-monitoring.yml +++ b/etc/kayobe/stackhpc-monitoring.yml @@ -24,6 +24,7 @@ stackhpc_enable_os_capacity: true # Enabling this will result in templating radosge_usage_exporter endpoint as # Prometheus scrape targets during deployment. stackhpc_enable_radosgw_usage_exporter: false +stackhpc_radosgw_usage_exporter_port: 9242 # Whether TLS certificate verification is enabled for the OpenStack Capacity # exporter during Keystone authentication. From c27a269972cbddf374b7179ae50f7b3cc6d8bcf7 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 22 May 2024 09:58:48 +0100 Subject: [PATCH 6/6] Add deploying rgw exporter at service deploy post hook This will not run by default as ``stackhpc_enable_radosgw_usage_exporter`` is initially set to false --- .../post.d/deploy-radosgw-usage-exporter.yml | 1 + 1 file changed, 1 insertion(+) create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-radosgw-usage-exporter.yml diff --git a/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-radosgw-usage-exporter.yml b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-radosgw-usage-exporter.yml new file mode 120000 index 000000000..3d939329a --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-radosgw-usage-exporter.yml @@ -0,0 +1 @@ +../../../ansible/deploy-radosgw-usage-exporter.yml \ No newline at end of file