From 66741ff14f91bb4322d452289ee437d2ded4d7f5 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 16 Dec 2020 11:38:20 +0100 Subject: [PATCH] day2: set noout at the bucket level instead It is safer to set the `noout` flag at the bucket level instead of the whole cluster. This commit updates the 3 playbooks where we set `noout` flag so it doesn't set it at the whole cluster level anymore. Signed-off-by: Guillaume Abrioux --- infrastructure-playbooks/cephadm-adopt.yml | 31 +++++++----- infrastructure-playbooks/rolling_update.yml | 47 +++++++++++++++++-- ...inerized-to-containerized-ceph-daemons.yml | 44 ++++++++++++----- 3 files changed, 95 insertions(+), 27 deletions(-) diff --git a/infrastructure-playbooks/cephadm-adopt.yml b/infrastructure-playbooks/cephadm-adopt.yml index 40e06a34cb..35f6dd8550 100644 --- a/infrastructure-playbooks/cephadm-adopt.yml +++ b/infrastructure-playbooks/cephadm-adopt.yml @@ -694,8 +694,8 @@ CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' -- name: set osd flags - hosts: "{{ osd_group_name|default('osds') }}" +- name: set osd flag nodeep-scrub + hosts: "{{ mon_group_name|default('mons') }}[0]" become: true gather_facts: false any_errors_fatal: True @@ -707,7 +707,6 @@ command: "{{ ceph_cmd }} --cluster {{ cluster }} osd pool ls detail -f json" register: pool_list run_once: true - delegate_to: "{{ groups[mon_group_name][0] }}" changed_when: false check_mode: false @@ -715,7 +714,6 @@ command: "{{ ceph_cmd }} --cluster {{ cluster }} balancer status -f json" register: balancer_status_adopt run_once: true - delegate_to: "{{ groups[mon_group_name][0] }}" changed_when: false check_mode: false @@ -738,7 +736,6 @@ cluster: "{{ cluster }}" pg_autoscale_mode: false with_items: "{{ pools_pgautoscaler_mode }}" - delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true when: - pools_pgautoscaler_mode is defined @@ -755,7 +752,6 @@ with_items: - noout - nodeep-scrub - delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true environment: CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" @@ -776,6 +772,12 @@ tasks_from: container_binary.yml when: containerized_deployment | bool + - name: set osd flag noout + command: "{{ cephadm_cmd }} shell --fsid {{ fsid }} -- ceph --cluster {{ cluster }} osd set-group noout {{ inventory_hostname }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + - name: get osd list ceph_volume: cluster: "{{ cluster }}" @@ -850,8 +852,14 @@ environment: CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' -- name: unset osd flags - hosts: "{{ osd_group_name|default('osds') }}" + - name: unset osd flag noout + command: "{{ cephadm_cmd }} shell --fsid {{ fsid }} -- ceph --cluster {{ cluster }} osd unset-group noout {{ inventory_hostname }}" + changed_when: false + environment: + CEPHADM_IMAGE: '{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}' + +- name: unset osd flag nodeep-scrub + hosts: "{{ mon_group_name|default('mons') }}[0]" become: true gather_facts: false any_errors_fatal: True @@ -865,7 +873,6 @@ cluster: "{{ cluster }}" pg_autoscale_mode: true with_items: "{{ pools_pgautoscaler_mode }}" - delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true when: - pools_pgautoscaler_mode is defined @@ -882,8 +889,11 @@ with_items: - noout - nodeep-scrub - delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true + + - name: unset osd flag nodeep-scrub + command: "{{ cephadm_cmd }} shell --fsid {{ fsid }} -- ceph --cluster {{ cluster }} osd unset nodeep-scrub" + changed_when: false environment: CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" CEPH_CONTAINER_BINARY: "{{ container_binary }}" @@ -891,7 +901,6 @@ - name: re-enable balancer command: "{{ ceph_cmd }} --cluster {{ cluster }} balancer on" run_once: true - delegate_to: "{{ groups[mon_group_name][0] }}" changed_when: false when: (balancer_status_adopt.stdout | from_json)['active'] | bool diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml index 4532d4998b..0be31173c5 100644 --- a/infrastructure-playbooks/rolling_update.yml +++ b/infrastructure-playbooks/rolling_update.yml @@ -412,9 +412,8 @@ name: ceph-mgr -- name: set osd flags - hosts: "{{ osd_group_name | default('osds') }}" - tags: osds +- name: set osd flag nodeep-scrub + hosts: "{{ mon_group_name | default('mons') }}[0]" become: True gather_facts: false tasks: @@ -426,7 +425,6 @@ - name: set osd flags, disable autoscaler and balancer run_once: true - delegate_to: "{{ groups[mon_group_name][0] }}" block: - name: get pool list command: "{{ ceph_cmd }} --cluster {{ cluster }} osd pool ls detail -f json" @@ -475,6 +473,14 @@ - noout - nodeep-scrub + - name: set osd flag nodeep-scrub + ceph_osd_flag: + name: "nodeep-scrub" + cluster: "{{ cluster }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + - name: upgrade ceph osds cluster vars: health_osd_check_retries: 600 @@ -491,6 +497,17 @@ - import_role: name: ceph-facts + - name: set osd flag noout + ceph_osd_flag: + name: noout + level: bucket + bucket: "{{ inventory_hostname }}" + cluster: "{{ cluster }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + - name: get osd numbers - non container shell: if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi # noqa 306 register: osd_ids @@ -554,6 +571,18 @@ retries: "{{ health_osd_check_retries }}" delay: "{{ health_osd_check_delay }}" + - name: unset osd flag noout + ceph_osd_flag: + name: noout + state: absent + level: bucket + bucket: "{{ inventory_hostname }}" + cluster: "{{ cluster }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + - name: complete osd upgrade hosts: "{{ osd_group_name | default('osds') }}" @@ -601,6 +630,16 @@ changed_when: false when: (balancer_status_update.stdout | from_json)['active'] | bool + - name: unset osd flag nodeep-scrub + ceph_osd_flag: + name: "nodeep-scrub" + cluster: "{{ cluster }}" + state: absent + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + + - name: upgrade ceph mdss cluster, deactivate all rank > 0 hosts: "{{ mon_group_name | default('mons') }}[0]" tags: mdss diff --git a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml index d07e271d0b..a8d7f4f2be 100644 --- a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml +++ b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml @@ -213,7 +213,7 @@ name: ceph-mgr -- name: set osd flags +- name: set osd flag nodeep-scrub hosts: "{{ mon_group_name | default('mons') }}[0]" become: True tasks: @@ -258,16 +258,13 @@ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" CEPH_CONTAINER_BINARY: "{{ container_binary }}" - - name: set osd flags + - name: set osd flag nodeep-scrub ceph_osd_flag: - name: "{{ item }}" + name: "nodeep-scrub" cluster: "{{ cluster }}" environment: CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" CEPH_CONTAINER_BINARY: "{{ container_binary }}" - with_items: - - noout - - nodeep-scrub - name: switching from non-containerized to containerized ceph osd @@ -286,6 +283,21 @@ - import_role: name: ceph-defaults + - import_role: + name: ceph-facts + tasks_from: container_binary.yml + + - name: set osd flag noout + ceph_osd_flag: + name: noout + level: bucket + bucket: "{{ inventory_hostname }}" + cluster: "{{ cluster }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + - name: collect running osds shell: | set -o pipefail; @@ -398,8 +410,19 @@ delay: "{{ health_osd_check_delay }}" changed_when: false + - name: unset osd flag noout + ceph_osd_flag: + name: noout + state: absent + level: bucket + bucket: "{{ inventory_hostname }}" + cluster: "{{ cluster }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" -- name: unset osd flags +- name: unset osd flag nodeep-scrub hosts: "{{ mon_group_name | default('mons') }}[0]" become: True tasks: @@ -422,17 +445,14 @@ CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" CEPH_CONTAINER_BINARY: "{{ container_binary }}" - - name: unset osd flags + - name: set osd flag nodeep-scrub ceph_osd_flag: - name: "{{ item }}" + name: "nodeep-scrub" cluster: "{{ cluster }}" state: absent environment: CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" CEPH_CONTAINER_BINARY: "{{ container_binary }}" - with_items: - - noout - - nodeep-scrub - name: re-enable balancer command: "{{ ceph_cmd }} --cluster {{ cluster }} balancer on"