Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Explicitly set ETCDCTL_API and use ETCDCTL_ENDPOINTS #6327

Merged
merged 1 commit into from
Jul 1, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion roles/etcd/handlers/backup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@
- name: Backup etcd v3 data
command: >-
{{ bin_dir }}/etcdctl
--endpoints={{ etcd_access_addresses }}
snapshot save {{ etcd_backup_directory }}/snapshot.db
environment:
ETCDCTL_API: 3
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
Expand Down
24 changes: 18 additions & 6 deletions roles/etcd/tasks/configure.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
- name: Configure | Check if etcd cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'"
register: etcd_cluster_is_healthy
failed_when: false
changed_when: false
Expand All @@ -10,12 +10,14 @@
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"

- name: Configure | Check if etcd-events cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'"
register: etcd_events_cluster_is_healthy
failed_when: false
changed_when: false
Expand All @@ -25,6 +27,8 @@
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
Expand Down Expand Up @@ -70,7 +74,7 @@
when: is_etcd_master and etcd_events_cluster_setup

- name: Configure | Wait for etcd cluster to be healthy
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'"
register: etcd_cluster_is_healthy
until: etcd_cluster_is_healthy.rc == 0
retries: "{{ etcd_retries }}"
Expand All @@ -85,12 +89,14 @@
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"

- name: Configure | Wait for etcd-events cluster to be healthy
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'"
register: etcd_events_cluster_is_healthy
until: etcd_events_cluster_is_healthy.rc == 0
retries: "{{ etcd_retries }}"
Expand All @@ -105,12 +111,14 @@
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"

- name: Configure | Check if member is in etcd cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
register: etcd_member_in_cluster
ignore_errors: true
changed_when: false
Expand All @@ -119,12 +127,14 @@
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"

- name: Configure | Check if member is in etcd-events cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_access_address }}"
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
register: etcd_events_member_in_cluster
ignore_errors: true
changed_when: false
Expand All @@ -133,6 +143,8 @@
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
Expand Down
8 changes: 6 additions & 2 deletions roles/etcd/tasks/join_etcd-events_member.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
---
- name: Join Member | Add member to etcd-events cluster
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
register: member_add_result
until: member_add_result.rc == 0
retries: "{{ etcd_retries }}"
delay: "{{ retry_stagger | random + 3 }}"
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why didn't you set ETCDCTL_API here ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missed it, adding that in latest commit.

ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"

Expand All @@ -22,13 +24,15 @@
{%- endfor -%}

- name: Join Member | Ensure member is in etcd-events cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_events_access_address }}"
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_events_access_address }}"
register: etcd_events_member_in_cluster
changed_when: false
check_mode: no
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same for ETCDCTL_API here

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missed it, adding that in latest commit.

ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"

Expand Down
8 changes: 6 additions & 2 deletions roles/etcd/tasks/join_etcd_member.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
---
- name: Join Member | Add member to etcd cluster
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_peer_url }}"
register: member_add_result
until: member_add_result.rc == 0
retries: "{{ etcd_retries }}"
delay: "{{ retry_stagger | random + 3 }}"
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
Expand All @@ -23,13 +25,15 @@
{%- endfor -%}

- name: Join Member | Ensure member is in etcd cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
register: etcd_member_in_cluster
changed_when: false
check_mode: no
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
Expand Down
24 changes: 18 additions & 6 deletions roles/recover_control_plane/etcd/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
---
- name: Get etcd endpoint health
shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem --endpoints={{ etcd_access_addresses }} endpoint health"
shell: "{{ bin_dir }}/etcdctl endpoint health"
register: etcd_endpoint_health
ignore_errors: true
changed_when: false
check_mode: no
environment:
- ETCDCTL_API: 3
ETCDCTL_API: 3
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
when:
- groups['broken_etcd']

Expand Down Expand Up @@ -53,21 +57,29 @@
- "item.rc != 0 and not 'No such file or directory' in item.stderr"

- name: Get etcd cluster members
shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem member list"
shell: "{{ bin_dir }}/etcdctl member list"
register: member_list
changed_when: false
check_mode: no
environment:
- ETCDCTL_API: 3
ETCDCTL_API: 3
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
when:
- groups['broken_etcd']
- not healthy
- has_quorum

- name: Remove broken cluster members
shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem --endpoints={{ etcd_access_addresses }} member remove {{ item[1].replace(' ','').split(',')[0] }}"
shell: "{{ bin_dir }}/etcdctl member remove {{ item[1].replace(' ','').split(',')[0] }}"
environment:
- ETCDCTL_API: 3
ETCDCTL_API: 3
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
with_nested:
- "{{ groups['broken_etcd'] }}"
- "{{ member_list.stdout_lines }}"
Expand Down
8 changes: 6 additions & 2 deletions roles/remove-node/remove-etcd-node/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,16 @@
- inventory_hostname in groups['etcd']

- name: Lookup etcd member id
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep {{ node_ip }} | cut -d: -f1"
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep {{ node_ip }} | cut -d: -f1"
register: etcd_member_id
ignore_errors: true
changed_when: false
check_mode: no
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
Expand All @@ -22,7 +24,7 @@
- inventory_hostname in groups['etcd']

- name: Remove etcd member from cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member remove {{ etcd_member_id.stdout }}"
shell: "{{ bin_dir }}/etcdctl --no-sync member remove {{ etcd_member_id.stdout }}"
register: etcd_member_in_cluster
ignore_errors: false
retries: 6
Expand All @@ -33,6 +35,8 @@
tags:
- facts
environment:
ETCDCTL_API: 2
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem"
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
Expand Down