Skip to content

Commit

Permalink
Fixup recover control plane playbook
Browse files Browse the repository at this point in the history
  • Loading branch information
floryut committed Sep 4, 2023
1 parent 3ca8e6b commit a4bc3ee
Show file tree
Hide file tree
Showing 4 changed files with 11 additions and 11 deletions.
2 changes: 1 addition & 1 deletion .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ variables:
MITOGEN_ENABLE: "false"
ANSIBLE_LOG_LEVEL: "-vv"
RECOVER_CONTROL_PLANE_TEST: "false"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
TERRAFORM_VERSION: 1.3.7
PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"

Expand Down
4 changes: 2 additions & 2 deletions .gitlab-ci/packet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -316,12 +316,12 @@ packet_ubuntu20-calico-ha-recover:
when: on_success
variables:
RECOVER_CONTROL_PLANE_TEST: "true"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"

packet_ubuntu20-calico-ha-recover-noquorum:
stage: deploy-part3
extends: .packet_periodic
when: on_success
variables:
RECOVER_CONTROL_PLANE_TEST: "true"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:],kube_control_plane[1:]"
RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
14 changes: 7 additions & 7 deletions roles/recover_control_plane/etcd/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
when:
- inventory_hostname in groups['broken_etcd']
- groups['broken_etcd']

- name: Set healthy fact
set_fact:
healthy: "{{ etcd_endpoint_health.stderr is match('Error: unhealthy cluster') }}"
when:
- inventory_hostname in groups['broken_etcd']
- groups['broken_etcd']

- name: Set has_quorum fact
set_fact:
has_quorum: "{{ etcd_endpoint_health.stdout_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}"
when:
- inventory_hostname in groups['broken_etcd']
- groups['broken_etcd']

- name: Recover lost etcd quorum
include_tasks: recover_lost_quorum.yml
Expand All @@ -40,7 +40,7 @@
with_items: "{{ groups['broken_etcd'] }}"
ignore_errors: true # noqa ignore-errors
when:
- inventory_hostname in groups['broken_etcd']
- groups['broken_etcd']
- has_quorum

- name: Delete old certificates
Expand All @@ -56,7 +56,7 @@
loop: "{{ delete_old_cerificates.results }}"
changed_when: false
when:
- inventory_hostname in groups['broken_etcd']
- groups['broken_etcd']
- "item.rc != 0 and not 'No such file or directory' in item.stderr"

- name: Get etcd cluster members
Expand All @@ -71,7 +71,7 @@
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
when:
- inventory_hostname in groups['broken_etcd']
- groups['broken_etcd']
- not healthy
- has_quorum

Expand All @@ -87,7 +87,7 @@
- "{{ groups['broken_etcd'] }}"
- "{{ member_list.stdout_lines }}"
when:
- inventory_hostname in groups['broken_etcd']
- groups['broken_etcd']
- not healthy
- has_quorum
- hostvars[item[0]]['etcd_member_name'] == item[1].replace(' ', '').split(',')[2]
2 changes: 1 addition & 1 deletion tests/scripts/testcases_run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ fi
# Test control plane recovery
if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then
ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube_control_plane:!fake_hosts recover-control-plane.yml
ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit "etcd:kube_control_plane:!fake_hosts" recover-control-plane.yml
fi

# Test collection build and install by installing our collection, emptying our repository, adding
Expand Down

0 comments on commit a4bc3ee

Please sign in to comment.