Skip to content

Commit

Permalink
refactor(upgrade): Modified the cspc pool and csi volume upgrade test…
Browse files Browse the repository at this point in the history 
… case (#513)

* refactor(upgrade): Modified the cspc pool and csi volume upgrade test case

Signed-off-by: nsathyaseelan <sathyaseelan.n@mayadata.io>

* Modified the task name

Signed-off-by: nsathyaseelan <sathyaseelan.n@mayadata.io>
  • Loading branch information
@nsathyaseelan
nsathyaseelan authored Sep 7, 2020
1 parent 29b4940 commit 0438012
Show file tree
Hide file tree
Showing 4 changed files with 105 additions and 104 deletions.
5 changes: 2 additions & 3 deletions providers/openebs/cspc_csi_upgrade/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,11 @@

| Parameters | Description |
| ----------------------- | ---------------------------------------------------------- |
| FROM_VERSION | Old version which needs to be upgraded |
| FROM_VERSION | Old version which needs to be upgraded |
| TO_VERSION | New version to upgrade |
| OPERATOR_NS | Namespace where the openebs is deployed |
| CSI_NAMESPACE | Namespace where OpenEBS csi operator is deployed |
| UPGRADE_IMAGE_TAG | Image tag for upgrade job |
| UPGRADE_TO_CI | In case of upgrade to ci set the image tag to `ci` |
| NODE_OS | Supported operating system for worker nodes in case of upgrading csi provisioner |
| CSPC_POOL_UPGRADE | Set the value as `true` to Upgrade cspc pool |
| CSPC_VOLUME_UPGRADE | Set the value as `true` to Upgrade csi volumes |
| CSTOR_OPERATOR_UPGRADE | Set the value as `true` to upgrade the cstor opertor and csi provisioner |
14 changes: 5 additions & 9 deletions providers/openebs/cspc_csi_upgrade/run_litmus_test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,15 @@ spec:

# Namespace where OpenEBS is deployed
- name: OPERATOR_NS
value: ""
value: "openebs"

# Namespace where OpenEBS csi operator is deployed
- name: CSI_NAMESPACE
value: "openebs"

# Image tag for Upgrade Job
- name: UPGRADE_IMAGE_TAG
value: ""

# UPGRADE TO CI image tag then the value should be v1.x.x-ci
- name: UPGRADE_TO_CI
value: ""

# SUpported os for worker nodes ubuntu-16.04, ubuntu-18-04, centos
- name: NODE_OS
value: ""

# set to "true" if you want to upgrade cStor CSPC pool
- name: CSPC_POOL_UPGRADE
Expand Down
183 changes: 94 additions & 89 deletions providers/openebs/cspc_csi_upgrade/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,14 @@
register: cspc_version
failed_when: "cspc_version.stdout != from_version"

- name: Record the namespace for csi plugin pods
set_fact:
csi_ns: kube-system
when: from_version == '1.10.0' or from_version == '1.11.0' or from_version == '1.12.0'

- name: Check if the csi daemonset is in expected version
shell: >
kubectl get daemonset -n kube-system
kubectl get daemonset -n {{ csi_ns }}
-o jsonpath='{.items[?(@.metadata.labels.app=="openebs-cstor-csi-node")].metadata.labels.openebs\.io\/version}'
args:
executable: /bin/bash
Expand All @@ -48,12 +53,12 @@

- name: Check if the csi controller is in expected version
shell: >
kubectl get sts -n kube-system
kubectl get sts -n {{ csi_ns }}
-o jsonpath='{.items[?(@.metadata.labels.name=="openebs-cstor-csi-controller")].metadata.labels.openebs\.io\/version}'
args:
executable: /bin/bash
register: csi_sts_version
failed_when: "csi_sts_version.stdout != from_version"
failed_when: "csi_sts_version.stdout != from_version"

- name: Downloading cstor Operator
get_url:
Expand Down Expand Up @@ -90,16 +95,34 @@
regexp: ':ci'
replace: ":{{ to_version }}"

- name: Applying rbac rule
shell: "kubectl apply -f https://raw.githubusercontent.com/openebs/cstor-operators/master/deploy/rbac.yaml"
args:
executable: /bin/bash

- name: Applying all crds
shell: "kubectl apply -f https://raw.githubusercontent.com/openebs/cstor-operators/master/deploy/crds/all_cstor_crds.yaml"
args:
executable: /bin/bash

when: "'OK' not in operator_file.msg"

- name: Enable auto-remount feature to recover volumes from readonly state
replace:
path: "{{ cstor_operator }}"
regexp: 'value: "false"'
after: '- name: REMOUNT'
replace: 'value: "true"'
when: "'OK' in operator_file.msg"

- name: Applying cspc operator
shell: "kubectl apply -f {{ cstor_operator }}"
args:
executable: /bin/bash

- name: Checking OpenEBS-CVC-Operator is running
shell: >
kubectl get pods -n {{ operator_ns }}
kubectl get pods -n {{ operator_ns }}
-o jsonpath='{.items[?(@.metadata.labels.name=="cvc-operator")].status.phase}'
register: cvc_status
until: "'Running' in cvc_status.stdout"
Expand All @@ -115,76 +138,56 @@
delay: 5
retries: 120

- block:
- name: Downloading the csi-operator file
get_url:
url: "https://raw.githubusercontent.com/openebs/cstor-csi/master/deploy/csi-operator.yaml"
dest: "{{ playbook_dir }}/csi-operator.yaml"
force: yes
register: status
until: "'OK' in status.msg"
delay: 5
retries: 3

- name: Update the desired image tag for openebs-plugin
replace:
path: "{{ playbook_dir }}/csi-operator.yaml"
regexp: "cstor-csi-driver:ci"
replace: "cstor-csi-driver:{{ to_version }}"

- name: Update the label tag for openebs-plugin
replace:
path: "{{ playbook_dir }}/csi-operator.yaml"
regexp: "openebs.io/version: dev"
replace: "openebs.io/version: {{ to_version }}"

- name: Deploy CSI Driver if OS is either centos or ubuntu-16.04
shell: >
kubectl apply -f csi-operator.yaml
args:
executable: /bin/bash
register: deploy_status
failed_when: "deploy_status.rc != 0"

when: node_os == "ubuntu-16.04" or node_os == "centos"
- name: Checking OpenEBS admission server is running
shell: >
kubectl get pods -n {{ operator_ns }}
-o jsonpath='{.items[?(@.metadata.labels.app=="cstor-admission-webhook")].status.phase}'
register: admission_status
until: "'Running' in admission_status.stdout"
delay: 5
retries: 120

- block:

- name: Downloading the csi-operator file
get_url:
url: "https://raw.githubusercontent.com/openebs/cstor-csi/master/deploy/csi-operator-ubuntu-18.04.yaml"
dest: "{{ playbook_dir }}/csi-operator-ubuntu-18.04.yaml"
force: yes
register: status
until: "'OK' in status.msg"
delay: 5
retries: 3

- name: Update the desired image tag for openebs-plugin
replace:
path: "{{ playbook_dir }}/csi-operator-ubuntu-18.04.yaml"
regexp: "cstor-csi-driver:ci"
replace: "cstor-csi-driver:{{ to_version }}"

- name: Update the label tag for openebs-plugin
replace:
path: "{{ playbook_dir }}/csi-operator-ubuntu-18.04.yaml"
regexp: "openebs.io/version: dev"
replace: "openebs.io/version: {{ to_version }}"

- name: Deploy CSI Driver if OS is either centos or ubuntu-18.04
shell: >
kubectl apply -f csi-operator-ubuntu-18.04.yaml
args:
executable: /bin/bash
register: deploy_status
failed_when: "deploy_status.rc != 0"

when: node_os == "ubuntu-18.04"
- name: Downloading the csi-operator file
get_url:
url: "{{ csi_operator_link }}"
dest: "{{ playbook_dir }}/{{ csi_operator }}"
force: yes
register: status
until: "'OK' in status.msg"
delay: 5
retries: 3

- name: Update the desired image tag for openebs-plugin
replace:
path: "{{ playbook_dir }}/{{ csi_operator }}"
regexp: "cstor-csi-driver:ci"
replace: "cstor-csi-driver:{{ to_version }}"

- name: Update the label tag for openebs-plugin
replace:
path: "{{ playbook_dir }}/{{ csi_operator }}"
regexp: "openebs.io/version: dev"
replace: "openebs.io/version: {{ to_version }}"

- name: Enable auto-remount feature to recover volumes from readonly state
replace:
path: "{{ playbook_dir }}/{{ csi_operator }}"
regexp: 'value: "false"'
after: '- name: REMOUNT'
replace: 'value: "true"'

- name: Applying cspc operator
shell: "kubectl apply -f {{ playbook_dir }}/{{ csi_operator }}"
args:
executable: /bin/bash

when: "'OK' not in operator_file.msg"

- name: check if csi-controller pod is running
shell: >
kubectl get pods -n kube-system -l app=openebs-cstor-csi-controller
kubectl get pods -n {{ csi_ns }} -l app=openebs-cstor-csi-controller
--no-headers -o custom-columns=:status.phase
args:
executable: /bin/bash
Expand All @@ -195,15 +198,15 @@

- name: Obtain the desired number of openebs-csi-node pods
shell: >
kubectl get ds -n kube-system openebs-cstor-csi-node --no-headers
kubectl get ds -n {{ csi_ns }} openebs-cstor-csi-node --no-headers
-o custom-columns=:status.desiredNumberScheduled
args:
executable: /bin/bash
register: desired_count

- name: Check if the desired count matches the ready pods
command: >
kubectl get ds -n kube-system openebs-cstor-csi-node --no-headers
kubectl get ds -n {{ csi_ns }} openebs-cstor-csi-node --no-headers
-o custom-columns=:status.numberReady
args:
executable: /bin/bash
Expand All @@ -212,6 +215,22 @@
delay: 5
retries: 50

- block:

- name: Remove the csi controler in kube-system namespace
shell: kubectl delete statefulset -n kube-system openebs-cstor-csi-controller
args:
executable: /bin/bash
register: csi_ctrl

- name: Remove the csi cstor node daemonset in kube-system namespace
shell: kubectl delete daemonset -n kube-system openebs-cstor-csi-node
args:
executable: /bin/bash
register: csi_daemonset

when: from_version == '1.10.0' or from_version == '1.11.0' or from_version == '1.12.0'

when: cstor_operator_upgrade == "true"

- name: Check if the cStor cvc operator is updated to new version
Expand All @@ -234,7 +253,7 @@

- name: Check if the csi daemonset is updated to new version
shell: >
kubectl get daemonset -n kube-system
kubectl get daemonset -n {{ operator_ns }}
-o jsonpath='{.items[?(@.metadata.labels.app=="openebs-cstor-csi-node")].metadata.labels.openebs\.io\/version}'
args:
executable: /bin/bash
Expand All @@ -243,12 +262,12 @@

- name: Check if the csi controller is updated to new version
shell: >
kubectl get sts -n kube-system
kubectl get sts -n {{ operator_ns }}
-o jsonpath='{.items[?(@.metadata.labels.name=="openebs-cstor-csi-controller")].metadata.labels.openebs\.io\/version}'
args:
executable: /bin/bash
register: new_csi_sts_version
failed_when: "new_csi_sts_version.stdout != to_version"
failed_when: "new_csi_sts_version.stdout != to_version"

- name: Obtain the service account name
shell: kubectl get deploy -n {{ operator_ns }} -l name=maya-apiserver -o jsonpath="{.items[*].spec.template.spec.serviceAccount}"
Expand All @@ -275,7 +294,7 @@

- name: Check the cspi are in ONLINE state before upgrade the pool
shell: >
kubectl get cspi -n openebs -l openebs.io/cstor-pool-cluster={{ item }} -o custom-columns=:.status.phase --no-headers
kubectl get cspi -n {{ operator_ns }} -l openebs.io/cstor-pool-cluster={{ item }} -o custom-columns=:.status.phase --no-headers
register: cspi_status
until: "((cspi_status.stdout_lines|unique)|length) == 1 and 'ONLINE' in cspi_status.stdout"
retries: 30
Expand All @@ -285,7 +304,7 @@

- name: Check the cspi is not in readOnly state before upgrade the pool
shell: >
kubectl get cspi -n openebs -l openebs.io/cstor-pool-cluster={{ item }} -o custom-columns=:.status.readOnly --no-headers
kubectl get cspi -n {{ operator_ns }} -l openebs.io/cstor-pool-cluster={{ item }} -o custom-columns=:.status.readOnly --no-headers
register: cspi_ro_status
until: "((cspi_ro_status.stdout_lines|unique)|length) == 1 and 'false' in cspi_ro_status.stdout"
retries: 30
Expand All @@ -312,13 +331,6 @@
with_items:
- "{{ cspc_name.stdout_lines }}"

- name: Adding image tag to upgrade the pools into CSPC job spec
lineinfile:
path: ./cstor-cspc-upgrade-job.yml
insertbefore: '"--v=4"'
line: ' - "--to-version-image-tag={{ upgrade_to_ci }}"'
when: upgrade_to_ci != ""

- name: Display cspc.yml for verification
debug: var=item
with_file:
Expand Down Expand Up @@ -424,13 +436,6 @@
with_items:
- "{{ pv_name.stdout_lines }}"

- name: Adding upgrade ci image tag in upgrade cstor volume job spec
lineinfile:
path: ./cstor-csi-volume-upgrade-job.yml
insertbefore: '"--v=4"'
line: ' - "--to-version-image-tag={{ upgrade_to_ci }}"'
when: upgrade_to_ci != ""

- name: Create the job to upgrade the cstor volume
shell: kubectl apply -f cstor-csi-volume-upgrade-job.yml
args:
Expand Down
7 changes: 4 additions & 3 deletions providers/openebs/cspc_csi_upgrade/test_vars.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ test_name: cstor-cspc-csi-upgrade
from_version: "{{ lookup('env','FROM_VERSION') }}"
to_version: "{{ lookup('env','TO_VERSION') }}"
upgrade_image_tag: "{{ lookup('env', 'UPGRADE_IMAGE_TAG') }}"
upgrade_to_ci: "{{ lookup('env', 'UPGRADE_TO_CI') }}"
cstor_operator: cstor-operator.yaml
csi_operator: csi-operator.yaml
cstor_operator_link: "https://raw.githubusercontent.com/openebs/cstor-operators/master/deploy/cstor-operator.yaml"
new_cstor_operator: "https://raw.githubusercontent.com/openebs/charts/gh-pages/{{ lookup('env','TO_VERSION') }}/cstor-operator-{{ lookup('env','TO_VERSION') }}.yaml"
csi_operator_link: "https://raw.githubusercontent.com/openebs/cstor-operators/master/deploy/csi-operator.yaml"
new_cstor_operator: "https://raw.githubusercontent.com/openebs/charts/gh-pages/{{ lookup('env','TO_VERSION') }}/cstor-operator.yaml"
cspc_pool_upgrade: "{{ lookup('env', 'CSPC_POOL_UPGRADE') }}"
csi_volume_upgrade: "{{ lookup('env', 'CSI_VOLUME_UPGRADE') }}"
cstor_operator_upgrade: "{{ lookup('env', 'CSTOR_OPERATOR_UPGRADE') }}"
node_os: "{{ lookup('env','NODE_OS') }}"
csi_ns: "{{ lookup('env', 'CSI_NAMESPACE') }}"

0 comments on commit 0438012

Please sign in to comment.