From 45a6f03cf87ad1fc9df16103996c39912dfa08ca Mon Sep 17 00:00:00 2001
From: Richard Lau <rlau@redhat.com>
Date: Fri, 28 Jan 2022 13:23:32 +0000
Subject: [PATCH] ansible: add RHEL 8

Extend Ansible and Jenkins scripts for Red Hat Enterprise Linux 8.
Also add new `release-builder` role, for setting up ssh config and
keys to upload to the staging server, and changes to make the
playbook idempotent.
---
 ansible/README.md                             |  6 +--
 ansible/inventory.yml                         |  5 +++
 ansible/playbooks/jenkins/worker/create.yml   |  2 +
 ansible/roles/baselayout/tasks/main.yml       | 10 +++++
 .../baselayout/tasks/partials/repo/rhel8.yml  | 14 +++++++
 ansible/roles/baselayout/vars/main.yml        |  8 ++++
 ansible/roles/bootstrap/handlers/main.yml     |  4 ++
 ansible/roles/bootstrap/tasks/main.yml        |  1 +
 .../bootstrap/tasks/partials/rhel8-s390x.yml  | 42 +++++++++++++++++++
 .../roles/bootstrap/tasks/partials/rhel8.yml  |  9 ++++
 ansible/roles/gn/tasks/main.yml               |  3 +-
 ansible/roles/gn/vars/main.yml                | 12 ++++++
 ansible/roles/java-base/tasks/main.yml        |  1 +
 ansible/roles/java-base/vars/main.yml         |  1 +
 .../roles/jenkins-worker/handlers/main.yml    | 15 +++++++
 ansible/roles/jenkins-worker/tasks/main.yml   |  6 ++-
 ansible/roles/jenkins-worker/vars/main.yml    |  2 +-
 ansible/roles/package-upgrade/vars/main.yml   |  2 +-
 .../roles/release-builder/files/ssh_config    |  4 ++
 .../release-builder/meta/argument_specs.yml   |  5 +++
 ansible/roles/release-builder/meta/main.yml   |  4 ++
 ansible/roles/release-builder/tasks/main.yml  | 42 +++++++++++++++++++
 ansible/roles/release-builder/vars/main.yml   |  5 +++
 jenkins/scripts/VersionSelectorScript.groovy  |  3 ++
 jenkins/scripts/select-compiler.sh            | 23 ++++++++++
 25 files changed, 222 insertions(+), 7 deletions(-)
 create mode 100644 ansible/roles/baselayout/tasks/partials/repo/rhel8.yml
 create mode 100644 ansible/roles/bootstrap/handlers/main.yml
 create mode 100644 ansible/roles/bootstrap/tasks/partials/rhel8-s390x.yml
 create mode 100644 ansible/roles/bootstrap/tasks/partials/rhel8.yml
 create mode 100644 ansible/roles/gn/vars/main.yml
 create mode 100644 ansible/roles/jenkins-worker/handlers/main.yml
 create mode 100644 ansible/roles/release-builder/files/ssh_config
 create mode 100644 ansible/roles/release-builder/meta/argument_specs.yml
 create mode 100644 ansible/roles/release-builder/meta/main.yml
 create mode 100644 ansible/roles/release-builder/tasks/main.yml
 create mode 100644 ansible/roles/release-builder/vars/main.yml

diff --git a/ansible/README.md b/ansible/README.md
index bfc98133e..e405d586d 100644
--- a/ansible/README.md
+++ b/ansible/README.md
@@ -234,10 +234,10 @@ debugging problems, see the
 
 Unsorted stuff of things we need to do/think about
 
-- [ ] playbook: copy keys and config to release machines
+- [x] playbook: copy keys and config to release machines
 - [ ] avoid messing with keys on machines that has multiple usage such as jump
       hosts (or set up a new jump host)
-- [ ] copy release (staging) keys to release machines
+- [x] copy release (staging) keys to release machines
 - [ ] backup host: generate config, install rsnapshot
 - [ ] switch to slaveLog for all jenkins instances lacking stdout redirection
       (note: this depends on init type!)
@@ -262,7 +262,7 @@ Unsorted stuff of things we need to do/think about
       setup/raspberry-pi/README.md, some of these can be automated)
 - [ ] epel-release for centos - required for centos7 on packet.net arm64
       before ccache can be installed
-- [ ] make .ssh/config and .ssh/id_rsa for release machines, adding config
+- [x] make .ssh/config and .ssh/id_rsa for release machines, adding config
       for `node-www` and record host key for node-www
 - [ ] add explicit ARCH and DESTCPU for release machines (RV: I'm adding
       "arm64" manually for both to force the right thing, from memory I've
diff --git a/ansible/inventory.yml b/ansible/inventory.yml
index 3247e5247..1a8af4d80 100644
--- a/ansible/inventory.yml
+++ b/ansible/inventory.yml
@@ -46,6 +46,7 @@ hosts:
             ip: 169.48.19.173
             server_jobs: 6
         rhel7-s390x-1: {ip: 148.100.86.101, user: linux1}
+        rhel8-s390x-1: {ip: 148.100.84.27, user: linux1}
 
     - iinthecloud:
         ibmi73-ppc64_be-1: {ip: 65.183.160.62, user: nodejs}
@@ -152,6 +153,10 @@ hosts:
         rhel7-s390x-2: {ip: 148.100.86.117, user: linux1, build_test_v8: yes}
         rhel7-s390x-3: {ip: 148.100.86.28, user: linux1, build_test_v8: yes}
         rhel7-s390x-4: {ip: 148.100.86.94, user: linux1, build_test_v8: yes}
+        rhel8-s390x-1: {ip: 148.100.84.112, user: linux1, build_test_v8: yes}
+        rhel8-s390x-2: {ip: 148.100.84.240, user: linux1, build_test_v8: yes}
+        rhel8-s390x-3: {ip: 148.100.84.56, user: linux1, build_test_v8: yes}
+        rhel8-s390x-4: {ip: 148.100.84.150, user: linux1, build_test_v8: yes}
         ubuntu1804-x64-1: {ip: 52.117.26.14, alias: jenkins-workspace-6}
         ubuntu1804-x64-2: {ip: 50.97.245.9}
 
diff --git a/ansible/playbooks/jenkins/worker/create.yml b/ansible/playbooks/jenkins/worker/create.yml
index 17c057be1..5310fb96c 100644
--- a/ansible/playbooks/jenkins/worker/create.yml
+++ b/ansible/playbooks/jenkins/worker/create.yml
@@ -16,6 +16,8 @@
     - { role: 'benchmarking',
         when: is_benchmark is defined and is_benchmark|bool == True }
     - jenkins-worker
+    - { role: release-builder,
+        when: '"release" in group_names' }
 
   pre_tasks:
 # Requires `secret: XXX` to be in the ansible/host_vars/HOST
diff --git a/ansible/roles/baselayout/tasks/main.yml b/ansible/roles/baselayout/tasks/main.yml
index f7cb77c01..7e74b2d55 100644
--- a/ansible/roles/baselayout/tasks/main.yml
+++ b/ansible/roles/baselayout/tasks/main.yml
@@ -141,6 +141,16 @@
     state: link
     src: "/usr/local/bin/python2"
 
+# Required for V8 builds
+- name: rhel8 | update python package alternatives
+  community.general.alternatives:
+    link: /usr/bin/python
+    name: python
+    path: /usr/bin/python2
+  when:
+    - os == "rhel8"
+    - build_test_v8|default(False)
+
 - name: smartos17 | update gcc symlinks
   when: os == "smartos17"
   file:
diff --git a/ansible/roles/baselayout/tasks/partials/repo/rhel8.yml b/ansible/roles/baselayout/tasks/partials/repo/rhel8.yml
new file mode 100644
index 000000000..094696678
--- /dev/null
+++ b/ansible/roles/baselayout/tasks/partials/repo/rhel8.yml
@@ -0,0 +1,14 @@
+---
+
+# Red Hat Enterprise Linux 8
+
+- name: install GPG key for EPEL 8
+  become: yes
+  ansible.builtin.rpm_key:
+    key: https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8
+    state: present
+
+- name: install EPEL 8
+  ansible.builtin.dnf:
+    name: https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+    state: present
diff --git a/ansible/roles/baselayout/vars/main.yml b/ansible/roles/baselayout/vars/main.yml
index fb7ef11e2..9ba2cfd12 100644
--- a/ansible/roles/baselayout/vars/main.yml
+++ b/ansible/roles/baselayout/vars/main.yml
@@ -144,6 +144,14 @@ packages: {
     'gcc-c++,sudo,git,zip,unzip,iptables-services,GConf2-devel,openssl-devel,python3',
   ],
 
+  rhel8_s390x: [
+    'GConf2-devel,python2' # Needed for V8 builds
+  ],
+
+  rhel8: [
+    'ccache,cmake,gcc-c++,gcc-toolset-11,git,make,python3',
+  ],
+
   smartos: [
     'gccmakedep',
     'git',
diff --git a/ansible/roles/bootstrap/handlers/main.yml b/ansible/roles/bootstrap/handlers/main.yml
new file mode 100644
index 000000000..1aa2ad684
--- /dev/null
+++ b/ansible/roles/bootstrap/handlers/main.yml
@@ -0,0 +1,4 @@
+- name: restart iptables
+  ansible.builtin.service:
+    name: iptables
+    state: restarted
diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml
index c10e878a5..f457a43f4 100644
--- a/ansible/roles/bootstrap/tasks/main.yml
+++ b/ansible/roles/bootstrap/tasks/main.yml
@@ -10,6 +10,7 @@
       loop_var: bootstrap_include
     with_first_found:
       - files:
+        - "{{ role_path }}/tasks/partials/{{ os }}-{{ arch }}.yml"
         - "{{ role_path }}/tasks/partials/{{ os }}.yml"
         - "{{ role_path }}/tasks/partials/{{ os|stripversion }}.yml"
         skip: true
diff --git a/ansible/roles/bootstrap/tasks/partials/rhel8-s390x.yml b/ansible/roles/bootstrap/tasks/partials/rhel8-s390x.yml
new file mode 100644
index 000000000..42fb155f6
--- /dev/null
+++ b/ansible/roles/bootstrap/tasks/partials/rhel8-s390x.yml
@@ -0,0 +1,42 @@
+---
+
+# Red Hat Enterprise Linux 8 on s390x (LinuxONE)
+
+- name: run common RHEL 8 tasks
+  ansible.builtin.include_tasks: rhel8.yml
+
+- name: Firewall | install iptables-services
+  ansible.builtin.dnf:
+    name: iptables-services
+    state: present
+
+- name: Firewall | enable iptables
+  ansible.builtin.systemd:
+    enabled: yes
+    name: iptables
+
+- name: Firewall | remove firewalld
+  ansible.builtin.dnf:
+    name: firewalld
+    state: absent
+
+- name: Firewall | add rule to allow accepting multicast
+  lineinfile:
+    dest: /etc/sysconfig/iptables
+    insertafter: ":OUTPUT ACCEPT.*]"
+    line: "-A INPUT -m pkttype --pkt-type multicast -j ACCEPT"
+  notify: restart iptables
+
+- name: Firewall | add basic rule to allow communication locally
+  lineinfile:
+    dest: /etc/sysconfig/iptables
+    insertafter: ":OUTPUT ACCEPT.*]"
+    line: "-A INPUT -s 127.0.0.1/32 -d 127.0.0.1/32 -j ACCEPT"
+  notify: restart iptables
+
+- name: Firewall | add additional rule to allow communication from 127.0.0.2
+  lineinfile:
+    dest: /etc/sysconfig/iptables
+    insertafter: ":OUTPUT ACCEPT.*]"
+    line: "-A INPUT -s 127.0.0.2/32 -d 127.0.0.1/32 -j ACCEPT"
+  notify: restart iptables
diff --git a/ansible/roles/bootstrap/tasks/partials/rhel8.yml b/ansible/roles/bootstrap/tasks/partials/rhel8.yml
new file mode 100644
index 000000000..2be829594
--- /dev/null
+++ b/ansible/roles/bootstrap/tasks/partials/rhel8.yml
@@ -0,0 +1,9 @@
+---
+
+# Red Hat Enterprise Linux 8
+
+- name: register Red Hat subscription
+  community.general.redhat_subscription:
+    activationkey: "{{ type }}"
+    org_id: "{{ rh_org }}"
+    state: present
diff --git a/ansible/roles/gn/tasks/main.yml b/ansible/roles/gn/tasks/main.yml
index d1f8337e8..875aaa964 100644
--- a/ansible/roles/gn/tasks/main.yml
+++ b/ansible/roles/gn/tasks/main.yml
@@ -8,6 +8,7 @@
   ansible.builtin.git:
     dest: "{{ gn_git_dir }}"
     repo: "https://gn.googlesource.com/gn"
+    version: "{{ gn_version|default(omit) }}"
   become: "{{ gn_user|default(omit)|bool }}"
   become_user: "{{ gn_user|default(omit) }}"
   register: gn_git
@@ -23,7 +24,7 @@
 - name: build gn
   ansible.builtin.shell: |
     python3 build/gen.py && \
-    . /opt/rh/devtoolset-8/enable && \
+    {{ gn_select_compiler }} && \
     {{ gn_dest_dir }}/ninja -C out && \
     out/gn_unittests
   args:
diff --git a/ansible/roles/gn/vars/main.yml b/ansible/roles/gn/vars/main.yml
new file mode 100644
index 000000000..909a5bd19
--- /dev/null
+++ b/ansible/roles/gn/vars/main.yml
@@ -0,0 +1,12 @@
+---
+
+compiler: {
+  'centos7': '. /opt/rh/devtoolset-8/enable',
+  'rhel7': '. /opt/rh/devtoolset-8/enable',
+  'rhel8': '. /opt/rh/gcc-toolset-11/enable'
+}
+
+gn_select_compiler: "{{ compiler[os]|default(compiler[os|stripversion])|default('true') }}"
+# Pin gn for now so we can still build older versions of V8 in Node.js 14.
+# Refs: https://github.com/nodejs/node/pull/40689#issuecomment-956303875
+gn_version: 69ec4fc
diff --git a/ansible/roles/java-base/tasks/main.yml b/ansible/roles/java-base/tasks/main.yml
index d84716440..bc1fe57c4 100644
--- a/ansible/roles/java-base/tasks/main.yml
+++ b/ansible/roles/java-base/tasks/main.yml
@@ -31,6 +31,7 @@
 # as appropriate -- try to use generic os family if available.
 
 - name: Check if java is already installed
+  changed_when: no
   check_mode: no
   raw: "{{ java_path[os] | default(\"java\") }} -version"
   register: java
diff --git a/ansible/roles/java-base/vars/main.yml b/ansible/roles/java-base/vars/main.yml
index cf29e156b..4949f8247 100644
--- a/ansible/roles/java-base/vars/main.yml
+++ b/ansible/roles/java-base/vars/main.yml
@@ -15,6 +15,7 @@ packages: {
   'ibmi': 'openjdk-11-ea',
   'macos': 'adoptopenjdk8',
   'rhel7': 'java-1.8.0-openjdk',
+  'rhel8': 'java-17-openjdk',
   'smartos': 'openjdk8',
   'ubuntu': 'openjdk-8-jre-headless',
   'ubuntu1404': 'oracle-java8-installer',
diff --git a/ansible/roles/jenkins-worker/handlers/main.yml b/ansible/roles/jenkins-worker/handlers/main.yml
new file mode 100644
index 000000000..6ffdba3ec
--- /dev/null
+++ b/ansible/roles/jenkins-worker/handlers/main.yml
@@ -0,0 +1,15 @@
+---
+
+#
+# generic handlers for jenkins-worker
+#
+
+- name: restart Jenkins
+  ansible.builtin.service:
+    name: jenkins
+    state: restarted
+  when:
+    - not os|startswith("aix")
+    - not os|startswith("ibmi")
+    - not os|startswith("macos")
+    - not os|startswith("zos")
diff --git a/ansible/roles/jenkins-worker/tasks/main.yml b/ansible/roles/jenkins-worker/tasks/main.yml
index 28ab3f188..8d575bf40 100644
--- a/ansible/roles/jenkins-worker/tasks/main.yml
+++ b/ansible/roles/jenkins-worker/tasks/main.yml
@@ -131,6 +131,7 @@
     mode: 0644
     timeout: 60
     force: yes
+  notify: restart Jenkins
 
 - name: Resolver | ipnodes needs to be present on zos, similar to hosts file
   when: os|startswith("zos")
@@ -208,6 +209,7 @@
   shell: "sh {{ home }}/{{ server_user }}/gen_runAsciiBash.sh"
 
 - name: render init script into place
+  notify: restart Jenkins
   template:
     src: "{{ jenkins.src }}"
     dest: "{{ jenkins.dest }}"
@@ -334,12 +336,14 @@
 
 
 - name: enable jenkins at startup - general
+  ansible.builtin.service:
+    name: jenkins
+    enabled: yes
   when:
     - not os|startswith("aix")
     - not os|startswith("ibmi")
     - not os|startswith("macos")
     - not os|startswith("zos")
-  service: name=jenkins state=restarted enabled=yes
 
 - name: Unload org.nodejs.osx.jenkins.plist from launchctl
   when: os|startswith("macos")
diff --git a/ansible/roles/jenkins-worker/vars/main.yml b/ansible/roles/jenkins-worker/vars/main.yml
index 9c12d88f4..b9fe350cb 100644
--- a/ansible/roles/jenkins-worker/vars/main.yml
+++ b/ansible/roles/jenkins-worker/vars/main.yml
@@ -12,7 +12,7 @@ init: {
   ibmi: 'ibmi73',
   macos: 'macos',
   rhel7: 'rhel7',
-  systemd: ['centos7', 'debian8', 'debian9', 'debian10', 'fedora', 'ubuntu1604', 'ubuntu1804'],
+  systemd: ['centos7', 'debian8', 'debian9', 'debian10', 'fedora', 'rhel8', 'ubuntu1604', 'ubuntu1804'],
   svc: 'smartos',
   upstart: ['ubuntu12', 'ubuntu1404'],
   zos_start: 'zos'
diff --git a/ansible/roles/package-upgrade/vars/main.yml b/ansible/roles/package-upgrade/vars/main.yml
index 186cfcb34..592441387 100644
--- a/ansible/roles/package-upgrade/vars/main.yml
+++ b/ansible/roles/package-upgrade/vars/main.yml
@@ -7,7 +7,7 @@
   pm: {
     'yum': ['centos', 'rhel7', 'aix', 'ibmi'],
     'apt': ['debian', 'ubuntu'],
-    'dnf': 'fedora',
+    'dnf': ['fedora', 'rhel8'],
     'pkg': 'freebsd',
     'pkgin': 'smartos',
     'chocolatey': 'win',
diff --git a/ansible/roles/release-builder/files/ssh_config b/ansible/roles/release-builder/files/ssh_config
new file mode 100644
index 000000000..2058e09d2
--- /dev/null
+++ b/ansible/roles/release-builder/files/ssh_config
@@ -0,0 +1,4 @@
+Host node-www
+  HostName direct.nodejs.org
+  User staging
+  IdentityFile ~/.ssh/id_rsa
diff --git a/ansible/roles/release-builder/meta/argument_specs.yml b/ansible/roles/release-builder/meta/argument_specs.yml
new file mode 100644
index 000000000..f371432e4
--- /dev/null
+++ b/ansible/roles/release-builder/meta/argument_specs.yml
@@ -0,0 +1,5 @@
+---
+
+argument_specs:
+  main:
+    short_description: Set up specific to hosts that build releases.
diff --git a/ansible/roles/release-builder/meta/main.yml b/ansible/roles/release-builder/meta/main.yml
new file mode 100644
index 000000000..984ee87b7
--- /dev/null
+++ b/ansible/roles/release-builder/meta/main.yml
@@ -0,0 +1,4 @@
+---
+
+dependencies:
+  - role: user-create
diff --git a/ansible/roles/release-builder/tasks/main.yml b/ansible/roles/release-builder/tasks/main.yml
new file mode 100644
index 000000000..5fd7e79b2
--- /dev/null
+++ b/ansible/roles/release-builder/tasks/main.yml
@@ -0,0 +1,42 @@
+---
+
+# Set up release hosts to be able to upload to the staging server.
+# Requires access to the secrets repository. User should have already
+# been prompted for GPG credentials during the inventory load.
+
+- name: create .ssh directory
+  ansible.builtin.file:
+    dest: "{{ home }}/{{ server_user }}/.ssh"
+    mode: "0700"
+    owner: "{{ server_user }}"
+    group: "{{ server_user }}"
+    state: directory
+
+- name: copy key to access staging server
+  ansible.builtin.copy:
+    content: "{{ lookup('pipe', 'gpg -d ' + staging_key | quote) }}"
+    dest: "{{ home }}/{{ server_user }}/.ssh/id_rsa"
+    mode: "0600"
+    owner: "{{ server_user }}"
+    group: "{{ server_user }}"
+  vars:
+    staging_key: "{{ secrets_repo_root }}/build/release/staging_id_rsa_private.key"
+
+- name: write ssh_config
+  ansible.builtin.copy:
+    dest: "{{ home }}/{{ server_user }}/.ssh/config"
+    src: "{{ role_path }}/files/ssh_config"
+    mode: "0600"
+    owner: "{{ server_user }}"
+    group: "{{ server_user }}"
+
+- name: write known_hosts
+  ansible.builtin.known_hosts:
+    name: direct.nodejs.org
+    key: "{{ lookup('pipe', 'gpg -d ' + known_hosts | quote) }}"
+    path: "{{ home }}/{{ server_user }}/.ssh/known_hosts"
+    state: present
+  become: yes
+  become_user: "{{ server_user }}"
+  vars:
+    known_hosts: "{{ secrets_repo_root }}/build/release/known_hosts"
diff --git a/ansible/roles/release-builder/vars/main.yml b/ansible/roles/release-builder/vars/main.yml
new file mode 100644
index 000000000..fc936ad72
--- /dev/null
+++ b/ansible/roles/release-builder/vars/main.yml
@@ -0,0 +1,5 @@
+---
+
+# The path to the secrets repository. 
+# See also `ansible/plugins/inventory/nodejs_yaml.py`.
+secrets_repo_root: "{{ lookup('env', 'NODE_BUILD_SECRETS')|default('../../../../../secrets', True) }}"
diff --git a/jenkins/scripts/VersionSelectorScript.groovy b/jenkins/scripts/VersionSelectorScript.groovy
index ba20598b6..cf43db153 100644
--- a/jenkins/scripts/VersionSelectorScript.groovy
+++ b/jenkins/scripts/VersionSelectorScript.groovy
@@ -30,6 +30,9 @@ def buildExclusions = [
   [ /debian8-x86/,                    anyType,     gte(10) ], // 32-bit linux for <10 only
   [ /debian8/,                        anyType,     gte(13) ],
   [ /debian9/,                        anyType,     gte(16) ],
+  [ /rhel7/,                          releaseType, gte(18) ],
+  [ /rhel8/,                          releaseType, lt(18)  ],
+  [ /rhel8/,                          anyType,     lt(14)  ],
   [ /^ubuntu1804/,                    anyType,     lt(10)  ], // probably temporary
   [ /^ubuntu1404-32/,                 anyType,     gte(10) ], // 32-bit linux for <10 only
   [ /^ubuntu1404-64/,                 anyType,     gte(12) ],
diff --git a/jenkins/scripts/select-compiler.sh b/jenkins/scripts/select-compiler.sh
index 927c15141..fa976ba8a 100644
--- a/jenkins/scripts/select-compiler.sh
+++ b/jenkins/scripts/select-compiler.sh
@@ -32,6 +32,29 @@ if [ -z ${NODEJS_MAJOR_VERSION+x} ]; then
   NODEJS_MAJOR_VERSION="$(echo "$NODE_VERSION" | cut -d . -f 1)"
 fi
 
+# Linux distros should be arch agnostic
+case $NODE_NAME in
+  *rhel8*)
+    case "$CONFIG_FLAGS" in
+      *--enable-lto*)
+        echo "Setting compiler for Node.js $NODEJS_MAJOR_VERSION (LTO) on" `cat /etc/redhat-release`
+        . /opt/rh/gcc-toolset-11/enable
+        export CC="ccache gcc"
+        export CXX="ccache g++"
+        echo "Selected compiler:" `${CXX} -dumpversion`  
+        return
+        ;;
+      *)
+        echo "Setting compiler for Node.js $NODEJS_MAJOR_VERSION on" `cat /etc/redhat-release`
+        # Default gcc on RHEL 8 is gcc 8
+        echo "Compiler left as system default:" `g++ -dumpversion`
+        return
+        ;;
+    esac
+    return
+    ;;
+esac
+
 if [ "$SELECT_ARCH" = "PPC64LE" ]; then
   # Set default
   export COMPILER_LEVEL="4.8"