Enable agent multi-arch (openshift#1518)

01_install_requirements.sh

@@ -62,10 +62,10 @@ case $DISTRO in
     elif [[ $DISTRO == "rhel8" ]]; then
       # Enable EPEL for python3-passlib and python3-bcrypt required by metal3-dev-env
       sudo dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-      if sudo subscription-manager repos --list-enabled 2>&1 | grep "ansible-2-for-rhel-8-x86_64-rpms"; then
+      if sudo subscription-manager repos --list-enabled 2>&1 | grep "ansible-2-for-rhel-8-$(uname -m)-rpms"; then
         # The packaged 2.x ansible is too old for compatibility with metal3-dev-env
         sudo dnf erase -y ansible
-        sudo subscription-manager repos --disable=ansible-2-for-rhel-8-x86_64-rpms
+        sudo subscription-manager repos --disable=ansible-2-for-rhel-8-$(uname -m)-rpms
       fi
     fi
     # Note recent ansible needs python >= 3.8 so we install 3.9 here

04_setup_ironic.sh

@@ -122,7 +122,7 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then
         _tmpfiles="$_tmpfiles $OLM_DIR"
 
         pushd $OLM_DIR
-        curl -O https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${VERSION}.0/opm-linux.tar.gz
+        curl -O https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/ocp/${VERSION}.0/opm-linux.tar.gz
         tar xf opm-linux.tar.gz
         sudo mv -f opm /usr/local/bin
         popd

README.md

@@ -73,6 +73,46 @@ There are variable defaults set in both the `common.sh` and the `ocp_install_env
 scripts, which may be important to override for your particular environment. You can
 set override values in your `config_$USER.sh` script.
 
+### Configuration for 64-bit ARM 
+64-bit ARM systems are supported for agent and BMIPI on 64-bit ARM hosts, emulating ARM on x86 (or vice versa) is not supported. You will also need to explictly set `OPENSHIFT_RELEASE_IMAGE` with a 64-bit ARM payload from https://arm64.ocp.releases.ci.openshift.org/ 
+
+BMIPI cannot ipxe using ipv6, and the upstream vbmc, sushy-tools and ironic  containers are only build for 64-bit x86. You must build your own vbmc, sushy-tools and then extract ironic from the release payload:
+
+Build a 64-bit ARM version of vbmc and sushy-tools (must be executed on a 64-bit ARM system):
+
+```
+podman build -f https://raw.githubusercontent.com/metal3-io/ironic-image/main/resources/vbmc/Dockerfile . -t <yourbuild>/vbmc:arm
+
+podman build -f https://raw.githubusercontent.com/metal3-io/ironic-image/main/resources/sushy-tools/Dockerfile . -t <yourbuild>/sushy-tools:arm
+```
+
+Override the sushy-tools and vbmc images in your `config_$USER.sh`
+
+```
+export SUSHY_TOOLS_IMAGE=<yourbuild>/sushy-tools:arm
+
+export VBMC_IMAGE=<yourbuild>/vbmc:arm
+```
+
+Lastly, you'll need the ironic container from the OCP release payload:
+
+```
+oc adm release info quay.io/openshift-release-dev/ocp-release:4.13.0-ec.4-aarch64 --image-for ironic
+
+quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cbddb6cdfa138bce5e8cf3ed59287024cf39ec0822b1b84dd125b96d8b871b20
+```
+
+Pull the Container
+
+```
+podman pull quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cbddb6cdfa138bce5e8cf3ed59287024cf39ec0822b1b84dd125b96d8b871b20 --authfile=pullsecret.json 
+```
+
+Override the ironic image in your `config_$USER.sh`
+```
+export IRONIC_IMAGE=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cbddb6cdfa138bce5e8cf3ed59287024cf39ec0822b1b84dd125b96d8b871b20
+```
+
 ### Baremetal
 
 For baremetal test setups where you don't require the VM fake-baremetal nodes,

agent/roles/manifests/tasks/main.yml

@@ -3,6 +3,10 @@
     ca_bundle_crt: "{{ lookup('file', mirror_path + '/ca-bundle.crt') | to_json }}"
   when: mirror_images
 
+- name: Set GoCPUArchitecture
+  set_fact:
+    goCPUArchitecture: "{{ lookup('env', 'ARCH') }}"
+
 - name: Get VIPs when not using SNO
   set_fact:
     ingress_vips: "{{ lookup('env', 'INGRESS_VIPS') }}"

agent/roles/manifests/templates/infraenv_yaml.j2

@@ -7,6 +7,7 @@ spec:
   clusterRef:
     name: {{ cluster_name }} 
     namespace: {{ cluster_namespace }}
+  cpuArchitecture: {{ ansible_architecture }}
   pullSecretRef:
     name: pull-secret
   sshAuthorizedKey: {{ ssh_pub_key }}

agent/roles/manifests/templates/install-config_yaml.j2

@@ -4,8 +4,9 @@ compute:
 - hyperthreading: Enabled
   name: worker
   replicas: {{ num_workers }} 
+  architecture: {{ goCPUArchitecture }}
 controlPlane:
-  architecture: amd64
+  architecture: {{ goCPUArchitecture }}
   hyperthreading: Enabled
   name: master
   replicas: {{ num_masters }} 

common.sh

@@ -473,7 +473,7 @@ if [[ -n "$MIRROR_IMAGES" || -z "${IP_STACK:-}" || "$IP_STACK" = "v6" || "$IP_ST
       pushd ${WORKING_DIR}
       release_candidate=`oc-mirror list releases --channel=candidate-${OPENSHIFT_RELEASE_STREAM} | tail -1`
       popd
-      export OPENSHIFT_RELEASE_TAG="${release_candidate}-x86_64"
+      export OPENSHIFT_RELEASE_TAG="${release_candidate}-$(uname -m)"
    fi
 
    # We're going to be using a locally modified release image

config_example.sh

@@ -32,6 +32,7 @@ set -x
 # Select a different release stream from which to pull the latest image, if the
 # image name is not specified.
 # Default: 4.14
+# NOTE: Do not use for arm64, instead override OPENSHIFT_RELEASE_IMAGE  
 #
 #export OPENSHIFT_RELEASE_STREAM=4.14
 
@@ -40,11 +41,13 @@ set -x
 # If using ga then set OPENSHIFT_VERSION to the required version.
 # Default: nightly
 # Choices: ci, nightly, ga
+# NOTE: Do not use for arm64, instead override OPENSHIFT_RELEASE_IMAGE  
 #
 #export OPENSHIFT_RELEASE_TYPE=nightly
 
 # OPENSHIFT_VERSION -
 # Set the Openshift version. If unset defaults to $OPENSHIFT_RELEASE_STREAM.
+# NOTE: Do not use for arm64, instead override OPENSHIFT_RELEASE_IMAGE  
 #
 #export OPENSHIFT_VERSION=4.12
 
@@ -195,6 +198,7 @@ set -x
 # IP_STACK -
 # IP stack for the cluster. "v4v6" is ipv4-primary dual stack, "v6v4" is
 # ipv6-primary dual stack.
+# NOTE: arm64 does not support v6
 # Default: "v6"
 # Choices: "v4", "v6", "v4v6", "v6v4"
 #
@@ -203,6 +207,7 @@ set -x
 # HOST_IP_STACK -
 # IP stack for the host.
 # Default: If unset, defaults to ${IP_STACK}.
+# NOTE: arm64 does not support v6
 # Choices: "v4", "v6", "v4v6"
 #
 #export HOST_IP_STACK=v4v6