Skip to content

Commit

Permalink
[DOCA]: Run tests in containers on ubuntu22.04
Browse files Browse the repository at this point in the history
Migrate test processes from static bare metal server to k8s
dynamic containers on ub22.04

Issue: HPCINFRA-2196
Signed-off-by: Daniel Pressler <danielpr@nvidia.com>
  • Loading branch information
dpressle committed Jul 7, 2024
1 parent a2d0d54 commit f52386e
Show file tree
Hide file tree
Showing 7 changed files with 226 additions and 170 deletions.
2 changes: 1 addition & 1 deletion .ci/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

// load pipeline functions
// Requires pipeline-github-lib plugin to load library from github
@Library('github.com/Mellanox/ci-demo@stable')
@Library('github.com/Mellanox/ci-demo@stable_media')
def matrix = new com.mellanox.cicd.Matrix()

matrix.main()
26 changes: 0 additions & 26 deletions .ci/dockerfiles/Dockerfile.rhel8.6

This file was deleted.

54 changes: 54 additions & 0 deletions .ci/dockerfiles/Dockerfile.ubuntu22.04
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
ARG ARCH=x86_64
FROM harbor.mellanox.com/hpcx/$ARCH/ubuntu22.04/builder:mofed-5.7-0.1.9.0 AS build
# ARG _UID=6213
# ARG _GID=101
# ARG _LOGIN=swx-jenkins
# ARG _HOME=/var/home/$_LOGIN

# RUN echo "${_LOGIN} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
# echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
# mkdir -p ${_HOME} && \
# groupadd -f -g "$_GID" "$_LOGIN" && \
# useradd -u "$_UID" -g "$_GID" -s /bin/bash -m -d ${_HOME} "${_LOGIN}" && \
# chown -R ${_LOGIN} ${_HOME} && \
# mkdir /build && chown -R ${_LOGIN} /build

# ENTRYPOINT [ "/bin/bash", "--login", "--rcfile", "/etc/bashrc", "-c" ]

FROM build AS tests
RUN apt-get update && \
apt-get install -y \
net-tools unzip iproute2 wget \
&& apt-get clean && rm -rf /var/lib/apt/lists/*

FROM tests AS vg
RUN apt-get update && \
apt-get install -y \
valgrind \
&& apt-get clean && rm -rf /var/lib/apt/lists/*

FROM tests AS test
RUN apt-get update && \
apt-get install -y \
openssh-server psmisc \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
# setup ssh server and passwordless login for root for tests flows (verifyer.pl)
RUN mkdir -p /var/run/sshd ~/.ssh && \
rm -rf ~/.ssh/id_rsa* && ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa && \
cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys && \
sed -i 's|#PermitRootLogin.*|PermitRootLogin without-password|g' /etc/ssh/sshd_config && \
sed -i 's|#PasswordAuthentication.*|PasswordAuthentication no|g' /etc/ssh/sshd_config && \
echo "Host *" >> ~/.ssh/config && \
echo " StrictHostKeyChecking no" >> ~/.ssh/config && \
echo " UserKnownHostsFile /dev/null" >> ~/.ssh/config && \
echo " LogLevel ERROR" >> ~/.ssh/config

FROM tests AS gtest

FROM build AS static
# for tidy tests
RUN pip3 install compiledb --no-cache-dir
RUN apt-get update && \
apt-get install -y \
cppcheck clang-tools clang-tidy \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
167 changes: 93 additions & 74 deletions .ci/matrix_job.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ registry_auth: 1daaea28-800e-425f-a91f-3bd3e9136eea
registry_path: /swx-infra/media

kubernetes:
privileged: false
privileged: true
cloud: swx-k8s-spray
nodeSelector: 'beta.kubernetes.io/os=linux'
namespace: xlio-ci
limits: '{memory: 8Gi, cpu: 7000m}'
requests: '{memory: 8Gi, cpu: 7000m}'
limits: '{memory: 10Gi, cpu: 10000m}'
requests: '{memory: 10Gi, cpu: 10000m}'

credentials:
- {credentialsId: '925b0900-e273-4042-bc7c-facaefae0727', usernameVariable: 'XLIO_COV_USER', passwordVariable: 'XLIO_COV_PASSWORD'}
Expand All @@ -34,46 +34,85 @@ volumes:
- {mountPath: /var/home/swx-jenkins, hostPath: /labhome/swx-jenkins}

runs_on_dockers:
# mofed
- {name: 'ub20.04-mofed-x86_64', url: 'harbor.mellanox.com/swx-infra/x86_64/ubuntu20.04/builder:mofed-5.2-2.2.0.0', category: 'base', arch: 'x86_64'}
- {name: 'ub20.04-mofed-aarch64', url: 'harbor.mellanox.com/swx-infra/aarch64/ubuntu20.04/builder:mofed-5.2-1.0.4.0', category: 'base', arch: 'aarch64'}
- {name: 'ub22.04-mofed-x86_64', url: 'harbor.mellanox.com/hpcx/x86_64/ubuntu22.04/builder:mofed-5.7-0.1.1.0', category: 'base', arch: 'x86_64'}
- {name: 'rhel8.6-mofed-x86_64', url: 'harbor.mellanox.com/hpcx/x86_64/rhel8.6/builder:mofed-5.6-0.4.5.0', category: 'base', arch: 'x86_64'}
- {name: 'ub24.04-mofed-x86_64', url: 'harbor.mellanox.com/hpcx/x86_64/ubuntu24.04/builder:mofed-24.04-0.6.6.0', category: 'base', arch: 'x86_64'}
- {name: 'ub24.04-mofed-aarch64', url: 'harbor.mellanox.com/hpcx/aarch64/ubuntu24.04/builder:mofed-24.04-0.6.6.0', category: 'base', arch: 'aarch64'}
# - {name: 'oracle8.6-mofed-x86_64', url: 'harbor.mellanox.com/rivermax/base_oraclelinux8.6:mofed-5.9-0.3.4.0', category: 'base', arch: 'x86_64'}
# build
- {
file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
arch: 'x86_64',
name: 'ubuntu22.04',
uri: 'xlio/$arch/$name/build',
tag: '20240805',
build_args: '--no-cache --target build',
category: 'base'
}
- {
file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
arch: 'aarch64',
name: 'ubuntu22.04',
uri: 'xlio/$arch/ubuntu22.04/build',
tag: '20240805',
build_args: '--target build --no-cache --build-arg ARCH=aarch64 --build-arg REPO_ARCH=arm64',
category: 'base'
}
# tool
- {name: 'toolbox', url: 'harbor.mellanox.com/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
- {name: 'blackduck', url: 'harbor.mellanox.com/toolbox/ngci-centos:7.9.2009.2', category: 'tool', arch: 'x86_64'}
- {name: 'header-check', url: 'harbor.mellanox.com/toolbox/header_check:0.0.14', category: 'tool', arch: 'x86_64', tag: '0.0.14'}
# static tests
- {file: '.ci/dockerfiles/Dockerfile.rhel8.6',
- {
file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
arch: 'x86_64',
name: 'xlio_static.cppcheck',
uri: '$arch/$name',
tag: '20240703',
build_args: '--no-cache',
name: 'static',
uri: 'xlio/$arch/ubuntu22.04/$name',
tag: '20240805',
build_args: '--no-cache --target static',
category: 'tool'
}
- {file: '.ci/dockerfiles/Dockerfile.rhel8.6',
}
# tests
- {
file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
arch: 'x86_64',
name: 'test',
uri: 'xlio/$arch/ubuntu22.04/$name',
tag: '20240805',
build_args: '--no-cache --target test',
category: 'tests',
annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p1' }],
limits: '{memory: 8Gi, cpu: 8000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p1: 1}',
requests: '{memory: 8Gi, cpu: 8000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p1: 1}',
caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
runAsUser: '0',
runAsGroup: '0'
}
- {
file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
arch: 'x86_64',
name: 'xlio_static.csbuild',
uri: '$arch/$name',
tag: '20240703',
build_args: '--no-cache',
category: 'tool'
}
- {file: '.ci/dockerfiles/Dockerfile.rhel8.6',
name: 'vg',
uri: 'xlio/$arch/ubuntu22.04/$name',
tag: '20240805',
build_args: '--no-cache --target vg',
category: 'tool',
annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p2' }],
limits: '{memory: 8Gi, cpu: 8000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p2: 1}',
requests: '{memory: 8Gi, cpu: 8000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p2: 1}',
caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
runAsUser: '0',
runAsGroup: '0'
}
- {
file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
arch: 'x86_64',
name: 'xlio_static.tidy',
uri: '$arch/$name',
tag: '20240703',
build_args: '--no-cache',
category: 'tool'
}

runs_on_agents:
- {nodeLabel: 'beni09', category: 'base'}
name: 'gtest',
uri: 'xlio/$arch/ubuntu22.04/$name',
tag: '20240805',
build_args: '--no-cache --target gtest',
category: 'tests',
annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p1@net1,sriov-cx6dx-p2@net2' }],
limits: '{memory: 8Gi, cpu: 8000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p1: 1, mellanox.com/sriov_cx6dx_p2: 1}',
requests: '{memory: 8Gi, cpu: 8000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p1: 1, mellanox.com/sriov_cx6dx_p2: 1}',
caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
runAsUser: '0',
runAsGroup: '0'
}

matrix:
axes:
Expand Down Expand Up @@ -120,10 +159,6 @@ steps:

- name: Build
enable: ${do_build}
containerSelector:
- "{category: 'base'}"
agentSelector:
- "{category: 'base'}"
run: |
[ "x${do_build}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_build=${action} ./contrib/test_jenkins.sh
Expand All @@ -138,9 +173,7 @@ steps:
- name: Service
enable: ${do_service}
containerSelector:
- "{category: 'base', variant:1}"
agentSelector:
- "{category: 'base', variant:1}"
- "{category: 'base'}"
run: |
[ "x${do_service}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_tool=${action} ./contrib/test_jenkins.sh
Expand All @@ -156,8 +189,6 @@ steps:
enable: ${do_package}
containerSelector:
- "{category: 'base'}"
agentSelector:
- "{nodeLabel: 'skip-agent'}"
run: |
[ "x${do_package}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_rpm=${action} ./contrib/test_jenkins.sh
Expand Down Expand Up @@ -240,15 +271,13 @@ steps:
archiveTap-onfail: |
jenkins/**/*.tap
- name: Cppcheck
enable: ${do_cppcheck}
- name: Tidy
enable: ${do_tidy}
containerSelector:
- "{name: 'xlio_static.cppcheck', category: 'tool', variant: 1}"
agentSelector:
- "{nodeLabel: 'skip-agent'}"
- "{name: 'static', category: 'tool'}"
run: |
[ "x${do_cppcheck}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_cppcheck=${action} ./contrib/test_jenkins.sh
[ "x${do_tidy}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_tidy=${action} ./contrib/test_jenkins.sh
parallel: false
onfail: |
./.ci/artifacts.sh
Expand All @@ -257,15 +286,13 @@ steps:
archiveTap-onfail: |
jenkins/**/*.tap
- name: Csbuild
enable: ${do_csbuild}
- name: Cppcheck
enable: ${do_cppcheck}
containerSelector:
- "{name: 'xlio_static.csbuild', category: 'tool', variant: 1}"
agentSelector:
- "{nodeLabel: 'skip-agent'}"
- "{name: 'static', category: 'tool'}"
run: |
[ "x${do_csbuild}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_csbuild=${action} ./contrib/test_jenkins.sh
[ "x${do_cppcheck}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_cppcheck=${action} ./contrib/test_jenkins.sh
parallel: false
onfail: |
./.ci/artifacts.sh
Expand All @@ -274,15 +301,13 @@ steps:
archiveTap-onfail: |
jenkins/**/*.tap
- name: Tidy
enable: ${do_tidy}
- name: Csbuild
enable: ${do_csbuild}
containerSelector:
- "{name: 'xlio_static.tidy', category: 'tool', variant: 1}"
agentSelector:
- "{nodeLabel: 'skip-agent'}"
- "{name: 'static', category: 'tool'}"
run: |
[ "x${do_tidy}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_tidy=${action} ./contrib/test_jenkins.sh
[ "x${do_csbuild}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_csbuild=${action} ./contrib/test_jenkins.sh
parallel: false
onfail: |
./.ci/artifacts.sh
Expand All @@ -294,9 +319,7 @@ steps:
- name: Test
enable: ${do_test}
containerSelector:
- "{name: 'skip-container'}"
agentSelector:
- "{nodeLabel: 'beni09'}"
- "{name: 'test', category: 'tests'}"
run: |
[ "x${do_test}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_run=${action} ./contrib/test_jenkins.sh
Expand All @@ -311,9 +334,7 @@ steps:
- name: Gtest
enable: ${do_gtest}
containerSelector:
- "{name: 'skip-container'}"
agentSelector:
- "{nodeLabel: 'beni09'}"
- "{name: 'gtest', category: 'tests'}"
run: |
[ "x${do_gtest}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_gtest=${action} ./contrib/test_jenkins.sh
Expand All @@ -330,9 +351,7 @@ steps:
- name: Valgrind
enable: ${do_valgrind}
containerSelector:
- "{name: 'skip-container'}"
agentSelector:
- "{nodeLabel: 'beni09'}"
- "{name: 'vg', category: 'tool'}"
run: |
[ "x${do_valgrind}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_vg=${action} ./contrib/test_jenkins.sh
Expand Down Expand Up @@ -404,4 +423,4 @@ pipeline_stop:
failFast: false

taskName: '${flags}/${name}/${axis_index}'
taskName: '${flags}/${arch}/${name}/${axis_index}'
Loading

0 comments on commit f52386e

Please sign in to comment.