Skip to content

Commit

Permalink
issue HPCINFRA-1968: Compile with DOCA
Browse files Browse the repository at this point in the history
Signed-off-by: Daniel Pressler <danielpr@nvidia.com>
  • Loading branch information
dpressle committed Jun 9, 2024
1 parent 1e18c6a commit 148e509
Show file tree
Hide file tree
Showing 9 changed files with 303 additions and 90 deletions.
2 changes: 1 addition & 1 deletion .ci/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

// load pipeline functions
// Requires pipeline-github-lib plugin to load library from github
@Library('github.com/Mellanox/ci-demo@stable')
@Library('github.com/Mellanox/ci-demo@master')
def matrix = new com.mellanox.cicd.Matrix()

matrix.main()
53 changes: 53 additions & 0 deletions .ci/dockerfiles/Dockerfile.ubuntu22.04
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
ARG ARCH=x86_64
FROM harbor.mellanox.com/hpcx/$ARCH/ubuntu22.04/core:latest
ARG _UID=6213
ARG _GID=101
ARG _LOGIN=swx-jenkins
ARG _HOME=/var/home/$_LOGIN
ARG HTTP_NFS_ENDPOINT="http://nbu-nfs.mellanox.com"
ARG DOCA_REPO_BUILD="/auto/sw/release/doca/doca-repo-x86/doca-repo-2.6.0/last_stable"
ARG OS_NAME="ubuntu2204"
ARG REPO_ARCH="amd64"

RUN apt-get update && \
apt-get install -y \
sudo wget lynx unzip \
automake dh-make \
g++ make vim python3-pip curl \
git libglib2.0-dev \
cppcheck clang-tools clang-tidy \
net-tools psmisc \
&& apt-get clean && rm -rf /var/lib/apt/lists/*

RUN DOCA_REPO_LINK=$(lynx -dump -listonly ${HTTP_NFS_ENDPOINT}/${DOCA_REPO_BUILD} | awk '{ print $2 }' | grep doca-host | grep -iE ".rpm$|.deb$" | grep -iE "$OS_NAME.*$REPO_ARCH" || true) \
&& wget -q --no-check-certificate -O doca-repo.deb $DOCA_REPO_LINK \
&& apt-get -qq install -y ./doca-repo.deb \
&& apt-get -qq update \
&& mkdir -p /lib/modules \
&& apt-get -qq install -y --allow-downgrades ibacm ibacm infiniband-diags libibumad \
libibverbs-dev libibverbs1 libibverbs-dev librdmacm-dev flex \
librdmacm1 mlnx-ethtool mlnx-iproute2 mlnx-tools mstflint \
ofed-scripts rdma-core collectx-clxapi collectx-clxapidev \
python3-protobuf libgrpc-dev mlnx-dpdk-dev mlnx-dpdk \
ninja-build meson libjson-c-dev libjsoncpp-dev libzip-dev libgtest-dev \
build-essential

RUN pip3 install -U pip --no-cache-dir \
&& pip3 install argparse --no-cache-dir

# get ibdev2netdev for tests usage
RUN git clone https://github.com/Mellanox/container_scripts.git && \
cp container_scripts/ibdev2netdev /usr/local/sbin/ && \
rm -rf container_scripts

RUN echo "${_LOGIN} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
mkdir -p ${_HOME} && \
groupadd -f -g "$_GID" "$_LOGIN" && \
useradd -u "$_UID" -g "$_GID" -s /bin/bash -m -d ${_HOME} "${_LOGIN}" && \
chown -R ${_LOGIN} ${_HOME} && \
mkdir /build && chown -R ${_LOGIN} /build

ENV MODULEPATH /hpc/local/etc/modulefiles
USER "$_LOGIN"
ENTRYPOINT [ "/bin/bash", "--login", "--rcfile", "/etc/bashrc", "-c" ]
Binary file added .ci/ibdev2netdev
Binary file not shown.
196 changes: 132 additions & 64 deletions .ci/matrix_job.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@ registry_auth: 1daaea28-800e-425f-a91f-3bd3e9136eea
registry_path: /swx-infra/media

kubernetes:
privileged: false
namespace: 'xlio-ci'
cloud: swx-k8s-spray
nodeSelector: 'beta.kubernetes.io/os=linux'
namespace: xlio-ci
limits: '{memory: 8Gi, cpu: 7000m}'
requests: '{memory: 8Gi, cpu: 7000m}'

Expand All @@ -34,44 +33,112 @@ volumes:
- {mountPath: /var/home/swx-jenkins, hostPath: /labhome/swx-jenkins}

runs_on_dockers:
# mofed
- {name: 'ub20.04-mofed-x86_64', url: 'harbor.mellanox.com/swx-infra/x86_64/ubuntu20.04/builder:mofed-5.2-2.2.0.0', category: 'base', arch: 'x86_64'}
- {name: 'ub20.04-mofed-aarch64', url: 'harbor.mellanox.com/swx-infra/aarch64/ubuntu20.04/builder:mofed-5.2-1.0.4.0', category: 'base', arch: 'aarch64'}
- {name: 'ub22.04-mofed-x86_64', url: 'harbor.mellanox.com/hpcx/x86_64/ubuntu22.04/builder:mofed-5.7-0.1.1.0', category: 'base', arch: 'x86_64'}
- {name: 'rhel8.6-mofed-x86_64', url: 'harbor.mellanox.com/hpcx/x86_64/rhel8.6/builder:mofed-5.6-0.4.5.0', category: 'base', arch: 'x86_64'}
# - {name: 'oracle8.6-mofed-x86_64', url: 'harbor.mellanox.com/rivermax/base_oraclelinux8.6:mofed-5.9-0.3.4.0', category: 'base', arch: 'x86_64'}
# build
# - {
# file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
# arch: 'x86_64',
# limits: '{memory: 10Gi, cpu: 10000m}',
# requests: '{memory: 10Gi, cpu: 10000m}',
# name: 'ubuntu22.04-build-x86_64',
# uri: 'xlio/$arch/$name',
# tag: '20240805',
# build_args: '--no-cache',
# category: 'base'
# }
# - {
# file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
# # url: 'harbor.mellanox.com/swx-infra/media/xlio/aarch64/ubuntu22.04-aarch64:20240805',
# arch: 'aarch64',
# limits: '{memory: 10Gi, cpu: 10000m}',
# requests: '{memory: 10Gi, cpu: 10000m}',
# name: 'ubuntu22.04-build-aarch64',
# uri: 'xlio/$arch/$name',
# tag: '20240805',
# build_args: '--no-cache --build-arg ARCH=aarch64 --build-arg REPO_ARCH=arm64',
# category: 'base'
# }
# # static tests
# - {
# file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
# arch: 'x86_64',
# name: 'ubuntu22.04-static',
# uri: 'xlio/$arch/$name',
# tag: '20240805',
# build_args: '--no-cache',
# category: 'tool'
# }
# - {
# file: '.ci/dockerfiles/Dockerfile.rhel8.6',
# arch: 'x86_64',
# name: 'csbuild',
# uri: 'xlio/$arch/rhel8.6/$name',
# tag: '20240805',
# build_args: '--no-cache',
# category: 'tool'
# }
# - {
# file: '.ci/dockerfiles/Dockerfile.rhel8.6',
# arch: 'x86_64',
# name: 'tidy',
# uri: 'xlio/$arch/rhel8.6/$name',
# tag: '20240805',
# build_args: '--no-cache',
# category: 'tool'
# }
# tool
- {name: 'toolbox', url: 'harbor.mellanox.com/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
- {name: 'blackduck', url: 'harbor.mellanox.com/toolbox/ngci-centos:7.9.2009.2', category: 'tool', arch: 'x86_64'}
- {name: 'header-check', url: 'harbor.mellanox.com/toolbox/header_check:0.0.14', category: 'tool', arch: 'x86_64', tag: '0.0.14'}
# static tests
- {file: '.ci/dockerfiles/Dockerfile.rhel8.6',
arch: 'x86_64',
name: 'xlio_static.cppcheck',
uri: '$arch/$name',
tag: '20240703',
build_args: '--no-cache',
category: 'tool'
}
- {file: '.ci/dockerfiles/Dockerfile.rhel8.6',
arch: 'x86_64',
name: 'xlio_static.csbuild',
uri: '$arch/$name',
tag: '20240703',
build_args: '--no-cache',
category: 'tool'
}
- {file: '.ci/dockerfiles/Dockerfile.rhel8.6',
arch: 'x86_64',
name: 'xlio_static.tidy',
uri: '$arch/$name',
tag: '20240703',
build_args: '--no-cache',
category: 'tool'
}

runs_on_agents:
- {nodeLabel: 'beni09', category: 'base'}
# - {name: 'toolbox', url: 'harbor.mellanox.com/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
# - {name: 'blackduck', url: 'harbor.mellanox.com/toolbox/ngci-centos:7.9.2009.2', category: 'tool', arch: 'x86_64'}
# - {name: 'header-check', url: 'harbor.mellanox.com/toolbox/header_check:0.0.14', category: 'tool', arch: 'x86_64', tag: '0.0.14'}
# test
- {
# file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
arch: 'x86_64',
name: 'ubuntu22.04-tests',
# uri: 'xlio/$arch/$name',
url: ' harbor.mellanox.com/swx-infra/media/xlio/x86_64/ubuntu22.04-tests:20240805',
tag: '20240805',
build_args: '--no-cache',
category: 'base',
annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p1' }],
limits: '{memory: 8Gi, cpu: 8000m, mellanox.com/sriov_cx6dx_p1: 1}',
requests: '{memory: 8Gi, cpu: 8000m, mellanox.com/sriov_cx6dx_p1: 1}',
caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
runAsUser: '0',
runAsGroup: '0',
privileged: true
}
# - {
# # file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
# arch: 'x86_64',
# name: 'vg',
# url: 'harbor.mellanox.com/swx-infra/media/xlio/x86_64/ubuntu22.04-tests:20242905',
# # uri: 'xlio/$arch/ubuntu22.04/$name',
# tag: '20242905',
# build_args: '--no-cache',
# category: 'tests',
# annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p2' }],
# limits: '{memory: 8Gi, cpu: 8000m, mellanox.com/sriov_cx6dx_p2: 1}',
# requests: '{memory: 8Gi, cpu: 8000m, mellanox.com/sriov_cx6dx_p2: 1}',
# runAsUser: '0',
# runAsGroup: '0',
# privileged: true
# }
# - {
# # file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
# arch: 'x86_64',
# name: 'gtest',
# url: 'harbor.mellanox.com/swx-infra/media/xlio/x86_64/ubuntu22.04-tests:20242905',
# # uri: 'xlio/$arch/ubuntu22.04/$name',
# tag: '20242905',
# build_args: '--no-cache',
# category: 'tests',
# annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p2' }],
# limits: '{memory: 8Gi, cpu: 8000m, mellanox.com/sriov_cx6dx_p2: 1}',
# requests: '{memory: 8Gi, cpu: 8000m, mellanox.com/sriov_cx6dx_p2: 1}',
# runAsUser: '0',
# runAsGroup: '0',
# privileged: true
# }

matrix:
axes:
Expand All @@ -84,6 +151,7 @@ matrix:

env:
jenkins_opt_artifacts: 'none'
TEST_RUN_ADAPTER: 'ConnectX-6'

steps:
- name: Setup
Expand Down Expand Up @@ -121,7 +189,7 @@ steps:
containerSelector:
- "{category: 'base'}"
agentSelector:
- "{category: 'base'}"
- "{nodeLabel: 'skip-agent'}"
run: |
[ "x${do_build}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_build=${action} ./contrib/test_jenkins.sh
Expand All @@ -138,7 +206,7 @@ steps:
containerSelector:
- "{category: 'base', variant:1}"
agentSelector:
- "{category: 'base', variant:1}"
- "{nodeLabel: 'skip-agent'}"
run: |
[ "x${do_service}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_tool=${action} ./contrib/test_jenkins.sh
Expand All @@ -153,7 +221,7 @@ steps:
- name: Package
enable: ${do_package}
containerSelector:
- "{category: 'base'}"
- "{name: '.*build.*', category: 'base'}"
agentSelector:
- "{nodeLabel: 'skip-agent'}"
run: |
Expand Down Expand Up @@ -241,7 +309,7 @@ steps:
- name: Cppcheck
enable: ${do_cppcheck}
containerSelector:
- "{name: 'xlio_static.cppcheck', category: 'tool', variant: 1}"
- "{name: '.*.static', category: 'tool', variant: 1}"
agentSelector:
- "{nodeLabel: 'skip-agent'}"
run: |
Expand All @@ -258,7 +326,7 @@ steps:
- name: Csbuild
enable: ${do_csbuild}
containerSelector:
- "{name: 'xlio_static.csbuild', category: 'tool', variant: 1}"
- "{name: '.*.static', category: 'tool', variant: 1}"
agentSelector:
- "{nodeLabel: 'skip-agent'}"
run: |
Expand All @@ -275,7 +343,7 @@ steps:
- name: Tidy
enable: ${do_tidy}
containerSelector:
- "{name: 'xlio_static.tidy', category: 'tool', variant: 1}"
- "{name: '.*.static', category: 'tool', variant: 1}"
agentSelector:
- "{nodeLabel: 'skip-agent'}"
run: |
Expand All @@ -292,9 +360,9 @@ steps:
- name: Test
enable: ${do_test}
containerSelector:
- "{name: 'skip-container'}"
- "{name: '.*-tests', category: 'base'}"
agentSelector:
- "{nodeLabel: 'beni09'}"
- "{nodeLabel: 'skip-agent'}"
run: |
[ "x${do_test}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_run=${action} ./contrib/test_jenkins.sh
Expand All @@ -306,42 +374,42 @@ steps:
archiveTap-onfail: |
jenkins/**/*.tap
- name: Gtest
enable: ${do_gtest}
- name: Valgrind
enable: ${do_valgrind}
containerSelector:
- "{name: 'skip-container'}"
- "{name: '.*-tests', category: 'base'}"
agentSelector:
- "{nodeLabel: 'beni09'}"
- "{nodeLabel: 'skip-agent'}"
run: |
[ "x${do_gtest}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_gtest=${action} ./contrib/test_jenkins.sh
[ "x${do_valgrind}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_vg=${action} ./contrib/test_jenkins.sh
parallel: false
onfail: |
./.ci/artifacts.sh
archiveArtifacts-onfail: |
jenkins/**/arch-*.tar.gz
jenkins/**/arch-*.tar.gz,
jenkins/**/vg/*valgrind*.log
archiveTap-onfail: |
jenkins/**/*.tap
archiveJunit-onfail: |
jenkins/**/*.xml
- name: Valgrind
enable: ${do_valgrind}
- name: Gtest
enable: ${do_gtest}
containerSelector:
- "{name: 'skip-container'}"
- "{name: '.*-tests', category: 'base'}"
agentSelector:
- "{nodeLabel: 'beni09'}"
- "{nodeLabel: 'skip-agent'}"
run: |
[ "x${do_valgrind}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_vg=${action} ./contrib/test_jenkins.sh
[ "x${do_gtest}" == "xtrue" ] && action=yes || action=no
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_gtest=${action} ./contrib/test_jenkins.sh
parallel: false
onfail: |
./.ci/artifacts.sh
archiveArtifacts-onfail: |
jenkins/**/arch-*.tar.gz,
jenkins/**/vg/*valgrind*.log
jenkins/**/arch-*.tar.gz
archiveTap-onfail: |
jenkins/**/*.tap
archiveJunit-onfail: |
jenkins/**/*.xml
- name: Commit
enable: ${do_commit}
Expand Down
Loading

0 comments on commit 148e509

Please sign in to comment.