From 6fae17477e975caa8c7afb04118d636c59f95560 Mon Sep 17 00:00:00 2001 From: lguohan Date: Wed, 10 Feb 2021 08:37:30 -0800 Subject: [PATCH 1/8] [ci]: use build template (#1633) Signed-off-by: Guohan Lu --- .azure-pipelines/build-template.yml | 100 +++++++++++++ azure-pipelines.yml | 223 ++++------------------------ 2 files changed, 131 insertions(+), 192 deletions(-) create mode 100644 .azure-pipelines/build-template.yml diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml new file mode 100644 index 0000000000..7d8e12c3ab --- /dev/null +++ b/.azure-pipelines/build-template.yml @@ -0,0 +1,100 @@ +parameters: +- name: arch + type: string + values: + - amd64 + - armhf + - arm64 + +- name: pool + type: string + values: + - sonicbld + - default + default: default + +- name: timeout + type: number + default: 60 + +- name: sonic_slave + type: string + +- name: sairedis_artifact_name + type: string + +- name: swss_common_artifact_name + type: string + +- name: artifact_name + type: string + +jobs: +- job: + displayName: ${{ parameters.arch }} + timeoutInMinutes: ${{ parameters.timeout }} + + pool: + ${{ if ne(parameters.pool, 'default') }}: + name: ${{ parameters.pool }} + ${{ if eq(parameters.pool, 'default') }}: + vmImage: 'ubuntu-20.04' + + container: + image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest + + steps: + - script: | + sudo apt-get install -y libhiredis0.14 libhiredis-dev + sudo apt-get install -y libzmq5 libzmq3-dev + sudo apt-get install -qq -y \ + libhiredis-dev \ + libnl-3-dev \ + libnl-genl-3-dev \ + libnl-route-3-dev \ + libnl-nf-3-dev \ + swig3.0 + sudo apt-get install -y libdbus-1-3 + sudo apt-get install -y libteam-dev \ + libteam5 \ + libteamdctl0 + displayName: "Install dependencies" + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 9 + artifacts: ${{ parameters.swss_common_artifact_name }} + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/master' + displayName: "Download sonic swss common deb packages" + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 12 + artifacts: ${{ parameters.sairedis_artifact_name }} + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/master' + displayName: "Download sonic sairedis deb packages" + - script: | + sudo dpkg -i ${{ parameters.swss_common_artifact_name }}/libswsscommon_1.0.0_${{ parameters.arch }}.deb + sudo dpkg -i ${{ parameters.swss_common_artifact_name }}/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb + sudo dpkg -i ${{ parameters.sairedis_artifact_name }}/libsaivs_*.deb + sudo dpkg -i ${{ parameters.sairedis_artifact_name }}/libsaivs-dev_*.deb + sudo dpkg -i ${{ parameters.sairedis_artifact_name }}/libsairedis_*.deb + sudo dpkg -i ${{ parameters.sairedis_artifact_name }}/libsairedis-dev_*.deb + sudo dpkg -i ${{ parameters.sairedis_artifact_name }}/libsaimetadata_*.deb + sudo dpkg -i ${{ parameters.sairedis_artifact_name }}/libsaimetadata-dev_*.deb + sudo dpkg -i ${{ parameters.sairedis_artifact_name }}/syncd-vs_*.deb + workingDirectory: $(Pipeline.Workspace) + displayName: "Install sonic swss common and sairedis" + - checkout: self + submodules: true + - script: | + ./autogen.sh + dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb . + displayName: "Compile sonic swss" + - publish: $(System.DefaultWorkingDirectory)/ + artifact: ${{ parameters.artifact_name }} + displayName: "Archive swss debian packages" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 852253aa49..af3222caec 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,195 +8,34 @@ trigger: include: - "*" -jobs: -- job: - displayName: "amd64" - pool: - vmImage: 'ubuntu-20.04' - - container: - image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest - - steps: - - script: | - sudo apt-get install -y libhiredis0.14 libhiredis-dev - sudo apt-get install -y libzmq5 libzmq3-dev - sudo apt-get install -qq -y \ - libhiredis-dev \ - libnl-3-dev \ - libnl-genl-3-dev \ - libnl-route-3-dev \ - libnl-nf-3-dev \ - swig3.0 - sudo apt-get install -y libdbus-1-3 - sudo apt-get install -y libteam-dev \ - libteam5 \ - libteamdctl0 - displayName: "Install dependencies" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 9 - artifacts: sonic-swss-common - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - displayName: "Download sonic swss common deb packages" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 12 - artifacts: sonic-sairedis - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - displayName: "Download sonic sairedis deb packages" - - script: | - sudo dpkg -i sonic-swss-common/libswsscommon_1.0.0_amd64.deb - sudo dpkg -i sonic-swss-common/libswsscommon-dev_1.0.0_amd64.deb - sudo dpkg -i sonic-sairedis/libsaivs_*.deb - sudo dpkg -i sonic-sairedis/libsaivs-dev_*.deb - sudo dpkg -i sonic-sairedis/libsairedis_*.deb - sudo dpkg -i sonic-sairedis/libsairedis-dev_*.deb - sudo dpkg -i sonic-sairedis/libsaimetadata_*.deb - sudo dpkg -i sonic-sairedis/libsaimetadata-dev_*.deb - sudo dpkg -i sonic-sairedis/syncd-vs_*.deb - workingDirectory: $(Pipeline.Workspace) - displayName: "Install sonic swss common and sairedis" - - checkout: self - submodules: true - - script: | - ./autogen.sh - dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb . - displayName: "Compile sonic swss" - - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-swss - displayName: "Archive swss debian packages" - -- job: - displayName: "arm64" - timeoutInMinutes: 180 - pool: sonicbld - - container: - image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-arm64:latest - - steps: - - script: | - sudo apt-get install -y libhiredis0.14 libhiredis-dev - sudo apt-get install -y libzmq5 libzmq3-dev - sudo apt-get install -qq -y \ - libhiredis-dev \ - libnl-3-dev \ - libnl-genl-3-dev \ - libnl-route-3-dev \ - libnl-nf-3-dev \ - swig3.0 - sudo apt-get install -y libdbus-1-3 - sudo apt-get install -y libteam-dev \ - libteam5 \ - libteamdctl0 - displayName: "Install dependencies" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 9 - artifacts: sonic-swss-common.arm64 - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - displayName: "Download sonic swss common deb packages" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 12 - artifacts: sonic-sairedis.arm64 - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - displayName: "Download sonic sairedis deb packages" - - script: | - sudo dpkg -i sonic-swss-common.arm64/libswsscommon_1.0.0_arm64.deb - sudo dpkg -i sonic-swss-common.arm64/libswsscommon-dev_1.0.0_arm64.deb - sudo dpkg -i sonic-sairedis.arm64/libsaivs_*.deb - sudo dpkg -i sonic-sairedis.arm64/libsaivs-dev_*.deb - sudo dpkg -i sonic-sairedis.arm64/libsairedis_*.deb - sudo dpkg -i sonic-sairedis.arm64/libsairedis-dev_*.deb - sudo dpkg -i sonic-sairedis.arm64/libsaimetadata_*.deb - sudo dpkg -i sonic-sairedis.arm64/libsaimetadata-dev_*.deb - sudo dpkg -i sonic-sairedis.arm64/syncd-vs_*.deb - workingDirectory: $(Pipeline.Workspace) - displayName: "Install sonic swss common and sairedis" - - checkout: self - submodules: true - - script: | - ./autogen.sh - dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb . - displayName: "Compile sonic swss" - - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-swss.arm64 - displayName: "Archive swss debian packages" - -- job: - displayName: "armhf" - timeoutInMinutes: 180 - pool: sonicbld - - container: - image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-armhf:latest - - steps: - - script: | - sudo apt-get install -y libhiredis0.14 libhiredis-dev - sudo apt-get install -y libzmq5 libzmq3-dev - sudo apt-get install -qq -y \ - libhiredis-dev \ - libnl-3-dev \ - libnl-genl-3-dev \ - libnl-route-3-dev \ - libnl-nf-3-dev \ - swig3.0 - sudo apt-get install -y libdbus-1-3 - sudo apt-get install -y libteam-dev \ - libteam5 \ - libteamdctl0 - displayName: "Install dependencies" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 9 - artifacts: sonic-swss-common.armhf - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - displayName: "Download sonic swss common deb packages" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 12 - artifacts: sonic-sairedis.armhf - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - displayName: "Download sonic sairedis deb packages" - - script: | - sudo dpkg -i sonic-swss-common.armhf/libswsscommon_1.0.0_armhf.deb - sudo dpkg -i sonic-swss-common.armhf/libswsscommon-dev_1.0.0_armhf.deb - sudo dpkg -i sonic-sairedis.armhf/libsaivs_*.deb - sudo dpkg -i sonic-sairedis.armhf/libsaivs-dev_*.deb - sudo dpkg -i sonic-sairedis.armhf/libsairedis_*.deb - sudo dpkg -i sonic-sairedis.armhf/libsairedis-dev_*.deb - sudo dpkg -i sonic-sairedis.armhf/libsaimetadata_*.deb - sudo dpkg -i sonic-sairedis.armhf/libsaimetadata-dev_*.deb - sudo dpkg -i sonic-sairedis.armhf/syncd-vs_*.deb - workingDirectory: $(Pipeline.Workspace) - displayName: "Install sonic swss common and sairedis" - - checkout: self - submodules: true - - script: | - ./autogen.sh - dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb . - displayName: "Compile sonic swss" - - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-swss.armhf - displayName: "Archive swss debian packages" +stages: +- stage: Build + + jobs: + - template: .azure-pipelines/build-template.yml + parameters: + arch: amd64 + sonic_slave: sonic-slave-buster + swss_common_artifact_name: sonic-swss-common + sairedis_artifact_name: sonic-sairedis + artifact_name: sonic-swss + + - template: .azure-pipelines/build-template.yml + parameters: + arch: armhf + timeout: 240 + pool: sonicbld + sonic_slave: sonic-slave-buster-armhf + swss_common_artifact_name: sonic-swss-common.armhf + sairedis_artifact_name: sonic-sairedis.armhf + artifact_name: sonic-swss.armhf + + - template: .azure-pipelines/build-template.yml + parameters: + arch: arm64 + timeout: 240 + pool: sonicbld + sonic_slave: sonic-slave-buster-arm64 + swss_common_artifact_name: sonic-swss-common.arm64 + sairedis_artifact_name: sonic-sairedis.arm64 + artifact_name: sonic-swss.arm64 From 63edd4076b54685d056306cdc1ece0196ba10d65 Mon Sep 17 00:00:00 2001 From: Guohan Lu Date: Thu, 11 Feb 2021 02:01:23 -0800 Subject: [PATCH 2/8] [ci]: run vstest Signed-off-by: Guohan Lu --- .../build-docker-sonic-vs-template.yml | 85 ++++++++++++++++++ .azure-pipelines/build-template.yml | 1 + .azure-pipelines/build_and_install_module.sh | 87 +++++++++++++++++++ .azure-pipelines/docker-sonic-vs/Dockerfile | 16 ++++ .../test-docker-sonic-vs-template.yml | 70 +++++++++++++++ azure-pipelines.yml | 23 +++++ 6 files changed, 282 insertions(+) create mode 100644 .azure-pipelines/build-docker-sonic-vs-template.yml create mode 100755 .azure-pipelines/build_and_install_module.sh create mode 100644 .azure-pipelines/docker-sonic-vs/Dockerfile create mode 100644 .azure-pipelines/test-docker-sonic-vs-template.yml diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml new file mode 100644 index 0000000000..97e8afb394 --- /dev/null +++ b/.azure-pipelines/build-docker-sonic-vs-template.yml @@ -0,0 +1,85 @@ +parameters: +- name: arch + type: string + values: + - amd64 + - armhf + - arm64 + default: amd64 + +- name: timeout + type: number + default: 60 + +- name: swss_artifact_name + type: string + +- name: sairedis_artifact_name + type: string + +- name: swss_common_artifact_name + type: string + +- name: artifact_name + type: string + +jobs: +- job: + displayName: ${{ parameters.arch }} + timeoutInMinutes: ${{ parameters.timeout }} + + pool: + vmImage: 'ubuntu-20.04' + + steps: + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 9 + artifact: ${{ parameters.swss_common_artifact_name }} + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/master' + displayName: "Download sonic swss common deb packages" + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 12 + artifact: ${{ parameters.sairedis_artifact_name }} + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/master' + displayName: "Download sonic sairedis deb packages" + - task: DownloadPipelineArtifact@2 + inputs: + artifact: ${{ parameters.swss_artifact_name }} + displayName: "Download sonic swss artifact" + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 1 + artifact: sonic-buildimage.vs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/master' + displayName: "Download sonic buildimage" + - script: | + echo $(Build.DefinitionName).$(Build.BuildNumber) + + docker load < ../target/docker-sonic-vs.gz + + mkdir -p .azure-pipelines/docker-sonic-vs/debs + + cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs + + pushd .azure-pipelines + + docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs + + popd + + docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz + + - publish: $(Build.ArtifactStagingDirectory)/ + artifact: ${{ parameters.artifact_name }} + displayName: "Archive sonic docker vs image" diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 7d8e12c3ab..7cc6350e2f 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -5,6 +5,7 @@ parameters: - amd64 - armhf - arm64 + default: amd64 - name: pool type: string diff --git a/.azure-pipelines/build_and_install_module.sh b/.azure-pipelines/build_and_install_module.sh new file mode 100755 index 0000000000..4bd026cb52 --- /dev/null +++ b/.azure-pipelines/build_and_install_module.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# build and install team/vrf driver +# + +set -e + +source /etc/os-release + +function build_and_install_kmodule() +{ + if sudo modprobe team 2>/dev/null && sudo modprobe vrf 2>/dev/null && sudo modprobe macsec 2>/dev/null; then + echo "The module team, vrf and macsec exist." + return + fi + + [ -z "$WORKDIR" ] && WORKDIR=$(mktemp -d) + cd $WORKDIR + + KERNEL_RELEASE=$(uname -r) + KERNEL_MAINVERSION=$(echo $KERNEL_RELEASE | cut -d- -f1) + EXTRAVERSION=$(echo $KERNEL_RELEASE | cut -d- -f2) + LOCALVERSION=$(echo $KERNEL_RELEASE | cut -d- -f3) + VERSION=$(echo $KERNEL_MAINVERSION | cut -d. -f1) + PATCHLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f2) + SUBLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f3) + + # Install the required debian packages to build the kernel modules + apt-get install -y build-essential linux-headers-${KERNEL_RELEASE} autoconf pkg-config fakeroot + apt-get install -y flex bison libssl-dev libelf-dev + apt-get install -y libnl-route-3-200 libnl-route-3-dev libnl-cli-3-200 libnl-cli-3-dev libnl-3-dev + + # Add the apt source mirrors and download the linux image source code + cp /etc/apt/sources.list /etc/apt/sources.list.bk + sed -i "s/^# deb-src/deb-src/g" /etc/apt/sources.list + apt-get update + apt-get source linux-image-unsigned-$(uname -r) > source.log + + # Recover the original apt sources list + cp /etc/apt/sources.list.bk /etc/apt/sources.list + apt-get update + + # Build the Linux kernel module drivers/net/team and vrf + cd $(find . -maxdepth 1 -type d | grep -v "^.$") + make allmodconfig + mv .config .config.bk + cp /boot/config-$(uname -r) .config + grep NET_TEAM .config.bk >> .config + echo CONFIG_NET_VRF=m >> .config + echo CONFIG_MACSEC=m >> .config + make VERSION=$VERSION PATCHLEVEL=$PATCHLEVEL SUBLEVEL=$SUBLEVEL EXTRAVERSION=-${EXTRAVERSION} LOCALVERSION=-${LOCALVERSION} modules_prepare + make M=drivers/net/team + mv drivers/net/Makefile drivers/net/Makefile.bak + echo 'obj-$(CONFIG_NET_VRF) += vrf.o' > drivers/net/Makefile + echo 'obj-$(CONFIG_MACSEC) += macsec.o' >> drivers/net/Makefile + make M=drivers/net + + # Install the module + TEAM_DIR=$(echo /lib/modules/$(uname -r)/kernel/net/team) + NET_DIR=$(echo /lib/modules/$(uname -r)/kernel/net) + if [ ! -e "$TEAM_DIR/team.ko" ]; then + mkdir -p $TEAM_DIR + cp drivers/net/team/*.ko $TEAM_DIR/ + modinfo $TEAM_DIR/team.ko + depmod + modprobe team + fi + if [ ! -e "$NET_DIR/vrf.ko" ]; then + mkdir -p $NET_DIR + cp drivers/net/vrf.ko $NET_DIR/ + modinfo $NET_DIR/vrf.ko + depmod + modprobe vrf + fi + if [ ! -e "$NET_DIR/macsec.ko" ]; then + mkdir -p $NET_DIR + cp drivers/net/macsec.ko $NET_DIR/ + modinfo $NET_DIR/macsec.ko + depmod + modprobe macsec + fi + + cd /tmp + rm -rf $WORKDIR +} + +build_and_install_kmodule diff --git a/.azure-pipelines/docker-sonic-vs/Dockerfile b/.azure-pipelines/docker-sonic-vs/Dockerfile new file mode 100644 index 0000000000..f2bfb425b9 --- /dev/null +++ b/.azure-pipelines/docker-sonic-vs/Dockerfile @@ -0,0 +1,16 @@ +FROM docker-sonic-vs + +ARG docker_container_name + +ADD ["debs", "/debs"] + +RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb +RUN dpkg -i /debs/python-swsscommon_1.0.0_amd64.deb +RUN dpkg -i /debs/python3-swsscommon_1.0.0_amd64.deb + +RUN dpkg -i /debs/libsaimetadata_1.0.0_amd64.deb +RUN dpkg -i /debs/libsairedis_1.0.0_amd64.deb +RUN dpkg -i /debs/libsaivs_1.0.0_amd64.deb +RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb + +RUN dpkg -i /debs/swss_1.0.0_amd64.deb diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml new file mode 100644 index 0000000000..a91f6ab59c --- /dev/null +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -0,0 +1,70 @@ +parameters: +- name: timeout + type: number + default: 180 + +- name: log_artifact_name + type: string + +jobs: +- job: + displayName: vstest + timeoutInMinutes: ${{ parameters.timeout }} + + pool: + vmImage: 'ubuntu-20.04' + + steps: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: docker-sonic-vs + displayName: "Download docker sonic vs image" + + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 9 + artifact: sonic-swss-common.amd64.ubuntu20_04 + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/master' + displayName: "Download sonic swss common deb packages" + + - script: | + set -x + sudo .azure-pipelines/build_and_install_module.sh + + sudo apt-get install -y libhiredis0.14 + sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f + sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb + + # install packages for vs test + sudo apt-get install -y net-tools bridge-utils vlan + sudo apt-get install -y python3-pip + sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker==4.4.1 redis==3.3.4 flaky==3.7.0 + displayName: "Install dependencies" + + - script: | + set -x + sudo docker load -i ../docker-sonic-vs.gz + docker ps + ip netns list + pushd tests + sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) + displayName: "Run vs tests" + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/tr.xml' + testRunTitle: vstest + condition: always() + + - script: | + cp -r tests/log $(Build.ArtifactStagingDirectory)/ + displayName: "Collect logs" + condition: always() + + - publish: $(Build.ArtifactStagingDirectory)/ + artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt) + displayName: "Publish logs" + condition: always() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index af3222caec..4f2762d348 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,6 +20,10 @@ stages: sairedis_artifact_name: sonic-sairedis artifact_name: sonic-swss +- stage: BuildArm + dependsOn: Build + condition: succeeded('Build') + jobs: - template: .azure-pipelines/build-template.yml parameters: arch: armhf @@ -39,3 +43,22 @@ stages: swss_common_artifact_name: sonic-swss-common.arm64 sairedis_artifact_name: sonic-sairedis.arm64 artifact_name: sonic-swss.arm64 + +- stage: BuildDocker + dependsOn: Build + condition: succeeded('Build') + jobs: + - template: .azure-pipelines/build-docker-sonic-vs-template.yml + parameters: + swss_common_artifact_name: sonic-swss-common + sairedis_artifact_name: sonic-sairedis + swss_artifact_name: sonic-swss + artifact_name: docker-sonic-vs + +- stage: Test + dependsOn: BuildDocker + condition: succeeded('BuildDocker') + jobs: + - template: .azure-pipelines/test-docker-sonic-vs-template.yml + parameters: + log_artifact_name: log From 50bc34e734e9aa53ec36e858c846307fe5bbbc44 Mon Sep 17 00:00:00 2001 From: lguohan Date: Sat, 3 Apr 2021 12:08:37 -0700 Subject: [PATCH 3/8] [ci]: archive swss pytests (#1690) in other pipelines, they will download swss and pytests. this is make sure the swss.deb and pytests are matched. Signed-off-by: Guohan Lu --- .azure-pipelines/build-template.yml | 11 +++++++++++ azure-pipelines.yml | 1 + 2 files changed, 12 insertions(+) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 7cc6350e2f..ea5d41e36f 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -30,6 +30,10 @@ parameters: - name: artifact_name type: string +- name: archive_pytests + type: boolean + default: false + jobs: - job: displayName: ${{ parameters.arch }} @@ -93,9 +97,16 @@ jobs: - checkout: self submodules: true - script: | + set -x + tar czf pytest.tgz tests + cp -r pytest.tgz $(Build.ArtifactStagingDirectory)/ ./autogen.sh dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb . displayName: "Compile sonic swss" - publish: $(System.DefaultWorkingDirectory)/ artifact: ${{ parameters.artifact_name }} displayName: "Archive swss debian packages" + - publish: $(Build.ArtifactStagingDirectory)/ + artifact: sonic-swss-pytests + displayName: "Archive swss pytests" + condition: eq('${{ parameters.archive_pytests }}', true) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4f2762d348..5f34d4b045 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,6 +19,7 @@ stages: swss_common_artifact_name: sonic-swss-common sairedis_artifact_name: sonic-sairedis artifact_name: sonic-swss + archive_pytests: true - stage: BuildArm dependsOn: Build From 6e209c023e2d2a9daf483c572109cb4d5a6f1d1f Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Thu, 4 Mar 2021 20:43:23 -0800 Subject: [PATCH 4/8] [ci]: Purge swss before install (#1654) Observe persistent failure of newly added vs test on testing code change in intfmgrd after migrating to azure pipeline. #1521 passed in legacy Jenkins pipeline. What we found is that the code change in the swss compiled with PR change is not properly installed in vs docker for a direct installation using dpkg -i. This is confirmed by verifying pipeline artifacts that md5sum value of /user/bin/intfmgrd changes if we install swss deb aritfact into the docker vs artifact, while that of /usr/bin/orchangent stays unchanged. md5sum before swss install in vs docker # md5sum /usr/bin/orchagent 28307a7805ea6f3bc5057c0257bf46e6 /usr/bin/orchagent # md5sum /usr/bin/intfmgrd fa2b06e20be683286adb47c55635a86d /usr/bin/intfmgrd md5sum after swss install # dpkg -i swss_1.0.0_amd64.deb (Reading database ... 19180 files and directories currently installed.) Preparing to unpack swss_1.0.0_amd64.deb ... Unpacking swss (1.0.0) over (1.0.0) ... Setting up swss (1.0.0) ... # md5sum /usr/bin/orchagent 28307a7805ea6f3bc5057c0257bf46e6 /usr/bin/orchagent # md5sum /usr/bin/intfmgrd e959340709e7aedd7489e69dfd19768f /usr/bin/intfmgrd Signed-off-by: Wenda Ni --- .azure-pipelines/docker-sonic-vs/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/docker-sonic-vs/Dockerfile b/.azure-pipelines/docker-sonic-vs/Dockerfile index f2bfb425b9..d425a6ffea 100644 --- a/.azure-pipelines/docker-sonic-vs/Dockerfile +++ b/.azure-pipelines/docker-sonic-vs/Dockerfile @@ -13,4 +13,5 @@ RUN dpkg -i /debs/libsairedis_1.0.0_amd64.deb RUN dpkg -i /debs/libsaivs_1.0.0_amd64.deb RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb +RUN dpkg --purge swss RUN dpkg -i /debs/swss_1.0.0_amd64.deb From 74fefe9e62890e660fb5df97b76b563efcddd987 Mon Sep 17 00:00:00 2001 From: xumia Date: Sun, 6 Jun 2021 09:54:20 +0000 Subject: [PATCH 5/8] Test azp --- .../test-docker-sonic-vs-template.yml | 1 + azure-pipelines.yml | 47 ------------------- 2 files changed, 1 insertion(+), 47 deletions(-) diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index a91f6ab59c..65056dcb2f 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -17,6 +17,7 @@ jobs: steps: - task: DownloadPipelineArtifact@2 inputs: + runId: 17871 artifact: docker-sonic-vs displayName: "Download docker sonic vs image" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5f34d4b045..3723b56f49 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,53 +9,6 @@ trigger: - "*" stages: -- stage: Build - - jobs: - - template: .azure-pipelines/build-template.yml - parameters: - arch: amd64 - sonic_slave: sonic-slave-buster - swss_common_artifact_name: sonic-swss-common - sairedis_artifact_name: sonic-sairedis - artifact_name: sonic-swss - archive_pytests: true - -- stage: BuildArm - dependsOn: Build - condition: succeeded('Build') - jobs: - - template: .azure-pipelines/build-template.yml - parameters: - arch: armhf - timeout: 240 - pool: sonicbld - sonic_slave: sonic-slave-buster-armhf - swss_common_artifact_name: sonic-swss-common.armhf - sairedis_artifact_name: sonic-sairedis.armhf - artifact_name: sonic-swss.armhf - - - template: .azure-pipelines/build-template.yml - parameters: - arch: arm64 - timeout: 240 - pool: sonicbld - sonic_slave: sonic-slave-buster-arm64 - swss_common_artifact_name: sonic-swss-common.arm64 - sairedis_artifact_name: sonic-sairedis.arm64 - artifact_name: sonic-swss.arm64 - -- stage: BuildDocker - dependsOn: Build - condition: succeeded('Build') - jobs: - - template: .azure-pipelines/build-docker-sonic-vs-template.yml - parameters: - swss_common_artifact_name: sonic-swss-common - sairedis_artifact_name: sonic-sairedis - swss_artifact_name: sonic-swss - artifact_name: docker-sonic-vs - - stage: Test dependsOn: BuildDocker condition: succeeded('BuildDocker') From 56dda3b251760b2180220ca71e49bdac3dd76594 Mon Sep 17 00:00:00 2001 From: xumia Date: Sun, 6 Jun 2021 09:57:36 +0000 Subject: [PATCH 6/8] Fix bug --- azure-pipelines.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3723b56f49..7ef783791c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,8 +10,6 @@ trigger: stages: - stage: Test - dependsOn: BuildDocker - condition: succeeded('BuildDocker') jobs: - template: .azure-pipelines/test-docker-sonic-vs-template.yml parameters: From 23c9a96498fab6486f0cc32ffca4b8ab0348f3ba Mon Sep 17 00:00:00 2001 From: xumia Date: Sun, 6 Jun 2021 10:04:19 +0000 Subject: [PATCH 7/8] Add dependencies --- .../test-docker-sonic-vs-template.yml | 1 - azure-pipelines.yml | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 65056dcb2f..a91f6ab59c 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -17,7 +17,6 @@ jobs: steps: - task: DownloadPipelineArtifact@2 inputs: - runId: 17871 artifact: docker-sonic-vs displayName: "Download docker sonic vs image" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7ef783791c..aeb7efb2e2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,7 +9,32 @@ trigger: - "*" stages: +stages: +- stage: Build + jobs: + - template: .azure-pipelines/build-template.yml + parameters: + arch: amd64 + sonic_slave: sonic-slave-buster + swss_common_artifact_name: sonic-swss-common + sairedis_artifact_name: sonic-sairedis + artifact_name: sonic-swss + archive_pytests: true + +- stage: BuildDocker + dependsOn: Build + condition: succeeded('Build') + jobs: + - template: .azure-pipelines/build-docker-sonic-vs-template.yml + parameters: + swss_common_artifact_name: sonic-swss-common + sairedis_artifact_name: sonic-sairedis + swss_artifact_name: sonic-swss + artifact_name: docker-sonic-vs + - stage: Test + dependsOn: BuildDocker + condition: succeeded('BuildDocker') jobs: - template: .azure-pipelines/test-docker-sonic-vs-template.yml parameters: From e1f00dcbf4389a42f3b3d3f712e232ddee58c439 Mon Sep 17 00:00:00 2001 From: xumia Date: Sun, 6 Jun 2021 10:05:57 +0000 Subject: [PATCH 8/8] fix bug --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index aeb7efb2e2..2f632e72df 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,7 +8,6 @@ trigger: include: - "*" -stages: stages: - stage: Build jobs: