diff --git a/.azure-pipelines/test-docker-sonic-vs-template-test.yml b/.azure-pipelines/test-docker-sonic-vs-template-test.yml new file mode 100644 index 0000000000..74dfdaf4a0 --- /dev/null +++ b/.azure-pipelines/test-docker-sonic-vs-template-test.yml @@ -0,0 +1,221 @@ +parameters: +- name: timeout + type: number + default: 480 + +- name: log_artifact_name + type: string + +- name: gcov_artifact_name + type: string + +- name: sonic_slave + type: string + +- name: archive_gcov + type: boolean + default: false + +- name: docker_sonic_vs_name + type: string + default: docker-sonic-vs + +- name: swss_common_branch + type: string + default: '$(BUILD_BRANCH)' + +- name: sonic_buildimage_ubuntu20_04 + type: string + default: '$(BUILD_BRANCH)' + +- name: asan + type: boolean + default: false + +- name: num_ports + type: number + default: 0 + +- name: run_tests_pattern + type: string + default: "" + +jobs: +- job: + displayName: vstest + timeoutInMinutes: 2400 + + pool: sonic-common-test + + steps: + - script: | + ls -A1 | xargs -I{} sudo rm -rf {} + ip a show dev eth0 + displayName: "Clean workspace" + - checkout: self + - task: DownloadPipelineArtifact@2 + inputs: + project: build + buildType: specific + buildVersionToDownload: specific + artifact: ${{ parameters.docker_sonic_vs_name }} + pipelineId: 457461 + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download pre-stage built ${{ parameters.docker_sonic_vs_name }}" + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: Azure.sonic-swss-common + artifact: sonic-swss-common.amd64.ubuntu20_04 + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.swss_common_branch }}' + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download sonic swss common deb packages" + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: sonic-net.sonic-buildimage-ubuntu20.04 + artifact: sonic-buildimage.amd64.ubuntu20_04 + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.sonic_buildimage_ubuntu20_04 }}' + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download sonic buildimage ubuntu20.04 deb packages" + + - script: | + set -ex + # Install .NET CORE + curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + sudo apt-add-repository https://packages.microsoft.com/ubuntu/20.04/prod + sudo apt-get update + sudo apt-get install -y dotnet-sdk-7.0 + sudo dotnet tool install dotnet-reportgenerator-globaltool --tool-path /usr/bin + displayName: "Install .NET CORE" + + - script: | + set -ex + sudo .azure-pipelines/build_and_install_module.sh + + sudo apt-get install -y libhiredis0.14 libyang0.16 + sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb + sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi*.deb + sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f + sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb + + # install packages for vs test + sudo apt-get install -y net-tools bridge-utils vlan + sudo apt-get install -y python3-pip + sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0 + displayName: "Install dependencies" + + - script: | + set -ex + sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz + docker images + docker tag docker-sonic-vs:$(docker images | grep docker-sonic-vs | tail -n 1 | awk '{print $2}') docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} + docker ps + ip netns list + uname -a + sudo /sbin/ip link add Vrf1 type vrf table 1001 || { echo 'vrf command failed' ; exit 1; } + sudo /sbin/ip link del Vrf1 type vrf table 1001 + pushd tests + + params="" + if [ '${{ parameters.archive_gcov }}' == True ]; then + params=" ${params} --collect-coverage --force-recreate-dvs " + fi + if [ '${{ parameters.asan }}' == True ]; then + params=" ${params} --graceful-stop " + fi + if [ ${{ parameters.num_ports }} -gt 0 ]; then + params=" ${params} --num-ports=${{ parameters.num_ports }} " + fi + + all_tests=$(ls test_*.py) + all_tests="${all_tests} p4rt" + all_tests=$(ls test_*.py | head -n 10 | xargs) + all_tests="test_srv6.py test_fabric.py test_fabric_port.py" + + if [ -n '${{ parameters.run_tests_pattern }}' ]; then + all_tests=" $(ls ${{ parameters.run_tests_pattern }}) " + fi + + echo $all_tests | xargs -n 1 | xargs -P 4 -I TEST_MODULE sudo sh -c 'py.test -v --force-flaky --junitxml="$(echo TEST_MODULE | cut -d "." -f1)_tr.xml" '"$params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} TEST_MODULE" + displayName: "Run vs tests" + continueOnError: ${{ parameters.asan }} + + - script: | + set -ex + cp $(Build.ArtifactStagingDirectory)/download/coverage.info ./tests/ + cp $(Build.ArtifactStagingDirectory)/download/coverage.xml ./tests/ + reportgenerator -reporttypes:Cobertura -reports:tests/*coverage.xml -targetdir:. + mkdir $(Build.ArtifactStagingDirectory)/gcov + cp tests/*coverage.xml $(Build.ArtifactStagingDirectory)/gcov/ + cp Cobertura.xml $(Build.ArtifactStagingDirectory)/gcov/ + cp tests/*coverage.info $(Build.ArtifactStagingDirectory)/gcov/ + cp tests/*.gcda.tar $(Build.ArtifactStagingDirectory)/gcov/ + rm -rf $(Build.ArtifactStagingDirectory)/download + condition: true + displayName: "Collect coverage.xml" + + - task: PublishCodeCoverageResults@1 + condition: true + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/Cobertura.xml' + #reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/' + displayName: 'Publish test coverage' + + - script: + sleep 72000 + condition: true + displayName: "Sleep 72000" + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/*_tr.xml' + testRunTitle: vstest + condition: succeeded() + + - script: | + cp -r tests/log $(Build.ArtifactStagingDirectory)/ + + if [ '${{ parameters.asan }}' == True ]; then + cp -vr tests/log/*/log/asan $(Build.ArtifactStagingDirectory)/ + fi + + if [ '${{ parameters.archive_gcov }}' == True ]; then + sudo apt-get install -y lcov + cd $(Build.ArtifactStagingDirectory)/gcov_tmp/ + tar -zcvf sonic-gcov.tar.gz sonic-gcov/ + rm -rf sonic-gcov + fi + displayName: "Collect logs" + condition: always() + + - publish: $(Build.ArtifactStagingDirectory)/gcov_tmp + artifact: ${{ parameters.gcov_artifact_name }} + displayName: "Publish gcov output" + condition: and(succeeded(), eq('${{ parameters.archive_gcov }}', true)) + + - publish: $(Build.ArtifactStagingDirectory)/ + artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt) + displayName: "Publish logs" + condition: always() + + - publish: $(Build.ArtifactStagingDirectory)/asan + artifact: asan-reports + displayName: "Publish ASAN reports" + condition: eq('${{ parameters.asan }}', true) + + - script: | + if [ "$(ls -A $(Build.ArtifactStagingDirectory)/asan)" ]; then + echo "There are issues reported by ASAN" + exit 1 + else + echo "No issues reported by ASAN" + fi + displayName: "Check ASAN reports" + condition: eq('${{ parameters.asan }}', true) + continueOnError: true diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 362a106225..6bb9614f69 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,117 +35,11 @@ variables: value: $(Build.SourceBranchName) stages: -- stage: Build - - jobs: - - template: .azure-pipelines/build-template.yml - parameters: - arch: amd64 - sonic_slave: sonic-slave-bullseye - common_lib_artifact_name: common-lib - swss_common_artifact_name: sonic-swss-common - sairedis_artifact_name: sonic-sairedis - artifact_name: sonic-swss - archive_pytests: true - archive_gcov: true - -- stage: BuildAsan - dependsOn: [] - jobs: - - template: .azure-pipelines/build-template.yml - parameters: - arch: amd64 - sonic_slave: sonic-slave-bullseye - common_lib_artifact_name: common-lib - swss_common_artifact_name: sonic-swss-common - sairedis_artifact_name: sonic-sairedis - artifact_name: sonic-swss-asan - asan: true - -- stage: BuildArm - dependsOn: Build - condition: succeeded('Build') - jobs: - - template: .azure-pipelines/build-template.yml - parameters: - arch: armhf - timeout: 240 - pool: sonicbld-armhf - sonic_slave: sonic-slave-bullseye-armhf - common_lib_artifact_name: common-lib.armhf - swss_common_artifact_name: sonic-swss-common.armhf - sairedis_artifact_name: sonic-sairedis.armhf - artifact_name: sonic-swss.armhf - archive_gcov: false - - - template: .azure-pipelines/build-template.yml - parameters: - arch: arm64 - timeout: 240 - pool: sonicbld-arm64 - sonic_slave: sonic-slave-bullseye-arm64 - common_lib_artifact_name: common-lib.arm64 - swss_common_artifact_name: sonic-swss-common.arm64 - sairedis_artifact_name: sonic-sairedis.arm64 - artifact_name: sonic-swss.arm64 - archive_gcov: false - -- stage: BuildDocker - dependsOn: Build - condition: succeeded('Build') - jobs: - - template: .azure-pipelines/build-docker-sonic-vs-template.yml - parameters: - swss_common_artifact_name: sonic-swss-common - sairedis_artifact_name: sonic-sairedis - swss_artifact_name: sonic-swss - artifact_name: docker-sonic-vs - -- stage: BuildDockerAsan - dependsOn: BuildAsan - condition: succeeded('BuildAsan') - jobs: - - template: .azure-pipelines/build-docker-sonic-vs-template.yml - parameters: - swss_common_artifact_name: sonic-swss-common - sairedis_artifact_name: sonic-sairedis - swss_artifact_name: sonic-swss-asan - artifact_name: docker-sonic-vs-asan - asan: true - - stage: Test - dependsOn: BuildDocker - condition: succeeded('BuildDocker') jobs: - - template: .azure-pipelines/test-docker-sonic-vs-template.yml + - template: .azure-pipelines/test-docker-sonic-vs-template-test.yml parameters: log_artifact_name: log gcov_artifact_name: sonic-gcov sonic_slave: sonic-slave-bullseye archive_gcov: true - -- stage: TestAsan - dependsOn: BuildDockerAsan - condition: succeeded('BuildDockerAsan') - jobs: - - template: .azure-pipelines/test-docker-sonic-vs-template.yml - parameters: - log_artifact_name: log-asan - gcov_artifact_name: sonic-gcov - sonic_slave: sonic-slave-bullseye - docker_sonic_vs_name: docker-sonic-vs-asan - asan: true - -- stage: Gcov - dependsOn: Test - condition: in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues') - jobs: - - template: .azure-pipelines/gcov.yml - parameters: - arch: amd64 - sonic_slave: sonic-slave-bullseye - swss_common_artifact_name: sonic-swss-common - sairedis_artifact_name: sonic-sairedis - swss_artifact_name: sonic-swss - artifact_name: sonic-gcov - archive_gcov: true diff --git a/tests/conftest.py b/tests/conftest.py index 9264417214..14806ae001 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ import sys import tarfile import io +import traceback from typing import Dict, Tuple from datetime import datetime @@ -102,6 +103,11 @@ def pytest_addoption(parser): type=int, help="number of ports") + parser.addoption("--collect-coverage", + action="store_true", + default=False, + help="Collect the test coverage information") + def random_string(size=4, chars=string.ascii_uppercase + string.digits): return "".join(random.choice(chars) for x in range(size)) @@ -283,6 +289,7 @@ def __init__( newctnname: str = None, ctnmounts: Dict[str, str] = None, buffer_model: str = None, + collect_coverage: bool = False ): self.basicd = ["redis-server", "rsyslogd"] self.swssd = [ @@ -304,6 +311,7 @@ def __init__( self.dvsname = name self.vct = vct self.ctn = None + self.collect_coverage = collect_coverage self.cleanup = not keeptb @@ -440,10 +448,33 @@ def del_appl_db(self): if getattr(self, 'appldb', False): del self.appldb - def destroy(self) -> None: self.del_appl_db() + if self.collect_coverage: + try: + # Generate the gcda files + self.runcmd('killall5 -15') + time.sleep(1) + + # Stop the services to reduce the CPU comsuption + if self.cleanup: + self.runcmd('supervisorctl stop all') + + # Generate the converage info by lcov and copy to the host + cmd = f"docker exec {self.ctn.short_id} sh -c 'cd $BUILD_DIR; rm -rf **/.libs ./lib/libSaiRedis*; lcov -c --directory . --no-external --exclude tests --output-file /tmp/coverage.info; sed -i \"s#SF:$BUILD_DIR/#SF:#\" /tmp/coverage.info; lcov_cobertura /tmp/coverage.info -o /tmp/coverage.xml'" + subprocess.getstatusoutput(cmd) + cmd = f"docker exec {self.ctn.short_id} sh -c 'cd $BUILD_DIR; find . -name *.gcda -type f -exec tar -rf /tmp/gcda.tar {{}} \\;'" + subprocess.getstatusoutput(cmd) + cmd = f"docker cp {self.ctn.short_id}:/tmp/gcda.tar {self.ctn.short_id}.gcda.tar" + subprocess.getstatusoutput(cmd) + cmd = f"docker cp {self.ctn.short_id}:/tmp/coverage.info {self.ctn.short_id}.coverage.info" + subprocess.getstatusoutput(cmd) + cmd = f"docker cp {self.ctn.short_id}:/tmp/coverage.xml {self.ctn.short_id}.coverage.xml" + subprocess.getstatusoutput(cmd) + except: + traceback.print_exc() + # In case persistent dvs was used removed all the extra server link # that were created if self.persistent: @@ -451,10 +482,13 @@ def destroy(self) -> None: # persistent and clean-up flag are mutually exclusive elif self.cleanup: - self.ctn.remove(force=True) - self.ctn_sw.remove(force=True) - os.system(f"rm -rf {self.mount}") - self.destroy_servers() + try: + self.ctn.remove(force=True) + self.ctn_sw.remove(force=True) + os.system(f"rm -rf {self.mount}") + self.destroy_servers() + except docker.errors.NotFound: + print("Skipped the container not found error, the container has already removed.") def destroy_servers(self): for s in self.servers: @@ -1400,7 +1434,8 @@ def __init__( log_path=None, max_cpu=2, forcedvs=None, - topoFile=None + topoFile=None, + collect_coverage=False, ): self.ns = namespace self.chassbr = "br4chs" @@ -1414,6 +1449,7 @@ def __init__( self.log_path = log_path self.max_cpu = max_cpu self.forcedvs = forcedvs + self.collect_coverage = collect_coverage if self.ns is None: self.ns = random_string() @@ -1466,7 +1502,7 @@ def find_all_ctns(self): self.dvss[ctn.name] = DockerVirtualSwitch(ctn.name, self.imgname, self.keeptb, self.env, log_path=ctn.name, max_cpu=self.max_cpu, forcedvs=self.forcedvs, - vct=self) + vct=self, collect_coverage=self.collect_coverage) if self.chassbr is None and len(self.dvss) > 0: ret, res = self.ctn_runcmd(self.dvss.values()[0].ctn, "sonic-cfggen --print-data -j /usr/share/sonic/virtual_chassis/vct_connections.json") @@ -1537,6 +1573,8 @@ def handle_request(self): def destroy(self): self.verify_vct() + for dv in self.dvss.values(): + dv.destroy() if self.keeptb: return self.oper = "delete" @@ -1587,7 +1625,8 @@ def create_vct_ctn(self, ctndir): max_cpu=self.max_cpu, forcedvs=self.forcedvs, vct=self,newctnname=ctnname, - ctnmounts=vol) + ctnmounts=vol, + collect_coverage=self.collect_coverage) self.set_ctninfo(ctndir, ctnname, self.dvss[ctnname].pid) return @@ -1759,6 +1798,7 @@ def manage_dvs(request) -> str: buffer_model = request.config.getoption("--buffer_model") force_recreate = request.config.getoption("--force-recreate-dvs") graceful_stop = request.config.getoption("--graceful-stop") + collect_coverage = request.config.getoption("--collect-coverage") dvs = None curr_dvs_env = [] # lgtm[py/unused-local-variable] @@ -1790,7 +1830,7 @@ def update_dvs(log_path, new_dvs_env=[]): dvs.get_logs() dvs.destroy() - dvs = DockerVirtualSwitch(name, imgname, keeptb, new_dvs_env, log_path, max_cpu, forcedvs, buffer_model = buffer_model) + dvs = DockerVirtualSwitch(name, imgname, keeptb, new_dvs_env, log_path, max_cpu, forcedvs, buffer_model = buffer_model, collect_coverage=collect_coverage) curr_dvs_env = new_dvs_env @@ -1811,6 +1851,7 @@ def update_dvs(log_path, new_dvs_env=[]): if graceful_stop: dvs.stop_swss() dvs.stop_syncd() + dvs.get_logs() dvs.destroy() @@ -1839,13 +1880,14 @@ def vst(request): keeptb = request.config.getoption("--keeptb") imgname = request.config.getoption("--imgname") max_cpu = request.config.getoption("--max_cpu") + collect_coverage = request.config.getoption("--collect-coverage") log_path = vctns if vctns else request.module.__name__ dvs_env = getattr(request.module, "DVS_ENV", []) if not topo: # use ecmp topology as default topo = "virtual_chassis/chassis_supervisor.json" vct = DockerVirtualChassisTopology(vctns, imgname, keeptb, dvs_env, log_path, max_cpu, - forcedvs, topo) + forcedvs, topo, collect_coverage) yield vct vct.get_logs(request.module.__name__) vct.destroy() @@ -1858,13 +1900,14 @@ def vct(request): keeptb = request.config.getoption("--keeptb") imgname = request.config.getoption("--imgname") max_cpu = request.config.getoption("--max_cpu") + collect_coverage = request.config.getoption("--collect-coverage") log_path = vctns if vctns else request.module.__name__ dvs_env = getattr(request.module, "DVS_ENV", []) if not topo: # use ecmp topology as default topo = "virtual_chassis/chassis_with_ecmp_neighbors.json" vct = DockerVirtualChassisTopology(vctns, imgname, keeptb, dvs_env, log_path, max_cpu, - forcedvs, topo) + forcedvs, topo, collect_coverage) yield vct vct.get_logs(request.module.__name__) vct.destroy()