Skip to content

Commit

Permalink
[ci][asan] add DVS tests run with ASAN (#2441)
Browse files Browse the repository at this point in the history
This adds 3 new stages:

BuildAsan: same as 'Build' but with ENABLE_ASAN=y
BuildDockerAsan: same as 'BuildDocker' but uses asan-enabled swss and
adds swss-dbg package. It also uses asan-enabled docker-sonic-vs from
the sonic-buildimage ci.
TestAsan: same as 'Test' but uses asan-enabled docker and publishes
the asan reports if any. It also shows a warning if asan report dir is not empty
(which means that there is some memory usage issue in swss)

- What I did
Added new CI stages to run the DVS tests with ASAN and check the ASAN reports.

- Why I did it
To utilize ASAN for catching memory usage issues on PR.

- How I verified it
Run the CI on my fork.

Signed-off-by: Yakiv Huryk <yhuryk@nvidia.com>
  • Loading branch information
Yakiv-Huryk authored Sep 22, 2022
1 parent 04fbc8e commit 3d757a8
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 31 deletions.
21 changes: 15 additions & 6 deletions .azure-pipelines/build-docker-sonic-vs-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ parameters:
- name: artifact_name
type: string

- name: asan
type: boolean
default: false

jobs:
- job:
displayName: ${{ parameters.arch }}
Expand Down Expand Up @@ -66,27 +70,32 @@ jobs:
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/$(BUILD_BRANCH)'
path: $(Build.ArtifactStagingDirectory)/download
patterns: '**/target/docker-sonic-vs.gz'
displayName: "Download sonic-buildimage docker-sonic-vs"
patterns: '**/target/${{ parameters.artifact_name }}.gz'
displayName: "Download sonic-buildimage ${{ parameters.artifact_name }}"
- script: |
set -ex
echo $(Build.DefinitionName).$(Build.BuildNumber)
docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
docker load < $(Build.ArtifactStagingDirectory)/download/target/${{ parameters.artifact_name }}.gz
mkdir -p .azure-pipelines/docker-sonic-vs/debs
cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
pushd .azure-pipelines
docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs
build_args=""
if [ '${{ parameters.asan }}' == True ]; then
build_args="--build-arg need_dbg=y"
fi
docker build $build_args --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} docker-sonic-vs
popd
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
rm -rf $(Build.ArtifactStagingDirectory)/download
displayName: "Build docker-sonic-vs"
displayName: "Build ${{ parameters.artifact_name }}"
- publish: $(Build.ArtifactStagingDirectory)/
artifact: ${{ parameters.artifact_name }}
displayName: "Archive sonic docker vs image"
7 changes: 7 additions & 0 deletions .azure-pipelines/build-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ parameters:
- name: common_lib_artifact_name
type: string

- name: asan
type: boolean
default: false

jobs:
- job:
displayName: ${{ parameters.arch }}
Expand Down Expand Up @@ -139,6 +143,9 @@ jobs:
if [ '${{ parameters.archive_gcov }}' == True ]; then
export ENABLE_GCOV=y
fi
if [ '${{ parameters.asan }}' == True ]; then
export ENABLE_ASAN=y
fi
./autogen.sh
dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
displayName: "Compile sonic swss"
Expand Down
2 changes: 2 additions & 0 deletions .azure-pipelines/docker-sonic-vs/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
FROM docker-sonic-vs

ARG docker_container_name
ARG need_dbg

ADD ["debs", "/debs"]

Expand All @@ -18,6 +19,7 @@ RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb

RUN dpkg --purge swss
RUN dpkg -i /debs/swss_1.0.0_amd64.deb
RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/swss-dbg_1.0.0_amd64.deb ; fi

RUN apt-get update

Expand Down
84 changes: 59 additions & 25 deletions .azure-pipelines/test-docker-sonic-vs-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ parameters:
type: boolean
default: false

- name: docker_sonic_vs_name
type: string
default: docker-sonic-vs

- name: asan
type: boolean
default: false

jobs:
- job:
displayName: vstest
Expand All @@ -30,9 +38,9 @@ jobs:
- checkout: self
- task: DownloadPipelineArtifact@2
inputs:
artifact: docker-sonic-vs
artifact: ${{ parameters.docker_sonic_vs_name }}
path: $(Build.ArtifactStagingDirectory)/download
displayName: "Download pre-stage built docker-sonic-vs"
displayName: "Download pre-stage built ${{ parameters.docker_sonic_vs_name }}"
- task: DownloadPipelineArtifact@2
inputs:
source: specific
Expand Down Expand Up @@ -68,40 +76,46 @@ jobs:
sudo /sbin/ip link del Vrf1 type vrf table 1001
pushd tests
params=''
if [ '${{ parameters.archive_gcov }}' == True ]; then
all_tests=$(ls test_*.py)
all_tests="${all_tests} p4rt"
test_set=()
# Run 20 tests as a set.
for test in ${all_tests}; do
test_set+=("${test}")
if [ ${#test_set[@]} -ge 20 ]; then
test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
container_count=$(docker ps -q -a | wc -l)
if [ ${container_count} -gt 0 ]; then
./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
docker stop $(docker ps -q -a)
docker rm $(docker ps -q -a)
fi
test_set=()
fi
done
if [ ${#test_set[@]} -gt 0 ]; then
params='--keeptb'
fi
if [ '${{ parameters.asan }}' == True ]; then
params='--graceful-stop'
fi
all_tests=$(ls test_*.py)
all_tests="${all_tests} p4rt"
test_set=()
# Run 20 tests as a set.
for test in ${all_tests}; do
test_set+=("${test}")
if [ ${#test_set[@]} -ge 20 ]; then
test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
container_count=$(docker ps -q -a | wc -l)
if [ ${container_count} -gt 0 ]; then
if [ '${{ parameters.archive_gcov }}' == True ] && [ ${container_count} -gt 0 ]; then
./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
docker stop $(docker ps -q -a)
docker rm $(docker ps -q -a)
fi
test_set=()
fi
done
if [ ${#test_set[@]} -gt 0 ]; then
test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
echo "${test_set[*]}" | xargs sudo py.test -v $params --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
container_count=$(docker ps -q -a | wc -l)
if [ '${{ parameters.archive_gcov }}' == True ] && [ ${container_count} -gt 0 ]; then
./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
docker stop $(docker ps -q -a)
docker rm $(docker ps -q -a)
fi
else
sudo py.test -v --force-flaky --junitxml=tests_tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
fi
rm -rf $(Build.ArtifactStagingDirectory)/download
displayName: "Run vs tests"
continueOnError: ${{ parameters.asan }}
- task: PublishTestResults@2
inputs:
Expand All @@ -112,6 +126,10 @@ jobs:
- script: |
cp -r tests/log $(Build.ArtifactStagingDirectory)/
if [ '${{ parameters.asan }}' == True ]; then
cp -vr tests/log/*/log/asan $(Build.ArtifactStagingDirectory)/
fi
if [ '${{ parameters.archive_gcov }}' == True ]; then
sudo apt-get install -y lcov
cd $(Build.ArtifactStagingDirectory)/gcov_tmp/
Expand All @@ -130,3 +148,19 @@ jobs:
artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)
displayName: "Publish logs"
condition: always()

- publish: $(Build.ArtifactStagingDirectory)/asan
artifact: asan-reports
displayName: "Publish ASAN reports"
condition: eq('${{ parameters.asan }}', true)

- script: |
if [ "$(ls -A $(Build.ArtifactStagingDirectory)/asan)" ]; then
echo "There are issues reported by ASAN"
exit 1
else
echo "No issues reported by ASAN"
fi
displayName: "Check ASAN reports"
condition: eq('${{ parameters.asan }}', true)
continueOnError: true
37 changes: 37 additions & 0 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,19 @@ stages:
archive_pytests: true
archive_gcov: true

- stage: BuildAsan
dependsOn: []
jobs:
- template: .azure-pipelines/build-template.yml
parameters:
arch: amd64
sonic_slave: sonic-slave-buster
common_lib_artifact_name: common-lib
swss_common_artifact_name: sonic-swss-common
sairedis_artifact_name: sonic-sairedis
artifact_name: sonic-swss-asan
asan: true

- stage: BuildArm
dependsOn: Build
condition: succeeded('Build')
Expand Down Expand Up @@ -88,6 +101,18 @@ stages:
swss_artifact_name: sonic-swss
artifact_name: docker-sonic-vs

- stage: BuildDockerAsan
dependsOn: BuildAsan
condition: succeeded('BuildAsan')
jobs:
- template: .azure-pipelines/build-docker-sonic-vs-template.yml
parameters:
swss_common_artifact_name: sonic-swss-common
sairedis_artifact_name: sonic-sairedis
swss_artifact_name: sonic-swss-asan
artifact_name: docker-sonic-vs-asan
asan: true

- stage: Test
dependsOn: BuildDocker
condition: succeeded('BuildDocker')
Expand All @@ -99,6 +124,18 @@ stages:
sonic_slave: sonic-slave-buster
archive_gcov: true

- stage: TestAsan
dependsOn: BuildDockerAsan
condition: succeeded('BuildDockerAsan')
jobs:
- template: .azure-pipelines/test-docker-sonic-vs-template.yml
parameters:
log_artifact_name: log-asan
gcov_artifact_name: sonic-gcov
sonic_slave: sonic-slave-buster
docker_sonic_vs_name: docker-sonic-vs-asan
asan: true

- stage: Gcov
dependsOn: Test
condition: in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues')
Expand Down

0 comments on commit 3d757a8

Please sign in to comment.