[WA] Disable MLIR when building debug to workaround sanitizer issue (#…

…3200)
ROCm · Aug 16, 2024 · 4be2a03 · 4be2a03
1 parent a6a59c2
commit 4be2a03
Show file tree

Hide file tree

Showing 9 changed files with 26 additions and 11 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -33,14 +33,21 @@ def cmake_build(Map conf=[:]){
     def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "")
     def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
     def prefixpath = conf.get("prefixpath","/opt/rocm")
+    def build_type_debug = (conf.get("build_type",'release') == 'debug')
+    def code_conv_enabled = conf.get("codecov", false)
+
     def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON")
+    // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+    if (build_type_debug || code_conv_enabled)
+    {
+        mlir_args = " -DMIOPEN_USE_MLIR=OFF"
+    }
+
     def setup_args = mlir_args + " -DMIOPEN_GPU_SYNC=Off " + conf.get("setup_flags","")
     def build_fin = conf.get("build_fin", "OFF")
 
     setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
 
-    def build_type_debug = (conf.get("build_type",'release') == 'debug')
-
     //cmake_env can overwrite default CXX variables.
     def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")
 
@@ -72,7 +79,7 @@ def cmake_build(Map conf=[:]){
         test_flags = " --disable-verification-cache " + test_flags
     }
 
-    if(conf.get("codecov", false)){ //Need
+    if(code_conv_enabled){ //Need
         setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags} -fprofile-arcs -ftest-coverage' -DCODECOV_TEST=On " + setup_args
     }else if(build_type_debug){
         setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args

diff --git a/test/nightlies/JenkinsfileNightlyAux b/test/nightlies/JenkinsfileNightlyAux
@@ -46,7 +46,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
 

diff --git a/test/nightlies/JenkinsfileNightlyConv2D b/test/nightlies/JenkinsfileNightlyConv2D
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
 

diff --git a/test/nightlies/JenkinsfileNightlyConv2Daux b/test/nightlies/JenkinsfileNightlyConv2Daux
@@ -48,7 +48,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
 

diff --git a/test/nightlies/JenkinsfileNightlyFastFullConv2D b/test/nightlies/JenkinsfileNightlyFastFullConv2D
@@ -43,7 +43,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 12, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_ENABLE_LOGGING_CMD=1 MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
 

diff --git a/test/nightlies/JenkinsfileNightlyFusions b/test/nightlies/JenkinsfileNightlyFusions
@@ -46,7 +46,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
 

diff --git a/test/nightlies/JenkinsfileNightlyRNN b/test/nightlies/JenkinsfileNightlyRNN
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=Off -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
 

diff --git a/test/nightlies/JenkinsfileNightlyReduce b/test/nightlies/JenkinsfileNightlyReduce
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 6, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
 

diff --git a/test/nightlies/JenkinsfileNightlyTensorOps b/test/nightlies/JenkinsfileNightlyTensorOps
@@ -48,7 +48,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 6, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=Off -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"