From 7c6f2a4b1f8e246d311df397b1e302a72af2e659 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Wed, 20 Dec 2023 16:38:18 -0800 Subject: [PATCH 1/6] [HOTFIX] Fix build with -DMIOPEN_USE_COMPOSABLEKERNEL=Off after #2517. --- src/solver/conv_ck_igemm_fwd_bias_res_add_activ_fused.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/solver/conv_ck_igemm_fwd_bias_res_add_activ_fused.cpp b/src/solver/conv_ck_igemm_fwd_bias_res_add_activ_fused.cpp index 59700e9858..0f9c373b46 100644 --- a/src/solver/conv_ck_igemm_fwd_bias_res_add_activ_fused.cpp +++ b/src/solver/conv_ck_igemm_fwd_bias_res_add_activ_fused.cpp @@ -46,6 +46,8 @@ namespace miopen { namespace solver { namespace fusion { +#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL + using CK_OutLayout = ck::tensor_layout::convolution::NDHWGK; // DataType also applies to weights @@ -67,7 +69,6 @@ using DeviceOp = ck::tensor_operation::device::instance::DeviceOperationInstance ck::tensor_operation::element_wise:: ScaleAddScaleAddRelu>>; // end DeviceOperationInstanceFactory -#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL namespace { struct CKArgs From 623d1465c9374a75debb688201727b1e020e046d Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Wed, 20 Dec 2023 16:40:59 -0800 Subject: [PATCH 2/6] add -DMIOPEN_USE_COMPOSABLEKERNEL=Off stage --- Jenkinsfile | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e19a4ffacf..7bcc9735d7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -406,7 +406,7 @@ def CheckPerfDbValid(Map conf=[:]){ /// * The default compiler is usually not specified. /// BuildType := { Release* | Debug | Install } [ BuildTypeModifier ] /// * BuildTypeModifier := { NOCOMGR | Embedded | Static | Normal-Find | Fast-Find -/// CK | NOMLIR | Tensile | Tensile-Latest | Package | ... } +/// NOCK | NOMLIR | Tensile | Tensile-Latest | Package | ... } /// TestSet := { All | Smoke* | Performance Dataset } [ Codecov ] /// * "All" corresponds to "cmake -DMIOPEN_TEST_ALL=On". /// * "Smoke" (-DMIOPEN_TEST_ALL=Off) is the default and usually not specified. @@ -533,6 +533,7 @@ pipeline { Smoke_targets = " check MIOpenDriver" NOCOMGR_flags = " -DMIOPEN_USE_COMGR=Off" NOMLIR_flags = " -DMIOPEN_USE_MLIR=Off" + NOCK_flags = " -DMIOPEN_USE_COMPOSABLEKERNEL=Off" } triggers{ @@ -729,6 +730,23 @@ pipeline { buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ') } } + stage('Fp32 Hip Debug NOCK gfx908/gfx90a') { + when { + beforeAgent true + expression { params.TARGET_GFX908 || params.TARGET_GFX90A } + } + options { + retry(2) + } + agent{ label rocmnode("gfx908 || gfx90a") } + environment{ + // Can be removed altogether with when WORKAROUND_SWDEV_290754. + NOCK_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_CONV_PRECISE_ROCBLAS_TIMING=0 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" + } + steps{ + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCK_flags, build_cmd: NOCK_build_cmd, test_flags: ' --verbose ') + } + } stage('Fp32 Hip Debug Embedded Vega20') { when { beforeAgent true From 5c7130756614420fc7c37c780aa20af4f38cad12 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Wed, 20 Dec 2023 20:57:39 -0800 Subject: [PATCH 3/6] make NOCK stage anyAPU and build ONLY --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7bcc9735d7..f6d5e65a03 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -730,21 +730,21 @@ pipeline { buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ') } } - stage('Fp32 Hip Debug NOCK gfx908/gfx90a') { + stage('Fp32 Hip Debug NOCK AnyGPU Build-Only') { when { beforeAgent true - expression { params.TARGET_GFX908 || params.TARGET_GFX90A } + expression { params.TARGET_VEGA20 || params.TARGET_VEGA10 || params.TARGET_GFX908 || params.TARGET_GFX90A } } options { retry(2) } - agent{ label rocmnode("gfx908 || gfx90a") } + agent{ label rocmnode("vega || gfx908 || gfx90a") } environment{ // Can be removed altogether with when WORKAROUND_SWDEV_290754. - NOCK_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_CONV_PRECISE_ROCBLAS_TIMING=0 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" + NOCK_build_cmd = "make -j\$(nproc)" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCK_flags, build_cmd: NOCK_build_cmd, test_flags: ' --verbose ') + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCK_flags, build_cmd: NOCK_build_cmd ) } } stage('Fp32 Hip Debug Embedded Vega20') { From 82ae4839f2a844421c4c45fad99fcfec11b307d7 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 21 Dec 2023 09:22:23 -0800 Subject: [PATCH 4/6] Adopt recommended changes --- Jenkinsfile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f6d5e65a03..d0ab5e2107 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -407,7 +407,7 @@ def CheckPerfDbValid(Map conf=[:]){ /// BuildType := { Release* | Debug | Install } [ BuildTypeModifier ] /// * BuildTypeModifier := { NOCOMGR | Embedded | Static | Normal-Find | Fast-Find /// NOCK | NOMLIR | Tensile | Tensile-Latest | Package | ... } -/// TestSet := { All | Smoke* | Performance Dataset } [ Codecov ] +/// TestSet := { All | Smoke* | | Build-only } [ Codecov ] /// * "All" corresponds to "cmake -DMIOPEN_TEST_ALL=On". /// * "Smoke" (-DMIOPEN_TEST_ALL=Off) is the default and usually not specified. /// * "Codecov" is optional code coverage analysis. @@ -533,7 +533,6 @@ pipeline { Smoke_targets = " check MIOpenDriver" NOCOMGR_flags = " -DMIOPEN_USE_COMGR=Off" NOMLIR_flags = " -DMIOPEN_USE_MLIR=Off" - NOCK_flags = " -DMIOPEN_USE_COMPOSABLEKERNEL=Off" } triggers{ @@ -739,12 +738,8 @@ pipeline { retry(2) } agent{ label rocmnode("vega || gfx908 || gfx90a") } - environment{ - // Can be removed altogether with when WORKAROUND_SWDEV_290754. - NOCK_build_cmd = "make -j\$(nproc)" - } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCK_flags, build_cmd: NOCK_build_cmd ) + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", config_targets: "") } } stage('Fp32 Hip Debug Embedded Vega20') { From 8a976bbf2b600a875c13304a5c53cad121234c65 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 21 Dec 2023 09:26:45 -0800 Subject: [PATCH 5/6] rename config_targets to make_targets --- Jenkinsfile | 62 ++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d0ab5e2107..0417571e50 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ def show_node_info() { def cmake_build(Map conf=[:]){ def compiler = conf.get("compiler","/opt/rocm/llvm/bin/clang++") - def config_targets = conf.get("config_targets","check") + def make_targets = conf.get("make_targets","check") def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") def build_envs = "CTEST_PARALLEL_LEVEL=4 MIOPEN_CONV_PRECISE_ROCBLAS_TIMING=0 " + conf.get("build_env","") def prefixpath = conf.get("prefixpath","/opt/rocm") @@ -47,13 +47,13 @@ def cmake_build(Map conf=[:]){ def package_build = (conf.get("package_build","") == "true") if (package_build == true) { - config_targets = "package" + make_targets = "package" } def miopen_install_path = "${env.WORKSPACE}/install" if(conf.get("build_install","") == "true") { - config_targets = 'install ' + config_targets + make_targets = 'install ' + make_targets setup_args = " -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=${miopen_install_path}" + setup_args } else{ setup_args = ' -DBUILD_DEV=On' + setup_args @@ -110,7 +110,7 @@ def cmake_build(Map conf=[:]){ def setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args} .. ") // WORKAROUND_SWDEV_290754 // It seems like this W/A is not required since 4.5. - def build_cmd = conf.get("build_cmd", "LLVM_PATH=/opt/rocm/llvm ${build_envs} dumb-init make -j\$(nproc) ${config_targets}") + def build_cmd = conf.get("build_cmd", "LLVM_PATH=/opt/rocm/llvm ${build_envs} dumb-init make -j\$(nproc) ${make_targets}") def execute_cmd = conf.get("execute_cmd", "") def cmd = conf.get("cmd", """ @@ -149,7 +149,7 @@ def cmake_build(Map conf=[:]){ def cmake_fin_build_cmd(prefixpath){ def flags = "-DCMAKE_INSTALL_PREFIX=${prefixpath} -DCMAKE_BUILD_TYPE=release" def compiler = 'clang++' - def config_targets = "install" + def make_targets = "install" def compilerpath = "/opt/rocm/llvm/bin/" + compiler def configargs = "" if (prefixpath != "") @@ -164,7 +164,7 @@ def cmake_fin_build_cmd(prefixpath){ mkdir build cd build CXX=${compilerpath} cmake ${configargs} ${flags} .. - dumb-init make -j\$(nproc) ${config_targets} + dumb-init make -j\$(nproc) ${make_targets} """ return fin_cmd } @@ -613,7 +613,7 @@ pipeline { fin_flags = "-DMIOPEN_BACKEND=HIPNOGPU" } steps{ - buildHipClangJobAndReboot(setup_flags: fin_flags, config_targets: "all", build_fin: "ON", needs_gpu:false, needs_reboot:false, build_install: "true") + buildHipClangJobAndReboot(setup_flags: fin_flags, make_targets: "all", build_fin: "ON", needs_gpu:false, needs_reboot:false, build_install: "true") } } } @@ -633,7 +633,7 @@ pipeline { } agent{ label rocmnode("vega || gfx908 || gfx90a") } steps{ - buildHipClangJobAndReboot(config_targets: Smoke_targets) + buildHipClangJobAndReboot(make_targets: Smoke_targets) } } stage('Fp32 Hip Debug AnyGPU') { @@ -646,7 +646,7 @@ pipeline { } agent{ label rocmnode("vega || gfx908 || gfx90a") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', config_targets: Smoke_targets) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets) } } stage('Fp32 Hip Debug gfx908') { @@ -659,7 +659,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', config_targets: Smoke_targets) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets) } } stage('Fp32 Hip Debug gfx90a') { @@ -672,7 +672,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', config_targets: Smoke_targets) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets) } } stage('Fp32 Hip Debug gfx94X') { @@ -685,7 +685,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', config_targets: Smoke_targets) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets) } } } @@ -739,7 +739,7 @@ pipeline { } agent{ label rocmnode("vega || gfx908 || gfx90a") } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", config_targets: "") + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "") } } stage('Fp32 Hip Debug Embedded Vega20') { @@ -781,11 +781,11 @@ pipeline { } agent{ label rocmnode("vega || gfx908 || gfx90a") } environment{ - config_targets = "test_conv2d" + make_targets = "test_conv2d" execute_cmd = "MIOPEN_CONV_PRECISE_ROCBLAS_TIMING=0 bin/test_conv2d --disable-verification-cache" } steps{ - buildHipClangJobAndReboot(config_targets: config_targets, execute_cmd: execute_cmd, find_mode: "Normal") + buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal") } } stage('Fp32 Hip Fast-Find AnyGPU') { @@ -798,11 +798,11 @@ pipeline { } agent{ label rocmnode("vega || gfx908 || gfx90a") } environment{ - config_targets = "test_conv2d" + make_targets = "test_conv2d" execute_cmd = "MIOPEN_FIND_MODE=2 CTEST_PARALLEL_LEVEL=4 MIOPEN_CONV_PRECISE_ROCBLAS_TIMING=0 bin/test_conv2d --disable-verification-cache" } steps{ - buildHipClangJobAndReboot( config_targets: config_targets, execute_cmd: execute_cmd) + buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd) } } stage('Fp32 Hip AnyGPU') { @@ -835,7 +835,7 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, config_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) } } stage('Bf16 Hip Vega20') { @@ -848,7 +848,7 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, config_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) } } stage('Fp16 Hip gfx908') { @@ -861,7 +861,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, config_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) } } stage('Bf16 Hip gfx908') { @@ -874,7 +874,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, config_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) } } stage('Fp16 Hip gfx90a') { @@ -887,7 +887,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, config_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) } } stage('Bf16 Hip gfx90a') { @@ -900,7 +900,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, config_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) } } stage('Fp16 Hip gfx94X') { @@ -913,7 +913,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, config_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) } } stage('Bf16 Hip gfx94X') { @@ -926,7 +926,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, config_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) } } } @@ -952,11 +952,11 @@ pipeline { agent{ label rocmnode("gfx908") } environment{ setup_flags="-DMIOPEN_TEST_DBSYNC=1" - config_targets='test_db_sync' - execute_cmd='./bin/test_db_sync' + make_targets='test_db_sync' + execute_cmd='./bin/test_db_sync' } steps{ - buildHipClangJobAndReboot(lfs_pull: true, setup_flags: setup_flags, config_targets: config_targets, execute_cmd: execute_cmd, + buildHipClangJobAndReboot(lfs_pull: true, setup_flags: setup_flags, make_targets: make_targets, execute_cmd: execute_cmd, needs_gpu:false, needs_reboot:false, build_install: "true") } } @@ -971,11 +971,11 @@ pipeline { agent{ label rocmnode("gfx90a") } environment{ setup_flags="-DMIOPEN_TEST_DBSYNC=1" - config_targets='test_db_sync' - execute_cmd='./bin/test_db_sync' + make_targets='test_db_sync' + execute_cmd='./bin/test_db_sync' } steps{ - buildHipClangJobAndReboot(lfs_pull: true, setup_flags: setup_flags, config_targets: config_targets, execute_cmd: execute_cmd, + buildHipClangJobAndReboot(lfs_pull: true, setup_flags: setup_flags, make_targets: make_targets, execute_cmd: execute_cmd, needs_gpu:false, needs_reboot:false, build_install: "true") } } From fe1b539518be75c38471c6858416d2807f3b050c Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Thu, 21 Dec 2023 10:28:25 -0800 Subject: [PATCH 6/6] Extend GTest DISCOVERY_TIMEOUT to 5 mins --- test/gtest/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/gtest/CMakeLists.txt b/test/gtest/CMakeLists.txt index 2e80fb5eac..eb3a716232 100644 --- a/test/gtest/CMakeLists.txt +++ b/test/gtest/CMakeLists.txt @@ -30,7 +30,8 @@ function(add_gtest TEST_NAME) endif() if(NOT WIN32) # TODO: cannot run on Windows due to missing DLL dependencies # Enable CMake to discover the test binary - gtest_discover_tests(test_${TEST_NAME} PROPERTIES ENVIRONMENT "MIOPEN_USER_DB_PATH=${CMAKE_CURRENT_BINARY_DIR};MIOPEN_TEST_FLOAT_ARG=${MIOPEN_TEST_FLOAT_ARG};MIOPEN_TEST_ALL=${MIOPEN_TEST_ALL};MIOPEN_TEST_MLIR=${MIOPEN_TEST_MLIR};MIOPEN_TEST_COMPOSABLEKERNEL=${MIOPEN_TEST_COMPOSABLEKERNEL}") + # Extend GTest DISCOVERY_TIMEOUT to 5 mins + gtest_discover_tests(test_${TEST_NAME} DISCOVERY_TIMEOUT 300 PROPERTIES ENVIRONMENT "MIOPEN_USER_DB_PATH=${CMAKE_CURRENT_BINARY_DIR};MIOPEN_TEST_FLOAT_ARG=${MIOPEN_TEST_FLOAT_ARG};MIOPEN_TEST_ALL=${MIOPEN_TEST_ALL};MIOPEN_TEST_MLIR=${MIOPEN_TEST_MLIR};MIOPEN_TEST_COMPOSABLEKERNEL=${MIOPEN_TEST_COMPOSABLEKERNEL}") endif() if(WIN32)