hotfix: use hipModuleLoadData instead of hipModuleLoad (#1575)

ROCm · Aug 31, 2022 · b33ca97 · b33ca97
1 parent 6d62a2b
commit b33ca97
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 3 deletions.
diff --git a/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp b/Tensile/Source/lib/include/Tensile/hip/HipSolutionAdapter.hpp
@@ -87,6 +87,7 @@ namespace Tensile
             std::mutex m_access;
 
             std::vector<hipModule_t>                       m_modules;
+            std::vector<std::unique_ptr<char[]>>           m_moduleBuffers;
             std::unordered_map<std::string, hipFunction_t> m_kernels;
             bool                                           m_debug           = false;
             bool                                           m_debugSkipLaunch = false;

diff --git a/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp b/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp
@@ -28,6 +28,7 @@
 #include <hip/hip_runtime.h>
 
 #include <cstddef>
+#include <fstream>
 
 #include <Tensile/Debug.hpp>
 #include <Tensile/EmbeddedData.hpp>
@@ -81,9 +82,27 @@ namespace Tensile
 
         hipError_t SolutionAdapter::loadCodeObjectFile(std::string const& path)
         {
-            hipModule_t module;
+            hipModule_t             module;
+            std::unique_ptr<char[]> buffer;
+            std::ifstream           coFile(path, std::ifstream::binary);
+
+            // hipModuleLoad holds the file descriptor/handle which can result in a process
+            // running out of descriptors/handles. Use hipModuleLoadData as a workaround
+            if(coFile)
+            {
+                coFile.seekg(0, coFile.end);
+                auto length = coFile.tellg();
+                coFile.seekg(0, coFile.beg);
 
-            HIP_CHECK_RETURN(hipModuleLoad(&module, path.c_str()));
+                buffer = std::make_unique<char[]>(length);
+                coFile.read(buffer.get(), length);
+
+                HIP_CHECK_RETURN(hipModuleLoadData(&module, (void*)buffer.get()));
+            }
+            else
+            {
+                return hipErrorFileNotFound;
+            }
 
             if(m_debug)
                 std::cout << "loaded code object " << path << std::endl;
@@ -93,6 +112,9 @@ namespace Tensile
                 m_modules.push_back(module);
                 m_loadedModuleNames.push_back(concatenate("File ", path));
 
+                // hipModuleLoadData requires the buffer to outlive the module, so cache the buffer
+                m_moduleBuffers.push_back(std::move(buffer));
+
                 //Isolate filename
                 size_t start = path.rfind('/');
                 start        = (start == std::string::npos) ? 0 : start + 1;
@@ -242,7 +264,7 @@ namespace Tensile
                 for(auto ver : {"", "-xnack-", "-xnack+"})
                 {
                     std::string modifiedCOName = helperKernelName + ver + ".hsaco";
-                    err = loadCodeObjectFile(codeObjectDir + modifiedCOName);
+                    err                        = loadCodeObjectFile(codeObjectDir + modifiedCOName);
 
                     if(err == hipSuccess)
                         return err;