Merge pull request #205 from vin-huang/fix_asan_build

Auto-submit by Jenkins

Author: rocm-ci

CMakeLists.txt

@@ -33,6 +33,8 @@ option(BUILD_CODE_COVERAGE "Build with code coverage enabled" OFF)
 option(BUILD_ADDRESS_SANITIZER "Build with address sanitizer enabled" OFF)
 # Find CUDA if the user wants a CUDA version.
 option(BUILD_CUDA "Look for CUDA and use that as a backend if found" OFF)
+option(Tensile_SEPARATE_ARCHITECTURES "Tensile to use GPU architecture specific files?" ON)
+option(Tensile_LAZY_LIBRARY_LOADING "Tensile to load kernels on demand?" ON)
 
 # IF cuda backend disable clients tests.
 if(BUILD_CUDA)
@@ -241,6 +243,16 @@ set(hipsparselt_SOVERSION 0.2)
 # setup rocsparselt defines used for both the library and clients
 if( BUILD_WITH_TENSILE )
     list(APPEND TENSILE_DEFINES BUILD_WITH_TENSILE=1)
+    if(Tensile_SEPARATE_ARCHITECTURES)
+      list(APPEND TENSILE_DEFINES ROCSPARSELT_TENSILE_SEPARATE_ARCH=1)
+    else()
+      list(APPEND TENSILE_DEFINES ROCSPARSELT_TENSILE_SEPARATE_ARCH=0)
+    endif()
+    if(Tensile_LAZY_LIBRARY_LOADING)
+      list(APPEND TENSILE_DEFINES ROCSPARSELT_TENSILE_LAZY_LOAD=1)
+    else()
+      list(APPEND TENSILE_DEFINES ROCSPARSELT_TENSILE_LAZY_LOAD=0)
+    endif()
 else()
     list(APPEND TENSILE_DEFINES BUILD_WITH_TENSILE=0)
 endif()
@@ -349,7 +361,7 @@ set( HIPSPARSELTS_CONFIG_DIR "\${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL
 
 rocm_create_package(
     NAME ${package_name}
-    DESCRIPTION "Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library"
+    DESCRIPTION "ROCm Structured Sparsity Matrix Multiplication marshalling library"
     MAINTAINER "hipSPARSELt Maintainer <hipsparselt-maintainer@amd.com>"
     LDCONFIG
     LDCONFIG_DIR ${HIPSPARSELTS_CONFIG_DIR}

clients/CMakeLists.txt

@@ -29,7 +29,7 @@ if( NOT DEFINED CMAKE_CONFIGURATION_TYPES AND NOT DEFINED CMAKE_BUILD_TYPE )
   set( CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." )
 endif()
 
-project( hipsparselt-clients LANGUAGES CXX C  )
+project( hipsparselt-clients LANGUAGES CXX C Fortran )
 
 
 if( SKIP_LIBRARY )

clients/benchmarks/CMakeLists.txt

@@ -78,8 +78,10 @@ if (NOT WIN32)
     target_link_libraries( hipsparselt-bench PRIVATE lapack cblas )
   endif()
   list( APPEND COMMON_LINK_LIBS "-lm -lstdc++fs")
-  if (NOT BUILD_FORTRAN_CLIENTS)
-    list( APPEND COMMON_LINK_LIBS "-lgfortran -lflang -lflangrti") # for lapack
+  if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
+    list( APPEND COMMON_LINK_LIBS "-lgfortran") # for lapack
+  else()
+    list( APPEND COMMON_LINK_LIBS "-lflang -lflangrti") # for lapack
   endif()
 else()
   list( APPEND COMMON_LINK_LIBS "libomp")

clients/gtest/CMakeLists.txt

@@ -87,8 +87,10 @@ endif()
 
 list( APPEND COMMON_LINK_LIBS "-lm -lstdc++fs")
 
-if (NOT BUILD_FORTRAN_CLIENTS)
-  list( APPEND COMMON_LINK_LIBS "-lgfortran -lflang -lflangrti") # for lapack
+if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
+  list( APPEND COMMON_LINK_LIBS "-lgfortran") # for lapack
+else()
+  list( APPEND COMMON_LINK_LIBS "-lflang -lflangrti") # for lapack
 endif()
 
 #if (NOT WIN32)

cmake/Dependencies.cmake

@@ -35,15 +35,15 @@ find_package(ROCM 0.6 QUIET CONFIG PATHS ${CMAKE_PREFIX_PATH})
 if(NOT ROCM_FOUND)
   set(PROJECT_EXTERN_DIR ${CMAKE_CURRENT_BINARY_DIR}/extern)
   set(rocm_cmake_tag "master" CACHE STRING "rocm-cmake tag to download")
-  file(DOWNLOAD https://github.com/RadeonOpenCompute/rocm-cmake/archive/${rocm_cmake_tag}.zip
+  file(DOWNLOAD https://github.com/ROCm/rocm-cmake/archive/${rocm_cmake_tag}.zip
        ${PROJECT_EXTERN_DIR}/rocm-cmake-${rocm_cmake_tag}.zip STATUS status LOG log)
 
   list(GET status 0 status_code)
   list(GET status 1 status_string)
 
   if(NOT status_code EQUAL 0)
     message(FATAL_ERROR "error: downloading
-    'https://github.com/RadeonOpenCompute/rocm-cmake/archive/${rocm_cmake_tag}.zip' failed
+    'https://github.com/ROCm/rocm-cmake/archive/${rocm_cmake_tag}.zip' failed
     status_code: ${status_code}
     status_string: ${status_string}
     log: ${log}

docs/tutorials/install/linux.rst

@@ -46,7 +46,7 @@ instructions.
     * `hipSPARSE <https://github.com/ROCm/hipSPARSE>`_
     * `git <https://git-scm.com/>`_
     * `CMake <https://cmake.org/>`_ 3.5 or later
-    * `AMD ROCm <https://github.com/RadeonOpenCompute/ROCm>`_
+    * `AMD ROCm <https://github.com/ROCm/ROCm>`_
     * (Optional, for clients) `GoogleTest <https://github.com/google/googletest>`_
 
 Download hipSPARSELt

library/src/CMakeLists.txt

@@ -53,7 +53,12 @@ if(NOT BUILD_CUDA)
     if(PACKAGE_TENSILE_LIBRARY)
       set(Tensile_Options ${Tensile_Options} GENERATE_PACKAGE)
     endif()
-
+    if(Tensile_SEPARATE_ARCHITECTURES)
+      set(Tensile_Options ${Tensile_Options} SEPARATE_ARCHITECTURES)
+    endif()
+    if(Tensile_LAZY_LIBRARY_LOADING)
+      set(Tensile_Options ${Tensile_Options} LAZY_LIBRARY_LOADING)
+    endif()
     if(Tensile_BUILD_ID)
       set(Options ${Options} "--build-id=${Tensile_BUILD_ID}")
     endif()

library/src/hcc_detail/rocsparselt/src/tensile_host.cpp

@@ -41,6 +41,7 @@
 #include <Tensile/Contractions.hpp>
 #include <Tensile/EmbeddedLibrary.hpp>
 #include <Tensile/MasterSolutionLibrary.hpp>
+#include <Tensile/PlaceholderLibrary.hpp>
 #include <Tensile/Tensile.hpp>
 #include <Tensile/TensorDescriptor.hpp>
 #include <Tensile/Utils.hpp>
@@ -450,14 +451,94 @@ namespace
         return inputs;
     }
 
+    TensileLite::LazyLoadingInit getLazyLoadingArch(int deviceID)
+    {
+        hipDeviceProp_t deviceProperties;
+        HIP_CHECK_EXC(hipGetDeviceProperties(&deviceProperties, deviceID));
+        // strip out xnack/ecc from name
+        std::string deviceFullString(deviceProperties.gcnArchName);
+        std::string deviceString = deviceFullString.substr(0, deviceFullString.find(":"));
+
+        if(deviceString.find("gfx803") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx803;
+        }
+        else if(deviceString.find("gfx900") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx900;
+        }
+        else if(deviceString.find("gfx906") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx906;
+        }
+        else if(deviceString.find("gfx908") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx908;
+        }
+        else if(deviceString.find("gfx90a") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx90a;
+        }
+        else if(deviceString.find("gfx940") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx940;
+        }
+        else if(deviceString.find("gfx941") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx941;
+        }
+        else if(deviceString.find("gfx942") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx942;
+        }
+        else if(deviceString.find("gfx1010") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1010;
+        }
+        else if(deviceString.find("gfx1011") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1011;
+        }
+        else if(deviceString.find("gfx1012") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1012;
+        }
+        else if(deviceString.find("gfx1030") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1030;
+        }
+        else if(deviceString.find("gfx1100") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1100;
+        }
+        else if(deviceString.find("gfx1101") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1101;
+        }
+        else if(deviceString.find("gfx1102") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1102;
+        }
+        else if(deviceString.find("gfx1200") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1200;
+        }
+        else if(deviceString.find("gfx1201") != std::string::npos)
+        {
+            return TensileLite::LazyLoadingInit::gfx1201;
+        }
+        return TensileLite::LazyLoadingInit::None;
+    }
+
     /**************************************************
      * The TensileHost struct interfaces with Tensile *
      **************************************************/
     class TensileHost
     {
         // The library object
         std::shared_ptr<TensileLite::MasterSolutionLibrary<TensileLite::ContractionProblemGemm>> m_library;
-        std::shared_ptr<hipDeviceProp_t> m_deviceProp;
+        std::unordered_set<TensileLite::LazyLoadingInit>                  m_deviceSet;
+        std::unordered_map<std::string, std::shared_ptr<hipDeviceProp_t>> m_devicePropMap;
 
         // The adapter object. mutable is used to allow adapters to be modified
         // even when they are stored in a const vector which is immutable in size
@@ -508,9 +589,9 @@ namespace
             return m_library;
         }
 
-        auto& get_device_property() const
+        auto& get_device_property(const std::string& deviceName) const
         {
-            return m_deviceProp;
+            return m_devicePropMap.at(deviceName);
         }
 
         auto& get_adapters() const
@@ -576,7 +657,7 @@ namespace
 
             // only load modules for the current architecture
             auto dir = path + "/*" + processor + "*co";
-
+#if ROCSPARSELT_TENSILE_LAZY_LOAD == 0
             bool no_match = false;
 #ifdef WIN32
             std::replace(dir.begin(), dir.end(), '/', '\\');
@@ -630,28 +711,59 @@ namespace
                       << std::endl;
                 (void)once;
             }
-
+#endif // ROCSPARSELT_TENSILE_LAZY_LOAD == 0
             // We initialize a local static variable with a lambda function call to avoid
             // race conditions when multiple threads with different device IDs try to
             // initialize library. This ensures that only one thread initializes library,
             // and other threads trying to initialize library wait for it to complete.
             static int once = [&] {
+                // Determine library path
+                std::string tensileLibPath;
+#if ROCSPARSELT_TENSILE_LAZY_LOAD
+#ifdef TENSILE_YAML
+                tensileLibPath = path + "/TensileLibrary_lazy_" + processor + ".yaml";
+#else
+                tensileLibPath = path + "/TensileLibrary_lazy_" + processor + ".dat";
+#endif
+#else
 #ifdef TENSILE_YAML
-                path += "/TensileLibrary.yaml";
+                tensileLibPath = path + "/TensileLibrary_" + processor + ".yaml";
 #else
-                path += "/TensileLibrary.dat";
+                tensileLibPath = path + "/TensileLibrary_" + processor + ".dat";
+#endif
 #endif
-                if(!TestPath(path))
+                if(!TestPath(tensileLibPath))
                 {
-                    hipsparselt_cerr << "\nhipsparselt_error: Cannot read " << path << ": "
+                    hipsparselt_cerr << "\nhipsparselt_error: Cannot read " << tensileLibPath << ": "
                                      << strerror(errno) << std::endl;
                     //rocsparselt_abort();
                 }
 
-                auto lib = TensileLite::LoadLibraryFile<TensileLite::ContractionProblemGemm>(path);
+                // Get devices
+                hipDeviceProp_t prop;
+                int             count;
+                HIP_CHECK_EXC(hipGetDeviceCount(&count));
+                for(int devId = 0; devId < count; devId++)
+                {
+                    auto deviceArch = getLazyLoadingArch(devId);
+                    if(m_deviceSet.find(deviceArch) == m_deviceSet.end())
+                    {
+                        // populate the arch list for lazy loading
+                        m_deviceSet.insert(deviceArch);
+                        // populate device property map, used in finding solutions based on arch
+                        HIP_CHECK_EXC(hipGetDeviceProperties(&prop, devId));
+                        // strip out xnack/ecc from name
+                        std::string deviceFullString(prop.gcnArchName);
+                        std::string deviceString
+                            = deviceFullString.substr(0, deviceFullString.find(":"));
+                        m_devicePropMap[deviceString] = std::make_shared<hipDeviceProp_t>(prop);
+                    }
+                }
+
+                auto lib = TensileLite::LoadLibraryFile<TensileLite::ContractionProblemGemm>(tensileLibPath);
                 if(!lib)
                 {
-                    hipsparselt_cerr << "\nhipsparselt_error: Could not load " << path << std::endl;
+                    hipsparselt_cerr << "\nhipsparselt_error: Could not load " << tensileLibPath << std::endl;
                     return -1;
                 }
                 else
@@ -662,17 +774,15 @@ namespace
                 return 0;
             }();
 
+            static_cast<void>(adapter.initializeLazyLoading(processor, path));
+
+
             if(!m_library && once != 0)
             {
                 hipsparselt_cerr << "\nhipsparselt_error: Could not initialize Tensile library"
                                  << std::endl;
                 //rocsparselt_abort();
             }
-
-            hipDeviceProp_t prop;
-            THROW_IF_HIP_ERROR(hipGetDeviceProperties(&prop, deviceId));
-
-            m_deviceProp = std::make_shared<hipDeviceProp_t>(prop);
         }
     };
 
@@ -719,7 +829,7 @@ namespace
         if(library)
             *library = host.get_library();
         if(deviceProp)
-            *deviceProp = host.get_device_property();
+            *deviceProp = host.get_device_property(rocsparselt_internal_get_arch_name());
 
         return *adapter;
     }
@@ -919,6 +1029,11 @@ rocsparselt_status getBestSolutions(const RocsparseltContractionProblem<Ti, To,
     // auto &adapter =
     get_library_and_adapter(&library, &deviceProp, prob.handle->device);
 
+    if(!library)
+    {
+        return rocsparselt_status_invalid_pointer;
+    }
+
     hardware          = TensileLite::hip::GetDevice(*deviceProp);
     auto tensile_prob = ConstructTensileProblem(prob);
     // auto handle = prob.handle;

tensilelite_tag.txt

@@ -1 +1 @@
-d6449b6f17868f0d5801651cc85e237c345f017b
+510dcac724743ff35ec8c60270bc08505eddcfd7