Be able to share compiled OpenCL binary caches among identical GPUs

Baikal/Controllers/clw_scene_controller.cpp

@@ -50,11 +50,9 @@ namespace Baikal
         return CameraType::kPerspective;
     }
 
-
     ClwSceneController::ClwSceneController(CLWContext context, RadeonRays::IntersectionApi* api, const CLProgramManager *program_manager)
     : m_context(context)
     , m_api(api)
-    , m_default_material(UberV2Material::Create())
     , m_program_manager(program_manager)
     {
         auto acc_type = "fatbvh";
@@ -70,9 +68,9 @@ namespace Baikal
         m_api->SetOption("bvh.sah.num_bins", 16.f);
     }
 
-    Material::Ptr ClwSceneController::GetDefaultMaterial() const
+    Material::Ptr ClwSceneController::GetDefaultMaterial(Scene1 const& scene) const
     {
-        return m_default_material;
+        return scene.GetDefaultMaterial();
     }
 
     ClwSceneController::~ClwSceneController()
@@ -502,8 +500,12 @@ namespace Baikal
 
             shape.linearvelocity = float3(0.0f, 0.f, 0.f);
             shape.angularvelocity = float3(0.f, 0.f, 0.f, 1.f);
-            shape.material.offset = GetMaterialIndex(mat_collector, mesh->GetMaterial());
-            shape.material.layers = GetMaterialLayers(mesh->GetMaterial());
+
+            auto mesh_material = mesh->GetMaterial();
+            if (mesh_material == nullptr)
+                mesh_material = scene.GetDefaultMaterial();
+            shape.material.offset = GetMaterialIndex(mat_collector, mesh_material);
+            shape.material.layers = GetMaterialLayers(mesh_material);
 
             shape.volume_idx = GetVolumeIndex(vol_collector, mesh->GetVolumeMaterial());
 
@@ -563,8 +565,12 @@ namespace Baikal
 
             shape.linearvelocity = float3(0.0f, 0.f, 0.f);
             shape.angularvelocity = float3(0.f, 0.f, 0.f, 1.f);
-            shape.material.offset = GetMaterialIndex(mat_collector, mesh->GetMaterial());
-            shape.material.layers = GetMaterialLayers(mesh->GetMaterial());
+
+            auto mesh_material = mesh->GetMaterial();
+            if (mesh_material == nullptr)
+                mesh_material = scene.GetDefaultMaterial();
+            shape.material.offset = GetMaterialIndex(mat_collector, mesh_material);
+            shape.material.layers = GetMaterialLayers(mesh_material);
 
             shape.volume_idx = GetVolumeIndex(vol_collector, mesh->GetVolumeMaterial());
 
@@ -611,8 +617,12 @@ namespace Baikal
 
             shape.linearvelocity = float3(0.0f, 0.f, 0.f);
             shape.angularvelocity = float3(0.f, 0.f, 0.f, 1.f);
-            shape.material.offset = GetMaterialIndex(mat_collector, instance->GetMaterial());
-            shape.material.layers = std::static_pointer_cast<UberV2Material>(instance->GetMaterial())->GetLayers();
+
+            auto instance_material = instance->GetMaterial();
+            if (instance_material == nullptr)
+                instance_material = scene.GetDefaultMaterial();
+            shape.material.offset = GetMaterialIndex(mat_collector, instance_material);
+            shape.material.layers = GetMaterialLayers(instance_material);
 
             shape.volume_idx = GetVolumeIndex(vol_collector, instance->GetVolumeMaterial());
 
@@ -668,8 +678,12 @@ namespace Baikal
             current_shape->transform.m1 = { transform.m10, transform.m11, transform.m12, transform.m13 };
             current_shape->transform.m2 = { transform.m20, transform.m21, transform.m22, transform.m23 };
             current_shape->transform.m3 = { transform.m30, transform.m31, transform.m32, transform.m33 };
-            current_shape->material.offset = GetMaterialIndex(mat_collector, mesh->GetMaterial());
-            current_shape->material.layers = GetMaterialLayers(mesh->GetMaterial());
+
+            auto mesh_material = mesh->GetMaterial();
+            if (mesh_material == nullptr)
+                mesh_material = scene.GetDefaultMaterial();
+            current_shape->material.offset = GetMaterialIndex(mat_collector, mesh_material);
+            current_shape->material.layers = GetMaterialLayers(mesh_material);
 
             current_shape->volume_idx = GetVolumeIndex(volume_collector, mesh->GetVolumeMaterial());
 
@@ -691,8 +705,12 @@ namespace Baikal
             current_shape->transform.m1 = { transform.m10, transform.m11, transform.m12, transform.m13 };
             current_shape->transform.m2 = { transform.m20, transform.m21, transform.m22, transform.m23 };
             current_shape->transform.m3 = { transform.m30, transform.m31, transform.m32, transform.m33 };
-            current_shape->material.offset = GetMaterialIndex(mat_collector, mesh->GetMaterial());
-            current_shape->material.layers = GetMaterialLayers(mesh->GetMaterial());
+
+            auto mesh_material = mesh->GetMaterial();
+            if (mesh_material == nullptr)
+                mesh_material = scene.GetDefaultMaterial();
+            current_shape->material.offset = GetMaterialIndex(mat_collector, mesh_material);
+            current_shape->material.layers = GetMaterialLayers(mesh_material);
 
             current_shape->volume_idx = GetVolumeIndex(volume_collector, mesh->GetVolumeMaterial());
 
@@ -714,8 +732,12 @@ namespace Baikal
             current_shape->transform.m1 = { transform.m10, transform.m11, transform.m12, transform.m13 };
             current_shape->transform.m2 = { transform.m20, transform.m21, transform.m22, transform.m23 };
             current_shape->transform.m3 = { transform.m30, transform.m31, transform.m32, transform.m33 };
-            current_shape->material.offset = GetMaterialIndex(mat_collector, instance->GetMaterial());
-            current_shape->material.layers = GetMaterialLayers(std::static_pointer_cast<UberV2Material>(instance->GetMaterial()));
+
+            auto instance_material = instance->GetMaterial();
+            if (instance_material == nullptr)
+                instance_material = scene.GetDefaultMaterial();
+            current_shape->material.offset = GetMaterialIndex(mat_collector, instance_material);
+            current_shape->material.layers = GetMaterialLayers(instance_material);
 
             current_shape->volume_idx = GetVolumeIndex(volume_collector, instance->GetVolumeMaterial());
 
@@ -1275,8 +1297,8 @@ namespace Baikal
 
     int ClwSceneController::GetMaterialIndex(Collector const& collector, Material::Ptr material) const
     {
-        auto m = material ? material : m_default_material;
-        return ResolveMaterialPtr(m);
+        assert(material != nullptr);
+        return ResolveMaterialPtr(material);
     }
 
     int ClwSceneController::GetVolumeIndex(Collector const& collector, VolumeMaterial::Ptr volume) const
@@ -1392,8 +1414,8 @@ namespace Baikal
 
     int ClwSceneController::GetMaterialLayers(Material::Ptr material) const
     {
-        auto m = material ? material : m_default_material;
-        return std::static_pointer_cast<UberV2Material>(m)->GetLayers();
+        assert(material != nullptr);
+        return std::static_pointer_cast<UberV2Material>(material)->GetLayers();
     }
 
 }

Baikal/Controllers/clw_scene_controller.h

@@ -87,7 +87,7 @@ namespace Baikal
         // Update input map leafs only
         void UpdateLeafsData(Scene1 const& scene, Collector& input_map_leafs_collector, Collector& tex_collector, ClwScene& out) const override;
         // Get default material
-        Material::Ptr GetDefaultMaterial() const override;
+        Material::Ptr GetDefaultMaterial(Scene1 const& scene) const override;
         // If m_current_scene changes
         void UpdateCurrentScene(Scene1 const& scene, ClwScene& out) const override;
         // Update volume materiuals
@@ -128,8 +128,6 @@ namespace Baikal
         CLWContext m_context;
         // Intersection API
         RadeonRays::IntersectionApi* m_api;
-        // Default material
-        Material::Ptr m_default_material;
         // CL Program manager
         const CLProgramManager *m_program_manager;
         // Material to device material map

Baikal/Controllers/scene_controller.h

@@ -94,7 +94,7 @@ namespace Baikal
         // Update input map leafs only
         virtual void UpdateLeafsData(Scene1 const& scene, Collector& input_map_leafs_collector, Collector& tex_collector, CompiledScene& out) const = 0;
         // Default material
-        virtual Material::Ptr GetDefaultMaterial() const = 0;
+        virtual Material::Ptr GetDefaultMaterial(Scene1 const& scene) const = 0;
         // If m_current_scene changes
         virtual void UpdateCurrentScene(Scene1 const& scene, CompiledScene& out) const = 0;
         // Update volume materials only

Baikal/Controllers/scene_controller.inl

@@ -74,7 +74,7 @@ namespace Baikal
         auto shape_iter = scene->CreateShapeIterator();
         auto light_iter = scene->CreateLightIterator();
 
-        auto default_material = GetDefaultMaterial();
+        auto default_material = scene->GetDefaultMaterial();
         // Collect materials from shapes first
         m_material_collector.Collect(*shape_iter,
                               // This function adds all materials to resulting map

Baikal/SceneGraph/scene1.cpp

@@ -2,6 +2,7 @@
 #include "light.h"
 #include "camera.h"
 #include "iterator.h"
+#include "SceneGraph/uberv2material.h"
 
 #include <vector>
 #include <list>
@@ -24,6 +25,8 @@ namespace Baikal
         Baikal::Texture::Ptr m_background_texture;
         EnvironmentOverride m_environment_override;
 
+        Material::Ptr m_default_material;
+
         DirtyFlags m_dirty_flags;
         std::mutex m_scene_mutex;
     };
@@ -32,6 +35,7 @@ namespace Baikal
     : m_impl(new SceneImpl)
     {
         m_impl->m_camera = nullptr;
+        m_impl->m_default_material = UberV2Material::Create();
         ClearDirtyFlags();
     }
 
@@ -220,6 +224,15 @@ namespace Baikal
         return m_impl->m_environment_override;
     }
 
+    void Scene1::SetDefaultMaterial(Baikal::Material::Ptr material)
+    {
+        m_impl->m_default_material = material;
+    }
+    Baikal::Material::Ptr Scene1::GetDefaultMaterial() const
+    {
+        return m_impl->m_default_material;
+    }
+
     void Scene1::Acquire(std::uint32_t controller_id)
     {
         m_impl->m_scene_mutex.lock();

Baikal/SceneGraph/scene1.h

@@ -123,6 +123,10 @@ namespace Baikal
         void SetEnvironmentOverride(const EnvironmentOverride& env_override);
         const EnvironmentOverride& GetEnvironmentOverride() const;
 
+        // Default material
+        void SetDefaultMaterial(Baikal::Material::Ptr material);
+        Material::Ptr GetDefaultMaterial() const;
+
         // Forbidden stuff
         Scene1(Scene1 const&) = delete;
         Scene1& operator = (Scene1 const&) = delete;

Baikal/Utils/cl_program.cpp

@@ -102,7 +102,6 @@ inline void SaveBinaries(std::string const& name, std::vector<std::uint8_t>& dat
     }
 }
 
-
 CLProgram::CLProgram(const CLProgramManager *program_manager, uint32_t id, CLWContext context,
                      const std::string &program_name, const std::string &cache_path) :
     m_program_manager(program_manager),
@@ -219,6 +218,9 @@ bool CLProgram::IsHeaderNeeded(const std::string &header_name) const
 
 CLWProgram CLProgram::GetCLWProgram(const std::string &opts)
 {
+    static std::unordered_map<std::string, std::shared_ptr<std::mutex>> s_binary_cache_names;
+    static std::mutex s_binary_cache_map_mutex;
+
     // global dirty flag
     if (m_is_dirty)
     {
@@ -246,8 +248,24 @@ CLWProgram CLProgram::GetCLWProgram(const std::string &opts)
         cached_program_path.append(".bin");
 
         std::vector<std::uint8_t> binary;
-        if (LoadBinaries(cached_program_path, binary))
+
+        std::unique_lock<std::mutex> map_lock(s_binary_cache_map_mutex);
+        auto iter = s_binary_cache_names.find(filename);
+        if (iter == s_binary_cache_names.end())
+            iter = s_binary_cache_names.insert({ filename, std::make_shared<std::mutex>() }).first;
+        auto cache_mutex = iter->second;
+        assert(cache_mutex != nullptr);
+        map_lock.unlock();
+
+        // Other workers with the same binary cache requirement will be blocked here
+        std::unique_lock<std::mutex> cache_lock(*cache_mutex);
+        bool loaded = LoadBinaries(cached_program_path, binary);
+
+        if (loaded)
         {
+            // Binary cache existed, other workers can read it
+            cache_lock.unlock();
+
             // Create from binary
             std::size_t size = binary.size();
             auto binaries = &binary[0];

Baikal/Utils/cl_program.h

@@ -27,6 +27,7 @@ THE SOFTWARE.
 #include <set>
 #include <unordered_map>
 #include <unordered_set>
+#include <mutex>
 #include "CLWProgram.h"
 #include "CLWContext.h"
 
@@ -94,6 +95,5 @@ namespace Baikal
         uint32_t m_id;
         CLWContext m_context;
         std::set<std::string> m_included_headers; ///< Set of included headers
-
     };
 }