Add parallel model loading (#165)

* Add rate limiter read lock

* Remove SharedLibrary global lock on non-Windows

* Update copyright year

* Add parallel model loading

* Cleanup comments

* Retry during parallel model load/unload conflict

* Revert copyright year on unmodified file

* Update on ModelInitFn and ModelInstanceInitFn SharedLibrary Win lock

* Separate LockUnlockNodes to LockNodes and UnlockNodes

* Group ModelInfoMap functions

* Adjust dependency graph naming

* Refactor parallel load logic

* Add wait on cond var for conflict retry

* Adjust cond var naming

* Rename LoadUnloadModelsPoll to PollModels

* More comment on Writeback vs Assign

* Rename ConditionVariable to EventNotifier

* Refactor LoadUnloadModels

* Refactor conflict notify mechanism

* Hide the map var inside model info map

* Add log print for load/unload conflict

* Init rate limiter ptr to null once declared

Author: kthui

src/backend_model.cc

@@ -169,19 +169,28 @@ TritonModel::Create(
 
   TritonModel* raw_local_model = local_model.get();
 
-  // Model initialization is optional... The TRITONBACKEND_Model
-  // object is this TritonModel object. We must set set shared library
-  // path to point to the backend directory in case the backend
-  // library attempts to load additional shared libaries.
+  // Model initialization is optional... The TRITONBACKEND_Model object is this
+  // TritonModel object.
   if (backend->ModelInitFn() != nullptr) {
+    // We must set set shared library path to point to the backend directory in
+    // case the backend library attempts to load additional shared libaries.
+    // Currently, the set and reset function is effective only on Windows, so
+    // there is no need to set path on non-Windows.
+    // However, parallel model loading will not see any speedup on Windows and
+    // the global lock inside the SharedLibrary is a WAR.
+    // [FIXME] Reduce lock WAR on SharedLibrary (DLIS-4300)
+#ifdef _WIN32
     std::unique_ptr<SharedLibrary> slib;
     RETURN_IF_ERROR(SharedLibrary::Acquire(&slib));
     RETURN_IF_ERROR(slib->SetLibraryDirectory(backend->Directory()));
+#endif
 
     TRITONSERVER_Error* err = backend->ModelInitFn()(
         reinterpret_cast<TRITONBACKEND_Model*>(raw_local_model));
 
+#ifdef _WIN32
     RETURN_IF_ERROR(slib->ResetLibraryDirectory());
+#endif
     RETURN_IF_TRITONSERVER_ERROR(err);
   }
 

src/backend_model_instance.cc

@@ -1,4 +1,4 @@
-// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -326,14 +326,25 @@ TritonModelInstance::CreateInstance(
   // library path to point to the backend directory in case the
   // backend library attempts to load additional shared libaries.
   if (model->Backend()->ModelInstanceInitFn() != nullptr) {
+    // We must set set shared library path to point to the backend directory in
+    // case the backend library attempts to load additional shared libaries.
+    // Currently, the set and reset function is effective only on Windows, so
+    // there is no need to set path on non-Windows.
+    // However, parallel model loading will not see any speedup on Windows and
+    // the global lock inside the SharedLibrary is a WAR.
+    // [FIXME] Reduce lock WAR on SharedLibrary (DLIS-4300)
+#ifdef _WIN32
     std::unique_ptr<SharedLibrary> slib;
     RETURN_IF_ERROR(SharedLibrary::Acquire(&slib));
     RETURN_IF_ERROR(slib->SetLibraryDirectory(model->Backend()->Directory()));
+#endif
 
     TRITONSERVER_Error* err =
         model->Backend()->ModelInstanceInitFn()(triton_instance);
 
+#ifdef _WIN32
     RETURN_IF_ERROR(slib->ResetLibraryDirectory());
+#endif
     RETURN_IF_TRITONSERVER_ERROR(err);
   }
 

src/model.h

@@ -38,6 +38,10 @@ struct ModelIdentifier {
   {
   }
 
+  ModelIdentifier(const ModelIdentifier&) = default;
+
+  ModelIdentifier& operator=(const ModelIdentifier&) = default;
+
   bool operator<(const ModelIdentifier& rhs) const
   {
     if (namespace_ == rhs.namespace_) {