initial tensorrt ep commit

sherpa-onnx/csrc/session.cc

@@ -54,6 +54,61 @@ static Ort::SessionOptions GetSessionOptionsImpl(int32_t num_threads,
       }
       break;
     }
+    case Provider::kTRT: {
+
+      struct TrtPairs {
+        const char* op_keys;
+        const char* op_values;
+      };
+
+      std::vector<TrtPairs> trt_options = {
+        {"device_id","0"},
+        {"trt_max_workspace_size","2147483648"},
+        {"trt_max_partition_iterations","10"},
+        {"trt_min_subgraph_size","5"},
+        {"trt_fp16_enable","0"},
+        {"trt_detailed_build_log","0"},
+        {"trt_engine_cache_enable","1"},
+        {"trt_engine_cache_path","."},
+        {"trt_timing_cache_enable","1"},
+        {"trt_timing_cache_path","."}
+      };
+
+      // ToDo : Trt configs
+      // "trt_int8_enable",
+      // "trt_int8_use_native_calibration_table",
+      // "trt_dump_subgraphs",
+
+      std::vector<const char*> option_keys,option_values;
+      for (const TrtPairs& pair : trt_options) {
+        option_keys.emplace_back(pair.op_keys);
+        option_values.emplace_back(pair.op_values);
+      }
+
+      std::vector<std::string> available_providers =
+          Ort::GetAvailableProviders();
+      if (std::find(available_providers.begin(), available_providers.end(),
+                "TensorrtExecutionProvider") != available_providers.end()) {
+        const auto& api = Ort::GetApi();
+
+        OrtTensorRTProviderOptionsV2* tensorrt_options;
+        OrtStatus *statusC = api.CreateTensorRTProviderOptions(
+                                    &tensorrt_options);
+        OrtStatus *statusU = api.UpdateTensorRTProviderOptions(
+                tensorrt_options, option_keys.data(), option_values.data(),
+                option_keys.size());
+
+        if (statusC) { OrtStatusFailure(statusC,&os);}
+        else if(statusU) { OrtStatusFailure(statusU,&os);}
+        else {
+          auto *statusTrt = OrtSessionOptionsAppendExecutionProvider_Tensorrt(
+                              sess_opts, 0);
+          if(statusTrt) { OrtStatusFailure(statusTrt,&os);}
+        }
+
+        api.ReleaseTensorRTProviderOptions(tensorrt_options);
+      }
+    }
     case Provider::kCUDA: {
       if (std::find(available_providers.begin(), available_providers.end(),
                     "CUDAExecutionProvider") != available_providers.end()) {
@@ -116,56 +171,7 @@ static Ort::SessionOptions GetSessionOptionsImpl(int32_t num_threads,
 #endif
       break;
     }
-    case Provider::kTRT: {
-      SHERPA_ONNX_LOGE("Came for TRT");
-      std::vector<const char*> option_keys = {
-          "device_id",
-          "trt_max_workspace_size",
-          "trt_max_partition_iterations",
-          "trt_min_subgraph_size",
-          "trt_fp16_enable",
-          "trt_int8_enable",
-          "trt_int8_use_native_calibration_table",
-          "trt_dump_subgraphs",
-          "trt_detailed_build_log",
-          // below options are strongly recommended !
-          "trt_engine_cache_enable",
-          "trt_engine_cache_path",
-          "trt_timing_cache_enable",
-          "trt_timing_cache_path",
-      };
-      std::vector<const char*> option_values = {
-          "0",
-          "2147483648",
-          "10",
-          "5",
-          "0",
-          "0",
-          "0",
-          "1",
-          "1",
-          "1",
-          ".",
-          "1",
-          ".",  // can be same as the engine cache folder
-      };
-      std::vector<std::string> available_providers =
-          Ort::GetAvailableProviders();
-      if (std::find(available_providers.begin(), available_providers.end(),
-                    "TensorrtExecutionProvider") != available_providers.end()) {
-        const auto& api = Ort::GetApi();
-
-        OrtTensorRTProviderOptionsV2* tensorrt_options;
-        Ort::ThrowOnError(api.CreateTensorRTProviderOptions(&tensorrt_options));
-
-        Ort::ThrowOnError(api.UpdateTensorRTProviderOptions(tensorrt_options,
-                option_keys.data(), option_values.data(), option_keys.size()));
-
-        sess_opts.AppendExecutionProvider_TensorRT_V2(*tensorrt_options);
-      }
-    }
   }
-
   return sess_opts;
 }
 
@@ -214,4 +220,13 @@ Ort::SessionOptions GetSessionOptions(
   return GetSessionOptionsImpl(config.num_threads, config.provider);
 }
 
+void OrtStatusFailure(OrtStatus *status,std::ostringstream *os) {
+    const auto &api = Ort::GetApi();
+    const char *msg = api.GetErrorMessage(status);
+    SHERPA_ONNX_LOGE(
+      "Failed to enable TensorRT : %s."
+      "Available providers: %s. Fallback to cuda", msg, os->str().c_str());
+    api.ReleaseStatus(status);
+}
+
 }  // namespace sherpa_onnx

sherpa-onnx/csrc/session.h

@@ -47,6 +47,8 @@ Ort::SessionOptions GetSessionOptions(const AudioTaggingModelConfig &config);
 Ort::SessionOptions GetSessionOptions(
     const OfflinePunctuationModelConfig &config);
 
+void OrtStatusFailure(OrtStatus *status,std::ostringstream *os);
+
 }  // namespace sherpa_onnx
 
 #endif  // SHERPA_ONNX_CSRC_SESSION_H_