GPU Selection work

Author: RamesTheGeneric

BabbleApp/algo_settings_widget.py

@@ -16,6 +16,8 @@ def __init__(self, widget_id: Tab, main_config: BabbleSettingsConfig, osc_queue:
         self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-"
         self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-"
         self.gui_inference_threads = f"-THREADS{widget_id}-"
+        self.gui_backend = f"-BACKEND{widget_id}"
+        self.gui_gpu_index = f"GPUINDEX{widget_id}"
         self.main_config = main_config
         self.config = main_config.settings
         self.osc_queue = osc_queue
@@ -30,21 +32,35 @@ def __init__(self, widget_id: Tab, main_config: BabbleSettingsConfig, osc_queue:
                  size=(32),
                  tooltip="Name of the model file.",
              ),
+            sg.Text("Inference Threads:", background_color='#424042'),
+            sg.InputText(
+                self.config.gui_inference_threads,
+                key=self.gui_inference_threads,
+                size=(4),
+                tooltip = "How many threads to use for processing the model.",
+            ),
              ],
-            [sg.Checkbox(
-                "Use GPU (DirectML)",
+            [sg.Text("Backend:", background_color='#424042'),   # Replace with Dropdown once I have internet to view docs. 
+                sg.InputText(
+                    self.config.gui_backend,
+                    key=self.gui_backend,
+                    size=(4),
+                    tooltip = "Method to run the model.",
+            ),
+            sg.Text("GPU Index:", background_color='#424042'),   # Replace with Dropdown once I have internet to view docs. 
+                sg.InputText(
+                    self.config.gui_gpu_index,
+                    key=self.gui_gpu_index,
+                    size=(4),
+                    tooltip = "Select which device to run inference.",
+            ),
+            sg.Checkbox(
+                "Use GPU",
                 default=self.config.gui_use_gpu,
                 key=self.gui_use_gpu,
                 background_color='#424042',
                 tooltip="Toggle GPU execution.",
             ),
-            sg.Text("Inference Threads:", background_color='#424042'),
-                sg.InputText(
-                    self.config.gui_inference_threads,
-                    key=self.gui_inference_threads,
-                    size=(4),
-                    tooltip = "How many threads to use for processing the model.",
-            ),
             ],
             [sg.Text("Model output multiplier:", background_color='#424042'),
                 sg.InputText(
@@ -123,6 +139,14 @@ def render(self, window, event, values):
         if self.config.gui_use_gpu != values[self.gui_use_gpu]:
             self.config.gui_use_gpu = values[self.gui_use_gpu]
             changed = True
+
+        if self.config.gui_gpu_index != int(values[self.gui_gpu_index]):
+            self.config.gui_gpu_index = int(values[self.gui_gpu_index])
+            changed = True
+
+        if self.config.gui_backend != int(values[self.gui_backend]):
+            self.config.gui_backend = int(values[self.gui_backend])
+            changed = True
 
         if self.config.gui_inference_threads != int(values[self.gui_inference_threads]):
             self.config.gui_inference_threads = int(values[self.gui_inference_threads]) 

BabbleApp/babble_model_loader.py

@@ -16,7 +16,7 @@
 
 
 def run_model(self):
-    if not self.use_gpu:
+    if self.backend == 0:
         frame = cv2.resize(self.current_image_gray, (256, 256))
         # make it pil
         frame = Image.fromarray(frame)
@@ -38,7 +38,7 @@ def run_model(self):
             output[i] = max(min(output[i], 1), 0)
         self.output = output
 
-    else:
+    if self.backend == 1:
         frame = cv2.resize(self.current_image_gray, (256, 256))
         # make it pil
         frame = Image.fromarray(frame)

BabbleApp/babble_processor.py

@@ -78,24 +78,30 @@ def __init__(
 
         self.current_algo = CamInfoOrigin.MODEL
         self.model = self.settings.gui_model_file
+        self.backend = self.settings.gui_backend
         self.use_gpu = self.settings.gui_use_gpu
+        self.gpu_index = self.settings.gui_gpu_index
         self.output = []
         self.val_list = []
         self.calibrate_config = np.empty((1, 45))
         self.min_max_array = np.empty((2, 45))
 
         self.opts = ort.SessionOptions()
         self.opts.intra_op_num_threads = settings.gui_inference_threads
-        self.opts.inter_op_num_threads = settings.gui_inference_threads
+        self.opts.inter_op_num_threads = settings.gui_inference_threads # Figure out how to set openvino threads
         self.opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
-        if not self.use_gpu:
+        if self.backend == 0:   # OpenVino
+            if self.use_gpu: provider = f'GPU{self.gpu_index}'
+            else: provider = 'CPU'
             ie = IECore()
             net = ie.read_network(model=f'{self.model}openvino/model.xml', weights=f'{self.model}openvino/model.bin')
-            self.sess = ie.load_network(network=net, device_name='CPU')
+            self.sess = ie.load_network(network=net, device_name=provider)
             self.input_name = next(iter(net.input_info))
             self.output_name = next(iter(net.outputs))
-        else:
-            self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=['DmlExecutionProvider'])
+        if self.backend == 1:    # ONNX 
+            if self.use_gpu: provider = 'DmlExecutionProvider' # Figure out how to set ONNX gpu index
+            else: provider = "CPUExecutionProvider" 
+            self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=[provider]) 
             self.input_name = self.sess.get_inputs()[0].name
             self.output_name = self.sess.get_outputs()[0].name
 
@@ -105,7 +111,7 @@ def __init__(
             beta = float(self.settings.gui_speed_coefficient)  # 0.62
         except:
             print('\033[93m[WARN] OneEuroFilter values must be a legal number.\033[0m')
-            min_cutoff = 15.0004
+            min_cutoff = 10.0004
             beta = 0.62
         noisy_point = np.array([45])
         self.one_euro_filter = OneEuroFilter(

BabbleApp/config.py

@@ -33,10 +33,12 @@ class BabbleSettingsConfig(BaseModel):
     gui_osc_location: str = ""
     gui_multiply: int = 1
     gui_model_file: str = 'Models/MOS3E26MCJPEG/'
+    gui_backend: int = 0
     gui_use_gpu: bool = False
+    gui_gpu_index: int = 0
     gui_inference_threads: int = 1
     gui_use_red_channel: bool = False
-    calib_array: str = None
+    calib_array: str = ""
     gui_cam_resolution_x: int = 0    
     gui_cam_resolution_y: int = 0
     gui_cam_framerate: int = 0

BabbleApp/requirements.txt

@@ -6,3 +6,7 @@ PySimpleGUI
 python_osc
 torch
 torchvision
+pydantic
+pyserial
+colorama
+winotify

pyproject.toml

@@ -20,6 +20,7 @@ winotify = [
 ]
 onnxruntime = "^1.13.1"
 onnxruntime-directml = "^1.15.0"
+openvino = "^2023.0.1"
 colorama = "^0.4.6"
 taskipy = "^1.10.4"
 torch = [

test.py

@@ -0,0 +1,14 @@
+import onnxruntime as ort
+from openvino.inference_engine import IECore
+import openvino.inference_engine as ovie
+'''
+print(ort.get_device()) # Returns the current device
+print(ort.get_all_providers())
+print(ort.get_available_providers())
+#sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=['DmlExecutionProvider'])
+'''
+ie = IECore()
+ovie.list_devices
+#print(ie.get_devices())
+#net = ie.read_network(model=f'{self.model}openvino/model.xml', weights=f'{self.model}openvino/model.bin')
+#self.sess = ie.load_network(network=net, device_name='CPU')