Merge pull request #27 from RamesTheGeneric/main

Final Build Tweaks

Author: SummerSigh

.gitignore

@@ -11,7 +11,14 @@ BabbleApp/dist
 .vscode
 /testing
 /old_models
+/Lib
+/scripts
+/share
+
+
 scripts/example_build_app_and_installer.bat
 scripts/example_build_app_and_installer.bat
 scripts/installer.iss
 Glia_cap.py
+lazyass_minecraft_script.py
+scripts/example_build_app_and_installer.bat

BabbleApp/Models/EFFB0MVE99TIMM/onnx/model.onnx

@@ -17,12 +17,12 @@ def __init__(self, widget_id: Tab, main_config: BabbleSettingsConfig, osc_queue:
         self.gui_speed_coefficient = f"-SPEEDCOEFFICIENT{widget_id}-"
         self.gui_min_cutoff = f"-MINCUTOFF{widget_id}-"
         self.gui_inference_threads = f"-THREADS{widget_id}-"
-        self.gui_backend = f"-BACKEND{widget_id}"
+        self.gui_runtime = f"-RUNTIME{widget_id}"
         self.gui_gpu_index = f"GPUINDEX{widget_id}"
         self.main_config = main_config
         self.config = main_config.settings
         self.osc_queue = osc_queue
-        self.backend_list = "OpenVino", "ONNX"
+        self.runtime_list = ("Default (ONNX)", "ONNX")
 
         # Define the window's contents
         self.general_settings_layout = [
@@ -42,15 +42,15 @@ def __init__(self, widget_id: Tab, main_config: BabbleSettingsConfig, osc_queue:
                 tooltip = "How many threads to use for processing the model.",
             ),
              ],
-            [sg.Text("Backend:", background_color='#424042'),   # Replace with Dropdown once I have internet to view docs. 
+            [sg.Text("Runtime:", background_color='#424042'),   # Replace with Dropdown once I have internet to view docs. 
                 sg.OptionMenu(
-                    self.backend_list,
-                    self.config.gui_backend,
-                    key=self.gui_backend,
+                    self.runtime_list,
+                    self.config.gui_runtime,
+                    key=self.gui_runtime,
                 ),
                 #sg.InputText(
-                #    self.config.gui_backend,
-                #    key=self.gui_backend,
+                #    self.config.gui_runtime,
+                #    key=self.gui_runtime,
                 #    size=(4),
                 #    tooltip = "Method to run the model.",
 
@@ -149,26 +149,27 @@ def render(self, window, event, values):
         if self.config.gui_use_gpu != values[self.gui_use_gpu]:
             self.config.gui_use_gpu = values[self.gui_use_gpu]
             changed = True
+        if values[self.gui_gpu_index] != '':
+            if self.config.gui_gpu_index != int(values[self.gui_gpu_index]):
+                self.config.gui_gpu_index = int(values[self.gui_gpu_index])
+                changed = True
 
-        if self.config.gui_gpu_index != int(values[self.gui_gpu_index]):
-            self.config.gui_gpu_index = int(values[self.gui_gpu_index])
-            changed = True
-
-        if self.config.gui_backend != str(values[self.gui_backend]):
-            self.config.gui_backend = str(values[self.gui_backend])
+        if self.config.gui_runtime != str(values[self.gui_runtime]):
+            self.config.gui_runtime = str(values[self.gui_runtime])
             changed = True
 
-        if self.config.gui_inference_threads != int(values[self.gui_inference_threads]):
-            self.config.gui_inference_threads = int(values[self.gui_inference_threads]) 
-            changed = True
-
-        if self.config.gui_min_cutoff != values[self.gui_min_cutoff]:
-            self.config.gui_min_cutoff = values[self.gui_min_cutoff]
-            changed = True
-
-        if self.config.gui_speed_coefficient != values[self.gui_speed_coefficient]:
-            self.config.gui_speed_coefficient = values[self.gui_speed_coefficient]
-            changed = True
+        if values[self.gui_inference_threads] != '':
+            if self.config.gui_inference_threads != int(values[self.gui_inference_threads]):
+                self.config.gui_inference_threads = int(values[self.gui_inference_threads]) 
+                changed = True
+        if values[self.gui_min_cutoff] != '':
+            if self.config.gui_min_cutoff != values[self.gui_min_cutoff]:
+                self.config.gui_min_cutoff = values[self.gui_min_cutoff]
+                changed = True
+        if values[self.gui_speed_coefficient] != '':
+            if self.config.gui_speed_coefficient != values[self.gui_speed_coefficient]:
+                self.config.gui_speed_coefficient = values[self.gui_speed_coefficient]
+                changed = True
 
         if changed:
             self.main_config.save()

BabbleApp/algo_settings_widget.py

@@ -1,7 +1,6 @@
 import os
 import json
 os.environ["OMP_NUM_THREADS"] = "1"
-from openvino.inference_engine import IECore
 import onnxruntime as ort
 import time
 import PySimpleGUI as sg
@@ -16,29 +15,7 @@
 
 
 def run_model(self):
-    if self.backend == "OpenVino":
-        frame = cv2.resize(self.current_image_gray, (256, 256))
-        # make it pil
-        frame = Image.fromarray(frame)
-        # make it grayscale
-        frame = to_grayscale(frame)
-        # make it a tensor
-        frame = transforms.ToTensor()(frame)
-        # make it a batch
-        frame = frame.unsqueeze(0)
-        # make it a numpy array
-        frame = frame.numpy()
-        out = self.sess.infer(inputs={self.input_name: frame})
-        #out = self.sess.run([self.output_name], {self.input_name: frame})
-        #end = time.time()
-        output = out[self.output_name]
-        output = output[0]
-        output = self.one_euro_filter(output)
-        for i in range(len(output)):  # Clip values between 0 - 1
-            output[i] = max(min(output[i], 1), 0)
-        self.output = output
-
-    if self.backend == "ONNX":
+    if self.runtime == "ONNX" or self.runtime == "Default (ONNX)":
         frame = cv2.resize(self.current_image_gray, (256, 256))
         # make it pil
         frame = Image.fromarray(frame)

BabbleApp/babble_model_loader.py

@@ -78,7 +78,7 @@ def __init__(
 
         self.current_algo = CamInfoOrigin.MODEL
         self.model = self.settings.gui_model_file
-        self.backend = self.settings.gui_backend
+        self.runtime = self.settings.gui_runtime
         self.use_gpu = self.settings.gui_use_gpu
         self.gpu_index = self.settings.gui_gpu_index
         self.output = []
@@ -88,24 +88,13 @@ def __init__(
 
         self.opts = ort.SessionOptions()
         self.opts.intra_op_num_threads = settings.gui_inference_threads
-        self.opts.inter_op_num_threads = settings.gui_inference_threads # Figure out how to set openvino threads
         self.opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
-        if self.backend == "OpenVino":   # OpenVino
-            if self.use_gpu: provider = f'GPU.{self.gpu_index}'
-            else: provider = 'CPU'
-            ie = IECore()
-            net = ie.read_network(model=f'{self.model}openvino/model.xml', weights=f'{self.model}openvino/model.bin')
-            self.sess = ie.load_network(network=net, device_name=provider)
-            self.input_name = next(iter(net.input_info))
-            self.output_name = next(iter(net.outputs))
-        if self.backend == "ONNX":    # ONNX 
-            if self.use_gpu: provider = 'DmlExecutionProvider' # Figure out how to set ONNX gpu index
-            else: provider = "CPUExecutionProvider" 
-            self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=[provider]) 
+        if self.runtime == "ONNX" or self.runtime == "Default (ONNX)":    # ONNX 
+            if self.use_gpu: provider = 'DmlExecutionProvider' 
+            else: provider = "CPUExecutionProvider"  # Build onnxruntime to get both DML and OpenVINO
+            self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=[provider], provider_options=[{'device_id': self.gpu_index}]) 
             self.input_name = self.sess.get_inputs()[0].name
             self.output_name = self.sess.get_outputs()[0].name
-        
-
         try:
             min_cutoff = float(self.settings.gui_min_cutoff)  # 15.5004
             beta = float(self.settings.gui_speed_coefficient)  # 0.62

BabbleApp/babble_processor.py

@@ -161,17 +161,12 @@ def main():
         ],
     ]
 
-
     if config.cam_display_id in [Tab.CAM]:
         cams[0].start()
     if config.cam_display_id in [Tab.SETTINGS]:
         settings[0].start()
     if config.cam_display_id in [Tab.ALGOSETTINGS]:
         settings[1].start()
-    '''
-    if config.cam_display_id in [Tab.CALIBSETTINGS]:
-        settings[2].start()
-    '''
 
     # the cam needs to be running before it is passed to the OSC
     if config.settings.gui_ROSC:

BabbleApp/babbleapp.py

@@ -7,7 +7,7 @@ a = Analysis(['babbleapp.py'],
              pathex=[],
              binaries=[],
              datas=[("Audio/*", "Audio"), ("Images/*", "Images/"), ("Models/*", "Models/")],
-             hiddenimports=['cv2', 'numpy', 'PySimpleGui', 'onnxruntime-directml'],
+             hiddenimports=[],
              hookspath=[],
              hooksconfig={},
              runtime_hooks=[],

BabbleApp/babbleapp.spec

@@ -32,11 +32,11 @@ class BabbleSettingsConfig(BaseModel):
     gui_ROSC: bool = False
     gui_osc_location: str = ""
     gui_multiply: int = 1
-    gui_model_file: str = 'Models/ENEXTXXE100D4/'
-    gui_backend: str = "OpenVino"
+    gui_model_file: str = 'Models/EFFB0MVE99TIMM/'
+    gui_runtime: str = "ONNX"
     gui_use_gpu: bool = False
     gui_gpu_index: int = 0
-    gui_inference_threads: int = 1
+    gui_inference_threads: int = 2
     gui_use_red_channel: bool = False
     calib_array: str = "[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]]"
     gui_cam_resolution_x: int = 0    

BabbleApp/config.py

@@ -224,18 +224,18 @@ def render(self, window, event, values):
         if self.config.gui_osc_location != values[self.gui_osc_location]:
             self.config.gui_osc_location = values[self.gui_osc_location]
             changed = True
-
-        if self.config.gui_cam_resolution_x != int(values[self.gui_cam_resolution_x]):
-            self.config.gui_cam_resolution_x = int(values[self.gui_cam_resolution_x])
-            changed = True
-
-        if self.config.gui_cam_resolution_y != int(values[self.gui_cam_resolution_y]):
-            self.config.gui_cam_resolution_y = int(values[self.gui_cam_resolution_y])
-            changed = True
-
-        if self.config.gui_cam_framerate != int(values[self.gui_cam_framerate]):
-            self.config.gui_cam_framerate = int(values[self.gui_cam_framerate])
-            changed = True
+        if values[self.gui_cam_resolution_x] != '':
+            if self.config.gui_cam_resolution_x != int(values[self.gui_cam_resolution_x]):
+                self.config.gui_cam_resolution_x = int(values[self.gui_cam_resolution_x])
+                changed = True
+        if values[self.gui_cam_resolution_y] != '':
+            if self.config.gui_cam_resolution_y != int(values[self.gui_cam_resolution_y]):
+                self.config.gui_cam_resolution_y = int(values[self.gui_cam_resolution_y])
+                changed = True
+        if values[self.gui_cam_framerate] != '':
+            if self.config.gui_cam_framerate != int(values[self.gui_cam_framerate]):
+                self.config.gui_cam_framerate = int(values[self.gui_cam_framerate])
+                changed = True
 
         if self.config.gui_use_red_channel != bool(values[self.gui_use_red_channel]):
             self.config.gui_use_red_channel = bool(values[self.gui_use_red_channel])

BabbleApp/general_settings_widget.py

@@ -1,4 +1,5 @@
 import numpy as np
+from math import log
 from enum import IntEnum
 from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC
 
@@ -7,43 +8,65 @@ class CamId(IntEnum):
     SETTINGS = 1
 
 class cal():
+
     def cal_osc(self, array):
         #print(self.calibration_frame_counter)
         if self.calibration_frame_counter == 0:
+            if not self.config.use_n_calibration:
+                self.calibration_frame_counter = None
+                values = np.array(self.val_list)
 
-            self.calibration_frame_counter = None
-            values = np.array(self.val_list)
+                # Initialize the min_max_array with shape (2, num_outputs)
+                num_outputs = values.shape[1]
+                self.min_max_array = np.zeros((2, num_outputs))
 
-            # Initialize the min_max_array with shape (2, num_outputs)
-            num_outputs = values.shape[1]
-            self.min_max_array = np.zeros((2, num_outputs))
+                # Iterate over each output index
+                for i in range(num_outputs):
+                    # Calculate the lower and upper thresholds for the current index
+                    lower_threshold = np.percentile(values[:, i], 1)
+                    upper_threshold = np.percentile(values[:, i], 99)
 
-            # Iterate over each output index
-            for i in range(num_outputs):
-                # Calculate the lower and upper thresholds for the current index
-                lower_threshold = np.percentile(values[:, i], 1)
-                upper_threshold = np.percentile(values[:, i], 99)
+                    # Filter out values within the thresholds for the current index
+                    filtered_values = values[(values[:, i] >= lower_threshold) & (values[:, i] <= upper_threshold), i]
 
-                # Filter out values within the thresholds for the current index
-                filtered_values = values[(values[:, i] >= lower_threshold) & (values[:, i] <= upper_threshold), i]
+                    # Extract the minimum and maximum values for the current index
+                    min_value = np.min(filtered_values)
+                    max_value = np.max(filtered_values)
+
+                    # Store the min and max values in the min_max_array
+                    self.min_max_array[0, i] = min_value
+                    self.min_max_array[1, i] = max_value
+                self.settings.calib_array = np.array2string(self.min_max_array, separator=',')
+                self.config_class.save()
+                print("[INFO] Calibration completed.")
+
+                PlaySound('Audio/completed.wav', SND_FILENAME | SND_ASYNC)
+
+            if self.config.use_n_calibration:
+                self.calibration_frame_counter = None
+                values = np.array(self.val_list)
+                deadzone_value = -0.01   # Maybe adjust from app????
+                # Initialize the min_max_array with shape (2, num_outputs)
+                num_outputs = values.shape[1]
+                self.min_max_array = np.zeros((2, num_outputs))
+                lower_threshold = np.clip([np.mean(values, axis = 0) + deadzone_value], 0, 1)
+                upper_threshold = np.ones((1, num_outputs)) # We don't need to adjust the max values. 
+                print(lower_threshold)
+                print(upper_threshold)
+                self.min_max_array = np.array([lower_threshold, upper_threshold])
+                self.settings.calib_array = np.array2string(self.min_max_array, separator=',')
+                self.config_class.save()
+                print("[INFO] Calibration completed.")
 
-                # Extract the minimum and maximum values for the current index
-                min_value = np.min(filtered_values)
-                max_value = np.max(filtered_values)
 
-                # Store the min and max values in the min_max_array
-                self.min_max_array[0, i] = min_value
-                self.min_max_array[1, i] = max_value
-            self.settings.calib_array = np.array2string(self.min_max_array, separator=',')
-            self.config_class.save()
-            print("[INFO] Calibration completed.")
 
-            PlaySound('Audio/completed.wav', SND_FILENAME | SND_ASYNC)
 
-        elif self.calibration_frame_counter != None:
 
+
+        elif self.calibration_frame_counter != None:
             self.val_list.append(array)
             self.calibration_frame_counter -= 1
+        
 
         if self.settings.calib_array is not None and self.config.use_calibration:
             self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',')
@@ -70,10 +93,9 @@ def cal_osc(self, array):
             for i, value in enumerate(array):
                 min_value = self.min_max_array[0, i]
 
-
                 calibrated_value = (value - min_value) / (1.0 - min_value)
 
                 calibrated_array[i] = calibrated_value
             array = calibrated_array
-
-        return array
+        #array[4] = log((np.clip(array[4]*10,0,10))+1.0, 11) Log Filter: Move to filter system. 
+        return np.clip(array,0,1)   # Clamp outputs between 0-1