Skip to content

Commit 7c204ed

Browse files
SummerSighSummerSigh
committed
Refresh Repo
1 parent 8382145 commit 7c204ed

13 files changed

+232
-511
lines changed

‎.gitattributes

-2
This file was deleted.

‎.gitignore

-159
This file was deleted.

‎BabbleCPU.py

+232
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
import os
2+
os.environ["OMP_NUM_THREADS"] = "1"
3+
import onnxruntime as ort
4+
import time
5+
import PySimpleGUI as sg
6+
import cv2
7+
from pythonosc import udp_client
8+
from torchvision.transforms.functional import to_grayscale
9+
import PIL.Image as Image
10+
from torchvision import transforms
11+
from threading import Thread
12+
13+
14+
class WebcamVideoStream:
15+
def __init__(self, src=0):
16+
# initialize the video camera stream and read the first frame
17+
# from the stream
18+
self.stream = cv2.VideoCapture(src)
19+
(self.grabbed, self.frame) = self.stream.read()
20+
# initialize the variable used to indicate if the thread should
21+
# be stopped
22+
self.stopped = False
23+
def start(self):
24+
# start the thread to read frames from the video stream
25+
Thread(target=self.update, args=()).start()
26+
return self
27+
def update(self):
28+
# keep looping infinitely until the thread is stopped
29+
while True:
30+
# if the thread indicator variable is set, stop the thread
31+
if self.stopped:
32+
return
33+
# otherwise, read the next frame from the stream
34+
(self.grabbed, self.frame) = self.stream.read()
35+
def read(self):
36+
# return the frame most recently read
37+
return self.frame
38+
def stop(self):
39+
# indicate that the thread should be stopped
40+
self.stopped = True
41+
42+
def onesizefitsallminmaxarray(): # Some predefined ranges in stored values instead of the generated ones
43+
stored_values = [
44+
[0.2, 0.6], # CheekPuff
45+
[0, 1], # cheekSquintLeft
46+
[0, 1], # cheekSquintRight
47+
[0, 1], # noseSneerLeft
48+
[0, 1], # noseSneerRight
49+
[0.1, 0.8], # jawOpen
50+
[0.1, 0.8], # jawForward
51+
[0.05, 0.5], # jawLeft
52+
[0.05, 0.5], # jawRight
53+
[0.1, 1], # mouthFunnel
54+
[0.03, 0.7], # mouthPucker
55+
[0.05, 0.5], # mouthLeft
56+
[0.05, 0.5], # mouthRight
57+
[0, 0.3], # mouthRollUpper
58+
[0, 0.3], # mouthRollLower
59+
[0.03, 0.5], # mouthShrugUpper
60+
[0.03, 0.5], # mouthShrugLower
61+
[0, 0.3], # mouthClose
62+
[0.1, 1], # mouthSmileLeft
63+
[0.1, 1], # mouthSmileRight
64+
[0, 0.4], # mouthFrownLeft
65+
[0, 0.4], # mouthFrownRight
66+
[0, 1], # mouthDimpleLeft
67+
[0, 1], # mouthDimpleRight
68+
[0.02, 0.8], # mouthUpperUpLeft
69+
[0.02, 0.8], # mouthUpperUpRight
70+
[0.02, 0.8], # mouthLowerDownLeft
71+
[0.02, 0.8], # mouthLowerDownRight
72+
[0, 1], # mouthPressLeft
73+
[0, 1], # mouthPressRight
74+
[0, 1], # mouthStretchLeft
75+
[0, 1], # mouthStretchRight
76+
[0.1, 0.8], # tongueOut
77+
]
78+
return stored_values
79+
80+
81+
82+
def makeminmaxarray(amount):
83+
stored_values = []
84+
for i in range(amount):
85+
stored_values.append([0,0])
86+
return stored_values
87+
88+
89+
def minmax(stored_values, value):
90+
if stored_values[0] == 0 and stored_values[1] == 0:
91+
stored_values[0] = value
92+
stored_values[1] = value
93+
array = (0, value, 1)
94+
else:
95+
if value < stored_values[0]: stored_values[0] = value
96+
if value > stored_values[1]: stored_values[1] = value
97+
array = (stored_values[0], value, stored_values[1])
98+
return array
99+
100+
def normalize_value(array):
101+
normalized_value = ((array[1] - array[0]) / (array[2] - array[0]))
102+
return normalized_value
103+
104+
stored_values = makeminmaxarray(33)
105+
stored_values = onesizefitsallminmaxarray()
106+
107+
108+
"""
109+
Demo program that allows a user to type in a webcam url or number and displays it using OpenCV
110+
"""
111+
112+
# ---------------- Create Form ----------------
113+
sg.theme('DarkAmber') # Add a touch of color
114+
# All the stuff inside your window.
115+
layout = [
116+
[sg.Image(key = '-IMAGE-')],
117+
[sg.Text('Enter Webcam URL or Number (Defualts to 0)'), sg.Input(key = '-URL-', size = (30, 1))],
118+
[sg.Text('Enter Onnx model name (Defualts to v1.onnx)'), sg.Input(key = '-MODEL-', size = (30, 1))],
119+
[sg.Text('OSC Location Address'), sg.Input(key = '-LOC-', size = (30, 1))],
120+
[sg.Text('Output Mutiplier (defualts to 1. Please use 100 if you are using the unity demo.)'), sg.Input(key = '-MULT-', size = (30, 1))],
121+
[sg.Text('Enter OSC IP (Defualts to 127.0.0.1)'), sg.Input(key = '-OSC-', size = (30, 1))],
122+
[sg.Text('Enter OSC Port (Defualts to 9000)'), sg.Input(key = '-PORT-', size = (30, 1))],
123+
[sg.Text('Press Ok to start the webcam')],
124+
[sg.Checkbox('Flip 180', key = '-FLIP180-')],
125+
[sg.Checkbox('No Calibration', key = '-CAL-')],
126+
[sg.Button('Start'), sg.Button('Stop')],
127+
]
128+
129+
# Create the Window
130+
window = sg.Window('Project BabbleV1.0', layout, location = (800, 400))
131+
# ---------------- Event Loop ----------------
132+
while True:
133+
event, values = window.read(timeout = 20)
134+
OSCip= values['-OSC-'] #VR Chat OSC ip
135+
if OSCip == '':
136+
OSCip="127.0.0.1"
137+
OSCport= values['-PORT-'] #VR Chat OSC port
138+
if OSCport == '':
139+
OSCport = 9000
140+
multi = values['-MULT-']
141+
if multi == '':
142+
multi = 1
143+
else:
144+
multi = int(multi)
145+
client = udp_client.SimpleUDPClient(OSCip, OSCport)
146+
model = values['-MODEL-']
147+
if model == '':
148+
model = 'v1.onnx'
149+
flip180 = values['-FLIP180-']
150+
location = values['-LOC-']
151+
sess = ort.InferenceSession(model, providers=['CPUExecutionProvider'])
152+
input_name = sess.get_inputs()[0].name
153+
output_name = sess.get_outputs()[0].name
154+
if event == sg.WIN_CLOSED or event == 'Stop':
155+
break
156+
if event == 'Start':
157+
url = values['-URL-']
158+
if url[0] in '0123456789':
159+
url = int(url)
160+
elif url == '':
161+
url = 0
162+
steamer = WebcamVideoStream(url).start()
163+
while True:
164+
frame = steamer.read()
165+
if flip180:
166+
frame = cv2.flip(frame, 0)
167+
frame2 = frame
168+
frame = cv2.resize(frame, (256, 256))
169+
#make it pil
170+
frame = Image.fromarray(frame)
171+
#make it grayscale
172+
frame = to_grayscale(frame)
173+
#make it a tensor
174+
frame = transforms.ToTensor()(frame)
175+
#make it a batch
176+
frame = frame.unsqueeze(0)
177+
#make it a numpy array
178+
frame = frame.numpy()
179+
#print te amout of outputs that are zero
180+
out = sess.run([output_name], {input_name: frame})
181+
end = time.time()
182+
output = out[0]
183+
output = output[0]
184+
output = [0 if x < 0 else x for x in output]
185+
if values['-CAL-'] == False:
186+
array = []
187+
for i in range(len(output)):
188+
yeah = minmax(stored_values[i], output[i])
189+
value = normalize_value(yeah)
190+
array.append(value)
191+
else:
192+
array = output
193+
client.send_message(location + "/cheekPuff", array[0] * multi)
194+
client.send_message(location + "/cheekSquintLeft", output[1] * multi)
195+
client.send_message(location + "/cheekSquintRight", output[2] * multi)
196+
client.send_message(location + "/noseSneerLeft", output[3] * multi)
197+
client.send_message(location + "/noseSneerRight", output[4] * multi)
198+
client.send_message(location + "/jawOpen", array[5] * multi)
199+
client.send_message(location + "/jawForward", array[6] * multi)
200+
client.send_message(location + "/jawLeft", array[7] * multi)
201+
client.send_message(location + "/jawRight", array[8] * multi)
202+
client.send_message(location + "/mouthFunnel", array[9] * multi)
203+
client.send_message(location + "/mouthPucker", array[10] * multi)
204+
client.send_message(location + "/mouthLeft", array[11] * multi)
205+
client.send_message(location + "/mouthRight", array[12] * multi)
206+
client.send_message(location + "/mouthRollUpper", array[13] * multi)
207+
client.send_message(location + "/mouthRollLower", array[14] * multi)
208+
client.send_message(location + "/mouthShrugUpper", array[15] * multi)
209+
client.send_message(location + "/mouthShrugLower", array[16] * multi)
210+
client.send_message(location + "/mouthClose", output[17] * multi)
211+
client.send_message(location + "/mouthSmileLeft", array[18] * multi)
212+
client.send_message(location + "/mouthSmileRight", array[19] * multi)
213+
client.send_message(location + "/mouthFrownLeft", array[20] * multi)
214+
client.send_message(location + "/mouthFrownRight", array[21] * multi)
215+
client.send_message(location + "/mouthDimpleLeft", array[22] * multi)
216+
client.send_message(location + "/mouthDimpleRight", array[23] * multi)
217+
client.send_message(location + "/mouthUpperUpLeft", array[24] * multi)
218+
client.send_message(location + "/mouthUpperUpRight", array[25] * multi)
219+
client.send_message(location + "/mouthLowerDownLeft", array[26] * multi)
220+
client.send_message(location + "/mouthLowerDownRight", array[27] * multi)
221+
client.send_message(location + "/mouthPressLeft", array[28] * multi)
222+
client.send_message(location + "/mouthPressRight", array[29] * multi)
223+
client.send_message(location + "/mouthStretchLeft", array[30] * multi)
224+
client.send_message(location + "/mouthStretchRight", array[31] * multi)
225+
client.send_message(location + "/tongueOut", array[32] * multi)
226+
imgbytes = cv2.imencode('.png', frame2)[1].tobytes() # ditto
227+
window['-IMAGE-'].update(data=imgbytes)
228+
event, values = window.read(timeout = 20)
229+
if event == sg.WIN_CLOSED or event == 'Stop':
230+
break
231+
steamer.stop()
232+
window.close()

‎BestTrainNeck1.pt

-8.87 MB
Binary file not shown.

0 commit comments

Comments
 (0)