Skip to content

Commit 3c83094

Browse files
hasibiqbal207hasibiqbal207
committed
Integrated to classifiers into one
1 parent 347c943 commit 3c83094

File tree

8 files changed

+131
-116
lines changed

8 files changed

+131
-116
lines changed

‎MainWindow.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ def __init__(self):
5454

5555
self.setWindowTitle(WINDOW_TITLE)
5656
self.main_image_label.setScaledContents(True)
57-
self.gaze_image_label.setScaledContents(True)
57+
self.left_gaze_label.setScaledContents(True)
5858
# self.setWindowFlags(QtCore.Qt.FramelessWindowHint)
5959
# self.b = QtGui.QPushButton("exit", self, clicked=self.close)
6060

‎UI/MainWindow.ui‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@
203203
</layout>
204204
</item>
205205
<item>
206-
<widget class="QLabel" name="image_info_textlabel">
206+
<widget class="QLabel" name="left_gaze_label">
207207
<property name="minimumSize">
208208
<size>
209209
<width>270</width>
@@ -224,29 +224,29 @@
224224
</font>
225225
</property>
226226
<property name="text">
227-
<string>Not initialized yet</string>
227+
<string>Eye Gaze Left</string>
228228
</property>
229229
<property name="scaledContents">
230230
<bool>false</bool>
231231
</property>
232232
<property name="alignment">
233-
<set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
233+
<set>Qt::AlignCenter</set>
234234
</property>
235235
<property name="margin">
236236
<number>19</number>
237237
</property>
238238
</widget>
239239
</item>
240240
<item>
241-
<widget class="QLabel" name="gaze_image_label">
241+
<widget class="QLabel" name="right_gaze_label">
242242
<property name="minimumSize">
243243
<size>
244244
<width>30</width>
245245
<height>70</height>
246246
</size>
247247
</property>
248248
<property name="text">
249-
<string>Eye Gaze not found</string>
249+
<string>Eye Gaze Right</string>
250250
</property>
251251
<property name="alignment">
252252
<set>Qt::AlignCenter</set>

‎inputs/Controller.py‎

Lines changed: 89 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
from PyQt5.QtGui import QImage, QPixmap
33

44
import MainWindow
5-
from inputs.BlinkDetector import BlinkDetector
65
from inputs.FaceDetector import FaceDetector
76
from inputs.GazeDetectorCnn import GazeDetector
87
from inputs.Speech import Speech
@@ -19,7 +18,6 @@
1918
class Controller:
2019
def __init__(self, main_window, giveOutput):
2120
self.gazeImg = None
22-
self.blink_detector = BlinkDetector()
2321
self.face_detector = FaceDetector()
2422
self.gaze_detector = GazeDetector()
2523
# self.speech_detector = Speech()
@@ -28,91 +26,100 @@ def __init__(self, main_window, giveOutput):
2826
self.cap = cv2.VideoCapture(0)
2927

3028
def getInput(self):
31-
dicBlink = None
32-
dicGaze = None
33-
dicHead = None
3429
label = ""
35-
self.drawImg = None
36-
img = None
3730

3831
# Blink
39-
if True:
40-
_, img = self.cap.read()
41-
dicBlink = self.blink_detector.processImage(img, self.main_window.eyeThreshold.value())
42-
43-
drawImg = dicBlink["img"]
44-
45-
label = label + \
46-
"Blink Detector : " + "\n" \
47-
"\t" + "both : " + str(dicBlink["both"]) + "\n" \
48-
"\t" + "left : " + str(
49-
dicBlink["left"]) + "\n" \
50-
"\t" + "right : " + str(dicBlink["right"]) + "\n" \
51-
"\t" + "leftEAR : " + str(
52-
dicBlink["leftEAR"]) + "\n" \
53-
"\t" + "rightEAR :" + str(dicBlink["rightEAR"]) + "\n" \
54-
"\t" + "bothTotal : " + str(
55-
dicBlink["bothTotal"]) + "\n" \
56-
"\t" + "leftTotal : " + str(dicBlink["leftTotal"]) + "\n" \
57-
"\t" + "rightTotal : " + str(
58-
dicBlink["rightTotal"]) + "\n" \
59-
\
60-
# Head
61-
if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.HEAD_HELP:
62-
dicHead = self.face_detector.processImage(img, drawImg)
63-
64-
label = label + \
65-
"Head Detector : " + "\n" \
66-
"\t" + "direction : " + str(dicHead["direction"]) + "\n"
67-
68-
# GAZE
69-
if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.EYE_HELP \
70-
and self.main_window.current_mode in [MainWindow.MODE.CHAIR, MainWindow.MODE.KEYBOARD]:
71-
72-
dicGaze = self.gaze_detector.processImage(img)
73-
74-
label = label + \
75-
"Gaze Detector : " + "\n" \
76-
"\t" + "Gaze : " + str(dicGaze["direction"]) + "\n"
77-
78-
if dicGaze["img"] is not None:
79-
self.gazeImg = dicGaze["img"]
80-
self.gazeImg = toQImage(self.gazeImg)
81-
self.gazeImg = self.gazeImg.rgbSwapped()
82-
83-
if self.gazeImg is not None:
84-
self.main_window.gaze_image_label.setPixmap(QPixmap.fromImage(self.gazeImg))
85-
else:
86-
self.main_window.gaze_image_label.setText("None")
87-
88-
outImage = toQImage(drawImg)
32+
# if True:
33+
# _, img = self.cap.read()
34+
# dicBlink = self.blink_detector.processImage(img, self.main_window.eyeThreshold.value())
35+
#
36+
# drawImg = dicBlink["img"]
37+
#
38+
# label = label + \
39+
# "Blink Detector : " + "\n" \
40+
# "\t" + "both : " + str(dicBlink["both"]) + "\n" \
41+
# "\t" + "left : " + str(
42+
# dicBlink["left"]) + "\n" \
43+
# "\t" + "right : " + str(dicBlink["right"]) + "\n" \
44+
# "\t" + "leftEAR : " + str(
45+
# dicBlink["leftEAR"]) + "\n" \
46+
# "\t" + "rightEAR :" + str(dicBlink["rightEAR"]) + "\n" \
47+
# "\t" + "bothTotal : " + str(
48+
# dicBlink["bothTotal"]) + "\n" \
49+
# "\t" + "leftTotal : " + str(dicBlink["leftTotal"]) + "\n" \
50+
# "\t" + "rightTotal : " + str(
51+
# dicBlink["rightTotal"]) + "\n" \
52+
# \
53+
# # Head
54+
# if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.HEAD_HELP:
55+
# dicHead = self.face_detector.processImage(img, drawImg)
56+
#
57+
# label = label + \
58+
# "Head Detector : " + "\n" \
59+
# "\t" + "direction : " + str(dicHead["direction"]) + "\n"
60+
#
61+
# # GAZE
62+
# if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.EYE_HELP \
63+
# and self.main_window.current_mode in [MainWindow.MODE.CHAIR, MainWindow.MODE.KEYBOARD]:
64+
#
65+
# dicGaze = self.gaze_detector.processImage(img)
66+
#
67+
# label = label + \
68+
# "Gaze Detector : " + "\n" \
69+
# "\t" + "Gaze : " + str(dicGaze["direction"]) + "\n"
70+
#
71+
# if dicGaze["img"] is not None:
72+
# self.gazeImg = dicGaze["img"]
73+
# self.gazeImg = toQImage(self.gazeImg)
74+
# self.gazeImg = self.gazeImg.rgbSwapped()
75+
#
76+
# if self.gazeImg is not None:
77+
# self.main_window.gaze_image_label.setPixmap(QPixmap.fromImage(self.gazeImg))
78+
# else:
79+
# self.main_window.gaze_image_label.setText("None")
80+
81+
82+
# Blink and Gaze
83+
_, img = self.cap.read()
84+
dicGaze = self.gaze_detector.processImage(img)
85+
86+
outImage = toQImage(dicGaze["img"])
8987
outImage = outImage.rgbSwapped()
9088
self.main_window.main_image_label.setPixmap(QPixmap.fromImage(outImage))
91-
self.main_window.image_info_textlabel.setText(label)
92-
93-
if dicBlink is not None:
94-
if dicBlink["left"]:
95-
if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.HEAD_HELP:
96-
self.face_detector.initPos(dicHead["face"])
97-
return
98-
self.giveOutput("blinkleft")
99-
return
100-
elif dicBlink["right"]:
101-
self.giveOutput("blinkright")
102-
return
103-
elif dicBlink["both"]:
104-
self.giveOutput("blinkboth")
105-
return
106-
107-
if dicHead is not None:
108-
if dicHead["direction"] != "not initialized":
109-
self.giveOutput(dicHead["direction"])
110-
return
111-
112-
if dicGaze is not None:
113-
if dicGaze["direction"] != "not initialized":
114-
self.giveOutput(dicGaze["direction"])
115-
return
89+
90+
if dicGaze["gazeleft"] is not None:
91+
outImage = toQImage(dicGaze["gazeleft"])
92+
outImage = outImage.rgbSwapped()
93+
self.main_window.left_gaze_label.setPixmap(QPixmap.fromImage(outImage))
94+
95+
if dicGaze["gazeright"] is not None:
96+
outImage = toQImage(dicGaze["gazeright"])
97+
outImage = outImage.rgbSwapped()
98+
self.main_window.right_gaze_label.setPixmap(QPixmap.fromImage(outImage))
99+
100+
# if dicBlink is not None:
101+
# if dicBlink["left"]:
102+
# if self.main_window.selectMethodComboBox.currentIndex() == MainWindow.METHOD.HEAD_HELP:
103+
# self.face_detector.initPos(dicHead["face"])
104+
# return
105+
# self.giveOutput("blinkleft")
106+
# return
107+
# elif dicBlink["right"]:
108+
# self.giveOutput("blinkright")
109+
# return
110+
# elif dicBlink["both"]:
111+
# self.giveOutput("blinkboth")
112+
# return
113+
114+
# if dicHead is not None:
115+
# if dicHead["direction"] != "not initialized":
116+
# self.giveOutput(dicHead["direction"])
117+
# return
118+
#
119+
# if dicGaze is not None:
120+
# if dicGaze["direction"] != "not initialized":
121+
# self.giveOutput(dicGaze["direction"])
122+
# return
116123

117124

118125
def toQImage(raw_img):

‎inputs/GazeDetectorCnn.py‎

Lines changed: 36 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@ def __init__(self):
2626
self.predictor = dlib.shape_predictor(PREDICTOR_PATH)
2727
# cap = cv2.VideoCapture(0)
2828

29-
3029
def processImage(self, image):
3130

3231
ret = {
3332
"direction": None,
3433
"img": None,
34+
"gaze": None
3535
}
3636

3737
# a = datetime.datetime.now()
@@ -73,7 +73,6 @@ def processImage(self, image):
7373
roi2 = image[y:y + h, x:x + w]
7474
roi2 = imutils.resize(roi2, width=250, height=250, inter=cv2.INTER_CUBIC)
7575

76-
7776
if roi1 is not None:
7877
cv2.imwrite('temp_right.jpg', roi1)
7978
img = cv2.imread('temp_right.jpg')
@@ -82,6 +81,7 @@ def processImage(self, image):
8281
classes_r81 = self.model_right.predict_classes(img)
8382
classes_r82 = self.model_right.predict(img)
8483
class_no = classes_r81.item(0)
84+
ret["gazeleft"] = roi1
8585

8686
if roi2 is not None:
8787
cv2.imwrite('temp_left.jpg', roi2)
@@ -91,55 +91,63 @@ def processImage(self, image):
9191
classes_l81 = self.model_left.predict(img)
9292
classes_l82 = self.model_left.predict(img)
9393
class_no = classes_l81.item(0)
94+
ret["gazeright"] = roi2
9495

95-
ret["img"] = roi1
96+
ret["img"] = image
9697

9798
if roi1 is not None or roi2 is not None:
98-
self.percent(classes_l81, classes_r82, roi1, ret)
99+
self.percent(classes_l81, classes_r82, ret)
99100

100101
return ret
101102

102-
def percent(self, values_left, values_right, image, ret):
103+
def percent(self, values_left, values_right, ret):
103104
x_l = values_left.item(0)
104105
y_l = values_left.item(1)
105106
z_l = values_left.item(2)
107+
b_l = values_left.item(3)
106108

107109
x_r = values_right.item(0)
108110
y_r = values_right.item(1)
109111
z_r = values_right.item(2)
112+
b_r = values_right.item(3)
110113

111-
total_l = x_l + y_l + z_l
112-
total_r = x_r + y_r + z_r
114+
total_l = b_l + x_l + y_l + z_l
115+
total_r = b_r + x_r + y_r + z_r
113116

114117
x_l = float("{0:.2f}".format(x_l / total_l))
115118
y_l = float("{0:.2f}".format(y_l / total_l))
116119
z_l = float("{0:.2f}".format(z_l / total_l))
120+
b_l = float("{0:.2f}".format(b_l / total_l))
117121

118122
x_r = float("{0:.2f}".format(x_r / total_r))
119123
y_r = float("{0:.2f}".format(y_r / total_r))
120124
z_r = float("{0:.2f}".format(z_r / total_r))
125+
b_r = float("{0:.2f}".format(b_r / total_r))
121126

122127
x = (x_l + x_r) / 2
123128
y = (y_l + y_r) / 2
124129
z = (z_l + z_r) / 2
125-
# print("Prob of Class 0 is : ", x, "\nProb of Class 1 is : ", y, "\nProb of Class 2 is : ", z)
126-
127-
if (x >= 0.80):
128-
cv2.putText(image, "LEFT", (15, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 155, thickness=2)
129-
ret["direction"] = "gazeleft"
130-
elif (z >= 0.80):
131-
cv2.putText(image, "RIGHT", (25, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 130, thickness=2)
132-
ret["direction"] = "gazeright"
133-
elif (y >= 0.80):
134-
cv2.putText(image, "Middle", (40, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 255, thickness=2)
135-
ret["direction"] = "gazecenter"
136-
137-
def reset(self):
138-
self.init = [0, 0]
139-
self.coordinate = [0, 0]
140-
self.dir_c = 0
141-
self.dir_l = 0
142-
self.dir_r = 0
143-
144-
def closeAll(self):
145-
cv2.destroyAllWindows()
130+
b = (b_l + b_r) / 2
131+
print("Prob of Class 0 is : ", x, "\nProb of Class 1 is : ", y, "\nProb of Class 2 is : ", z,
132+
"\nProb of Class 3 is : ", b)
133+
134+
if x >= 0.80:
135+
cv2.putText(ret["img"], "Blink", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 155, thickness=3)
136+
ret["blink"] = "both"
137+
elif x_l >= 0.80 and x_r <= 0.50:
138+
cv2.putText(ret["img"], "LEFT Blink", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 255, thickness=3)
139+
ret["blink"] = "left"
140+
elif x_l <= 0.50 and x_r >= 0.80:
141+
cv2.putText(ret["img"], "RIGHT Blink", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 255, thickness=3)
142+
ret["blink"] = "right"
143+
else:
144+
ret["blink"] = "none"
145+
if y >= 0.80:
146+
cv2.putText(ret["img"], "LEFT", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 255, thickness=3)
147+
ret["gazedirection"] = "left"
148+
elif z >= 0.80:
149+
cv2.putText(ret["img"], "Middle", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 130, thickness=3)
150+
ret["gazedirection"] = "center"
151+
elif b >= 0.80:
152+
cv2.putText(ret["img"], "RIGHT", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 255, thickness=3)
153+
ret["gazedirection"] = "right"

‎inputs/lefteyemodel.h5‎

488 Bytes
Binary file not shown.

‎inputs/righteyemodel.h5‎

488 Bytes
Binary file not shown.

‎temp_left.jpg‎

-494 Bytes
Loading

‎temp_right.jpg‎

1.47 KB
Loading

0 commit comments

Comments
 (0)