Skip to content

Commit 19b671d

Browse files
amalnanavatiamalnanavati
authored
Override AutoContinue if moving back from mouth (#119)
* Override auto-continue when moving from mouth * Add exception handling in webrtc server
1 parent b0e748f commit 19b671d

File tree

3 files changed

+89
-74
lines changed

3 files changed

+89
-74
lines changed

‎feedingwebapp/server.js

Lines changed: 79 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -24,82 +24,90 @@ app.use(bodyParser.urlencoded({ extended: true }))
2424
app.use(cors())
2525

2626
app.post('/subscribe', async ({ body }, res) => {
27-
console.log('got subscriber on IP', body.ip, 'for topic', body.topic)
28-
29-
// Configure the peer connection
30-
const peer = new webrtc.RTCPeerConnection({
31-
iceServers: [
32-
{
33-
urls: 'stun:stun1.l.google.com:19302'
34-
}
35-
]
36-
})
37-
38-
// Close any old peers on the same IP address
39-
const topic = body.topic
40-
const key = body.ip + ':' + topic
41-
if (key in subscribePeers && subscribePeers[key] && subscribePeers[key].connectionState !== 'closed') {
42-
const senders = subscribePeers[key].getSenders()
43-
senders.forEach((sender) => subscribePeers[key].removeTrack(sender))
44-
subscribePeers[key].close()
27+
try {
28+
console.log('got subscriber on IP', body.ip, 'for topic', body.topic)
29+
30+
// Configure the peer connection
31+
const peer = new webrtc.RTCPeerConnection({
32+
iceServers: [
33+
{
34+
urls: 'stun:stun1.l.google.com:19302'
35+
}
36+
]
37+
})
38+
39+
// Close any old peers on the same IP address
40+
const topic = body.topic
41+
const key = body.ip + ':' + topic
42+
if (key in subscribePeers && subscribePeers[key] && subscribePeers[key].connectionState !== 'closed') {
43+
const senders = subscribePeers[key].getSenders()
44+
senders.forEach((sender) => subscribePeers[key].removeTrack(sender))
45+
subscribePeers[key].close()
46+
}
47+
subscribePeers[key] = peer
48+
49+
const desc = new webrtc.RTCSessionDescription(body.sdp)
50+
await peer.setRemoteDescription(desc)
51+
52+
// Add the publisher's video stream to the subscriber's peer connection
53+
if (topic in senderStream) {
54+
senderStream[topic].getTracks().forEach((track) => peer.addTrack(track, senderStream[topic]))
55+
}
56+
57+
// Create an answer to the publisher's offer
58+
const answer = await peer.createAnswer()
59+
await peer.setLocalDescription(answer)
60+
const payload = {
61+
sdp: peer.localDescription
62+
}
63+
64+
// Send the answer to the publisher
65+
res.json(payload)
66+
} catch (err) {
67+
console.error('Failed to process subscriber, exception: ' + err.message)
4568
}
46-
subscribePeers[key] = peer
47-
48-
const desc = new webrtc.RTCSessionDescription(body.sdp)
49-
await peer.setRemoteDescription(desc)
50-
51-
// Add the publisher's video stream to the subscriber's peer connection
52-
if (topic in senderStream) {
53-
senderStream[topic].getTracks().forEach((track) => peer.addTrack(track, senderStream[topic]))
54-
}
55-
56-
// Create an answer to the publisher's offer
57-
const answer = await peer.createAnswer()
58-
await peer.setLocalDescription(answer)
59-
const payload = {
60-
sdp: peer.localDescription
61-
}
62-
63-
// Send the answer to the publisher
64-
res.json(payload)
6569
})
6670

6771
app.post('/publish', async ({ body }, res) => {
68-
console.log('got publisher on IP', body.ip, 'for topic', body.topic)
69-
70-
// Configure the peer connection
71-
const peer = new webrtc.RTCPeerConnection({
72-
iceServers: [
73-
{
74-
urls: 'stun:stun1.l.google.com:19302'
75-
}
76-
]
77-
})
78-
79-
// Close any old peers on the same IP address
80-
const topic = body.topic
81-
const key = body.ip + ':' + topic
82-
if (key in publishPeers && publishPeers[key] && publishPeers[key].connectionState !== 'closed') {
83-
const senders = publishPeers[key].getSenders()
84-
senders.forEach((sender) => publishPeers[key].removeTrack(sender))
85-
publishPeers[key].close()
72+
try {
73+
console.log('got publisher on IP', body.ip, 'for topic', body.topic)
74+
75+
// Configure the peer connection
76+
const peer = new webrtc.RTCPeerConnection({
77+
iceServers: [
78+
{
79+
urls: 'stun:stun1.l.google.com:19302'
80+
}
81+
]
82+
})
83+
84+
// Close any old peers on the same IP address
85+
const topic = body.topic
86+
const key = body.ip + ':' + topic
87+
if (key in publishPeers && publishPeers[key] && publishPeers[key].connectionState !== 'closed') {
88+
const senders = publishPeers[key].getSenders()
89+
senders.forEach((sender) => publishPeers[key].removeTrack(sender))
90+
publishPeers[key].close()
91+
}
92+
publishPeers[key] = peer
93+
94+
// Send the publisher's video stream to all subscribers on that topic
95+
peer.ontrack = (e) => handleTrackEvent(e, topic)
96+
97+
// Create an answer to the publisher's offer
98+
const desc = new webrtc.RTCSessionDescription(body.sdp)
99+
await peer.setRemoteDescription(desc)
100+
const answer = await peer.createAnswer()
101+
await peer.setLocalDescription(answer)
102+
const payload = {
103+
sdp: peer.localDescription
104+
}
105+
106+
// Send the answer to the publisher
107+
res.json(payload)
108+
} catch (err) {
109+
console.error('Failed to process publisher, exception: ' + err.message)
86110
}
87-
publishPeers[key] = peer
88-
89-
// Send the publisher's video stream to all subscribers on that topic
90-
peer.ontrack = (e) => handleTrackEvent(e, topic)
91-
92-
// Create an answer to the publisher's offer
93-
const desc = new webrtc.RTCSessionDescription(body.sdp)
94-
await peer.setRemoteDescription(desc)
95-
const answer = await peer.createAnswer()
96-
await peer.setLocalDescription(answer)
97-
const payload = {
98-
sdp: peer.localDescription
99-
}
100-
101-
// Send the answer to the publisher
102-
res.json(payload)
103111
})
104112

105113
function handleTrackEvent(e, topic) {

‎feedingwebapp/src/Pages/GlobalState.jsx

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,11 +103,13 @@ export const SETTINGS_STATE = {
103103
*/
104104
export const useGlobalState = create(
105105
persist(
106-
(set) => ({
106+
(set, get) => ({
107107
// The current app page
108108
appPage: APP_PAGE.Home,
109109
// The app's current meal state
110110
mealState: MEAL_STATE.U_PreMeal,
111+
// The app's previous meal state
112+
prevMealState: null,
111113
// The timestamp when the robot transitioned to its current meal state
112114
mealStateTransitionTime: Date.now(),
113115
// The currently displayed settings page
@@ -152,6 +154,7 @@ export const useGlobalState = create(
152154
set(() => {
153155
let retval = {
154156
mealState: mealState,
157+
prevMealState: get().mealState,
155158
mealStateTransitionTime: Date.now(),
156159
biteTransferPageAtFace: false // Reset this flag when the meal state changes
157160
}

‎feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const DetectingFace = (props) => {
2222
// Keep track of whether a mouth has been detected or not
2323
const [mouthDetected, setMouthDetected] = useState(false)
2424
// Get the relevant global variables
25+
const prevMealState = useGlobalState((state) => state.prevMealState)
2526
const setMealState = useGlobalState((state) => state.setMealState)
2627
const setMoveToMouthActionGoal = useGlobalState((state) => state.setMoveToMouthActionGoal)
2728
const faceDetectionAutoContinue = useGlobalState((state) => state.faceDetectionAutoContinue)
@@ -84,11 +85,14 @@ const DetectingFace = (props) => {
8485
face_detection: message
8586
})
8687
// Automatically move on to the next stage if a face is detected
87-
if (faceDetectionAutoContinue) {
88+
// If the app got to this screen after moving away from the user's mouth,
89+
// don't auto-continue. Only do so if it gets to this page from
90+
// R_MovingToStagingConfiguration
91+
if (faceDetectionAutoContinue && prevMealState !== MEAL_STATE.R_MovingFromMouth) {
8892
moveToMouthCallback()
8993
}
9094
},
91-
[faceDetectionAutoContinue, moveToMouthCallback, setMoveToMouthActionGoal]
95+
[faceDetectionAutoContinue, moveToMouthCallback, prevMealState, setMoveToMouthActionGoal]
9296
)
9397

9498
/** Get the full page view

0 commit comments

Comments
 (0)