Skip to content

Commit 2e8f3d3

Browse files
amalnanavatiamalnanavati
authored
Allow Users to Zoom Into Bite Selection Video (#120)
* Allow users to zoom into the bite selection video * Rearrange display on DetectingFace page * Fix bug in scale factor * Proper error return in siganlling server * Lower robot browser's memory footprint by half * Add Segfault monitor --------- Co-authored-by: Amal Nanavati <amaln@cs.washington.edu>
1 parent 90c95e2 commit 2e8f3d3

File tree

9 files changed

+205
-48
lines changed

9 files changed

+205
-48
lines changed

‎feedingwebapp/package-lock.json

Lines changed: 29 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎feedingwebapp/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
"react-toastify": "^9.0.7",
3333
"roslib": "github:personalrobotics/roslibjs",
3434
"rosreact": "^0.2.0",
35+
"segfault-handler": "^1.3.0",
3536
"styled-components": "^5.3.9",
3637
"web-vitals": "^2.1.4",
3738
"webpack": "^5.82.1",

‎feedingwebapp/server.js

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
* Each subscriber sees the video stream from the publisher.
77
*/
88

9+
const SegfaultHandler = require('segfault-handler')
10+
SegfaultHandler.registerHandler('crash.log')
911
const express = require('express')
1012
const app = express()
1113
const bodyParser = require('body-parser')
@@ -65,6 +67,7 @@ app.post('/subscribe', async ({ body }, res) => {
6567
res.json(payload)
6668
} catch (err) {
6769
console.error('Failed to process subscriber, exception: ' + err.message)
70+
res.sendStatus(500)
6871
}
6972
})
7073

@@ -107,10 +110,12 @@ app.post('/publish', async ({ body }, res) => {
107110
res.json(payload)
108111
} catch (err) {
109112
console.error('Failed to process publisher, exception: ' + err.message)
113+
res.sendStatus(500)
110114
}
111115
})
112116

113117
function handleTrackEvent(e, topic) {
118+
console.log('Handle track for publisher')
114119
senderStream[topic] = e.streams[0]
115120
}
116121

‎feedingwebapp/src/Pages/GlobalState.jsx

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,9 @@ export const useGlobalState = create(
138138
// this is the state we transition to after R_MovingFromMouth. In practice,
139139
// it is either R_MovingAbovePlate, R_MovingToRestingPosition, or R_DetectingFace.
140140
mostRecentBiteDoneResponse: MEAL_STATE.R_DetectingFace,
141+
// How much the video on the Bite Selection page should be zoomed in.
142+
biteSelectionZoom: 1.0,
143+
141144
// Settings values
142145
// stagingPosition: SETTINGS.stagingPosition[0],
143146
// biteInitiation: SETTINGS.biteInitiation[0],
@@ -196,6 +199,10 @@ export const useGlobalState = create(
196199
setBiteTransferPageAtFace: (biteTransferPageAtFace) =>
197200
set(() => ({
198201
biteTransferPageAtFace: biteTransferPageAtFace
202+
})),
203+
setBiteSelectionZoom: (biteSelectionZoom) =>
204+
set(() => ({
205+
biteSelectionZoom: biteSelectionZoom
199206
}))
200207
// setStagingPosition: (stagingPosition) =>
201208
// set(() => ({

‎feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ const BiteSelection = (props) => {
3636
// Get the relevant global variables
3737
const setMealState = useGlobalState((state) => state.setMealState)
3838
const setBiteAcquisitionActionGoal = useGlobalState((state) => state.setBiteAcquisitionActionGoal)
39+
const biteSelectionZoom = useGlobalState((state) => state.biteSelectionZoom)
40+
const setBiteSelectionZoom = useGlobalState((state) => state.setBiteSelectionZoom)
3941
// Get icon image for move to mouth
4042
let moveToStagingConfigurationImage = MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingToStagingConfiguration]
4143

@@ -458,7 +460,7 @@ const BiteSelection = (props) => {
458460
height: '100%'
459461
}}
460462
>
461-
<VideoFeed pointClicked={imageClicked} webrtcURL={props.webrtcURL} />
463+
<VideoFeed pointClicked={imageClicked} webrtcURL={props.webrtcURL} zoom={biteSelectionZoom} setZoom={setBiteSelectionZoom} />
462464
</View>
463465
</View>
464466
<View
@@ -592,6 +594,8 @@ const BiteSelection = (props) => {
592594
imageClicked,
593595
props.debug,
594596
props.webrtcURL,
597+
biteSelectionZoom,
598+
setBiteSelectionZoom,
595599
debugButton
596600
])
597601

‎feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,10 @@ const DetectingFace = (props) => {
3838
let otherDimension = isPortrait ? 'row' : 'column'
3939
// Font size for text
4040
let textFontSize = 3
41-
let buttonWidth = 30
42-
let buttonHeight = 18
43-
let iconWidth = 28
44-
let iconHeight = 16
41+
// let buttonWidth = 22
42+
let buttonHeight = 14
43+
let iconWidth = 20
44+
let iconHeight = 12
4545
let sizeSuffix = isPortrait ? 'vh' : 'vw'
4646

4747
/**
@@ -130,10 +130,10 @@ const DetectingFace = (props) => {
130130
width: '100%'
131131
}}
132132
>
133-
<View style={{ flex: 5, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
133+
<View style={{ flex: 6, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
134134
<DetectingFaceSubcomponent faceDetectedCallback={faceDetectedCallback} webrtcURL={props.webrtcURL} />
135135
</View>
136-
<View style={{ flex: 3, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
136+
<View style={{ flex: 2, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
137137
<p className='transitionMessage' style={{ marginBottom: '0px', fontSize: textFontSize.toString() + sizeSuffix }}>
138138
{mouthDetected ? 'Continue' : 'Continue without detection'}
139139
</p>
@@ -144,7 +144,7 @@ const DetectingFace = (props) => {
144144
size='lg'
145145
onClick={moveToMouthCallback}
146146
style={{
147-
width: buttonWidth.toString() + sizeSuffix,
147+
width: '90%',
148148
height: buttonHeight.toString() + sizeSuffix,
149149
display: 'flex',
150150
justifyContent: 'center',
@@ -180,8 +180,8 @@ const DetectingFace = (props) => {
180180
size='lg'
181181
onClick={moveToRestingCallback}
182182
style={{
183-
width: (buttonWidth / 2).toString() + sizeSuffix,
184-
height: (buttonHeight / 2).toString() + sizeSuffix,
183+
width: '90%',
184+
height: ((buttonHeight * 2) / 3).toString() + sizeSuffix,
185185
display: 'flex',
186186
justifyContent: 'center',
187187
alignContent: 'center'
@@ -191,7 +191,7 @@ const DetectingFace = (props) => {
191191
src={moveToRestingImage}
192192
alt='move_to_resting_image'
193193
className='center'
194-
style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: (iconHeight / 2).toString() + sizeSuffix }}
194+
style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: ((iconHeight * 2) / 3).toString() + sizeSuffix }}
195195
/>
196196
</Button>
197197
</View>
@@ -206,8 +206,8 @@ const DetectingFace = (props) => {
206206
size='lg'
207207
onClick={moveAbovePlateCallback}
208208
style={{
209-
width: (buttonWidth / 2).toString() + sizeSuffix,
210-
height: (buttonHeight / 2).toString() + sizeSuffix,
209+
width: '90%',
210+
height: ((buttonHeight * 2) / 3).toString() + sizeSuffix,
211211
display: 'flex',
212212
justifyContent: 'center',
213213
alignContent: 'center'
@@ -217,7 +217,7 @@ const DetectingFace = (props) => {
217217
src={moveAbovePlateImage}
218218
alt='move_above_plate_image'
219219
className='center'
220-
style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: (iconHeight / 2).toString() + sizeSuffix }}
220+
style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: ((iconHeight * 2) / 3).toString() + sizeSuffix }}
221221
/>
222222
</Button>
223223
</View>
@@ -239,7 +239,6 @@ const DetectingFace = (props) => {
239239
props.webrtcURL,
240240
textFontSize,
241241
buttonHeight,
242-
buttonWidth,
243242
sizeSuffix,
244243
iconHeight,
245244
iconWidth,

‎feedingwebapp/src/Pages/Home/MealStates/DetectingFaceSubcomponent.jsx

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
// React Imports
2-
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
2+
import React, { useCallback, useEffect, useRef, useState } from 'react'
33
import PropTypes from 'prop-types'
44
import { useMediaQuery } from 'react-responsive'
55
import { View } from 'react-native'
66

77
// Local Imports
88
import { useROS, createROSService, createROSServiceRequest, subscribeToROSTopic, unsubscribeFromROSTopic } from '../../../ros/ros_helpers'
99
import '../Home.css'
10-
import { convertRemToPixels } from '../../../helpers'
1110
import { MEAL_STATE } from '../../GlobalState'
1211
import { FACE_DETECTION_IMG_TOPIC, FACE_DETECTION_TOPIC, FACE_DETECTION_TOPIC_MSG, ROS_SERVICE_NAMES } from '../../Constants'
1312
import VideoFeed from '../VideoFeed'
@@ -29,10 +28,6 @@ const DetectingFaceSubcomponent = (props) => {
2928
// conidered valid. NOTE: This must match the values in the MoveToMouth tree.
3029
const min_face_distance = 0.4
3130
const max_face_distance = 1.25
32-
// Margin for the video feed and between the mask buttons. Note this cannot
33-
// be re-defined per render, otherwise it messes up re-rendering order upon
34-
// resize in VideoFeed.
35-
const margin = useMemo(() => convertRemToPixels(1), [])
3631

3732
/**
3833
* Connect to ROS, if not already connected. Put this in useRef to avoid
@@ -130,14 +125,7 @@ const DetectingFaceSubcomponent = (props) => {
130125
height: '100%'
131126
}}
132127
>
133-
<VideoFeed
134-
marginTop={margin}
135-
marginBottom={margin}
136-
marginLeft={margin}
137-
marginRight={margin}
138-
topic={FACE_DETECTION_IMG_TOPIC}
139-
webrtcURL={props.webrtcURL}
140-
/>
128+
<VideoFeed topic={FACE_DETECTION_IMG_TOPIC} webrtcURL={props.webrtcURL} />
141129
</View>
142130
</>
143131
)

0 commit comments

Comments
 (0)