Skip to content

Commit 16d1e00

Browse files
committed
Add reload button to video feed and improve WebRTC reconnection
1 parent 961bce4 commit 16d1e00

File tree

9 files changed

+195
-164
lines changed

9 files changed

+195
-164
lines changed

feedingwebapp/server.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ app.post('/subscribe', async ({ body }, res) => {
3838
// Close any old peers on the same IP address
3939
const topic = body.topic
4040
const key = body.ip + ':' + topic
41-
if (key in subscribePeers) {
41+
if (key in subscribePeers && subscribePeers[key] && subscribePeers[key].connectionState !== 'closed') {
4242
const senders = subscribePeers[key].getSenders()
4343
senders.forEach((sender) => subscribePeers[key].removeTrack(sender))
4444
subscribePeers[key].close()
@@ -79,7 +79,7 @@ app.post('/publish', async ({ body }, res) => {
7979
// Close any old peers on the same IP address
8080
const topic = body.topic
8181
const key = body.ip + ':' + topic
82-
if (key in publishPeers) {
82+
if (key in publishPeers && publishPeers[key] && publishPeers[key].connectionState !== 'closed') {
8383
const senders = publishPeers[key].getSenders()
8484
senders.forEach((sender) => publishPeers[key].removeTrack(sender))
8585
publishPeers[key].close()

feedingwebapp/src/App.jsx

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ function App() {
6060

6161
// Get the WebRTC signalling server URL
6262
const webrtcURL = 'http://'.concat(window.location.hostname, ':', process.env.REACT_APP_SIGNALLING_SERVER_PORT)
63-
console.log(process.env)
6463

6564
// Get the debug flag
6665
const debug = process.env.REACT_APP_DEBUG === 'true'

feedingwebapp/src/Pages/Header/LiveVideoModal.jsx

Lines changed: 14 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// React imports
2-
import React, { useMemo, useRef } from 'react'
2+
import React from 'react'
33
import { useMediaQuery } from 'react-responsive'
44
// The Modal is a screen that appears on top of the main app, and can be toggled
55
// on and off.
@@ -10,7 +10,6 @@ import PropTypes from 'prop-types'
1010

1111
// Local imports
1212
import { CAMERA_FEED_TOPIC } from '../Constants'
13-
import { convertRemToPixels } from '../../helpers'
1413
import VideoFeed from '../Home/VideoFeed'
1514

1615
/**
@@ -19,12 +18,6 @@ import VideoFeed from '../Home/VideoFeed'
1918
* TODO: Consider what will happen if the connection to ROS isn't working.
2019
*/
2120
function LiveVideoModal(props) {
22-
// Variables to render the VideoFeed
23-
const modalBodyRef = useRef(null)
24-
// Margin for the video feed and between the mask buttons. Note this cannot
25-
// be re-defined per render, otherwise it messes up re-rendering order upon
26-
// resize in VideoFeed.
27-
const margin = useMemo(() => convertRemToPixels(1), [])
2821
// Flag to check if the current orientation is portrait
2922
const isPortrait = useMediaQuery({ query: '(orientation: portrait)' })
3023
// Text font size for portrait and landscape orientations
@@ -47,19 +40,19 @@ function LiveVideoModal(props) {
4740
Live Video
4841
</Modal.Title>
4942
</Modal.Header>
50-
<Modal.Body ref={modalBodyRef} style={{ overflow: 'hidden' }}>
51-
<center>
52-
<VideoFeed
53-
topic={CAMERA_FEED_TOPIC}
54-
updateRateHz={10}
55-
parent={modalBodyRef}
56-
marginTop={margin}
57-
marginBottom={margin}
58-
marginLeft={margin}
59-
marginRight={margin}
60-
webrtcURL={props.webrtcURL}
61-
/>
62-
</center>
43+
<Modal.Body
44+
style={{
45+
flex: 1,
46+
display: 'flex',
47+
flexDirection: 'column',
48+
alignItems: 'center',
49+
justifyContent: 'center',
50+
width: '100%',
51+
height: '100%',
52+
overflow: 'hidden'
53+
}}
54+
>
55+
<VideoFeed topic={CAMERA_FEED_TOPIC} updateRateHz={10} webrtcURL={props.webrtcURL} />
6356
</Modal.Body>
6457
</Modal>
6558
)

feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,6 @@ const BiteSelection = (props) => {
3939
// Get icon image for move to mouth
4040
let moveToStagingConfigurationImage = MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingToStagingConfiguration]
4141

42-
// Reference to the DOM element of the parent of the video feed
43-
const videoParentRef = useRef(null)
4442
// Margin for the video feed and between the mask buttons. Note this cannot
4543
// be re-defined per render, otherwise it messes up re-rendering order upon
4644
// resize in VideoFeed.
@@ -473,7 +471,6 @@ const BiteSelection = (props) => {
473471
<h5 style={{ textAlign: 'center', fontSize: textFontSize }}>Click on image to select food.</h5>
474472
</View>
475473
<View
476-
ref={videoParentRef}
477474
style={{
478475
flex: 9,
479476
alignItems: 'center',
@@ -482,7 +479,7 @@ const BiteSelection = (props) => {
482479
height: '100%'
483480
}}
484481
>
485-
<VideoFeed parent={videoParentRef} pointClicked={imageClicked} webrtcURL={props.webrtcURL} />
482+
<VideoFeed pointClicked={imageClicked} webrtcURL={props.webrtcURL} />
486483
</View>
487484
</View>
488485
<View
@@ -565,7 +562,6 @@ const BiteSelection = (props) => {
565562
actionStatusText,
566563
renderMaskButtons,
567564
skipAcquisisitionButton,
568-
videoParentRef,
569565
imageClicked,
570566
props.debug,
571567
props.webrtcURL,

feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,6 @@ const DetectingFace = (props) => {
5858
// be re-defined per render, otherwise it messes up re-rendering order upon
5959
// resize in VideoFeed.
6060
const margin = useMemo(() => convertRemToPixels(1), [])
61-
// Reference to the DOM element of the parent of the video feed
62-
const videoParentRef = useRef(null)
6361

6462
/**
6563
* Connect to ROS, if not already connected. Put this in useRef to avoid
@@ -218,7 +216,6 @@ const DetectingFace = (props) => {
218216
</p>
219217
</View>
220218
<View
221-
ref={videoParentRef}
222219
style={{
223220
flex: 9,
224221
alignItems: 'center',
@@ -227,7 +224,6 @@ const DetectingFace = (props) => {
227224
}}
228225
>
229226
<VideoFeed
230-
parent={videoParentRef}
231227
marginTop={margin}
232228
marginBottom={margin}
233229
marginLeft={margin}

feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// React Imports
2-
import React, { useCallback, useMemo, useRef } from 'react'
2+
import React, { useCallback, useMemo } from 'react'
33
// PropTypes is used to validate that the used props are in fact passed to this
44
// Component
55
import PropTypes from 'prop-types'
@@ -31,8 +31,6 @@ const PlateLocator = (props) => {
3131
// Indicator of how to arrange screen elements based on orientation
3232
let dimension = isPortrait ? 'column' : 'row'
3333

34-
// Variables to render the VideoFeed
35-
const videoParentRef = useRef(null)
3634
// Margin for the video feed and between the mask buttons. Note this cannot
3735
// be re-defined per render, otherwise it messes up re-rendering order upon
3836
// resize in VideoFeed.
@@ -178,15 +176,8 @@ const PlateLocator = (props) => {
178176
// Render the component
179177
return (
180178
<View style={{ flex: 'auto', flexDirection: dimension, justifyContent: 'center', alignItems: 'center', width: '100%', height: '100%' }}>
181-
<View ref={videoParentRef} style={{ flex: 5, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
182-
<VideoFeed
183-
parent={videoParentRef}
184-
marginTop={margin}
185-
marginBottom={margin}
186-
marginLeft={margin}
187-
marginRight={margin}
188-
webrtcURL={props.webrtcURL}
189-
/>
179+
<View style={{ flex: 5, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
180+
<VideoFeed marginTop={margin} marginBottom={margin} marginLeft={margin} marginRight={margin} webrtcURL={props.webrtcURL} />
190181
</View>
191182
<View style={{ flex: 5, alignItems: 'center', justifyContent: 'center' }}>
192183
{directionalArrows()}

feedingwebapp/src/Pages/Home/VideoFeed.jsx

Lines changed: 88 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
// React Imports
22
import React, { useCallback, useEffect, useRef, useState } from 'react'
3+
import Button from 'react-bootstrap/Button'
34
// PropTypes is used to validate that the used props are in fact passed to this Component
45
import PropTypes from 'prop-types'
6+
import { View } from 'react-native'
57

68
// Local Imports
79
import { CAMERA_FEED_TOPIC, REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../Constants'
810
import { useWindowSize } from '../../helpers'
9-
import { createPeerConnection, closePeerConnection } from '../../webrtc/webrtc_helpers'
11+
import { WebRTCConnection } from '../../webrtc/webrtc_helpers'
1012

1113
/**
1214
* Takes in an imageWidth and imageHeight, and returns a width and height that
@@ -80,38 +82,49 @@ const VideoFeed = (props) => {
8082
const [imgWidth, setImgWidth] = useState(0)
8183
const [imgHeight, setImgHeight] = useState(0)
8284
const [scaleFactor, setScaleFactor] = useState(0.0)
85+
const [refreshCount, setRefreshCount] = useState(0)
8386

8487
// Ref for the video element
8588
const videoRef = useRef(null)
89+
const parentRef = useRef(null)
90+
91+
// Rendering variables
92+
let textFontSize = '2.5vh'
8693

8794
/**
8895
* Create the peer connection
8996
*/
9097
useEffect(() => {
9198
// Create the peer connection
92-
console.log('Creating peer connection', createPeerConnection, props.webrtcURL)
93-
const peer = createPeerConnection(props.webrtcURL + '/subscribe', props.topic, (event) => {
94-
console.log('Got track event', event)
95-
if (event.streams && event.streams[0]) {
96-
videoRef.current.srcObject = event.streams[0]
97-
console.log('video', videoRef.current)
98-
}
99+
console.log('Creating peer connection', props.webrtcURL, refreshCount)
100+
const webRTCConnection = new WebRTCConnection({
101+
url: props.webrtcURL + '/subscribe',
102+
topic: props.topic,
103+
onTrackAdded: (event) => {
104+
console.log('Got track event', event)
105+
if (event.streams && event.streams[0]) {
106+
videoRef.current.srcObject = event.streams[0]
107+
console.log('video', videoRef.current)
108+
}
109+
},
110+
transceiverKind: 'video',
111+
transceiverOptions: { direction: 'recvonly' }
99112
})
100-
peer.addTransceiver('video', { direction: 'recvonly' })
101113

102114
return () => {
103-
closePeerConnection(peer)
115+
webRTCConnection.close()
104116
}
105-
}, [props.topic, props.webrtcURL, videoRef])
117+
}, [props.topic, props.webrtcURL, refreshCount, videoRef])
106118

107119
// Callback to resize the image based on the parent width and height
108120
const resizeImage = useCallback(() => {
109-
if (!props.parent.current) {
121+
console.log('Resizing image', parentRef.current)
122+
if (!parentRef.current) {
110123
return
111124
}
112125
// Get the width and height of the parent DOM element
113-
let parentWidth = props.parent.current.clientWidth
114-
let parentHeight = props.parent.current.clientHeight
126+
let parentWidth = parentRef.current.clientWidth
127+
let parentHeight = parentRef.current.clientHeight
115128

116129
// Calculate the width and height of the video feed
117130
let {
@@ -133,12 +146,7 @@ const VideoFeed = (props) => {
133146
setImgWidth(childWidth)
134147
setImgHeight(childHeight)
135148
setScaleFactor(childScaleFactor)
136-
}, [props.parent, props.marginTop, props.marginBottom, props.marginLeft, props.marginRight])
137-
138-
// When the component is first mounted, resize the image
139-
useEffect(() => {
140-
resizeImage()
141-
}, [resizeImage])
149+
}, [parentRef, props.marginTop, props.marginBottom, props.marginLeft, props.marginRight])
142150

143151
/** When the resize event is triggered, the elements have not yet been laid out,
144152
* and hence the parent width/height might not be accurate yet based on the
@@ -150,6 +158,11 @@ const VideoFeed = (props) => {
150158
}, [resizeImage])
151159
useWindowSize(resizeImageNextEventCycle)
152160

161+
// When the component is first mounted, resize the image
162+
useEffect(() => {
163+
resizeImageNextEventCycle()
164+
}, [resizeImageNextEventCycle])
165+
153166
// The callback for when the image is clicked.
154167
const imageClicked = useCallback(
155168
(event) => {
@@ -172,32 +185,65 @@ const VideoFeed = (props) => {
172185
// Render the component
173186
return (
174187
<>
175-
<video
176-
playsInline
177-
autoPlay
178-
muted
179-
ref={videoRef}
180-
alt='Live video feed from the robot'
188+
<View
189+
ref={parentRef}
190+
style={{
191+
flex: 4,
192+
alignItems: 'center',
193+
justifyContent: 'center',
194+
width: '100%',
195+
height: '100%'
196+
}}
197+
>
198+
<video
199+
playsInline
200+
autoPlay
201+
muted
202+
ref={videoRef}
203+
alt='Live video feed from the robot'
204+
style={{
205+
width: imgWidth,
206+
height: imgHeight,
207+
display: 'block',
208+
alignItems: 'center',
209+
justifyContent: 'center'
210+
}}
211+
onClick={props.pointClicked ? imageClicked : null}
212+
/>
213+
</View>
214+
<View
181215
style={{
182-
width: imgWidth,
183-
height: imgHeight,
184-
display: 'block',
216+
flex: 1,
217+
// TODO: consider replacing 'center' with 'end' if the margin is 0.
185218
alignItems: 'center',
186-
justifyContent: 'center'
219+
justifyContent: 'start',
220+
width: '100%',
221+
height: '100%'
187222
}}
188-
onClick={props.pointClicked ? imageClicked : null}
189-
/>
223+
>
224+
<Button
225+
variant='warning'
226+
className='mx-2 mb-2 btn-huge'
227+
size='lg'
228+
style={{
229+
fontSize: textFontSize,
230+
width: '60%',
231+
color: 'black'
232+
}}
233+
onClick={() => setRefreshCount(refreshCount + 1)}
234+
>
235+
Reload Video
236+
</Button>
237+
</View>
190238
</>
191239
)
192240
}
193241
VideoFeed.propTypes = {
194-
// The ref to the parent DOM element. Null if the component is not yet mounted
195-
parent: PropTypes.object.isRequired,
196242
// The margins around the video feed
197-
marginTop: PropTypes.number.isRequired,
198-
marginBottom: PropTypes.number.isRequired,
199-
marginLeft: PropTypes.number.isRequired,
200-
marginRight: PropTypes.number.isRequired,
243+
marginTop: PropTypes.number,
244+
marginBottom: PropTypes.number,
245+
marginLeft: PropTypes.number,
246+
marginRight: PropTypes.number,
201247
// The topic of the video feed
202248
topic: PropTypes.string.isRequired,
203249
/**
@@ -211,6 +257,10 @@ VideoFeed.propTypes = {
211257
webrtcURL: PropTypes.string.isRequired
212258
}
213259
VideoFeed.defaultProps = {
260+
marginTop: 0,
261+
marginBottom: 0,
262+
marginLeft: 0,
263+
marginRight: 0,
214264
topic: CAMERA_FEED_TOPIC
215265
}
216266

0 commit comments

Comments
 (0)