Consolidated infra changes from RSS Demo (#122)

* Add robot browser logs

* Add webrtc logs

* Run server in pm2 to restart on crash

* MaskButton should allow for changing num masks

* Formatting

* Made Auto-Continue automatically checked

---------

Co-authored-by: Amal Nanavati <amaln@uw.edu>
Co-authored-by: Amal Nanavati <amaln@cs.washington.edu>

Author: egordon

feedingwebapp/README.md

@@ -13,6 +13,7 @@ The overall user flow for this robot can be seen below.
 ## Dependencies
 - [Node.js](https://nodejs.org/en/download/package-manager)
 - [`serve` must be globally installed](https://create-react-app.dev/docs/deployment/) (ideally with `sudo`): `sudo npm install -g serve`
+- [`pm2` must be globally installed](https://pm2.keymetrics.io/docs/usage/quick-start/): `npm install pm2@latest -g`
 
 ## Getting Started in Computer
 
@@ -23,7 +24,7 @@ The overall user flow for this robot can be seen below.
 4. Source the directory: `source install/setup.bash`
 5. Navigate to the web app folder: `cd feeding_web_interface/feedingwebapp`
 6. Install web app dependencies: `npm install --legacy-peer-deps`
-    1. You may also have to run `npx playwright install`; you might be prompted to run that after `node --env-file=.env start_robot_browser.js`
+    1. You may also have to run `npx playwright install`; you might be prompted to run that after `node start_robot_browser.js`
 * Consider checking out the Troubleshooting section if there are errors in this process. 
 If your workspace has already been built, you should run `source install/setup.bash`. If this is your first time building your workspace, you should `source /opt/ros/humble/setup.bash` and then run `colcon build` followed by `source install/setup.bash`. For both of the above cases, you must be in the main directory of your workspace (e.g., `src` should be a subfolder).
 
@@ -33,7 +34,7 @@ If your workspace has already been built, you should run `source install/setup.b
     - If you're not running the robot code alongside the app, set `REACT_APP_DEBUG=true` in `.env` to be able to move past screens where the app is waiting on the robot. The default is `REACT_APP_DEBUG=false`.
     - If users will be accessing the app on a device other than the device running ROS, change `REACT_APP_ROS_SERVER_HOSTNAME` in `.env` to be the hostname of the device running ROS. Ensure that device is configured so that ports 8080 (web_video_server default) and 9090 (rosbridge default) can be accessed.
 3. Start the app: `npm start`
-4. Start the WebRTC signalling server: `node --env-file=.env server.js`
+4. Start the WebRTC signalling server: `pm2 start server.js` (see [here](https://pm2.keymetrics.io/docs/usage/quick-start/) for `pm2`` instructions)
 5. Start the headless robot browser: `node start_robot_browser.js`
 6. Use a web browser to navigate to `localhost:3000` to see the application.
 

feedingwebapp/package-lock.json

@@ -13,6 +13,7 @@
     "bootstrap": "^5.1.3",
     "caniuse-lite": "^1.0.30001579",
     "cors": "^2.8.5",
+    "dotenv": "^16.4.1",
     "express": "^4.18.2",
     "mdb-react-ui-kit": "^3.0.0",
     "minimist": "^1.2.8",

feedingwebapp/package.json

@@ -8,6 +8,7 @@
 
 const SegfaultHandler = require('segfault-handler')
 SegfaultHandler.registerHandler('crash.log')
+require('dotenv').config()
 const express = require('express')
 const app = express()
 const bodyParser = require('body-parser')
@@ -40,86 +41,86 @@ app.post('/subscribe', async ({ body }, res) => {
       ]
     })
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: created peer connection object")
+      console.log(Date(Date.now()).toString(), 'subscribe: created peer connection object')
     }
 
     // Close any old peers on the same IP address
     const topic = body.topic
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: got topic", topic)
+      console.log(Date(Date.now()).toString(), 'subscribe: got topic', topic)
     }
     const key = body.ip + ':' + topic
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: got key", key)
+      console.log(Date(Date.now()).toString(), 'subscribe: got key', key)
     }
     if (key in subscribePeers && subscribePeers[key] && subscribePeers[key].connectionState !== 'closed') {
       if (debug) {
-        console.log(Date(Date.now()).toString(), "subscribe: peer for key already exists")
+        console.log(Date(Date.now()).toString(), 'subscribe: peer for key already exists')
       }
       const senders = subscribePeers[key].getSenders()
       if (debug) {
-        console.log(Date(Date.now()).toString(), "subscribe: got senders for peer for key")
+        console.log(Date(Date.now()).toString(), 'subscribe: got senders for peer for key')
       }
       senders.forEach((sender) => subscribePeers[key].removeTrack(sender))
       if (debug) {
-        console.log(Date(Date.now()).toString(), "subscribe: removed tracks")
+        console.log(Date(Date.now()).toString(), 'subscribe: removed tracks')
       }
       subscribePeers[key].close()
       if (debug) {
-        console.log(Date(Date.now()).toString(), "subscribe: closed peer connection")
+        console.log(Date(Date.now()).toString(), 'subscribe: closed peer connection')
       }
     }
     subscribePeers[key] = peer
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: set new peer connection")
+      console.log(Date(Date.now()).toString(), 'subscribe: set new peer connection')
     }
 
     const desc = new webrtc.RTCSessionDescription(body.sdp)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: created desc")
+      console.log(Date(Date.now()).toString(), 'subscribe: created desc')
     }
     await peer.setRemoteDescription(desc)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: set remote desc")
+      console.log(Date(Date.now()).toString(), 'subscribe: set remote desc')
     }
 
     // Add the publisher's video stream to the subscriber's peer connection
     if (topic in senderStream) {
       if (debug) {
-        console.log(Date(Date.now()).toString(), "subscribe: adding topics from publisher")
+        console.log(Date(Date.now()).toString(), 'subscribe: adding topics from publisher')
       }
       senderStream[topic].getTracks().forEach((track) => peer.addTrack(track, senderStream[topic]))
       if (debug) {
-        console.log(Date(Date.now()).toString(), "subscribe: added topics from publisher")
+        console.log(Date(Date.now()).toString(), 'subscribe: added topics from publisher')
       }
     }
 
     // Create an answer to the publisher's offer
     const answer = await peer.createAnswer()
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: created answer")
+      console.log(Date(Date.now()).toString(), 'subscribe: created answer')
     }
     await peer.setLocalDescription(answer)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: set local description")
+      console.log(Date(Date.now()).toString(), 'subscribe: set local description')
     }
     const payload = {
-      sdp: answer
+      sdp: peer.localDescription
     }
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: created payload")
+      console.log(Date(Date.now()).toString(), 'subscribe: created payload')
     }
 
     // Send the answer to the publisher
     res.json(payload)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: set payload")
+      console.log(Date(Date.now()).toString(), 'subscribe: set payload')
     }
   } catch (err) {
     console.error(Date(Date.now()).toString(), 'subscribe: Failed to process subscriber, exception: ' + err.message)
     res.sendStatus(500)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "subscribe: sent error status 500")
+      console.log(Date(Date.now()).toString(), 'subscribe: sent error status 500')
     }
   }
 })
@@ -137,80 +138,80 @@ app.post('/publish', async ({ body }, res) => {
       ]
     })
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: create peer")
+      console.log(Date(Date.now()).toString(), 'publish: create peer')
     }
 
     // Close any old peers on the same IP address
     const topic = body.topic
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: got topic", topic)
+      console.log(Date(Date.now()).toString(), 'publish: got topic', topic)
     }
     const key = body.ip + ':' + topic
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: got key", key)
+      console.log(Date(Date.now()).toString(), 'publish: got key', key)
     }
     if (key in publishPeers && publishPeers[key] && publishPeers[key].connectionState !== 'closed') {
       if (debug) {
-        console.log(Date(Date.now()).toString(), "publish: found existing publisher for key", key)
+        console.log(Date(Date.now()).toString(), 'publish: found existing publisher for key', key)
       }
       const senders = publishPeers[key].getSenders()
       if (debug) {
-        console.log(Date(Date.now()).toString(), "publish: got senders for old key")
+        console.log(Date(Date.now()).toString(), 'publish: got senders for old key')
       }
       senders.forEach((sender) => publishPeers[key].removeTrack(sender))
       if (debug) {
-        console.log(Date(Date.now()).toString(), "publish: removed tracks for old key")
+        console.log(Date(Date.now()).toString(), 'publish: removed tracks for old key')
       }
       publishPeers[key].close()
       if (debug) {
-        console.log(Date(Date.now()).toString(), "publish: closed old peer connection")
+        console.log(Date(Date.now()).toString(), 'publish: closed old peer connection')
       }
     }
     publishPeers[key] = peer
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: added new peer")
+      console.log(Date(Date.now()).toString(), 'publish: added new peer')
     }
 
     // Send the publisher's video stream to all subscribers on that topic
     peer.ontrack = (e) => handleTrackEvent(e, topic)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: handled track events")
+      console.log(Date(Date.now()).toString(), 'publish: handled track events')
     }
 
     // Create an answer to the publisher's offer
     const desc = new webrtc.RTCSessionDescription(body.sdp)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: got desc")
+      console.log(Date(Date.now()).toString(), 'publish: got desc')
     }
     await peer.setRemoteDescription(desc)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: set remote description")
+      console.log(Date(Date.now()).toString(), 'publish: set remote description')
     }
     const answer = await peer.createAnswer()
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: got answer")
+      console.log(Date(Date.now()).toString(), 'publish: got answer')
     }
     await peer.setLocalDescription(answer)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: set local description")
+      console.log(Date(Date.now()).toString(), 'publish: set local description')
     }
     const payload = {
-      sdp: answer
+      sdp: peer.localDescription
     }
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: created payload")
+      console.log(Date(Date.now()).toString(), 'publish: created payload')
     }
 
     // Send the answer to the publisher
     res.json(payload)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: set json payload")
+      console.log(Date(Date.now()).toString(), 'publish: set json payload')
     }
   } catch (err) {
     console.error(Date(Date.now()).toString(), 'publish: Failed to process publisher, exception: ' + err.message)
     res.sendStatus(500)
     if (debug) {
-      console.log(Date(Date.now()).toString(), "publish: sent error status 500")
+      console.log(Date(Date.now()).toString(), 'publish: sent error status 500')
     }
   }
 })

feedingwebapp/server.js

@@ -128,7 +128,7 @@ export const useGlobalState = create(
       // Flag to indicate robot motion trough teleoperation interface
       teleopIsMoving: false,
       // Flag to indicate whether to auto-continue after face detection
-      faceDetectionAutoContinue: false,
+      faceDetectionAutoContinue: true,
       // Whether the settings bite transfer page is currently at the user's face
       // or not. This is in the off-chance that the mealState is not at the user's
       // face, the settings page is, and the user refreshes -- the page should

feedingwebapp/src/Pages/GlobalState.jsx

@@ -39,9 +39,9 @@ const DetectingFace = (props) => {
   // Font size for text
   let textFontSize = 3
   // let buttonWidth = 22
-  let buttonHeight = 14
+  let buttonHeight = 12
   let iconWidth = 20
-  let iconHeight = 12
+  let iconHeight = 10
   let sizeSuffix = isPortrait ? 'vh' : 'vw'
 
   /**

feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx

@@ -46,16 +46,20 @@ function MaskButton(props) {
   const drawCircle = useCallback(() => {
     // Get the canvas
     const canvas = canvasRef.current
-    // Get the context
-    const ctx = canvas.getContext('2d')
-    // Clear the canvas
-    ctx.clearRect(0, 0, canvas.width, canvas.height)
-    // Draw a red filled circle
-    let radius = 5
-    ctx.beginPath()
-    ctx.arc(canvas.width / 2, canvas.height / 2, radius, 0, 2 * Math.PI)
-    ctx.fillStyle = 'red'
-    ctx.fill()
+    // Canvas might be null if the previous render had more MaskButtons
+    // than the current render.
+    if (canvas !== null) {
+      // Get the context
+      const ctx = canvas.getContext('2d')
+      // Clear the canvas
+      ctx.clearRect(0, 0, canvas.width, canvas.height)
+      // Draw a red filled circle
+      let radius = 5
+      ctx.beginPath()
+      ctx.arc(canvas.width / 2, canvas.height / 2, radius, 0, 2 * Math.PI)
+      ctx.fillStyle = 'red'
+      ctx.fill()
+    }
   }, [])
 
   // Draw a red filled circle on the middle of the canvas

feedingwebapp/src/buttons/MaskButton.jsx

@@ -43,6 +43,7 @@ export class WebRTCConnection {
 
       this.peerConnection.onnegotiationneeded = async () => {
         try {
+          console.log('onnegotiationneeded')
           const offer = await this.peerConnection.createOffer()
           await this.peerConnection.setLocalDescription(offer)
           const ip = await this.getIPAddress()
@@ -61,6 +62,7 @@ export class WebRTCConnection {
       }
 
       this.peerConnection.oniceconnectionstatechange = () => {
+        console.log('oniceconnectionstatechange')
         if (!this.peerConnection) throw new Error('peerConnection is undefined')
         if (this.peerConnection.iceConnectionState === 'failed') {
           this.peerConnection.restartIce()

feedingwebapp/src/webrtc/webrtc_helpers.js

@@ -41,6 +41,9 @@ if (argv.port) {
     ]
   })
   const page = await browser.newPage()
+  page.on('console', (msg) => {
+    console.log(msg)
+  })
 
   while (num_tries < max_tries) {
     try {