Revamp Bite Selection Visualization (#108)

* Remove dependency on web video server

* Unsubscribe before unload

* Display mask middle and resize masks

* Re-render canvas on resize

---------

Co-authored-by: Amal Nanavati <amaln@cs.washington.edu>

Author: amalnanavati

feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx

@@ -52,6 +52,8 @@ const BiteSelection = (props) => {
   let textFontSize = isPortrait ? '2.5vh' : '2vw'
   // Indicator of how to arrange screen elements based on orientation
   let dimension = isPortrait ? 'column' : 'row'
+  // Whether to scale all the masks equally or not
+  const scaleMasksEqually = false
 
   /**
    * Create a local state variable to store the detected masks, the
@@ -284,17 +286,6 @@ const BiteSelection = (props) => {
     if (actionStatus.actionStatus === ROS_ACTION_STATUS_SUCCEED) {
       // If we have a result and there are detected items
       if (actionResult && actionResult.detected_items && actionResult.detected_items.length > 0) {
-        // Get the size of the largest mask
-        let [maxWidth, maxHeight] = [0, 0]
-        for (let detected_item of actionResult.detected_items) {
-          if (detected_item.roi.width > maxWidth) {
-            maxWidth = detected_item.roi.width
-          }
-          if (detected_item.roi.height > maxHeight) {
-            maxHeight = detected_item.roi.height
-          }
-        }
-
         // Get the allotted space per mask
         let parentWidth, parentHeight
         if (maskButtonParentRef.current) {
@@ -320,10 +311,26 @@ const BiteSelection = (props) => {
          * Determine how much to scale the masks so that the largest mask fits
          * into the alloted space.
          */
-        let widthScaleFactor = allottedSpaceWidth / maxWidth
-        let heightScaleFactor = allottedSpaceHeight / maxHeight
-        let maskScaleFactor = Math.min(widthScaleFactor, heightScaleFactor)
-        // maskScaleFactor  = Math.min(maskScaleFactor, 1.0)
+        // Get the size of the largest mask
+        let [maxWidth, maxHeight] = [0, 0]
+        if (scaleMasksEqually) {
+          for (let detected_item of actionResult.detected_items) {
+            if (detected_item.roi.width > maxWidth) {
+              maxWidth = detected_item.roi.width
+            }
+            if (detected_item.roi.height > maxHeight) {
+              maxHeight = detected_item.roi.height
+            }
+          }
+        }
+        // Create a list to contain the scale factors for each mask
+        let maskScaleFactors = []
+        for (let detected_item of actionResult.detected_items) {
+          let widthScaleFactor = allottedSpaceWidth / (scaleMasksEqually ? maxWidth : detected_item.roi.width)
+          let heightScaleFactor = allottedSpaceHeight / (scaleMasksEqually ? maxHeight : detected_item.roi.height)
+          let maskScaleFactor = Math.min(widthScaleFactor, heightScaleFactor)
+          maskScaleFactors.push(maskScaleFactor)
+        }
 
         return (
           <View style={{ flexDirection: 'row', justifyContent: 'center', alignItems: 'center', width: '100%', height: '100%' }}>
@@ -334,7 +341,7 @@ const BiteSelection = (props) => {
                   buttonSize={buttonSize}
                   maskSrc={'data:image/jpeg;base64,' + detected_item.mask.data}
                   invertMask={true}
-                  maskScaleFactor={maskScaleFactor}
+                  maskScaleFactor={maskScaleFactors[i]}
                   maskBoundingBox={detected_item.roi}
                   onClick={foodItemClicked}
                   value={i.toString()}
@@ -345,7 +352,7 @@ const BiteSelection = (props) => {
         )
       }
     }
-  }, [actionStatus, actionResult, foodItemClicked, isPortrait, windowSize, margin])
+  }, [actionStatus, actionResult, foodItemClicked, isPortrait, windowSize, margin, scaleMasksEqually])
 
   /** Get the button for continue without acquiring bite
    *

feedingwebapp/src/buttons/MaskButton.jsx

@@ -1,8 +1,9 @@
 // React imports
-import React from 'react'
+import React, { useCallback, useEffect, useRef } from 'react'
 import Button from 'react-bootstrap/Button'
 // PropTypes is used to validate that the used props are in fact passed to this Component
 import PropTypes from 'prop-types'
+import { useWindowSize } from '../helpers'
 
 /**
  * A component that renders an image and a mask on top of it, all within a
@@ -28,6 +29,9 @@ import PropTypes from 'prop-types'
  *
  */
 function MaskButton(props) {
+  // Get a reference for the canvas
+  const canvasRef = useRef(null)
+
   // Get the properties
   let buttonSize = props.buttonSize
   let imgSrc = props.imgSrc
@@ -38,6 +42,34 @@ function MaskButton(props) {
   let onClick = props.onClick
   let value = props.value
 
+  // Draw a red filled circle on the middle of the canvas
+  const drawCircle = useCallback(() => {
+    // Get the canvas
+    const canvas = canvasRef.current
+    // Get the context
+    const ctx = canvas.getContext('2d')
+    // Clear the canvas
+    ctx.clearRect(0, 0, canvas.width, canvas.height)
+    // Draw a red filled circle
+    let radius = 5
+    ctx.beginPath()
+    ctx.arc(canvas.width / 2, canvas.height / 2, radius, 0, 2 * Math.PI)
+    ctx.fillStyle = 'red'
+    ctx.fill()
+  }, [])
+
+  // Draw a red filled circle on the middle of the canvas
+  useEffect(() => {
+    // Dummy log necessary to have useEffect re-run when the props change
+    console.log('props.maskBoundingBox', props.maskBoundingBox)
+
+    // Draw the circle
+    drawCircle()
+  }, [drawCircle, props.maskBoundingBox])
+
+  // Redraw the circle when the window size changes
+  useWindowSize(drawCircle)
+
   return (
     <Button
       style={{ backgroundColor: invertMask ? 'white' : 'black', borderColor: invertMask ? 'black' : 'white', borderWidth: 'medium' }}
@@ -88,6 +120,17 @@ function MaskButton(props) {
               pointerEvents: 'none'
             }}
           />
+          <canvas
+            ref={canvasRef}
+            width={maskBoundingBox.width * maskScaleFactor}
+            height={maskBoundingBox.height * maskScaleFactor}
+            style={{
+              position: 'absolute',
+              top: '0px',
+              display: 'block',
+              pointerEvents: 'none'
+            }}
+          />
         </div>
       </div>
     </Button>