diff --git a/docs/source/includes/tags/bitmask.md b/docs/source/includes/tags/bitmask.md new file mode 100644 index 000000000000..00fb11246fd6 --- /dev/null +++ b/docs/source/includes/tags/bitmask.md @@ -0,0 +1,12 @@ +### Parameters + +| Param | Type | Default | Description | +| --- | --- | --- | --- | +| name | string | | Name of the element | +| toName | string | | Name of the image to label | +| [choice] | single \| multiple | single | Configure whether the data labeler can select one or multiple labels | +| [maxUsages] | number | | Maximum number of times a label can be used per task | +| [showInline] | boolean | true | Show labels in the same visual line | +| [smart] | boolean | | Show smart tool for interactive pre-annotations | +| [smartOnly] | boolean | | Only show smart tool for interactive pre-annotations | + diff --git a/docs/source/includes/tags/bitmasklabels.md b/docs/source/includes/tags/bitmasklabels.md new file mode 100644 index 000000000000..d7c46fdfaf25 --- /dev/null +++ b/docs/source/includes/tags/bitmasklabels.md @@ -0,0 +1,10 @@ +### Parameters + +| Param | Type | Default | Description | +| --- | --- | --- | --- | +| name | string | | Name of the element | +| toName | string | | Name of the image to label | +| [choice] | single \| multiple | single | Configure whether the data labeler can select one or multiple labels | +| [maxUsages] | number | | Maximum number of times a label can be used per task | +| [showInline] | boolean | true | Show labels in the same visual line | + diff --git a/web/libs/core/src/lib/utils/feature-flags/ff.ts b/web/libs/core/src/lib/utils/feature-flags/ff.ts index 4dc7193148fe..6a2952e27f5d 100644 --- a/web/libs/core/src/lib/utils/feature-flags/ff.ts +++ b/web/libs/core/src/lib/utils/feature-flags/ff.ts @@ -1,4 +1,4 @@ -import { FF_SAMPLE_DATASETS } from "./flags"; +import { FF_BITMASK, FF_SAMPLE_DATASETS } from "./flags"; const FEATURE_FLAGS = window.APP_SETTINGS?.feature_flags || {}; @@ -12,6 +12,7 @@ const FLAGS_OVERRIDE: Record = { // Add your flags overrides as following: // [FF_FLAG_NAME]: boolean [FF_SAMPLE_DATASETS]: true, + [FF_BITMASK]: true, }; /** @@ -19,7 +20,8 @@ const FLAGS_OVERRIDE: Record = { */ export const isActive = (id: string) => { const defaultValue = window.APP_SETTINGS?.feature_flags_default_value === true; - const isSentryOSS = window?.APP_SETTINGS?.sentry_environment === "opensource"; + const isSentryOSS = + window?.APP_SETTINGS?.sentry_environment === "opensource" || process.env.NODE_ENV === "development"; if (isSentryOSS && id in FLAGS_OVERRIDE) return FLAGS_OVERRIDE[id]; if (id in FEATURE_FLAGS) return FEATURE_FLAGS[id] ?? defaultValue; diff --git a/web/libs/core/src/lib/utils/feature-flags/flags.ts b/web/libs/core/src/lib/utils/feature-flags/flags.ts index 640264dc12a9..4bb638da7c02 100644 --- a/web/libs/core/src/lib/utils/feature-flags/flags.ts +++ b/web/libs/core/src/lib/utils/feature-flags/flags.ts @@ -97,3 +97,10 @@ export const FF_MULTICHANNEL_TS = "fflag_feat_front_bros58_timeseries_multichann * Enables enterprise email notifications */ export const FF_ENTERPRISE_EMAIL_NOTIFICATIONS = "fflag_feat_front_fit_183_email_notifications_short"; + +/** + * Bitmask is a new image segmentation tool for masking that allows pixel-perfect editing + * + * @link https://app.launchdarkly.com/projects/default/flags/fflag_front_feat_bros_87_pixel_wise_16062025_short + */ +export const FF_BITMASK = "fflag_front_feat_bros_87_pixel_wise_16062025_short"; diff --git a/web/libs/editor/src/components/ImageView/Image.jsx b/web/libs/editor/src/components/ImageView/Image.jsx index c9872954e4f6..76a52f9a174a 100644 --- a/web/libs/editor/src/components/ImageView/Image.jsx +++ b/web/libs/editor/src/components/ImageView/Image.jsx @@ -5,6 +5,8 @@ import { FF_LSDV_4711, isFF } from "../../utils/feature-flags"; import messages from "../../utils/messages"; import { ErrorMessage } from "../ErrorMessage/ErrorMessage"; import "./Image.scss"; +import { ff } from "@humansignal/core"; +import { FF_BITMASK } from "@humansignal/core/lib/utils/feature-flags"; /** * Coordinates in relative mode belong to a data domain consisting of percentages in the range from 0 to 100 @@ -84,7 +86,16 @@ if (isFF(FF_LSDV_4711)) imgDefaultProps.crossOrigin = "anonymous"; const ImageRenderer = observer( forwardRef(({ src, onLoad, imageTransform, isLoaded }, ref) => { const imageStyles = useMemo(() => { - const style = imageTransform ?? {}; + // We can't just skip rendering the image because we need its dimensions to be set + // so we just hide it with 0x0 dimensions. + // + // Real dimension will still be available via `naturalWidth` and `naturalHeight` + const style = ff.isActive(FF_BITMASK) + ? { + width: 0, + height: 0, + } + : imageTransform; return { ...style, maxWidth: "unset", visibility: isLoaded ? "visible" : "hidden" }; }, [imageTransform, isLoaded]); diff --git a/web/libs/editor/src/components/ImageView/ImageView.jsx b/web/libs/editor/src/components/ImageView/ImageView.jsx index f53d2c590067..53676809256c 100644 --- a/web/libs/editor/src/components/ImageView/ImageView.jsx +++ b/web/libs/editor/src/components/ImageView/ImageView.jsx @@ -1,5 +1,5 @@ -import { Component, createRef, forwardRef, Fragment, memo, useEffect, useRef, useState } from "react"; -import { Group, Layer, Line, Rect, Stage } from "react-konva"; +import { Component, createRef, forwardRef, Fragment, memo, useEffect, useMemo, useRef, useState } from "react"; +import { Group, Layer, Line, Rect, Stage, Image as KonvaImage, Circle } from "react-konva"; import { observer } from "mobx-react"; import { getEnv, getRoot, isAlive } from "mobx-state-tree"; @@ -32,6 +32,9 @@ import { } from "../../utils/feature-flags"; import { Pagination } from "../../common/Pagination/Pagination"; import { Image } from "./Image"; +import { isHoveringNonTransparentPixel } from "../../regions/BitmaskRegion/utils"; +import { ff } from "@humansignal/core"; +import { FF_BITMASK } from "@humansignal/core/lib/utils/feature-flags"; Konva.showWarnings = false; @@ -43,21 +46,25 @@ if (isFF(FF_LSDV_4711)) imgDefaultProps.crossOrigin = "anonymous"; const splitRegions = (regions) => { const brushRegions = []; const shapeRegions = []; - const l = regions.length; - let i = 0; - - for (i; i < l; i++) { - const region = regions[i]; - - if (region.type === "brushregion") { - brushRegions.push(region); - } else { - shapeRegions.push(region); + const bitmaskRegions = []; + + for (const region of regions) { + switch (region.type) { + case "brushregion": + brushRegions.push(region); + break; + case "bitmaskregion": + bitmaskRegions.push(region); + break; + default: + shapeRegions.push(region); + break; } } return { brushRegions, + bitmaskRegions, shapeRegions, }; }; @@ -66,37 +73,51 @@ const Region = memo(({ region, showSelected = false }) => { return useObserver(() => Tree.renderItem(region, region.annotation, true)); }); -const RegionsLayer = memo(({ regions, name, useLayers, showSelected = false }) => { +const RegionsLayer = memo(({ regions, name, useLayers, showSelected = false, smoothing = true }) => { const content = regions.map((el) => ); - return useLayers === false ? content : {content}; -}); - -const Regions = memo(({ regions, useLayers = true, chunkSize = 15, suggestion = false, showSelected = false }) => { - return ( - - {(chunkSize ? chunks(regions, chunkSize) : regions).map((chunk, i) => ( - - ))} - + return useLayers === false ? ( + content + ) : ( + + {content} + ); }); +const Regions = memo( + ({ regions, useLayers = true, chunkSize = 15, suggestion = false, showSelected = false, smoothing = true }) => { + return ( + + {(chunkSize ? chunks(regions, chunkSize) : regions).map((chunk, i) => ( + + ))} + + ); + }, +); + const DrawingRegion = observer(({ item }) => { const { drawingRegion } = item; if (!drawingRegion) return null; if (item.multiImage && item.currentImage !== drawingRegion.item_index) return null; - const Wrapper = drawingRegion && drawingRegion.type === "brushregion" ? Fragment : Layer; + const isBrush = drawingRegion.type === "brushregion"; + const Wrapper = drawingRegion && isBrush ? Fragment : Layer; - return {drawingRegion ? : drawingRegion}; + return ( + + {drawingRegion ? : drawingRegion} + + ); }); const SELECTION_COLOR = "#40A9FF"; @@ -435,6 +456,57 @@ const Crosshair = memo( }), ); +const PixelGridLayer = observer(({ item }) => { + const ZOOM_THRESHOLD = 20; + + const visible = item.zoomScale > ZOOM_THRESHOLD; + const { naturalWidth, naturalHeight } = item.currentImageEntity; + const { stageWidth, stageHeight } = item; + const imageSmallerThanStage = naturalWidth < stageWidth || naturalHeight < stageHeight; + + const step = 1 / (item.imageIsSmallerThanStage ? 1 : item.stageToImageRatio); // image pixel + + const { verticalPoints, horizontalPoints } = useMemo(() => { + const vPts = []; + const hPts = []; + + // Grid starts from image origin (0, 0) + for (let x = 0; x <= stageWidth; x += step) { + vPts.push(x, 0, x, stageHeight, x, 0); + } + + for (let y = 0; y <= stageHeight; y += step) { + hPts.push(0, y, stageWidth, y, 0, y); + } + + return { verticalPoints: vPts, horizontalPoints: hPts }; + }, [stageWidth, stageHeight]); + + return ( + + + + + ); +}); /** * Component that creates an overlay on top * of the image to support Magic Wand tool @@ -505,6 +577,33 @@ export default observer( return; } } + + // We can only handle Bitmask selection here because the way it works -- it overlays an + // entire stage with a single image that is not click-through, and there is no particular + // shape we can click on. Here we're relying on cursor position and non-transparent pixels + // of the mask to detect cursor-region collision. + if (ff.isActive(FF_BITMASK)) { + const selectedRegion = item.selectedRegions.find((r) => r.type === "bitmaskregion"); + const currentTool = item.getToolsManager().findSelectedTool().toolName; + + // We want to avoid weird behavior here with drawing while selecting another region + // so we just do nothing when clicked outside AND we have a tool selected + if (selectedRegion && currentTool === "BitmaskTool") { + return; + } + + const hoveredRegion = item.regs.find((reg) => { + if (reg.type !== "bitmaskregion") return false; + if (reg.selected) return false; + + return isHoveringNonTransparentPixel(reg); + }); + + if (hoveredRegion) { + hoveredRegion.onClickRegion(e); + return; + } + } return item.event("click", evt, x, y); }; @@ -561,15 +660,21 @@ export default observer( } const isRightElementToCatchToolInteractions = (el) => { + // Bitmask is like Brush, so treat it the same + // The only difference is that Bitmask doesn't have a group inside + if (el.nodeType === "Layer" && !isMoveTool && el.attrs?.name === "bitmask") { + return true; + } + // It could be ruler ot segmentation if (el.nodeType === "Group") { - if ("ruler" === el?.attrs?.name) { + if (el?.attrs?.name === "ruler") { return true; } // segmentation is specific for Brushes // but click interaction on the region covers the case of the same MoveTool interaction here, // so it should ignore move tool interaction to prevent conflicts - if (!isMoveTool && "segmentation" === el?.attrs?.name) { + if (!isMoveTool && el?.attrs?.name === "segmentation") { return true; } } @@ -716,6 +821,31 @@ export default observer( } else { item.event("mousemove", e, e.evt.offsetX, e.evt.offsetY); } + + // Handle Bitmask hover + // We can only do it here due to Bitmask implementation. See `self.handleOnClick` method + // for a full explanation. + if (!e.evt.ctrlKey && !e.evt.shiftKey && ff.isActive(FF_BITMASK)) { + if (item.regs.some((r) => r.isDrawing)) return; + requestAnimationFrame(() => { + for (const region of item.regs) { + region.setHighlight(false); + region.updateCursor(false); + } + for (const region of item.regs) { + if (region.type !== "bitmaskregion") continue; + + const checkHover = !region.selected && !region.isDrawing; + const hovered = checkHover && isHoveringNonTransparentPixel(region); + + if (hovered) { + // region.setHighlight(true); + region.updateCursor(true); + break; + } + } + }); + } }; updateCrosshair = (e) => { @@ -749,23 +879,37 @@ export default observer( * Handle to zoom */ handleZoom = (e) => { - /** - * Disable if user doesn't use ctrl - */ - if (e.evt && !e.evt.ctrlKey) { - return; - } - if (e.evt && e.evt.ctrlKey) { - /** - * Disable scrolling page - */ + if (e.evt?.ctrlKey || e.evt?.metaKey || e.evt?.shiftKey) { e.evt.preventDefault(); - } - if (e.evt) { + const { item } = this.props; const stage = item.stageRef; - item.handleZoom(e.evt.deltaY, stage.getPointerPosition()); + item.handleZoom(e.evt.deltaY, stage.getPointerPosition(), e.evt.ctrlKey || e.evt.metaKey); + } else if (e.evt) { + // Two fingers scroll + const { item } = this.props; + + const maxScrollX = Math.round(item.stageWidth * item.zoomScale) - item.stageWidth; + const maxScrollY = Math.round(item.stageHeight * item.zoomScale) - item.stageHeight; + + const newPos = { + x: Math.min(0, Math.ceil(item.zoomingPositionX - e.evt.deltaX)), + y: Math.min(0, Math.ceil(item.zoomingPositionY - e.evt.deltaY)), + }; + + // Calculate scroll boundaries to allow scrolling the page when reaching stage edges + const withinX = newPos.x !== 0 && newPos.x > -maxScrollX && item.zoomScale !== 1; + const withinY = newPos.y !== 0 && newPos.y > -maxScrollY && item.zoomScale !== 1; + + // Detect scroll direction + const scrollingX = Math.abs(e.evt.deltaX) > Math.abs(e.evt.deltaY); + const scrollingY = Math.abs(e.evt.deltaY) > Math.abs(e.evt.deltaX); + + if (withinX && scrollingX) e.evt.preventDefault(); + if (withinY && scrollingY) e.evt.preventDefault(); + + item.setZoomPosition(newPos.x, newPos.y); } }; @@ -1103,6 +1247,7 @@ const EntireStage = observer( item.setStageRef(ref); }} className={[styles["image-element"], ...imagePositionClassnames].join(" ")} + style={{ cursor: "none" }} width={size.width} height={size.height} scaleX={item.zoomScale} @@ -1127,35 +1272,112 @@ const EntireStage = observer( }, ); +const ImageLayer = observer(({ item }) => { + const imageEntity = item.currentImageEntity; + const image = useMemo(() => { + const ent = item.currentImageEntity; + + if (ent && ent.downloaded) { + const img = new window.Image(); + img.src = ent.currentSrc; + img.width = Number.parseInt(ent.naturalWidth); + img.height = Number.parseInt(ent.naturalHeight); + return img; + } + return null; + }, [imageEntity?.downloaded]); + + const { width, height } = useMemo(() => { + return { + width: Math.min(imageEntity.naturalWidth, item.stageWidth), + height: Math.min(imageEntity.naturalHeight, item.stageHeight), + }; + }, [imageEntity.naturalWidth, imageEntity.naturalHeight, item.stageWidth, item.stageHeight]); + + return image ? ( + <> + + + + + ) : null; +}); + +const CursorLayer = observer(({ item, tool }) => { + const [[x, y], setCursorPosition] = useState([0, 0]); + const [visible, setVisible] = useState(false); + + useEffect(() => { + if (!item.stageRef) return; + const stage = item.stageRef; + const onMouseMove = (e) => { + const { x, y } = stage.getPointerPosition(); + const { x: deltaX, y: deltaY } = stage.position(); + const { x: scaleX, y: scaleY } = stage.scale(); + setCursorPosition([(x - deltaX) / scaleX, (y - deltaY) / scaleY]); + }; + const onMouseEnter = () => { + setVisible(true); + }; + const onMouseLeave = () => { + setVisible(false); + }; + + stage.on("mousemove", onMouseMove); + stage.on("mouseenter", onMouseEnter); + stage.on("mouseleave", onMouseLeave); + + return () => { + stage.off("mousemove", onMouseMove); + stage.off("mouseenter", onMouseEnter); + stage.off("mouseleave", onMouseLeave); + }; + }, [item.stageRef]); + + const size = useMemo(() => { + return (item.imageIsSmallerThanStage ? tool.strokeWidth : tool.strokeWidth / item.stageToImageRatio) + 2; + }, [tool.strokeWidth, item.imageIsSmallerThanStage, item.stageToImageRatio]); + + return visible ? ( + + + + + ) : null; +}); + const StageContent = observer(({ item, store, state, crosshairRef }) => { if (!isAlive(item)) return null; if (!store.task || !item.currentSrc) return null; - const regions = item.regs; + // Keep selected or highlighted region on top + const regions = [...item.regs].sort((r) => (r.highlighted || r.selected ? 1 : -1)); const paginationEnabled = !!item.isMultiItem; const wrapperClasses = [styles.wrapperComponent, item.images.length > 1 ? styles.withGallery : styles.wrapper]; + const tool = item.getToolsManager().findSelectedTool(); if (paginationEnabled) wrapperClasses.push(styles.withPagination); - const { brushRegions, shapeRegions } = splitRegions(regions); + const { brushRegions, shapeRegions, bitmaskRegions } = splitRegions(regions); - const { brushRegions: suggestedBrushRegions, shapeRegions: suggestedShapeRegions } = splitRegions(item.suggestions); + const { + brushRegions: suggestedBrushRegions, + shapeRegions: suggestedShapeRegions, + bitmaskRegions: suggestedBitmaskRegions, + } = splitRegions(item.suggestions); const renderableRegions = Object.entries({ brush: brushRegions, shape: shapeRegions, + bitmask: bitmaskRegions, suggestedBrush: suggestedBrushRegions, + suggestedBismask: suggestedBitmaskRegions, suggestedShape: suggestedShapeRegions, }); return ( <> - {/* Hack to keep stage in place when there's no regions */} - {regions.length === 0 && ( - - - - )} + {ff.isActive(ff.FF_BITMASK) && } {item.grid && item.sizeUpdated && } {isFF(FF_LSDV_4930) ? : null} @@ -1171,6 +1393,7 @@ const StageContent = observer(({ item, store, state, crosshairRef }) => { regions={list} useLayers={isBrush === false} suggestion={isSuggestion} + smoothing={item.smoothing} /> ) : ( @@ -1178,6 +1401,7 @@ const StageContent = observer(({ item, store, state, crosshairRef }) => { })} + {ff.isActive(ff.FF_BITMASK) && item.smoothing === false && } {item.crosshair && ( { height={isFF(FF_ZOOM_OPTIM) ? item.containerHeight : item.stageHeight} /> )} + + {tool && tool.toolName.match(/bitmask/i) && } ); }); diff --git a/web/libs/editor/src/components/InteractiveOverlays/BoundingBox.js b/web/libs/editor/src/components/InteractiveOverlays/BoundingBox.js index def6596b2d27..b8f5e0023a53 100644 --- a/web/libs/editor/src/components/InteractiveOverlays/BoundingBox.js +++ b/web/libs/editor/src/components/InteractiveOverlays/BoundingBox.js @@ -134,7 +134,8 @@ const _detect = (region) => { case "ellipseregion": case "polygonregion": case "keypointregion": - case "brushregion": { + case "brushregion": + case "bitmaskregion": { const bbox = region.bboxCoordsCanvas; return bbox diff --git a/web/libs/editor/src/components/Node/Node.tsx b/web/libs/editor/src/components/Node/Node.tsx index 53154c9ea6b5..9e2c6d58692d 100644 --- a/web/libs/editor/src/components/Node/Node.tsx +++ b/web/libs/editor/src/components/Node/Node.tsx @@ -108,6 +108,12 @@ const NodeViews = { altIcon: IconBrushToolSmart, }), + BitmaskRegionModel: NodeView({ + name: "Brush", + icon: IconBrushTool, + altIcon: IconBrushToolSmart, + }), + ChoicesModel: NodeView({ name: "Classification", icon: ApartmentOutlined, diff --git a/web/libs/editor/src/components/TaskSummary/labelings.tsx b/web/libs/editor/src/components/TaskSummary/labelings.tsx index 235f56ae26a0..19b7cc67a760 100644 --- a/web/libs/editor/src/components/TaskSummary/labelings.tsx +++ b/web/libs/editor/src/components/TaskSummary/labelings.tsx @@ -51,6 +51,7 @@ export const renderers: Record = { timeserieslabels: LabelsRenderer, paragraphlabels: LabelsRenderer, timelinelabels: LabelsRenderer, + bitmasklabels: LabelsRenderer, number: (results) => { if (!results.length) return "-"; diff --git a/web/libs/editor/src/regions/Area.js b/web/libs/editor/src/regions/Area.js index c20ac28f56a5..253b8140cd62 100644 --- a/web/libs/editor/src/regions/Area.js +++ b/web/libs/editor/src/regions/Area.js @@ -15,6 +15,7 @@ import { TimelineRegionModel } from "./TimelineRegion"; import { TimeSeriesRegionModel } from "./TimeSeriesRegion"; import { ParagraphsRegionModel } from "./ParagraphsRegion"; import { VideoRectangleRegionModel } from "./VideoRectangleRegion"; +import { BitmaskRegionModel } from "./BitmaskRegion"; // general Area type for classification Results which doesn't belong to any real Area const ClassificationArea = types.compose( @@ -52,6 +53,7 @@ const Area = types.union( // `sequence` and `ranges` are used for video regions !sn.sequence && !sn.ranges && + !sn.imageDataURL && sn.value && Object.values(sn.value).length <= 1 ) @@ -84,6 +86,7 @@ const Area = types.union( EllipseRegionModel, PolygonRegionModel, BrushRegionModel, + BitmaskRegionModel, VideoRectangleRegionModel, ClassificationArea, ); diff --git a/web/libs/editor/src/regions/BitmaskRegion.jsx b/web/libs/editor/src/regions/BitmaskRegion.jsx new file mode 100644 index 000000000000..7227211f491d --- /dev/null +++ b/web/libs/editor/src/regions/BitmaskRegion.jsx @@ -0,0 +1,500 @@ +import { useMemo } from "react"; +import { Group, Image, Line, Rect } from "react-konva"; +import { isAlive, types } from "mobx-state-tree"; + +import Registry from "../core/Registry"; +import NormalizationMixin from "../mixins/Normalization"; +import RegionsMixin from "../mixins/Regions"; +import { defaultStyle } from "../core/Constants"; +import { guidGenerator } from "../core/Helpers"; +import { AreaMixin } from "../mixins/AreaMixin"; +import IsReadyMixin from "../mixins/IsReadyMixin"; +import { KonvaRegionMixin } from "../mixins/KonvaRegion"; +import { ImageModel } from "../tags/object/Image"; +import { FF_DEV_3793, isFF } from "../utils/feature-flags"; +import { AliveRegion } from "./AliveRegion"; +import { RegionWrapper } from "./RegionWrapper"; +import { BitmaskDrawing, getCanvasPixelBounds } from "./BitmaskRegion/utils"; +import chroma from "chroma-js"; +import { generateMultiShapeOutline } from "./BitmaskRegion/contour"; +import { observe } from "mobx"; +import { LabelOnMask } from "../components/ImageView/LabelOnRegion"; + +/** + * Bitmask masking region + */ +const Model = types + .model({ + id: types.optional(types.identifier, guidGenerator), + pid: types.optional(types.string, guidGenerator), + type: "bitmaskregion", + object: types.late(() => types.reference(ImageModel)), + + /** + * Used to restore an image from the result or from a drawing region + * @type {string} + */ + imageDataURL: types.maybeNull(types.optional(types.string, "")), + }) + .volatile(() => ({ + /** + * Determines node opacity. Can be any number between 0 and 1 + */ + opacity: 0.6, + needsUpdate: 1, + hideable: true, + + /** @type {Layer} */ + layerRef: undefined, + + /** @type {Image} */ + imageRef: undefined, + + /** @type {HTMLCanvasElement} */ + bitmaskRef: null, + + /** @type {CanvasRenderingContext2D} */ + bitmaskCtx: null, + + /** @type {{x: number, y: number}} */ + lastPos: { x: 0, y: 0 }, + + /** @type {OffscreenCanvas} */ + offscreenCanvas: null, + + /** @type {number[][]} */ + outline: [], + + bbox: null, + })) + .views((self) => { + return { + get parent() { + return isAlive(self) ? self.object : null; + }, + get colorParts() { + const style = self.style?.strokecolor || self.tag?.strokecolor || defaultStyle?.strokecolor; + + return style ? chroma(style).rgb() : []; + }, + get strokeColor() { + return chroma(self.colorParts).hex(); + }, + get bboxCoordsCanvas() { + if (self.offscreenCanvas) { + return self.bbox; + } + }, + + get scale() { + return self.parent?.imageIsSmallerThanStage ? 1 : self.drawingOffset.scale; + }, + + /** + * Brushes are processed in pixels, so percentages are derived values for them, + * unlike for other tools. + */ + get bboxCoords() { + const bbox = self.bbox; + + if (!bbox) return null; + if (!isFF(FF_DEV_3793)) return bbox; + + return { + left: self.parent.canvasToInternalX(bbox.left), + top: self.parent.canvasToInternalY(bbox.top), + right: self.parent.canvasToInternalX(bbox.right), + bottom: self.parent.canvasToInternalY(bbox.bottom), + }; + }, + + get dimensions() { + const image = self.parent; + return { + stageWidth: image?.stageWidth ?? 0, + stageHeight: image?.stageHeight ?? 0, + imageWidth: image?.currentImageEntity.naturalWidth ?? 0, + imageHeight: image?.currentImageEntity.naturalHeight ?? 0, + }; + }, + + get drawingOffset() { + const { dimensions } = self; + + const scale = Math.min( + dimensions.stageWidth / dimensions.imageWidth, + dimensions.stageHeight / dimensions.imageHeight, + ); + + return { + offsetX: dimensions.imageWidth - dimensions.stageWidth / scale, + offsetY: dimensions.imageHeight - dimensions.stageHeight / scale, + scale, + }; + }, + + getImageDataURL() { + const canvas = self.getImageSnapshotCanvas(); + const imageDataURL = canvas.toDataURL("image/png"); + + return imageDataURL; + }, + }; + }) + .actions((self) => { + const lastPointX = -1; + const lastPointY = -1; + const disposers = []; + + return { + afterCreate() { + self.createCanvas(); + self.restoreFromImageDataURL(); // Only runs when the region is deserialized from result + + // We want to update color of the color mask dynamically + // so that changing label is reflected right away + disposers.push( + observe(self, "strokeColor", () => { + self.composeMask(); + }), + ); + + // The only way to track changes history is through current `imageDataURL` + // because we don't store points or re-render entire path from scratch + disposers.push( + observe(self, "imageDataURL", () => { + self.redraw(); + }), + ); + }, + + beforeDestroy() { + for (const disposer of disposers) { + disposer(); + } + }, + + setOutline(outline) { + self.outline = outline; + }, + + /** + * Restores image from a png data url (base64) + * Used when deserializing from result + */ + restoreFromImageDataURL() { + if (!self.imageDataURL) return; + async function renderDataURL() { + const context = self.offscreenCanvas.getContext("2d"); + const bitmask = self.bitmaskRef; + const image = new window.Image(); + + image.src = self.imageDataURL; + + try { + await image.decode(); + context.canvas.width = image.naturalWidth; + context.canvas.height = image.naturalHeight; + bitmask.width = image.naturalWidth; + bitmask.height = image.naturalHeight; + + context.drawImage(image, 0, 0); + + self.finalizeRegion(); + } catch (err) { + console.log(err); + } + } + renderDataURL(); + }, + + finalizeRegion() { + self.composeMask(); + self.generateOutline(); + self.updateBBox(); + }, + + updateImageURL() { + self.setImageDataURL(self.getImageDataURL()); + }, + + redraw() { + if (self.bitmaskRef && self.offscreenCanvas && self.imageDataURL) { + requestIdleCallback(() => { + const ctx = self.offscreenCanvas.getContext("2d"); + ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height); + self.restoreFromImageDataURL(); + }); + } + }, + + setBBox(bbox) { + self.bbox = bbox; + }, + + setImageDataURL(url) { + self.imageDataURL = url; + }, + + generateOutline() { + self.setOutline(generateMultiShapeOutline(self)); + }, + + canvasSize() { + if (!self.parent) return { width: 0, height: 0 }; + const ent = self.parent.currentImageEntity; + const scale = self.parent.imageIsSmallerThanStage ? 1 : self.parent.stageToImageRatio; + + return { + width: ent.naturalWidth / scale, + height: ent.naturalHeight / scale, + }; + }, + + createCanvas() { + const { width, height } = { + width: self.parent.currentImageEntity.naturalWidth, + height: self.parent.currentImageEntity.naturalHeight, + }; + if (!self.bitmaskRef) { + self.bitmaskRef = self.bitmaskRef ?? new OffscreenCanvas(width, height); + } + + if (!self.offscreenCanvas) { + self.offscreenCanvas = self.offscreenCanvas ?? new OffscreenCanvas(width, height); + } + + if (!self.bitmaskCtx) { + const ctx = self.bitmaskRef.getContext("2d"); + ctx.imageSmoothingEnabled = self.parent.smoothing; + self.bitmaskCtx = self.bitmaskCtx ?? ctx; + } + + return self.offscreenCanvas; + }, + + composeMask() { + const ctx = self.bitmaskRef.getContext("2d"); + + // Only clear if we're not in the middle of drawing + if (!self.isDrawing) { + ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height); + } + + ctx.globalCompositeOperation = "destination-atop"; + ctx.fillStyle = self.strokeColor; + ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height); + ctx.drawImage(self.offscreenCanvas, 0, 0); + + // Always batch draw to ensure visual updates + self.layerRef?.batchDraw(); + }, + + updateBBox() { + self.setBBox(getCanvasPixelBounds(self.offscreenCanvas, self.scale)); + }, + + /** + * @param {Group} ref + */ + setLayerRef(ref) { + if (!ref) return; + + const layer = ref.getParent(); + self.layerRef = layer; + }, + + setImageRef(ref) { + if (ref) self.imageRef = ref; + }, + + setLastPos(pos) { + self.lastPos = pos; + }, + + beginPath({ type, strokeWidth, opacity = self.opacity, x = 0, y = 0 }) { + self.object.annotation.pauseAutosave(); + + const { drawingOffset: offset } = self; + const ctx = self.offscreenCanvas.getContext("2d"); + + // Set up context properties once + ctx.fillStyle = type === "eraser" ? "white" : "black"; + ctx.globalCompositeOperation = type === "eraser" ? "destination-out" : "source-over"; + + self.setLastPos( + BitmaskDrawing.begin({ + ctx, + ...self.positionToStage(x, y), + brushSize: strokeWidth, + color: self.strokeColor, + eraserMode: type === "eraser", + }), + ); + + self.composeMask(); + }, + + addPoint(x, y, strokeWidth, options = { erase: false }) { + self.setLastPos( + BitmaskDrawing.draw({ + ctx: self.offscreenCanvas.getContext("2d"), + ...self.positionToStage(x, y), + brushSize: strokeWidth, + color: self.strokeColor, + lastPos: self.lastPos, + eraserMode: options.erase, + }), + ); + + // Only compose mask every few points or on significant changes + if (!self._lastComposeTime || Date.now() - self._lastComposeTime > 16) { + // ~60fps + self.composeMask(); + self._lastComposeTime = Date.now(); + } + }, + + positionToStage(x, y) { + const { drawingOffset: offset } = self; + const smaller = self.parent.imageIsSmallerThanStage; + const ratio = smaller ? 1 : offset.scale; + + return { + x: Math.floor(x / ratio + offset.offsetX), + y: Math.floor(y / ratio + offset.offsetY), + }; + }, + + endPath() { + const { annotation } = self.object; + + // we finalize the region and re-compute imageDataURL + // before enabling autosave to ensure that it's available + // for the draft at all times + self.finalizeRegion(); + self.updateImageURL(); + + // will resume in the next tick... + annotation.startAutosave(); + + self.notifyDrawingFinished(); + + // ...so we run this toggled function also delayed + annotation.autosave && setTimeout(() => annotation.autosave()); + }, + + updateImageSize(wp, hp, sw, sh) { + if (self.parent.stageWidth > 1 && self.parent.stageHeight > 1) { + self.finalizeRegion(); + + self.needsUpdate = self.needsUpdate + 1; + } + }, + + /** + * Prepared a bitmask for serialization/export + * @returns {HTMLCanvasElement} + */ + getImageSnapshotCanvas() { + const tempCanvas = document.createElement("canvas"); + tempCanvas.width = self.offscreenCanvas.width; + tempCanvas.height = self.offscreenCanvas.height; + const ctx = tempCanvas.getContext("2d"); + + // Convert back to black mask + ctx.globalCompositeOperation = "destination-atop"; + ctx.fillStyle = "black"; + ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height); + ctx.drawImage(self.offscreenCanvas, 0, 0); + return tempCanvas; + }, + + /** + * @param {object} options + * @param {boolean} [options.fast] + * @return {BrushRegionResult} + */ + serialize(options) { + const value = { imageDataURL: self.imageDataURL }; + return self.parent.createSerializedResult(self, value); + }, + }; + }); + +const BitmaskRegionModel = types.compose( + "BitmaskRegionModel", + RegionsMixin, + NormalizationMixin, + AreaMixin, + KonvaRegionMixin, + IsReadyMixin, + Model, +); + +const HtxBitmaskView = ({ item, setShapeRef }) => { + const highlightedRegions = item.parent?.regs.filter((r) => r.highlighted); + + const displayHighlight = useMemo(() => { + if (highlightedRegions?.length > 1) return false; + return item.highlighted || item.selected; + }, [item.highlighted, item.isDrawing, item.selected, highlightedRegions]); + + const ent = item.parent?.currentImageEntity; + const { width, height } = useMemo(() => { + if (!item.parent) return { width: 0, height: 0 }; + return item.canvasSize(); + }, [item.parent?.stageWidth, item.parent?.stageHeight, ent?.naturalWidth, ent?.naturalHeight]); + + const stage = item.parent?.stageRef; + + return ( + + + + {displayHighlight && } + + + + + {displayHighlight || + (highlightedRegions?.length > 1 && + item.outline.map((points, i) => ( + + )))} + + + + + + + ); +}; + +const HtxBitmask = AliveRegion(HtxBitmaskView, { + renderHidden: true, + shouldNotUsePortal: true, +}); + +Registry.addTag("bitmaskregion", BitmaskRegionModel, HtxBitmask); +Registry.addRegionType(BitmaskRegionModel, "image", (value) => "imageDataURL" in value); + +export { BitmaskRegionModel, HtxBitmask }; diff --git a/web/libs/editor/src/regions/BitmaskRegion/contour.ts b/web/libs/editor/src/regions/BitmaskRegion/contour.ts new file mode 100644 index 000000000000..402a9f26c901 --- /dev/null +++ b/web/libs/editor/src/regions/BitmaskRegion/contour.ts @@ -0,0 +1,133 @@ +import simplify from "simplify-js"; + +/** + * Generates outline contours from a pixel-based region/mask + * + * @param item - Object containing the canvas with pixel data and rendering properties + * @returns Array of flattened point coordinates forming contours around the shapes + */ +export function generateMultiShapeOutline(item: { + highlighted: boolean; + offscreenCanvas: HTMLCanvasElement; + drawingOffset: { scale: number }; + scale: number; +}) { + if (!item.offscreenCanvas) return []; + + const ctx = item.offscreenCanvas.getContext("2d"); + if (!ctx) return []; + + const { width, height } = item.offscreenCanvas; + const data = ctx.getImageData(0, 0, width, height).data; + + // Create a binary grid from the image data (1 for visible pixels, 0 for transparent) + const grid: number[][] = []; + for (let y = 0; y < height; y++) { + const row: number[] = []; + for (let x = 0; x < width; x++) { + const alpha = data[(y * width + x) * 4 + 3]; + row.push(alpha > 0 ? 1 : 0); + } + grid.push(row); + } + + const visited = Array.from({ length: height }, () => Array(width).fill(false)); + const dirs = [ + [1, 0], + [0, 1], + [-1, 0], + [0, -1], + ]; + + // Helper to check if two points are within 1 pixel (including diagonals) + const isNear = (x1: number, y1: number, x2: number, y2: number) => Math.abs(x1 - x2) <= 1 && Math.abs(y1 - y2) <= 1; + + /** + * Determines if a pixel is on the edge of a shape + * A pixel is an edge if it's non-transparent and has at least one transparent neighbor + */ + const isEdge = (x: number, y: number): boolean => { + if (grid[y][x] === 0) return false; + for (let dy = -1; dy <= 1; dy++) { + for (let dx = -1; dx <= 1; dx++) { + if (dx === 0 && dy === 0) continue; + const nx = x + dx; + const ny = y + dy; + if (nx < 0 || ny < 0 || nx >= width || ny >= height || grid[ny][nx] === 0) { + return true; + } + } + } + return false; + }; + + /** + * Traces a contour starting from a given point + * Uses a boundary-following algorithm to create a closed path + */ + const trace = (sx: number, sy: number) => { + const path = []; + const seen = new Set(); + let x = sx; + let y = sy; + let dir = 0; + let closed = false; + + for (let steps = 0; steps < 5000; steps++) { + path.push([x, y]); + seen.add(`${x},${y}`); + visited[y][x] = true; + let moved = false; + + for (let i = 0; i < 4; i++) { + const d = (dir + i) % 4; + const [dx, dy] = dirs[d]; + const nx = x + dx; + const ny = y + dy; + if (nx >= 0 && ny >= 0 && nx < width && ny < height && isEdge(nx, ny) && !seen.has(`${nx},${ny}`)) { + x = nx; + y = ny; + dir = d; + moved = true; + break; + } + } + + // Only close if we're back near the start and the path is long enough + if (!moved || (path.length > 10 && isNear(x, y, sx, sy))) { + closed = isNear(x, y, sx, sy) && path.length > 10; + break; + } + } + + // Only accept closed contours + if (closed) { + return path; + } + return []; + }; + + // Find and trace all contours in the image + const contours: number[][][] = []; + for (let y = 1; y < height - 1; y++) { + for (let x = 1; x < width - 1; x++) { + if (isEdge(x, y) && !visited[y][x]) { + const contour = trace(x, y); + if (contour.length > 5) { + contours.push(contour); + } + } + } + } + + // Scale and simplify the contours for rendering + const { scale } = item; + return contours.map((contour) => { + const simplified = simplify( + contour.map(([x, y]) => ({ x: x * scale, y: y * scale })), + 0.9, + true, + ); + return simplified.flatMap(({ x, y }) => [x, y]); + }); +} diff --git a/web/libs/editor/src/regions/BitmaskRegion/utils.ts b/web/libs/editor/src/regions/BitmaskRegion/utils.ts new file mode 100644 index 000000000000..7b37aceb4420 --- /dev/null +++ b/web/libs/editor/src/regions/BitmaskRegion/utils.ts @@ -0,0 +1,187 @@ +export const BitmaskDrawing = { + /** + * Draws initial point on the canvas + */ + begin({ + ctx, + x, + y, + brushSize = 10, + eraserMode = false, + }: { ctx: CanvasRenderingContext2D; x: number; y: number; brushSize: number; color: string; eraserMode: boolean }): { + x: number; + y: number; + } { + ctx.fillStyle = eraserMode ? "white" : "black"; + ctx.globalCompositeOperation = eraserMode ? "destination-out" : "source-over"; + + if (brushSize === 1) { + ctx.fillRect(x, y, 1, 1); + } else { + ctx.beginPath(); + ctx.arc(x + 0.5, y + 0.5, brushSize, 0, Math.PI * 2); + ctx.fill(); + } + return { x, y }; + }, + + /** + * Draws a line between last and current position + */ + draw({ + ctx, + x, + y, + brushSize = 10, + eraserMode = false, + lastPos, + }: { + ctx: CanvasRenderingContext2D; + x: number; + y: number; + brushSize: number; + color: string; + lastPos: { x: number; y: number }; + eraserMode: boolean; + }): { x: number; y: number } { + ctx.fillStyle = eraserMode ? "white" : "black"; + ctx.globalCompositeOperation = eraserMode ? "destination-out" : "source-over"; + + this.drawLine(ctx, lastPos.x, lastPos.y, x, y, brushSize); + return { x, y }; + }, + + /** + * Interpolation algorithm to connect separate + * dots on the canvas + */ + drawLine(ctx: CanvasRenderingContext2D, x0: number, y0: number, x1: number, y1: number, size: number) { + const dx = Math.abs(x1 - x0); + const dy = Math.abs(y1 - y0); + const sx = x0 < x1 ? 1 : -1; + const sy = y0 < y1 ? 1 : -1; + let err = dx - dy; + + while (true) { + if (size === 1) { + ctx.fillRect(x0, y0, 1, 1); + } else { + ctx.beginPath(); + ctx.arc(x0 + 0.5, y0 + 0.5, size, 0, Math.PI * 2); + ctx.fill(); + } + + if (x0 === x1 && y0 === y1) break; + const e2 = 2 * err; + if (e2 > -dy) { + err -= dy; + x0 += sx; + } + if (e2 < dx) { + err += dx; + y0 += sy; + } + } + }, +}; + +/** + * Checks if the mouse pointer is hovering over a non-transparent pixel in a canvas-based image. + * This function is used to determine if the user is interacting with a visible part of a bitmask region. + * + * @param item - An object containing references to the canvas layer, offscreen canvas, and image + * @param item.layerRef - Reference to the Konva layer containing the canvas + * @param item.offscreenCanvas - The offscreen canvas element containing the bitmask data + * @param item.imageRef - Reference to the Konva image element + * @param item.drawingOffset - Object containing offsetX and offsetY for drawing position + * @param item.scale - Scale factor of the image + * @returns {boolean} True if hovering over a non-transparent pixel, false otherwise + */ +export function isHoveringNonTransparentPixel(item: any) { + if (!item?.layerRef || !item?.offscreenCanvas || !item?.imageRef) { + return false; + } + + const stage = item.layerRef.getStage(); + const pointer = stage?.getPointerPosition(); + const ctx = item.offscreenCanvas.getContext("2d"); + + if (!pointer || !ctx) return false; + + try { + // Convert global pointer to image-local coordinates + const { drawingOffset: offset } = item; + const transform = item.imageRef.getAbsoluteTransform().copy().invert(); + const localPos = transform.point(pointer); + + const { width, height } = item.offscreenCanvas; + + // Convert to pixel coords in the canvas backing the image + const x = Math.floor(localPos.x / item.scale + offset.offsetX); + const y = Math.floor(localPos.y / item.scale + offset.offsetY); + + if (x < 0 || y < 0 || x >= width || y >= height) return false; + + const alpha = ctx.getImageData(x, y, 1, 1).data[3]; + return alpha > 0; + } catch (error) { + console.warn("Error checking pixel transparency:", error); + return false; + } +} + +/** + * Calculates the bounding box of non-transparent pixels in a canvas. + * This function scans the canvas pixel by pixel to find the minimum rectangle + * that contains all visible (non-transparent) pixels. + * + * @param canvas - The HTML canvas element to analyze + * @param scale - Scale factor to apply to the returned coordinates + * @returns {Object|null} An object containing the bounds of non-transparent pixels: + * - left: Leftmost x-coordinate of visible pixels + * - top: Topmost y-coordinate of visible pixels + * - right: Rightmost x-coordinate of visible pixels (exclusive) + * - bottom: Bottommost y-coordinate of visible pixels (exclusive) + * Returns null if no visible pixels are found + */ +export function getCanvasPixelBounds( + canvas: HTMLCanvasElement, + scale: number, +): { left: number; top: number; right: number; bottom: number } | null { + const ctx = canvas.getContext("2d"); + if (!ctx) return null; + + const { width, height } = canvas; + const imageData = ctx.getImageData(0, 0, width, height); + const data = imageData.data; + + let minX = width; + let minY = height; + let maxX = -1; + let maxY = -1; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const index = (y * width + x) * 4; + const alpha = data[index + 3]; // alpha channel + if (alpha > 0) { + if (x < minX) minX = x; + if (x > maxX) maxX = x; + if (y < minY) minY = y; + if (y > maxY) maxY = y; + } + } + } + + const hasVisiblePixels = maxX >= minX && maxY >= minY; + if (!hasVisiblePixels) return null; + + // Scale is applied to the points to compensate for + // the image being different size than the stage + return { + left: minX * scale, + top: minY * scale, + right: (maxX + 1) * scale, + bottom: (maxY + 1) * scale, + }; +} diff --git a/web/libs/editor/src/regions/Result.js b/web/libs/editor/src/regions/Result.js index f2038b59b767..c558f8381aa3 100644 --- a/web/libs/editor/src/regions/Result.js +++ b/web/libs/editor/src/regions/Result.js @@ -40,12 +40,14 @@ const Result = types "keypoint", "polygon", "brush", + "bitmask", "ellipse", "magicwand", "rectanglelabels", "keypointlabels", "polygonlabels", "brushlabels", + "bitmasklabels", "ellipselabels", "timeserieslabels", "timelinelabels", @@ -82,6 +84,7 @@ const Result = types brushlabels: types.maybe(types.array(types.string)), timeserieslabels: types.maybe(types.array(types.string)), timelinelabels: types.maybe(types.array(types.string)), // new one + bitmasklabels: types.maybe(types.array(types.string)), taxonomy: types.frozen(), // array of arrays of strings sequence: types.frozen(), }), diff --git a/web/libs/editor/src/regions/index.js b/web/libs/editor/src/regions/index.js index df562cac188a..a2f380b8c08e 100644 --- a/web/libs/editor/src/regions/index.js +++ b/web/libs/editor/src/regions/index.js @@ -2,6 +2,7 @@ import { types } from "mobx-state-tree"; import { AudioRegionModel } from "./AudioRegion"; import { BrushRegionModel, HtxBrush } from "./BrushRegion"; +import { BitmaskRegionModel, HtxBitmask } from "./BitmaskRegion"; import { ParagraphsRegionModel } from "./ParagraphsRegion"; import { TimeSeriesRegionModel } from "./TimeSeriesRegion"; import { HtxKeyPoint, KeyPointRegionModel } from "./KeyPointRegion"; @@ -17,6 +18,7 @@ import { VideoRectangleRegionModel } from "./VideoRectangleRegion"; const AllRegionsType = types.union( AudioRegionModel, BrushRegionModel, + BitmaskRegionModel, EllipseRegionModel, TimeSeriesRegionModel, KeyPointRegionModel, @@ -36,6 +38,7 @@ export { BrushRegionModel, EllipseRegionModel, HtxBrush, + HtxBitmask, HtxEllipse, HtxKeyPoint, HtxPolygon, diff --git a/web/libs/editor/src/tags/control/Bitmask.js b/web/libs/editor/src/tags/control/Bitmask.js new file mode 100644 index 000000000000..deda1b3d561f --- /dev/null +++ b/web/libs/editor/src/tags/control/Bitmask.js @@ -0,0 +1,72 @@ +import { types } from "mobx-state-tree"; + +import Registry from "../../core/Registry"; +import ControlBase from "./Base"; +import { AnnotationMixin } from "../../mixins/AnnotationMixin"; +import SeparatedControlMixin from "../../mixins/SeparatedControlMixin"; +import { ToolManagerMixin } from "../../mixins/ToolManagerMixin"; + +/** + * The `Bitmask` tag is used for image segmentation tasks where you want to apply a mask or use a brush to draw a region on the image. + * + * Use with the following data types: image. + * @example + * + * + * + * + * + * + * @name Bitmask + * @regions BitmaskRegion + * @meta_title Bitmask Tag for Image Segmentation Labeling + * @meta_description Customize Label Studio with brush tags for image segmentation labeling for machine learning and data science projects. + * @param {string} name - Name of the element + * @param {string} toName - Name of the image to label + * @param {single|multiple=} [choice=single] - Configure whether the data labeler can select one or multiple labels + * @param {number} [maxUsages] - Maximum number of times a label can be used per task + * @param {boolean} [showInline=true] - Show labels in the same visual line + * @param {boolean} [smart] - Show smart tool for interactive pre-annotations + * @param {boolean} [smartOnly] - Only show smart tool for interactive pre-annotations + */ + +const TagAttrs = types.model({ + toname: types.maybeNull(types.string), + strokewidth: types.optional(types.string, "15"), +}); + +const Model = types + .model({ + type: "bitmask", + removeDuplicatesNamed: "BitmaskErase", + }) + .views((self) => ({ + get hasStates() { + const states = self.states(); + + return states && states.length > 0; + }, + })) + .volatile(() => ({ + toolNames: ["Bitmask", "BitmaskErase"], + })); + +const BitmaskModel = types.compose( + "BitmaskModel", + ControlBase, + AnnotationMixin, + SeparatedControlMixin, + TagAttrs, + Model, + ToolManagerMixin, +); + +const HtxView = () => { + return null; +}; + +Registry.addTag("bitmask", BitmaskModel, HtxView); + +export { HtxView, BitmaskModel }; diff --git a/web/libs/editor/src/tags/control/BitmaskLabels.jsx b/web/libs/editor/src/tags/control/BitmaskLabels.jsx new file mode 100644 index 000000000000..df7101a0c5e4 --- /dev/null +++ b/web/libs/editor/src/tags/control/BitmaskLabels.jsx @@ -0,0 +1,64 @@ +import { observer } from "mobx-react"; +import { types } from "mobx-state-tree"; + +import LabelMixin from "../../mixins/LabelMixin"; +import Registry from "../../core/Registry"; +import SelectedModelMixin from "../../mixins/SelectedModel"; +import Types from "../../core/Types"; +import { HtxLabels, LabelsModel } from "./Labels/Labels"; +import ControlBase from "./Base"; +import { BitmaskModel } from "./Bitmask"; + +/** + * The `BitmaskLabels` tag for image segmentation tasks is used in the area where you want to apply a mask or use a brush to draw a region on the image. + * + * Bitmask operates on pixel level and outputs a png encoded in a Base64 data URL. + * + * Use with the following data types: image. + * @example + * + * + * + * + * + * + * @name BitmaskLabels + * @regions BitmaskRegion + * @meta_title Bitmask Label Tag for Image Segmentation Labeling + * @meta_description Customize Label Studio with brush label tags for image segmentation labeling for machine learning and data science projects. + * @param {string} name - Name of the element + * @param {string} toName - Name of the image to label + * @param {single|multiple=} [choice=single] - Configure whether the data labeler can select one or multiple labels + * @param {number} [maxUsages] - Maximum number of times a label can be used per task + * @param {boolean} [showInline=true] - Show labels in the same visual line + */ + +const Validation = types.model({ + controlledTags: Types.unionTag(["Image"]), +}); + +const ModelAttrs = types.model("BitmaskLabelsModel", { + type: "bitmaskregion", + children: Types.unionArray(["label", "header", "view", "hypertext"]), +}); + +const BitmaskLabelsModel = types.compose( + "BitmaskLabelsModel", + ControlBase, + LabelsModel, + ModelAttrs, + BitmaskModel, + Validation, + LabelMixin, + SelectedModelMixin.props({ _child: "LabelModel" }), +); + +const HtxBitmaskLabels = observer(({ item }) => { + return ; +}); + +Registry.addTag("bitmasklabels", BitmaskLabelsModel, HtxBitmaskLabels); + +export { HtxBitmaskLabels, BitmaskLabelsModel }; diff --git a/web/libs/editor/src/tags/control/Label.jsx b/web/libs/editor/src/tags/control/Label.jsx index 208e06764d8d..cd515c1e8865 100644 --- a/web/libs/editor/src/tags/control/Label.jsx +++ b/web/libs/editor/src/tags/control/Label.jsx @@ -83,6 +83,7 @@ const Model = types "TimelineLabels", "TimeSeriesLabels", "ParagraphLabels", + "BitmaskLabels", ]), }) .volatile((self) => { diff --git a/web/libs/editor/src/tags/control/index.js b/web/libs/editor/src/tags/control/index.js index 3276cff0a21e..493dbbf86f48 100644 --- a/web/libs/editor/src/tags/control/index.js +++ b/web/libs/editor/src/tags/control/index.js @@ -16,6 +16,8 @@ import { TimelineLabelsModel } from "./TimelineLabels"; import { VideoRectangleModel } from "./VideoRectangle"; import { BrushLabelsModel } from "./BrushLabels"; +import { BitmaskLabelsModel } from "./BitmaskLabels"; +import { BitmaskModel } from "./Bitmask"; import { BrushModel } from "./Brush"; import { EllipseLabelsModel } from "./EllipseLabels"; import { EllipseModel } from "./Ellipse"; @@ -43,6 +45,8 @@ export { HyperTextLabelsModel, LabelsModel, ParagraphLabelsModel, + BitmaskLabelsModel, + BitmaskModel, TimeSeriesLabelsModel, TimelineLabelsModel, VideoRectangleModel, diff --git a/web/libs/editor/src/tags/object/Image/Image.js b/web/libs/editor/src/tags/object/Image/Image.js index 16089cfe80c9..971452941a91 100644 --- a/web/libs/editor/src/tags/object/Image/Image.js +++ b/web/libs/editor/src/tags/object/Image/Image.js @@ -35,6 +35,9 @@ import { RELATIVE_STAGE_HEIGHT, RELATIVE_STAGE_WIDTH, SNAP_TO_PIXEL_MODE } from import MultiItemObjectBase from "../MultiItemObjectBase"; const IMAGE_PRELOAD_COUNT = 3; +const ZOOM_INTENSITY = 0.009; +const MIN_ZOOM = 0.1; +const MAX_ZOOM = 100; /** * The `Image` tag shows an image on the page. Use for all image annotation tasks to display an image on the labeling interface. @@ -138,6 +141,8 @@ const IMAGE_CONSTANTS = { keypointlabels: "keypointlabels", polygonlabels: "polygonlabels", brushlabels: "brushlabels", + bitmaskModel: "BitmaskModel", + bitmasklabels: "bitmasklabels", brushModel: "BrushModel", ellipselabels: "ellipselabels", }; @@ -279,10 +284,29 @@ const Model = types }[self.rotation]; }, + get stageToImageRatio() { + const ent = self.currentImageEntity; + return Math.min(ent.naturalWidth / self.stageWidth, ent.naturalHeight / self.stageHeight); + }, + + get imageIsSmallerThanStage() { + const ent = self.currentImageEntity; + return ent.naturalWidth < self.stageWidth || ent.naturalHeight < self.stageHeight; + }, + get stageScale() { return self.zoomScale; }, + get layerZoomScalePosition() { + return { + scaleX: self.zoomScale, + scaleY: self.zoomScale, + x: self.zoomingPositionX + self.alignmentOffset.x, + y: self.zoomingPositionY + self.alignmentOffset.y, + }; + }, + get hasTools() { return !!self.getToolsManager().allTools()?.length; }, @@ -400,6 +424,7 @@ const Model = types if ( item.type === IMAGE_CONSTANTS.rectanglelabels || item.type === IMAGE_CONSTANTS.brushlabels || + item.type === IMAGE_CONSTANTS.bitmasklabels || item.type === IMAGE_CONSTANTS.ellipselabels ) { returnedControl = item; @@ -914,17 +939,20 @@ const Model = types self.resetZoomPositionToCenter(); }, - handleZoom(val, mouseRelativePos = { x: self.canvasSize.width / 2, y: self.canvasSize.height / 2 }) { + handleZoom(val, mouseRelativePos = { x: self.canvasSize.width / 2, y: self.canvasSize.height / 2 }, pinch = false) { if (val) { let zoomScale = self.currentZoom; - zoomScale = val > 0 ? zoomScale * self.zoomBy : zoomScale / self.zoomBy; + const newZoom = clamp(self.currentZoom * Math.exp(val * (pinch ? -1 : 1) * ZOOM_INTENSITY), MIN_ZOOM, MAX_ZOOM); + zoomScale = pinch ? newZoom : val > 0 ? zoomScale * self.zoomBy : zoomScale / self.zoomBy; + if (self.negativezoom !== true && zoomScale <= 1) { self.setZoom(1); self.setZoomPosition(0, 0); self.updateImageAfterZoom(); return; } + if (zoomScale <= 1) { self.setZoom(zoomScale); self.setZoomPosition(0, 0); diff --git a/web/libs/editor/src/tags/object/Image/ImageEntity.js b/web/libs/editor/src/tags/object/Image/ImageEntity.js index b347c5785a3b..4a2ac7c74629 100644 --- a/web/libs/editor/src/tags/object/Image/ImageEntity.js +++ b/web/libs/editor/src/tags/object/Image/ImageEntity.js @@ -6,7 +6,7 @@ import { FF_IMAGE_MEMORY_USAGE, isFF } from "../../../utils/feature-flags"; const fileLoader = new FileLoader(); export const ImageEntity = types - .model({ + .model("ImageEntity", { id: types.identifier, src: types.string, index: types.number, diff --git a/web/libs/editor/src/tags/object/PagedView.jsx b/web/libs/editor/src/tags/object/PagedView.jsx index c2ff3aa2392a..36ce7bff16ac 100644 --- a/web/libs/editor/src/tags/object/PagedView.jsx +++ b/web/libs/editor/src/tags/object/PagedView.jsx @@ -38,6 +38,7 @@ const Model = types.model({ "polygonlabels", "keypointlabels", "brushlabels", + "bitmasklabels", "hypertextlabels", "timeserieslabels", "text", diff --git a/web/libs/editor/src/tags/visual/Collapse.jsx b/web/libs/editor/src/tags/visual/Collapse.jsx index 7d23d832dce9..c08c4fd45213 100644 --- a/web/libs/editor/src/tags/visual/Collapse.jsx +++ b/web/libs/editor/src/tags/visual/Collapse.jsx @@ -49,6 +49,7 @@ const PanelModel = types "keypoint", "brush", "rectanglelabels", + "bitmasklabels", "ellipselabels", "polygonlabels", "keypointlabels", diff --git a/web/libs/editor/src/tags/visual/View.jsx b/web/libs/editor/src/tags/visual/View.jsx index f3f8d47e0c98..7c90f835bcd1 100644 --- a/web/libs/editor/src/tags/visual/View.jsx +++ b/web/libs/editor/src/tags/visual/View.jsx @@ -94,6 +94,7 @@ const Model = types "polygon", "keypoint", "brush", + "bitmask", "magicwand", "rectanglelabels", "ellipselabels", @@ -102,6 +103,7 @@ const Model = types "brushlabels", "hypertextlabels", "timeserieslabels", + "bitmasklabels", "text", "audio", "image", diff --git a/web/libs/editor/src/tools/Bitmask.jsx b/web/libs/editor/src/tools/Bitmask.jsx new file mode 100644 index 000000000000..f24d3d284fd9 --- /dev/null +++ b/web/libs/editor/src/tools/Bitmask.jsx @@ -0,0 +1,243 @@ +import { observer } from "mobx-react"; +import { types } from "mobx-state-tree"; + +import BaseTool from "./Base"; +import ToolMixin from "../mixins/Tool"; +import { clamp, findClosestParent } from "../utils/utilities"; +import { DrawingTool } from "../mixins/DrawingTool"; +import { Tool } from "../components/Toolbar/Tool"; +import { Range } from "../common/Range/Range"; +import { NodeViews } from "../components/Node/Node"; + +const MIN_SIZE = 1; +const MAX_SIZE = 50; + +const IconDot = ({ size }) => { + return ( + + ); +}; + +const ToolView = observer(({ item }) => { + return ( + { + if (item.selected) return; + + item.manager.selectTool(item, true); + }} + controls={item.controls} + /> + ); +}); + +const _Tool = types + .model("BitmaskTool", { + strokeWidth: types.optional(types.number, 15), + group: "segmentation", + shortcut: "B", + smart: true, + unselectRegionOnToolChange: false, + }) + .volatile(() => ({ + canInteractWithRegions: false, + })) + .views((self) => ({ + get viewClass() { + return () => ; + }, + get iconComponent() { + return self.dynamic ? NodeViews.BitmaskRegionModel.altIcon : NodeViews.BitmaskRegionModel.icon; + }, + get tagTypes() { + return { + stateTypes: "bitmasklabels", + controlTagTypes: ["bitmasklabels", "bitmask"], + }; + }, + get controls() { + return [ + } + maxIcon={} + onChange={(value) => { + self.setStroke(value); + }} + />, + ]; + }, + get extraShortcuts() { + return { + "[": [ + "Decrease size", + () => { + self.setStroke(clamp(self.strokeWidth - 5, MIN_SIZE, MAX_SIZE)); + }, + ], + "]": [ + "Increase size", + () => { + self.setStroke(clamp(self.strokeWidth + 5, MIN_SIZE, MAX_SIZE)); + }, + ], + }; + }, + })) + .actions((self) => { + let brush; + let isFirstBrushStroke; + + return { + commitDrawingRegion() { + const { currentArea, control, obj } = self; + const value = { + imageDataURL: currentArea.getImageDataURL(), + }; + const newArea = self.annotation.createResult(value, currentArea.results[0].value.toJSON(), control, obj); + + currentArea.setDrawing(false); + self.applyActiveStates(newArea); + try { + self.deleteRegion(); + } catch (e) { + /* do nothing*/ + } + newArea.notifyDrawingFinished(); + return newArea; + }, + + setStroke(val) { + self.strokeWidth = val; + self.updateCursor(); + }, + + afterUpdateSelected() { + self.updateCursor(); + }, + + addPoint(x, y) { + brush.addPoint(x, y, self.strokeWidth || self.control.strokeWidth); + }, + + mouseupEv(_ev, _, [x, y]) { + if (self.mode !== "drawing") return; + self.addPoint(x, y); + self.mode = "viewing"; + brush.setDrawing(false); + brush.endPath(); + if (isFirstBrushStroke) { + const newBrush = self.commitDrawingRegion(); + self.obj.annotation.selectArea(newBrush); + self.annotation.history.unfreeze(); + self.obj.annotation.setIsDrawing(false); + } else { + self.annotation.history.unfreeze(); + self.obj.annotation.setIsDrawing(false); + } + }, + + mousemoveEv(ev, _, [x, y]) { + if (!self.isAllowedInteraction(ev)) return; + if (self.mode !== "drawing") return; + if ( + !findClosestParent( + ev.target, + (el) => el === self.obj.stageRef.content, + (el) => el.parentElement, + ) + ) + return; + + self.addPoint(x, y); + }, + + mousedownEv(ev, _, [x, y]) { + if (!self.isAllowedInteraction(ev)) return; + if ( + !findClosestParent( + ev.target, + (el) => el === self.obj.stageRef.content, + (el) => el.parentElement, + ) + ) + return; + const c = self.control; + const o = self.obj; + const hasHighlighted = o.regs.some((r) => r.highlighted); + + if (hasHighlighted) return; + brush = self.getSelectedShape; + + // prevent drawing when current image is + // different from image where the brush was started + if (o && brush && o.multiImage && o.currentImage !== brush.item_index) return; + + // Reset the timer if a user started drawing again + if (brush && brush.type === "bitmaskregion") { + self.annotation.history.freeze(); + self.mode = "drawing"; + isFirstBrushStroke = false; + brush.setDrawing(true); + self.obj.annotation.setIsDrawing(true); + } else { + if (!self.canStartDrawing()) return; + if (self.tagTypes.stateTypes === self.control.type && !self.control.isSelected) return; + self.annotation.history.freeze(); + self.mode = "drawing"; + isFirstBrushStroke = true; + self.obj.annotation.setIsDrawing(true); + brush = self.createDrawingRegion({ + imageDataURL: null, + }); + } + + brush.beginPath({ + type: "add", + strokeWidth: self.strokeWidth || c.strokeWidth, + x, + y, + }); + }, + }; + }); + +const BitmaskCursorMixin = types + .model("BitmaskCursorMixin") + .views((self) => ({ + get cursorStyleRule() { + const val = self.strokeWidth; + return "crosshair"; + }, + })) + .actions((self) => ({ + updateCursor() { + if (!self.selected || !self.obj?.stageRef) return; + const stage = self.obj.stageRef; + stage.container().style.cursor = self.cursorStyleRule; + }, + })); + +const Bitmask = types.compose(_Tool.name, ToolMixin, BaseTool, DrawingTool, BitmaskCursorMixin, _Tool); + +export { Bitmask, BitmaskCursorMixin }; diff --git a/web/libs/editor/src/tools/BitmaskErase.jsx b/web/libs/editor/src/tools/BitmaskErase.jsx new file mode 100644 index 000000000000..6541be786101 --- /dev/null +++ b/web/libs/editor/src/tools/BitmaskErase.jsx @@ -0,0 +1,172 @@ +import { observer } from "mobx-react"; +import { types } from "mobx-state-tree"; + +import BaseTool from "./Base"; +import ToolMixin from "../mixins/Tool"; +import { clamp, findClosestParent } from "../utils/utilities"; +import { DrawingTool } from "../mixins/DrawingTool"; +import { IconEraserTool } from "@humansignal/icons"; +import { Tool } from "../components/Toolbar/Tool"; +import { Range } from "../common/Range/Range"; +import { BitmaskCursorMixin } from "./Bitmask"; + +const MIN_SIZE = 1; +const MAX_SIZE = 50; + +const IconDot = ({ size }) => { + return ( + + ); +}; + +const ToolView = observer(({ item }) => { + return ( + { + if (item.selected) return; + + item.manager.selectTool(item, true); + }} + icon={item.iconClass} + controls={item.controls} + /> + ); +}); + +const _Tool = types + .model("BitmaskEraserTool", { + strokeWidth: types.optional(types.number, 10), + group: "segmentation", + unselectRegionOnToolChange: false, + }) + .volatile(() => ({ + index: 9999, + canInteractWithRegions: false, + })) + .views((self) => ({ + get viewClass() { + return () => ; + }, + get iconComponent() { + return IconEraserTool; + }, + get controls() { + return [ + } + maxIcon={} + onChange={(value) => { + self.setStroke(value); + }} + />, + ]; + }, + get extraShortcuts() { + return { + "[": [ + "Decrease size", + () => { + self.setStroke(clamp(self.strokeWidth - 5, MIN_SIZE, MAX_SIZE)); + }, + ], + "]": [ + "Increase size", + () => { + self.setStroke(clamp(self.strokeWidth + 5, MIN_SIZE, MAX_SIZE)); + }, + ], + }; + }, + })) + .actions((self) => { + let brush; + + return { + afterUpdateSelected() { + self.updateCursor(); + }, + + addPoint(x, y) { + brush.addPoint(Math.floor(x), Math.floor(y), self.strokeWidth, { erase: true }); + }, + + setStroke(val) { + self.strokeWidth = val; + self.updateCursor(); + }, + + mouseupEv() { + if (self.mode !== "drawing") return; + self.mode = "viewing"; + brush.endPath(); + }, + + mousemoveEv(ev, _, [x, y]) { + if (self.mode !== "drawing") return; + if ( + !findClosestParent( + ev.target, + (el) => el === self.obj.stageRef.content, + (el) => el.parentElement, + ) + ) + return; + + if (brush?.type === "bitmaskregion") { + self.addPoint(x, y); + } + }, + + mousedownEv(ev, _, [x, y]) { + if (!self.isAllowedInteraction(ev)) return; + if ( + !findClosestParent( + ev.target, + (el) => el === self.obj.stageRef.content, + (el) => el.parentElement, + ) + ) + return; + + brush = self.getSelectedShape; + if (!brush) return; + + if (brush && brush.type === "bitmaskregion") { + self.mode = "drawing"; + brush.beginPath({ + type: "eraser", + opacity: 1, + strokeWidth: self.strokeWidth, + x, + y, + }); + self.addPoint(x, y); + } + }, + }; + }); + +const BitmaskErase = types.compose(_Tool.name, ToolMixin, BaseTool, DrawingTool, BitmaskCursorMixin, _Tool); + +export { BitmaskErase }; diff --git a/web/libs/editor/src/tools/index.js b/web/libs/editor/src/tools/index.js index 7c0e85214b95..9c6eaba15438 100644 --- a/web/libs/editor/src/tools/index.js +++ b/web/libs/editor/src/tools/index.js @@ -2,7 +2,9 @@ // export { default as KeyPoint } from "./KeyPoint"; import { Brush } from "./Brush"; +import { Bitmask } from "./Bitmask"; import { Erase } from "./Erase"; +import { BitmaskErase } from "./BitmaskErase"; import { KeyPoint } from "./KeyPoint"; import { Polygon } from "./Polygon"; import { Rect, Rect3Point } from "./Rect"; @@ -16,7 +18,9 @@ import { Selection } from "./Selection"; export { Brush, + Bitmask, Erase, + BitmaskErase, KeyPoint, Polygon, Rect, diff --git a/web/libs/editor/tests/e2e/tests/regression-tests/bitmask.test.js b/web/libs/editor/tests/e2e/tests/regression-tests/bitmask.test.js new file mode 100644 index 000000000000..3a40c7134778 --- /dev/null +++ b/web/libs/editor/tests/e2e/tests/regression-tests/bitmask.test.js @@ -0,0 +1,258 @@ +Feature("Bitmask tool").tag("@regress"); + +const assert = require("assert"); + +const IMAGE = "https://data.heartex.net/open-images/train_0/mini/0030019819f25b28.jpg"; + +const config = ` + + + + +`; + +// Add cleanup hook +Before(async ({ I }) => { + I.amOnPage("/"); +}); + +Scenario("Basic Bitmask drawing", async ({ I, LabelStudio, AtImageView, AtLabels }) => { + const params = { + config, + data: { image: IMAGE }, + annotations: [{ id: 1, result: [] }], + }; + + I.amOnPage("/"); + LabelStudio.init(params); + LabelStudio.waitForObjectsReady(); + await AtImageView.lookForStage(); + + I.say("Select Bitmask tool"); + I.pressKey("B"); + AtLabels.clickLabel("Test"); + + I.say("Draw a simple mask"); + AtImageView.drawThroughPoints([ + [20, 20], + [20, 40], + [40, 40], + [40, 20], + [20, 20], + ]); + + I.say("Check if mask was created"); + const result = await LabelStudio.serialize(); + assert.strictEqual(result.length, 1); + assert.ok(result[0].value.imageDataURL); +}); + +Scenario("Bitmask eraser functionality", async ({ I, LabelStudio, AtImageView, AtLabels }) => { + const params = { + config, + data: { image: IMAGE }, + annotations: [{ id: 1, result: [] }], + }; + + I.amOnPage("/"); + LabelStudio.init(params); + LabelStudio.waitForObjectsReady(); + await AtImageView.lookForStage(); + + I.say("Select Bitmask tool and draw initial mask"); + I.pressKey("B"); + AtLabels.clickLabel("Test"); + AtImageView.drawThroughPoints([ + [20, 20], + [20, 40], + [40, 40], + [40, 20], + [20, 20], + ]); + + I.say("Switch to eraser and erase part of the mask"); + I.pressKey("E"); + AtImageView.drawThroughPoints([ + [25, 25], + [35, 35], + ]); + + I.say("Check if mask was modified"); + const result = await LabelStudio.serialize(); + assert.strictEqual(result.length, 1); + assert.ok(result[0].value.imageDataURL); +}); + +Scenario("Bitmask size controls", async ({ I, LabelStudio, AtImageView, AtLabels }) => { + const params = { + config, + data: { image: IMAGE }, + annotations: [{ id: 1, result: [] }], + }; + + I.amOnPage("/"); + LabelStudio.init(params); + LabelStudio.waitForObjectsReady(); + await AtImageView.lookForStage(); + + I.say("Select Bitmask tool"); + I.pressKey("B"); + AtLabels.clickLabel("Test"); + + I.say("Test size increase shortcut"); + I.pressKey("]"); + AtImageView.drawThroughPoints([[30, 30]]); + + I.say("Test size decrease shortcut"); + I.pressKey("["); + AtImageView.drawThroughPoints([[50, 50]]); + + I.say("Check if masks were created with different sizes"); + const result = await LabelStudio.serialize(); + assert.strictEqual(result.length, 1); + assert.ok(result[0].value.imageDataURL); +}); + +Scenario("Bitmask hover and selection", async ({ I, LabelStudio, AtImageView, AtLabels, AtOutliner }) => { + const params = { + config, + data: { image: IMAGE }, + annotations: [{ id: 1, result: [] }], + }; + + I.amOnPage("/"); + LabelStudio.init(params); + LabelStudio.waitForObjectsReady(); + await AtImageView.lookForStage(); + + I.say("Create initial mask"); + I.pressKey("B"); + AtLabels.clickLabel("Test"); + AtImageView.drawThroughPoints([ + [20, 20], + [20, 40], + [40, 40], + [40, 20], + [20, 20], + ]); + + I.say("Verify selection behavior"); + AtOutliner.seeRegions(1); + + I.say("Click on the region"); + AtImageView.clickAt(30, 30); + AtOutliner.seeSelectedRegion(); +}); + +Scenario("Verify Bitmask drawing content", async ({ I, LabelStudio, AtImageView, AtLabels }) => { + const params = { + config, + data: { image: IMAGE }, + annotations: [{ id: 1, result: [] }], + }; + + I.amOnPage("/"); + LabelStudio.init(params); + LabelStudio.waitForObjectsReady(); + await AtImageView.lookForStage(); + + I.say("Select Bitmask tool"); + I.pressKey("B"); + AtLabels.clickLabel("Test"); + + I.say("Draw a rectangle mask"); + AtImageView.drawThroughPoints([ + [20, 20], + [20, 40], + [40, 40], + [40, 20], + [20, 20], + ]); + + I.say("Verify mask content"); + const result = await LabelStudio.serialize(); + assert.strictEqual(result.length, 1); + assert.ok(result[0].value.imageDataURL); + + // Verify that the imageDataURL contains actual pixel data + const imageData = result[0].value.imageDataURL; + assert.ok(imageData.startsWith("data:image/png;base64,")); + + // Decode base64 and verify it's not empty + const base64Data = imageData.replace("data:image/png;base64,", ""); + const decodedData = Buffer.from(base64Data, "base64"); + assert.ok(decodedData.length > 0, "Decoded image data should not be empty"); +}); + +Scenario("Verify Bitmask pixel content", async ({ I, LabelStudio, AtImageView, AtLabels }) => { + const params = { + config, + data: { image: IMAGE }, + annotations: [{ id: 1, result: [] }], + }; + + I.amOnPage("/"); + LabelStudio.init(params); + LabelStudio.waitForObjectsReady(); + await AtImageView.lookForStage(); + + I.say("Select Bitmask tool"); + I.pressKey("B"); + AtLabels.clickLabel("Test"); + + I.say("Draw a rectangle mask"); + AtImageView.drawThroughPoints([ + [20, 20], + [20, 40], + [40, 40], + [40, 20], + [20, 20], + ]); + + // Wait for the drawing to be complete + await I.wait(0.5); + + I.say("Verify mask content and dimensions"); + const result = await LabelStudio.serialize(); + assert.strictEqual(result.length, 1); + assert.ok(result[0].value.imageDataURL); + + // Wait for the region to be fully processed + await I.wait(0.5); + + // Get all data we need before making assertions + const bbox = await I.executeScript(() => { + const region = Htx.annotationStore.selected.regions[0]; + if (!region) throw new Error("Region not found"); + if (!region.bboxCoordsCanvas) throw new Error("Bbox coordinates not available"); + return region.bboxCoordsCanvas; + }); + + // Define thresholds for assertions + const THRESHOLD = 5; + const EXPECTED_SIZE = 40; + + // Verify that the bbox exists + assert.ok(bbox, "Bounding box should exist"); + + // Calculate actual dimensions + const width = bbox.right - bbox.left; + const height = bbox.bottom - bbox.top; + + // Verify that the bbox has the expected size + assert.ok( + Math.abs(width - EXPECTED_SIZE) <= THRESHOLD, + `Width should be close to ${EXPECTED_SIZE} pixels (got ${width})`, + ); + + assert.ok( + Math.abs(height - EXPECTED_SIZE) <= THRESHOLD, + `Height should be close to ${EXPECTED_SIZE} pixels (got ${height})`, + ); + + // Verify that the bbox is roughly square + assert.ok( + Math.abs(width - height) <= THRESHOLD, + `Width and height should be similar (got width=${width}, height=${height})`, + ); +}); diff --git a/web/package.json b/web/package.json index 971b57088195..3f32fadf9437 100644 --- a/web/package.json +++ b/web/package.json @@ -107,6 +107,7 @@ "react-window-infinite-loader": "^1.0.5", "sanitize-html": "^2.14.0", "shadcn": "^2.1.8", + "simplify-js": "^1.2.4", "storybook": "^8.5.0", "strman": "^2.0.1", "tailwind-merge": "^2.6.0", diff --git a/web/yarn.lock b/web/yarn.lock index d06e86a716f8..bc3ca90c52ac 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -18810,6 +18810,11 @@ simple-xpath-position@^2.0.0: dependencies: get-document "^1.0.0" +simplify-js@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/simplify-js/-/simplify-js-1.2.4.tgz#7aab22d6df547ffd40ef0761ccd82b75287d45c7" + integrity sha512-vITfSlwt7h/oyrU42R83mtzFpwYk3+mkH9bOHqq/Qw6n8rtR7aE3NZQ5fbcyCUVVmuMJR6ynsAhOfK2qoah8Jg== + sinon@^17.0.1: version "17.0.1" resolved "https://registry.yarnpkg.com/sinon/-/sinon-17.0.1.tgz#26b8ef719261bf8df43f925924cccc96748e407a" @@ -19203,16 +19208,7 @@ string-length@^5.0.1: char-regex "^2.0.0" strip-ansi "^7.0.1" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -19253,7 +19249,7 @@ stringify-object@^5.0.0: is-obj "^3.0.0" is-regexp "^3.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -19267,13 +19263,6 @@ strip-ansi@^3.0.0: dependencies: ansi-regex "^2.0.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -20925,7 +20914,7 @@ workerpool@6.2.0: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.0.tgz#827d93c9ba23ee2019c3ffaff5c27fccea289e8b" integrity sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -20943,15 +20932,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"