How do I use SceneWidget to pick points and get them?

[Nightinghost]

I'm trying to write an app for experiment that can pick points using Open3D's GUI module. I already load an OBJ file using gui.SCeneWidget, and set the mouse interaction mode to Controls.PICK_POINTS, but it dosen’t work.

 self._scene = gui.SceneWidget()
 self._scene.scene = rendering.Open3DScene(w.renderer)
 self._scene.set_view_controls(gui.SceneWidget.Controls.PICK_POINTS)

Also，SceneWidget does not provide the method for Getting Picked Points.
I would appreciate it if someone could tell me how to pick points in SceneWidget and get them.

[hduonggithub]

I have the same question too. I found this example (https://github.com/intel-isl/Open3D/blob/master/examples/python/gui/mouse-and-point-coord.py) but it does not give me a correct point. I picked the corner (green circle), I received the coordinate and added as the red sphere with 3d_label. The green point and the red sphere did not match.

Any idea? Thanks

[hduonggithub]

Thanks @Nightinghost !

It is pity that we still cannot have a good solution from open3d guys. Anyone help, please!

[hduonggithub]

@Nightinghost Good news !

I finally figure out the bug (possibly).

Option 1: Use open3d method. Just replace the below line will fix the bug
world = self._scene.scene.camera.unproject(x, self._scene.frame.height-y, depth, self._scene.frame.width, self._scene.frame.height)

Option 2: use your matrix way which is the same from opend3d c++ code

• Convert your world (world = [x, y, depth]) to ndc unit
• Remove model from MVP multiplication: MVP = projection * view.
• Divide the final world coordinate to world[3]: world = world[:-1]/world[3]

[Nightinghost]

It worked (Option 1) ! I finally got the right points,。
Really thinks !

[glintec-education]

Can you share the snipped code please?

[Nightinghost]

Can you share the snipped code please?

Hope it will help you @glintec-education.

import open3d as o3d
import open3d.visualization.gui as gui
import open3d.visualization.rendering as rendering
import numpy as np


class App:

    _picked_indicates = []
    _picked_positions = []
    _picked_num = 0
    _picked_label3d = []

    mesh = None
    mesh_wireframe = None
    mesh_kdtree = None

    def __init__(self) -> None:
        gui.Application.instance.initialize()
        self.window = gui.Application.instance.create_window(
            "Pick Points", 800, 600)

        self._scene3d = gui.SceneWidget()
        self._scene3d.scene = rendering.Open3DScene(self.window.renderer)
        self._scene3d.set_on_mouse(self._mouse_event)

        self.window.add_child(self._scene3d)

        self.mesh = o3d.geometry.TriangleMesh.create_sphere()
        self.mesh_kdtree = o3d.geometry.KDTreeFlann(self.mesh)
        self.mesh_wireframe = o3d.geometry.LineSet.create_from_triangle_mesh(
            self.mesh)
        self.mesh_wireframe.paint_uniform_color([.7, .7, .7])
        self._scene3d.scene.add_geometry(
            "geometry", self.mesh, rendering.MaterialRecord())
        wireframe_material = rendering.MaterialRecord()
        wireframe_material.shader = 'unlitLine'
        self._scene3d.scene.add_geometry(
            "wireframe", self.mesh_wireframe, wireframe_material)
        bounds = self.mesh.get_axis_aligned_bounding_box()
        self._scene3d.setup_camera(60, bounds, bounds.get_center())

    def _mouse_event(self, event):
        if event.type == gui.MouseEvent.Type.BUTTON_DOWN and event.is_button_down(gui.MouseButton.LEFT) and event.is_modifier_down(gui.KeyModifier.CTRL):

            def depth_callback(depth_image):
                x = event.x - self._scene3d.frame.x
                y = event.y - self._scene3d.frame.y

                depth = np.asarray(depth_image)[y, x]

                if depth == 1.0:
                    return
                else:
                    world_coord = self._scene3d.scene.camera.unproject(
                        x, y, depth, self._scene3d.frame.width, self._scene3d.frame.height)
                    idx = self._calc_prefer_indicate(world_coord)
                    picked_point = np.asarray(self.mesh.vertices)[idx]

                    self._picked_num += 1
                    self._picked_indicates.append(idx)
                    self._picked_positions.append(picked_point)

                    print(
                        f"Picked point #{idx} at ({picked_point[0]}, {picked_point[1]}, {picked_point[2]})")

                    def draw_point():
                        label3d = self._scene3d.add_3d_label(
                            picked_point, "#"+str(self._picked_num))
                        self._picked_label3d.append(label3d)

                        sphere = o3d.geometry.TriangleMesh.create_sphere(
                            0.025)
                        sphere.paint_uniform_color([1, 0, 0])
                        sphere.translate(picked_point)
                        material = rendering.MaterialRecord()
                        material.shader = 'defaultUnlit'

                        self._scene3d.scene.add_geometry(
                            "sphere"+str(self._picked_num), sphere, material)
                        self._scene3d.force_redraw()

                    gui.Application.instance.post_to_main_thread(
                        self.window, draw_point)

            self._scene3d.scene.scene.render_to_depth_image(depth_callback)
            return gui.Widget.EventCallbackResult.HANDLED
        elif event.type == gui.MouseEvent.Type.BUTTON_DOWN and event.is_button_down(gui.MouseButton.RIGHT) and event.is_modifier_down(gui.KeyModifier.CTRL):
            if self._picked_num > 0:
                idx = self._picked_indicates.pop()
                point = self._picked_positions.pop()

                print(
                    f"Undo pick: #{idx} at ({point[0]}, {point[1]}, {point[2]})")

                self._scene3d.scene.remove_geometry(
                    'sphere'+str(self._picked_num))
                self._picked_num -= 1
                self._scene3d.remove_3d_label(self._picked_label3d.pop())
                self._scene3d.force_redraw()
            else:
                print('Undo nothing!')
            return gui.Widget.EventCallbackResult.HANDLED
        return gui.Widget.EventCallbackResult.IGNORED

    def _calc_prefer_indicate(self, point):
        [k, idx, _] = self.mesh_kdtree.search_knn_vector_3d(point, 1)
        return idx[0]

    def run(self):
        gui.Application.instance.run()


if __name__ == '__main__':
    app = App()
    app.run()