Why is there an increased memory usage when using o3d.geometry.Image(self.frame)?

[mbenmahdjoub]

Checklist

• I have searched for similar issues.
• For Python issues, I have tested with the latest development wheel.
• I have checked the release documentation and the latest documentation (for main branch).

My Question

I am trying to visualize a webcam feed on a plane in Open3D (testable Code below).
After I run the script, the memory usage starts increasing after 3-5minutes non stop.
When I comment the line modify_geometry_material, it works fine (no memory increase is noticed), although the texture is not updated until I move the camera.
I tried self.app.post_to_main_thread(self.vis, lambda: self.vis.modify_geometry_material(self.geometries[0]['name'], self.geometries[0]['material'])), the texture is update accordingly, but there is a an increase of memory usage after a few minutes.
Any ideas to achieve this real-time VideoCapture texture on a 3D plane?

import open3d as o3d
import numpy as np
import cv2
import open3d.visualization.rendering as rendering
import open3d.visualization.gui as gui
import time
import threading

def create_video_plane(width, height):
    # Vertices for a rectangle in the XY plane, centered at (0,0,0)
    vertices = np.array([
        [-width/2, -height/2, 0],
        [ width/2, -height/2, 0],
        [ width/2,  height/2, 0],
        [-width/2,  height/2, 0]
    ])
    triangles = np.array([
        [0, 1, 2],
        [2, 3, 0]
    ])
    # UV coordinates for each vertex (must match vertices order)
    uvs = np.array([
        [0, 0],  # vertex 0
        [1, 0],  # vertex 1
        [1, 1],  # vertex 2
        [0, 1],  # vertex 3
    ])
    mesh = o3d.geometry.TriangleMesh()
    mesh.vertices = o3d.utility.Vector3dVector(vertices)
    mesh.triangles = o3d.utility.Vector3iVector(triangles)
    mesh.triangle_uvs = o3d.utility.Vector2dVector([
        [0, 0], [1, 0], [1, 1],   # triangle 0
        [1, 1], [0, 1], [0, 0]    # triangle 1
    ])
    mesh.compute_vertex_normals()
    return mesh

class TestApp:
    def __init__(self):
        self.rgb_frame_width = 3840
        self.rgb_frame_height = 2160
        self.ratio = self.rgb_frame_width / self.rgb_frame_height
        self.window_width = 1280
        self.window_height = int(self.window_width / self.ratio)

        self.app = o3d.visualization.gui.Application.instance
        self.app.initialize()
        self.vis = o3d.visualization.O3DVisualizer("Test")
        self.app.add_window(self.vis)
        self.frame = None

        self.image_texture = o3d.geometry.Image(np.zeros((self.rgb_frame_height, self.rgb_frame_width, 3), dtype=np.uint8))
        self.plane = create_video_plane(self.rgb_frame_width, self.rgb_frame_height)
        self.plane.translate([0, 0, self.rgb_frame_width/100])
        self.image_added = False
        #VideoFrameSingleton().start(video_source=0, K=np.eye(3), dist_coeffs=np.zeros(5), fps=30, rgb_frame_width=self.rgb_frame_width, rgb_frame_height=self.rgb_frame_height)
        
        self.cap = cv2.VideoCapture(0)
        self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.rgb_frame_width)
        self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.rgb_frame_height)
        self.cap.set(cv2.CAP_PROP_FPS, 30)
        self.cap.set(cv2.CAP_PROP_AUTOFOCUS, 0)
        self.is_running = True
        self._thread = threading.Thread(target=self.video_loop, daemon=True)
        self._thread.start()

        video_material = rendering.MaterialRecord()
        video_material.shader = "defaultUnlit"
        video_material.base_color = [1.0, 1.0, 1.0, 1.0]
        video_material.albedo_img = self.image_texture
        self.geometries = []
        self.geometries.append({"name":"video_plane", "geometry":self.plane, "material":video_material})

        threading.Thread(target=self.update_thread, daemon=True).start()

        self.app.run()
    def video_loop(self):
        while self.is_running:
            ret, frame = self.cap.read()
            if ret:
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                frame = np.ascontiguousarray(frame)
                self.frame = frame  # Store only the raw frame
            time.sleep(1.0 / 30)
        

    def update_thread(self):
        while(self.frame is None):
            time.sleep(0.1)
        
        while(self.frame is not None):
            self.geometries[0]['material'].albedo_img = o3d.geometry.Image(self.frame)
            if not self.image_added:
                self.vis.add_geometry(self.geometries[0]['name'], self.geometries[0]['geometry'], self.geometries[0]['material'])
                self.image_added = True
            else:
                self.app.post_to_main_thread(self.vis, lambda: self.vis.modify_geometry_material(self.geometries[0]['name'], self.geometries[0]['material']))
                #self.vis.modify_geometry_material(self.geometries[0]['name'], self.geometries[0]['material'])
            time.sleep(1/20)
app = TestApp()

[Mike-YANG-11]

I have the same issue!