Black output when rendering top down image

[Kurokabe]

With kaolin==0.16.0, when I use easy_render with the camera above the center of my mesh, and the camera looking straight down, I get a black image.

from kaolin.render.easy_render import render_mesh, default_lighting
from kaolin.io.usd import import_mesh
from kaolin.render.camera import Camera
import torch
from torchvision.transforms import ToPILImage

mesh = import_mesh("my_mesh.usd", triangulate=True, with_materials=True).to("cuda")

mesh.vertices.min(axis=0), mesh.vertices.max(axis=0)
center = (mesh.vertices.min(axis=0)[0] + mesh.vertices.max(axis=0)[0]) / 2

offset = 0.0

camera_y_position = 3.0

eye = torch.tensor([center[0], camera_y_position , center[2]], dtype=torch.float32)
at = torch.tensor([center[0] + offset, 0, center[2]], dtype=torch.float32)

lighting = default_lighting().to("cuda")
camera = Camera.from_args(
                eye=eye,
                at=at,
                up=torch.tensor([0.0, 1.0, 0.0]),
                fov=45,  
                width=512, height=512,
                device='cuda'
            )

with torch.jit.optimized_execution(False):
    with torch.no_grad():
        render = render_mesh(
                camera,
                mesh,
                lighting,
            )
        

ToPILImage()(render["render"][0].transpose(0, 2))

But if I add a slight offset, e.g. offset = 0.01 to either the X or Z coordinate, then I get a proper rendering.

[Caenorst]

Hi @Kurokabe , Looking into it

EDIT: actually if you look at your code the up axis is aligned with forward, so the camera is basically undefined, if camera is looking perfectly down I suggest setting a different up axis.

[Kurokabe]

Thank you @Caenorst for your answer, I was able to fix it by setting a different up axis.

I have been experimenting with camera rotations with azimuth and elevation angles, but I am faced with weird results, if you can have a look:

Here is an example mesh:
chair.zip

And below the code to generate an animation:

from kaolin.render.easy_render import render_mesh, default_lighting
from kaolin.io.usd.mesh import import_mesh, SurfaceMesh
from kaolin.render.camera import Camera
import torch
from copy import deepcopy
from kaolin.ops.pointcloud import center_points
from torchvision.transforms import ToPILImage
from kaolin.ops.coords import spherical2cartesian
from kaolin.render.camera import blender_coords
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from IPython.display import HTML
import numpy as np

mesh = import_mesh("./chair.usd", triangulate=True, with_normals=True).to("cuda")

def center_mesh(mesh: SurfaceMesh, normalize: bool):
    mesh = deepcopy(mesh)
    mesh = mesh.to_batched()
    mesh.vertices = center_points(mesh.vertices, normalize=normalize)
    mesh = mesh.set_batching(SurfaceMesh.Batching.NONE)
    return mesh

def render(mesh, azimuth, elevation):
    mesh = center_mesh(mesh, normalize=True).to("cuda")

    # Convert azimuth and elevation to radians
    azimuth_tensor = torch.deg2rad(torch.tensor(azimuth, dtype=torch.float32))
    elevation_tensor = torch.deg2rad(torch.tensor(elevation, dtype=torch.float32))

    # Compute eye position
    x, y, z = spherical2cartesian(azimuth=azimuth_tensor, elevation=elevation_tensor, distance=2.0)
    eye = torch.tensor([x, y, z], dtype=torch.float32)

    # Camera target (always at origin)
    at = torch.tensor([0.0, 0.0, 0.0], dtype=torch.float32)

    # Forward vector (from eye to target)
    forward = -eye / torch.linalg.norm(eye)

    # Fixed world up vector
    world_up = torch.tensor([0.0, 0.0, 1.0], dtype=torch.float32)

    # Right and up vectors
    right = torch.cross(world_up, forward)
    right = right / torch.linalg.norm(right)  # Normalize right vector
    up = torch.cross(forward, right)  # Up vector from forward and right

    print(f"Eye: {eye}, Up: {up}")

    # Lighting and camera setup
    lighting = default_lighting().to("cuda")
    camera = Camera.from_args(eye=eye, at=at, up=up, fov=45, width=512, height=512, device="cuda")
    camera.extrinsics.change_coordinate_system(blender_coords())
    
    with torch.jit.optimized_execution(False):
        with torch.no_grad():
            render = render_mesh(camera, mesh, lighting)

    return ToPILImage()(render["render"][0].transpose(0, 2))

When I run the following to animate the azimuth between 0° and 360° :

frames = []
for angle in range(0, 360, 10): 
    frame = render(mesh, azimuth=angle, elevation=0)
    frames.append(frame)

fig, ax = plt.subplots()

def update(frame):
    ax.clear()
    ax.imshow(np.asarray(frames[frame]))
    ax.axis('off')  # Hide axes

ani = FuncAnimation(fig, update, frames=len(frames), interval=100)  # Interval in ms
HTML(ani.to_jshtml())

I get this result where as if the elevation was impacted:

And when I animate the elevation between 0° and 360° with the following:

frames = []
for angle in range(0, 360, 10): 
    frame = render(mesh, azimuth=0, elevation=angle)
    frames.append(frame)

fig, ax = plt.subplots()

def update(frame):
    ax.clear()
    ax.imshow(np.asarray(frames[frame]))
    ax.axis('off')  # Hide axes

ani = FuncAnimation(fig, update, frames=len(frames), interval=100)  # Interval in ms
HTML(ani.to_jshtml())

I get this result as if the azimuth is impacted

Furthermore, at 180°, there is a 'flip' that occurs as well.

The mesh is oriented with Z up and I've changed the camera coordinate system to also be with Z up, as in Blender

When looking at the eye and up vectors, they seem correct, but I don't understand the following (potential issue with the Camera?):

1. Why when moving the azimuth, the elevation is impacted and vice versa
2. Why does the flip occurs at 180°?

[github-actions]

Stale issue message

[github-actions]

Stale issue, please reopen if still relevant