Keypoint Annotation

[ErikDerGute]

Hello Guys,

for my thesis I need to train a neuronal network with a bunch of data (pictures). So I would like to use blenderproc for generating and annotating this data. The pictures contain some objects and for each object there several "keypoints". I mark my "keypoints" using a mesh object (circle) and set it as a child to my object. For annotating the objects I use the build in function "write_coco_annotations" and it works very well. Now my question is: is there a smart way to annotate also the keypoints via the build in function, or do I have to write my own "annotator"? I got the 2D pixel koordinates of my keypoints but I dont know how to to merge them with my other annotation information.

Here is my Code:

import blenderproc as bp
import numpy as np
import bpy
import os
import json
#import debugpy

#debugpy.listen(5678)
#debugpy.wait_for_client()

bp.init()

def load_blender_files_from_path(data_path):
    instruments = []
    file_name = []

    for name in os.listdir(data_path):

        if name.endswith(".blend"):
            file_name.append(os.path.join(data_path, name)) 
    
    for i in range(len(file_name)):
        objs = file_name[i]
        tmp = bp.loader.load_blend(objs, obj_types=['MESH'])

        checker = False
        for parts in tmp:
            if 'part' in parts.get_name():
                checker = True 
        print(checker)
        if checker is True:
            for counter, parts in enumerate(tmp):
                if 'part' in parts.get_name():
                    tmp_move_part = counter
                elif 'part' and 'kp' not in parts.get_name():
                    tmp_rigid_part = counter
                if 'nadel_halter' in parts.get_name():
                    max_rotation = 15
                else:
                    max_rotation = 55
            rotation = np.random.uniform(0, max_rotation, 1)

            tmp[tmp_move_part].set_rotation_euler([0, 0, np.deg2rad(rotation)])
            # get all parent objects
            key_points = tmp[tmp_move_part].get_children()
            for key_point in key_points:
                # set new parent to rigid part
                key_point.set_parent(tmp[tmp_rigid_part])
            tmp[tmp_rigid_part].join_with_other_objects([tmp[tmp_move_part]])
            instruments.extend([tmp[tmp_rigid_part]])
        else:
            for counter, parts in enumerate(tmp):
                if 'kp' not in parts.get_name():
                    adult_id = counter
            key_points = tmp[adult_id].get_children()
            for key_point in key_points:
                key_point.set_parent(tmp[adult_id])
            instruments.extend([tmp[adult_id]])



    return instruments

def generate_euler_rot_for_frame(frames, start_rot_alph, end_rot_alph, start_rot_bet, end_rot_bet, start_rot_gam, end_rot_gam):
    alph = end_rot_alph - start_rot_alph
    bet = end_rot_bet - start_rot_bet
    gam = end_rot_gam - start_rot_gam

    alph_frame = []
    bet_frame = [] 
    gam_frame = []   

    alph_disc = alph/(frames - 1)
    bet_disc = bet/(frames - 1)
    gam_disc = gam/(frames - 1)

    for i in range(frames):
        temp_alph = alph_disc * i
        temp_bet = bet_disc * i
        temp_gam = gam_disc * i

        alph_frame.append(temp_alph)
        bet_frame.append(temp_bet)
        gam_frame.append(temp_gam)

    return alph_frame, bet_frame, gam_frame 

def classify_objects(instrument_model):
    for j, obj in enumerate(instrument_model):  #select objects (parents)
        obj.set_cp("category_id", j + 1)        #set "category_id" for parents 
        tmp = obj.get_children()                #Select children
        
        for i, child_obj in enumerate(tmp):
            child_obj.set_cp("category_id", i + 1) #set "category_id" for children

def get_world_koordinates_from_child_objects(instrument_model):
    all_points = []

    for j, obj in enumerate(instrument_model):
        tmp = obj.get_children()
        tmp_name = obj.get_name()
        world_coordinates_kp = np.empty([len(tmp), 3])

        for i, child_obj in enumerate(tmp):
            tmp_coordinates = child_obj.get_location()
            world_coordinates_kp[i, :] = tmp_coordinates

        all_points.append((tmp_name, world_coordinates_kp))
    
    return all_points

def project_child_objects_to_pixel(key_points_koordinates):
    all_points_in_pixel = []
    for i in range(len(key_points_koordinates)):
        tmp = key_points_koordinates[i]
        tmp_name = tmp[0] + "_pixel"
        tmp_koordinates_pixel = bp.camera.project_points(tmp[1])
        all_points_in_pixel.append((tmp_name, tmp_koordinates_pixel))

    return all_points_in_pixel

data_path = "/Users/eriktzschoppe/Documents/Masterarbeit/3d_instruments_8_cls_keypoint"

#Load objects 
instrument_model = load_blender_files_from_path(data_path)
print(instrument_model)

#Set light
light = bp.types.Light(light_type="AREA")
light.set_location([0, 0, -0.25])
light.set_rotation_euler([np.pi, 0, 0])
light.set_energy(25)

#Generete object poses 
frames = 70

alph_frame, bet_frame, gam_frame = generate_euler_rot_for_frame(frames, 0, 2*np.pi, 0, 2*np.pi, 0, 2*np.pi)
#print(alph_frame)
#print(bet_frame)
#print(gam_frame)

#Set camera
frame_counter = 0
cam_pose = bp.math.build_transformation_mat([0, 0, -0.25], [np.pi, 0, 0])
bp.camera.add_camera_pose(cam_pose)

world_koordinates = get_world_koordinates_from_child_objects(instrument_model)

pixel_koordinates = project_child_objects_to_pixel(world_koordinates)

classify_objects(instrument_model)

##Generate frames of rotating object 

#for i in range (frames):
    #objs[0].set_rotation_euler([alph_frame[i], bet_frame[i], -1*gam_frame[i]], frame = frame_counter)
    #bp.camera.add_camera_pose(cam_pose, frame = frame_counter)
    #frame_counter = frame_counter + 1


bp.renderer.enable_normals_output()
bp.renderer.enable_segmentation_output(map_by=["category_id", "instance", "name"])

#Rendering
data_render = bp.renderer.render(output_key= 'colors')

#Writing Data
bp.writer.write_gif_animation("/Users/eriktzschoppe/Documents/Masterarbeit/output/", data_render)

bp.writer.write_hdf5("/Users/eriktzschoppe/Documents/Masterarbeit/output/", data_render)

bp.write_coco_annotations(
    output_dir="/Users/eriktzschoppe/Documents/Masterarbeit/output/",
    instance_segmaps=data_render["instance_segmaps"],
    instance_attribute_maps=data_render["instance_attribute_maps"],
    colors=data_render["colors"],
    color_file_format="JPEG")

Thank you you for helping me out

Best Erik