How to correctly apply annotation on the data augmentation image?

[cris-m]

I was working a data augmentation. Before starting the training, I wanted to visualize the augmented data. The problem I am facing is the bbox doesnot appear and some image the segmentation are well apply event after some transformation but other image the segmentation is not well apply. From the result I got it seems RandomBrightness , RandomContrast , RandomSaturation and RandomLighting are not apply. Here is what I did:

1. Customer Mapper

def custom_mapper(dataset_dict):
    # Deep copy dataset dictionary to avoid modifying the original
    dataset_dict = dataset_dict.copy()

    # Read image
    image = utils.read_image(dataset_dict["file_name"], format="BGR")

    # Define image transforms
    transform_list = [
        T.Resize((1000, 1500)),
        T.RandomFlip(prob=0.5),
        T.RandomRotation(angle=[-10, 10], sample_style="range"),
        T.RandomCrop("absolute", (1000, 1500)),
        T.RandomBrightness(0.9, 1.1),
        T.RandomContrast(0.9, 1.1),
        T.RandomSaturation(0.9, 1.1),
        T.RandomLighting(0.7)
    ]

    image, transforms = T.apply_transform_gens(transform_list, image)

    # Convert image to tensor and normalize
    image = torch.as_tensor(image.transpose(2, 0, 1).astype("float32")) / 255.0

    # Transform instance annotations
    annos = [utils.transform_instance_annotations(obj, transforms, image.shape[:2]) for obj in dataset_dict.pop("annotations") if not obj.get("iscrowd", 0)]
    instances = utils.annotations_to_instances(annos, image.shape[:2])
    dataset_dict["instances"] = utils.filter_empty_instances(instances)

    # Assign processed image to dataset dictionary
    dataset_dict["image"] = image
    dataset_dict["annotations"] = annos

    return dataset_dict

2. Visualize dataset sample

for sample in random.sample(train_dataset_dicts, 20):
    sample_processed = custom_mapper(sample)
    image = sample_processed["image"].permute(1, 2, 0).numpy()
    image = (image * 255).astype(int)

    # Create a visualizer instance and draw instances on the image
    visualizer = Visualizer(image[:, :, ::-1], metadata=train_metadata_dicts, scale=1)
    vis = visualizer.draw_dataset_dict(sample_processed)
    cv2_imshow(vis.get_image()[:, :, ::-1])

Here is some result:

When I apply the following transformation, I get some warning:

transform_list = [
    T.ResizeShortestEdge(
     [640, 672, 704, 736, 768, 800],
     max_size=1333,
     sample_style='choice'
    ),
    T.RandomBrightness(0.9, 1.1),
    T.RandomContrast(0.9, 1.1),
    T.RandomSaturation(0.9, 1.1),
    T.RandomRotation(angle=[-10, 10], sample_style="range"),
    T.RandomLighting(0.7),
    T.RandomFlip(prob=0.5)

Here is the error:

/usr/local/lib/python3.10/dist-packages/shapely/set_operations.py:133: RuntimeWarning: invalid value encountered in intersection
  return lib.intersection(a, b, **kwargs)

Any help is welcome !!

[cris-m]

I fixed the problem of bbox now I can apply the transformation on annotations but I still have the problem of with the segment which is not well aligned for some picture

def custom_mapper(dataset_dict):
    # Deep copy dataset dictionary to avoid modifying the original
    dataset_dict = dataset_dict.copy()

    # Read image
    image = utils.read_image(dataset_dict["file_name"], format="BGR")

    transform_list = [
        T.ResizeShortestEdge(
            [640, 672, 704, 736, 768, 800],
            max_size=1333,
            sample_style='choice'
        ),
        T.RandomBrightness(0.9, 1.1),
        T.RandomContrast(0.9, 1.1),
        T.RandomSaturation(0.9, 1.1),
        T.RandomRotation(angle=[-10, 10], sample_style="range"),
        T.RandomLighting(0.7),
        T.RandomFlip(prob=0.5),
    ]

    image, transforms = T.apply_transform_gens(transform_list, image)

    # Convert image to tensor and normalize
    image = torch.as_tensor(image.transpose(2, 0, 1).astype("float32")) / 255.0

    # Transform instance annotations
    annos = []
    for obj in dataset_dict.pop("annotations"):
        if not obj.get("iscrowd", 0):
            transformed_obj = utils.transform_instance_annotations(obj, transforms, image.shape[1:])
            annos.append(transformed_obj)

    instances = utils.annotations_to_instances(annos, image.shape[1:])
    dataset_dict["instances"] = utils.filter_empty_instances(instances)

    # Assign processed image to dataset dictionary
    dataset_dict["image"] = image
    dataset_dict["annotations"] = annos

    return dataset_dict