Achieving deterministic runs

[sguysc]

I'd like to be able to run the exact same run, given I didn't change any parameters of the simulation. Before I paste my code, my logic is that I have to set a specific seed for any Random operation to be repeatable, set a specific seed for the traffic manager, and in general to work in synchronous_mode=True so the lag in my computer won't interrupt(?). As you'll see, I log the x,y,z location of the ego vehicle, and run the simulation twice. It is similar, but not the same. Even when I remove all other actors. Even when I run without rendering to the screen. What can I do to make it repeatable (not in recording mode, actual live runs)?

Additional info: Carla 0.9.14 on Ubuntu 20.04, Python 3.8.

import random 
import numpy as np
import sys
import os
try:
    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/carla')
except IndexError:
    pass

import carla 
from agents.navigation.behavior_agent import BehaviorAgent  # pylint: disable=import-error


seed = 123
N_vehicles = 50
camera = None
telemetry = []

random.seed(seed)

try:
    # Connect the client and set up bp library and spawn points
    client = carla.Client('localhost', 2000) 
    client.set_timeout(60.0)

    world = client.get_world()
    bp_lib = world.get_blueprint_library() 
    spawn_points = world.get_map().get_spawn_points() 

    settings = world.get_settings()
    settings.synchronous_mode = True
    settings.fixed_delta_seconds = 0.10 
    world.apply_settings(settings)

    traffic_manager = client.get_trafficmanager()
    traffic_manager.set_random_device_seed(seed) 
    traffic_manager.set_synchronous_mode(True)

    # Spawn ego vehicle
    vehicle_bp = bp_lib.find('vehicle.audi.a2') 
    # breakpoint()
    vehicle = world.try_spawn_actor(vehicle_bp, random.choice(spawn_points))

    # Move spectator behind vehicle to motion
    spectator = world.get_spectator() 
    transform = carla.Transform(vehicle.get_transform().transform(carla.Location(x=-6,z=2.5)),vehicle.get_transform().rotation) 
    spectator.set_transform(transform)
    world.tick()

    # set the car's controls
    agent = BehaviorAgent(vehicle, behavior="normal")
    destination = random.choice(spawn_points).location
    agent.set_destination(destination)
    print('destination:')
    print(destination)
    print('current location:')
    print(vehicle.get_location())

    #Iterate this cell to find desired camera location
    camera_bp = bp_lib.find('sensor.camera.rgb') 

    # Spawn camera
    camera_init_trans = carla.Transform(carla.Location(z=2))
    camera = world.spawn_actor(camera_bp, camera_init_trans, attach_to=vehicle)

    # Callback stores sensor data in a dictionary for use outside callback                         
    def camera_callback(image, data_dict):
        data_dict['image'] = np.reshape(np.copy(image.raw_data), (image.height, image.width, 4))

    # Get gamera dimensions and initialise dictionary                       
    image_w = camera_bp.get_attribute("image_size_x").as_int()
    image_h = camera_bp.get_attribute("image_size_y").as_int()
    camera_data = {'image': np.zeros((image_h, image_w, 4))}

    # Start camera recording
    camera.listen(lambda image: camera_callback(image, camera_data))

    # Add traffic to the simulation
    SpawnActor = carla.command.SpawnActor
    SetAutopilot = carla.command.SetAutopilot
    FutureActor = carla.command.FutureActor

    vehicles_list, batch = [], []
    for i in range(N_vehicles): 
        ovehicle_bp = random.choice(bp_lib.filter('vehicle')) 
        npc = world.try_spawn_actor(ovehicle_bp, random.choice(spawn_points)) 
        # add it if it was successful
        if(npc):
           vehicles_list.append(npc)
    print(f'only {len(vehicles_list)} cars were spawned')
    world.tick()
    # Set the all vehicles in motion using the Traffic Manager
    for idx, v in enumerate(vehicles_list): 
        try:
            v.set_autopilot(True) 
        except:
            pass

    # Game loop
    while True:
        world.tick()
        pose = vehicle.get_location()
        telemetry.append([pose.x, pose.y, pose.z])

        # keep following the car
        transform = carla.Transform(vehicle.get_transform().transform(carla.Location(x=-6,z=2.5)),vehicle.get_transform().rotation) 
        spectator.set_transform(transform)

        if agent.done():
            print("The target has been reached, stopping the simulation")
            break
        
        control = agent.run_step()
        control.manual_gear_shift = False
        vehicle.apply_control(control)

finally:
    # Stop the camera when we've recorded enough data
    if(camera):
        camera.stop()
        camera.destroy()
    settings = world.get_settings()
    settings.synchronous_mode = False
    settings.fixed_delta_seconds = None
    world.apply_settings(settings)
    traffic_manager.set_synchronous_mode(True)

    if(vehicles_list):
        client.apply_batch([carla.command.DestroyActor(v) for v in vehicles_list])
    vehicle.destroy()
    
    np.savetxt('telemetry.txt', np.array(telemetry), delimiter=',')

[sguysc]

I have more information:
I found this so I made a few changes:

1. added reload_world() after setting the synchronization settings.
2. force small physical substeps:

settings = world.get_settings()
settings.synchronous_mode = True
settings.fixed_delta_seconds = 0.05 
settings.substepping = True
settings.max_substep_delta_time = 0.003125 
settings.max_substeps = 16 
world.apply_settings(settings)

3. not spawning any extra vehicles (to not overcomplicate the simulation)
4. providing an open-loop deterministic controls in the main loop:

if(t<20):
    # idle in the beginning so the actor can "settle" from being dropped
    control.throttle = 0.0
    control.steer = 0.0
else:
    control.throttle = 0.5
    control.steer = 0.3 * np.sin(2.*np.pi*1.0*t/fs)

Even with these changes, and running for only 200 steps, I still get non-determinism in the x-y location of the car.

What might I be missing here?

[olek-osikowicz]

Have you looked at 'determinism test' in PythonAPI testing suite.