Regarding how to run two Linux VMs on QEMU RISC-V #217

Dz0526 · 2025-05-23T15:38:29Z

Dz0526
May 23, 2025

Hello. Thank you for the wonderful Bao project.

I am trying to run two Linux VMs on the Bao Hypervisor, but it seems the CPU allocation for one of the VMs is not working correctly, and I would appreciate some advice.

I am allocating two cores to each VM.
One Linux VM outputs to UART, and the other Linux VM has a virtconsole allocated via MMIO.
I can confirm that Linux is booting on both VMs, but when I actually check the one outputting to UART, only one CPU is allocated.
I can see an error message when the Bao Hypervisor starts, but I couldn't find the same error string in the source code, so I'm not sure what the issue is.

Below, I will show my environment and config, logs.

qemu-system-riscv64 version: 8.2.2
bao-hypervisor version: main branch
Linux Image: I use bao-demos's buildroot

config.c

#include <config.h>

VM_IMAGE(linux_image, XSTR(BAO_DEMOS_WRKDIR_IMGS/linux.bin))
VM_IMAGE(linux_image2, XSTR(BAO_DEMOS_WRKDIR_IMGS/linux2.bin))

struct config config = {

  CONFIG_HEADER

  .vmlist_size = 2,
  .vmlist = (struct vm_config[]) {
    {
      .image = {
          .base_addr = 0x90200000,
          .load_addr = VM_IMAGE_OFFSET(linux_image),
          .size = VM_IMAGE_SIZE(linux_image)
        },

        .entry = 0x90200000,
        .cpu_affinity = 0x3,
        .platform = {
          .cpu_num = 2,

          .region_num = 1,
          .regions = (struct vm_mem_region[]) {
            {
              .base = 0x90000000, .size = 0x40000000, .place_phys = true, .phys = 0x90000000
            }
          },

          .dev_num = 1,
          .devs = (struct vm_dev_region[]) {
            {
              /* virtio devices */
              .pa = 0x10007000,
                .va = 0x10007000,
                .size = 0x2000,
                .interrupt_num = 2,
                .interrupts = (irqid_t[]) {
                  7,
                  8
                }
            },
          },

          .arch = {
            .irqc = {
              .plic = {
                .base = 0xc000000,
              }
            }
          }
        },
    }, {
      .image = {
        .base_addr = 0xE0200000,
        .load_addr = VM_IMAGE_OFFSET(linux_image2),
        .size = VM_IMAGE_SIZE(linux_image2)
      },

      .entry = 0xE0200000,
      .cpu_affinity = 0xC,

      .platform = {
        .cpu_num = 2,

        .region_num = 1,
        .regions = (struct vm_mem_region[]) {
          {
            .base = 0xE0000000, .size = 0x40000000, .place_phys = true, .phys = 0xE0000000
          }
        },

        .dev_num = 2,
        .devs = (struct vm_dev_region[]) {
          {
            /* virtio devices */
            .pa = 0x10000000,
              .va = 0x10000000,
              .size = 0x100,
              .interrupt_num = 1,
              .interrupts = (irqid_t[]) {
                10
              }
          }, {
            .pa = 0x10006000,
            .va = 0x10006000,
            .size = 0x1000,
            .interrupt_num = 1,
            .interrupts = (irqid_t[]) {
              6
            }
          },
        },

        .arch = {
          .irqc = {
            .plic = {
              .base = 0xc000000,
            }
          }
        }
      }
    }
  }
};

linux.dts

/dts-v1/;

/ {
	#address-cells = <0x2>;
	#size-cells = <0x2>;

	cpus {
		#address-cells = <0x1>;
		#size-cells = <0x0>;
		timebase-frequency = <10000000>;

		cpu@0 {
			device_type = "cpu";
			reg = <0x0>;
			status = "okay";
			compatible = "riscv";
			riscv,isa = "rv64imafdcsu";
			mmu-type = "riscv,sv48";

			cpu0_intc: interrupt-controller {
				#interrupt-cells = <0x1>;
				interrupt-controller;
				compatible = "riscv,cpu-intc";
			};
		};

		cpu@1 {
			device_type = "cpu";
			reg = <0x1>;
			status = "okay";
			compatible = "riscv";
			riscv,isa = "rv64imafdcsu";
			mmu-type = "riscv,sv48";

			cpu1_intc: interrupt-controller {
				#interrupt-cells = <0x1>;
				interrupt-controller;
				compatible = "riscv,cpu-intc";
			};
		};
	};

	memory@90000000 {
		device_type = "memory";
		reg = <0x0 0x90000000 0x0 0x40000000>;
	};

	plic: interrupt-controller@c000000 {
		riscv,ndev = <60>;
		reg = <0x0 0xc000000 0x0 0x4000000>;
		interrupts-extended = <
			&cpu0_intc 11 &cpu0_intc 9 
			&cpu1_intc 11 &cpu1_intc 9 
		>;
		interrupt-controller;
		compatible = "riscv,plic0";
		#interrupt-cells = <0x1>;
	};

	virtio_mmio@10008000 {
		interrupts = <0x8>;
		interrupt-parent = <&plic>;
		reg = <0x0 0x10008000 0x0 0x1000>;
		compatible = "virtio,mmio";
	};

	virtio_mmio@10007000 {
		interrupts = <0x7>;
		interrupt-parent = <&plic>;
		reg = <0x0 0x10007000 0x0 0x1000>;
		compatible = "virtio,mmio";
	};

	chosen {
		bootargs = "earlycon console=hvc0 ip=192.168.42.15";
	};

};

linux2.dts

/dts-v1/;

/ {
	#address-cells = <0x2>;
	#size-cells = <0x2>;

	cpus {
		#address-cells = <0x1>;
		#size-cells = <0x0>;
		timebase-frequency = <10000000>;

		cpu@0 {
			device_type = "cpu";
			reg = <0x0>;
			status = "okay";
			compatible = "riscv";
			riscv,isa = "rv64imafdcsu";
			mmu-type = "riscv,sv48";

			cpu0_intc: interrupt-controller {
				#interrupt-cells = <0x1>;
				interrupt-controller;
				compatible = "riscv,cpu-intc";
			};
		};

		cpu@1 {
			device_type = "cpu";
			reg = <0x1>;
			status = "okay";
			compatible = "riscv";
			riscv,isa = "rv64imafdcsu";
			mmu-type = "riscv,sv48";

			cpu1_intc: interrupt-controller {
				#interrupt-cells = <0x1>;
				interrupt-controller;
				compatible = "riscv,cpu-intc";
			};
		};
	};

	memory@e0000000 {
		device_type = "memory";
		reg = <0x0 0xe0000000 0x0 0x40000000>;
	};

	plic: interrupt-controller@c000000 {
		riscv,ndev = <60>;
		reg = <0x0 0xc000000 0x0 0x4000000>;
		interrupts-extended = <
			&cpu0_intc 11 &cpu0_intc 9 
			&cpu1_intc 11 &cpu1_intc 9 
		>;
		interrupt-controller;
		compatible = "riscv,plic0";
		#interrupt-cells = <0x1>;
	};

	serial@10000000 {
		interrupts = <0x0a>;
		interrupt-parent = <&plic>;
		clock-frequency = "\08@";
		reg = <0x00 0x10000000 0x00 0x100>;
		compatible = "ns16550a";
	};

	virtio_mmio@10006000 {
		interrupts = <0x06>;
		interrupt-parent = <&plic>;
		reg = <0x0 0x10006000 0x0 0x1000>;
		compatible = "virtio,mmio";
	};

	chosen {
		bootargs = "earlycon console=ttyS0 ip=192.168.42.14";
	};

};

QEMU command

	qemu-system-riscv64 -nographic\
		-M virt -cpu rv64 -m 10G -smp 4\
		-bios $(opensbi_image)\
		-device virtio-serial-device\
		-chardev pty,id=serial3\
		-device virtconsole,chardev=serial3\
		-device virtio-net-device,netdev=net0\
		-netdev user,id=net0,net=192.168.42.0/24,hostfwd=tcp:127.0.0.1:6666-:22\
		-device virtio-net-device,netdev=net1\
		-netdev user,id=net1,net=192.168.42.0/24,hostfwd=tcp:127.0.0.1:6667-:22\
		-S

Bao Hypervisor Boot log

Domain0 Boot HART           : 0
Domain0 HARTs               : 0*,1*,2*,3*
Domain0 Region00            : 0x0000000000100000-0x0000000000100fff M: (I,R,W) S/U: (R,W)
Domain0 Region01            : 0x0000000010000000-0x0000000010000fff M: (I,R,W) S/U: (R,W)
Domain0 Region02            : 0x0000000002000000-0x000000000200ffff M: (I,R,W) S/U: ()
Domain0 Region03            : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: ()
Domain0 Region04            : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: ()
Domain0 Region05            : 0x000000000c400000-0x000000000c5fffff M: (I,R,W) S/U: (R,W)
Domain0 Region06            : 0x000000000c000000-0x000000000c3fffff M: (I,R,W) S/U: (R,W)
Domain0 Region07            : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X)
Domain0 Next Address        : 0x0000000080200000
Domain0 Next Arg1           : 0x0000000080100000
Domain0 Next Mode           : S-mode
Domain0 SysReset            : yes
Domain0 SysSuspend          : yes

Boot HART ID                : 0
Boot HART Domain            : root
Boot HART Priv Version      : v1.12
Boot HART Base ISA          : rv64imafdch
Boot HART ISA Extensions    : sstc,zicntr,zihpm,zicboz,zicbom,sdtrig,svadu
Boot HART PMP Count         : 16
Boot HART PMP Granularity   : 2 bits
Boot HART PMP Address Bits  : 54
Boot HART MHPM Info         : 16 (0x0007fff8)
Boot HART Debug Triggers    : 2 triggers
Boot HART MIDELEG           : 0x0000000000001666
Boot HART MEDELEG           : 0x0000000000f0b509
Bao Hypervisor
BAO WARNING: trying to flush caches but the operation is not defined for this platform
BAO WARNING: trying to flush caches but the operation is not defined for this platform
BAO ERROR: no emulation handler for abort(0x1eff4370 at 0xe040108c)

.... Continue Linux Boot Log

Linux VM2 log

# dmesg | grep CPU
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=2, Nodes=1
[    0.000000] rcu: 	RCU restricting CPUs from NR_CPUS=64 to nr_cpu_ids=2.
[    0.046396] smp: Bringing up secondary CPUs ...
[    1.112645] CPU1: failed to come online
[    1.114996] smp: Brought up 1 node, 1 CPU

Initially, I thought the CPU reg values assigned in the Linux DeviceTree should be unique for each, so I had assigned 0x02 and 0x03 to Linux VM2. However, this caused an error like the one shown below, and the Linux VM wouldn't boot. That's why I changed to the current DeviceTree configuration.

Domain0 HARTs               : 0*,1*,2*,3*
Domain0 Region00            : 0x0000000000100000-0x0000000000100fff M: (I,R,W) S/U: (R,W)
Domain0 Region01            : 0x0000000010000000-0x0000000010000fff M: (I,R,W) S/U: (R,W)
Domain0 Region02            : 0x0000000002000000-0x000000000200ffff M: (I,R,W) S/U: ()
Domain0 Region03            : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: ()
Domain0 Region04            : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: ()
Domain0 Region05            : 0x000000000c400000-0x000000000c5fffff M: (I,R,W) S/U: (R,W)
Domain0 Region06            : 0x000000000c000000-0x000000000c3fffff M: (I,R,W) S/U: (R,W)
Domain0 Region07            : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X)
Domain0 Next Address        : 0x0000000080200000
Domain0 Next Arg1           : 0x0000000080100000
Domain0 Next Mode           : S-mode
Domain0 SysReset            : yes
Domain0 SysSuspend          : yes

Boot HART ID                : 0
Boot HART Domain            : root
Boot HART Priv Version      : v1.12
Boot HART Base ISA          : rv64imafdch
Boot HART ISA Extensions    : sstc,zicntr,zihpm,zicboz,zicbom,sdtrig,svadu
Boot HART PMP Count         : 16
Boot HART PMP Granularity   : 2 bits
Boot HART PMP Address Bits  : 54
Boot HART MHPM Info         : 16 (0x0007fff8)
Boot HART Debug Triggers    : 2 triggers
Boot HART MIDELEG           : 0x0000000000001666
Boot HART MEDELEG           : 0x0000000000f0b509
Bao Hypervisor
BAO WARNING: trying to flush caches but the operation is not defined for this platform
BAO WARNING: trying to flush caches but the operation is not defined for this platform
BAO ERROR: unkown synchronous exception (3)

My apologies for my broken English, but I would appreciate it if you could check it.

Dz0526 · 2025-05-29T17:38:51Z

Dz0526
May 29, 2025
Author

Hello,

Regarding this issue, it appears there was a problem with memory allocation. It now works correctly after reducing the memory size allocated to each Linux VM by half.

I would like to increase the total memory capacity for Bao itself, but I will investigate that separately at a later time. For now, since this particular issue has been resolved, I will close it.

Thank you again for the Bao project.

0 replies

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Regarding how to run two Linux VMs on QEMU RISC-V #217

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{editor}}'s edit

{{editor}}'s edit

Uh oh!

Replies: 1 comment

Uh oh!

{{title}}

Uh oh!

Select a reply

Uh oh!

Regarding how to run two Linux VMs on QEMU RISC-V #217

Uh oh!

Uh oh!

Dz0526 May 23, 2025

Replies: 1 comment

Uh oh!

Dz0526 May 29, 2025 Author

Dz0526
May 23, 2025

Dz0526
May 29, 2025
Author