BHyVe code reading
@syuu1228

https://github.com/lattera/bhyve

BHyVe

BHyVe

FreeBSDLinux KVM

Intel VT

Web Site

http://www.bhyve.org/

Intel VT-x, EPT (= Nehalem)

BIOS (disk)

:PCIvirtio-net, virtio-blk

pci passthrough(VT-d)

pci UART

paravirtual console/debug port

OS: FreeBSD 8, 9, 10

MSI

Local APIC

Linux KVM

Intel VTVMM

VMCS

CPUVMCS

VMLAUNCH

trapVMExit

VMExit

3

QEMU

QEMUKVMLinux kernel

IOCTL

Guest kernelUser program

VMLAUNCH

VMExit

Linux KVM

Intel VTVMM

VMCS

CPUVMCS

VMLAUNCH

trapVMExit

VMExit

3

KVMQEMU

BHyVe

/usr/sbin/bhyvevmm.koBSD kernel

IOCTL(VM_RUN)

Guest kernelUser program

VMLAUNCH

VMExit

BHyVe

Intel VTVMM

VMCS

CPUVMCS

VMLAUNCH

trapVMExit

VMExit

3

vmm.ko/usr/sbin/bhyve

/usr/sbin/bhyve

src/usr.sbin/bhyve/fbsdrun.c:669
fbsdrun_addcpu()CPU0src/usr.sbin/bhyve/fbsdrun.c:209
pthread_create(fbsdrun_start_thread)src/usr.sbin/bhyve/fbsdrun.c:195
vm_loop()src/usr.sbin/bhyve/fbsdrun.c:476
while(1) {vm_run();}

/usr/sbin/bhyve

src/usr.sbin/bhyve/fbsdrun.c:476
while(1) {
vm_run();src/lib/libvmmapi/vmmapi.c:265
ioctl(VM_RUN) vmm.koVMX non root mode

src/usr.sbin/bhyve/fbsdrun.c:494
handler[exitcode]() EXIT_REASON

OS

OS

BHyVeBIOS

HDDBIOS/usr/sbin/bhyve

BIOSFreeBSD
XendomU

/usr/sbin/bhyveload

/usr/sbin/bhyve

bhyveload - vm_create

src/usr.sbin/bhyveload/bhyveload.c:557
vm_create(vmname)/dev/vmm/%sdevice filesrc/lib/libvmmapi/vmmapi.c:85
sysctldevice filevmm.ko

bhyveload - vm_setup_memory

src/usr.sbin/bhyveload/bhyveload.c:570
vm_setup_memory()membasemmapsrc/lib/libvmmapi/vmmapi.c:139
vmm.koioctl(VM_MAP_MEMORY)
vmm.kommapmembase

bhyveload - vm_open

usr.sbin/bhyveload/bhyveload.c:564vm_open(vmname)/dev/vmm/%sopen()src/lib/libvmmapi/vmmapi.c:92
vm_open()src/lib/libvmmapi/vmmapi.c:67vm_device_open()

bhyveload userboot.so

usr.sbin/bhyveload/bhyveload.c:589FreeBSD

wrap

mmapioctlVMM

kload

Linuxkexec

bhyveload userboot.so

usr.sbin/bhyveload/bhyveload.c:589
dlopenuserboot.so

usr.sbin/bhyveload/bhyveload.c:594dlsymloader_main

usr.sbin/bhyveload/bhyveload.c:603
loader_main

bhyveload userboot.so

loader_mainboot2 cb_putc, cb_getc, cb_poll

cb_open, cb_close, cb_isdir, cb_read, cb_readdir, cb_seek, cb_stat

cb_diskread

cb_copyin, cb_copyout, cb_getmem

cb_setreg, cb_setmsr, cb_setcr, cb_setgdt, cb_exec

bhyveload cb_copyin, cb_copyout

src/usr.sbin/bhyveload/bhyveload.c:297
membasememcpy

src/usr.sbin/bhyveload/bhyveload.c:313
membasememcpy

bhyveload cb_setreg, cb_exec

src/usr.sbin/bhyveload/bhyveload.c:327
vm_set_register

src/usr.sbin/bhyveload/bhyveload.c:434
vm_setup_freebsd_registerssrc/lib/libvmmapi/vmmapi_freebsd.c:63
vm_set_register, vm_set_desc

bhyveload vm_setup_freebsd_registers

src/lib/libvmmapi/vmmapi_freebsd.c:63CR0 = PE | PG | NE #

CR4 = PAE | VMXE # PAEVMX

EFER = LME | LMA # long mode

GDT

CR3

RSP

IO

devicepcidevicebvmconsoledevicebvmdebugdevicemptable
ACPI
virtio.ko, if_vtnet.ko, virtio_pci.ko, virtio_blk.ko
XendomU

/usr/sbin/bhyvevmm.koBSD kernel

IOCTL return

Guest kernel

VMExit

IO

IOconsolePCInetblk

io emulation

/usr/sbin/bhyve IO emulation

src/usr.sbin/bhyve/fbsdrun.c:494
handler[exitcode]() EXIT_REASONsrc/usr.sbin/bhyve/fbsdrun.c:465IOVM_EXITCODE_INOUTvmexit_inoutsrc/usr.sbin/bhyve/fbsdrun.c:281
EAXemulate_inout()

/usr/sbin/bhyve IO emulation

src/usr.sbin/bhyve/inout.c:72
inout_handers[port].handler(in, port, bytes, eax)
port = 0x220console
src/usr.sbin/bhyve/consport.c:127src/usr.sbin/bhyve/consport.c:101
in = 1eax
in = 0eax

vmm.ko

sysctl

src/sys/amd64/vmm/vmm_dev.c:387hw.vmm.create(name)
/dev/vmm/${name}VM

hw.vmm.destroy(name)
/dev/vmm/${name}VM

/dev/vmm/${name}API

read/writesrc/sys/amd64/vmm/vmm_dev.c:184
offset =

mmapsrc/sys/amd64/vmm/vmm_dev.c:347

/dev/vmm/${name}ioctl (1)

src/sys/amd64/vmm/vmm_dev.c:144

VM_RUN: VMLAUNCH

VM_SET_PINNING/VM_GET_PINNING: CPU

VM_MAP_MEMORY:

VM_GET_MEMORY_SEG:

VM_SET_REGISTER/VM_GET_REGISTER:

/dev/vmm/${name}ioctl(2)

VM_SET_SEGMENT_DESCRIPTOR/VM_GET_SEGMENT_DESCRIPTOR:

VM_INJECT_EVENT:

VM_LAPIC_IRQ:

VM_SET_CAPABILITY/VM_GET_CAPABILITY: VT-x

VM_BIND_PPTDEV/VM_UNBIND_PPTDEV: PCI passthrough

VM_MAP_PPTDEV_MMIO: PCI passthrough

/dev/vmm/${name}ioctl (3)

VM_PPTDEV_MSI: PCI passthorugh

VM_INJECT_NMI:

VM_STATS:

VM_STAT_DESC: