Merge pull request #9 from KuangjuX/virtio

Add fdt parse dtb && `MachineMeta` structure
KuangjuX · Feb 11, 2023 · 500f93d · 500f93d
2 parents 71d2c44 + 8ec6404
commit 500f93d
Show file tree

Hide file tree

Showing 11 changed files with 110 additions and 47 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,4 @@ hyper.S
 guest.S
 guest_kernel
 .gdb_history
-fd.img
+fs.img
diff --git a/Makefile b/Makefile
@@ -27,7 +27,7 @@ KERNEL_ENTRY_PA := 0x80200000
 QEMUOPTS	= --machine virt -m 3G -bios $(BOOTLOADER) -nographic
 QEMUOPTS	+=-device loader,file=$(KERNEL_BIN),addr=$(KERNEL_ENTRY_PA)
 QEMUOPTS	+=-drive file=$(FS_IMG),if=none,format=raw,id=x0
-QEMUOPTS	+=-device virtio-blk-device,drive=x0
+QEMUOPTS	+=-device virtio-blk-device,drive=x0,bus=virtio-mmio-bus.0
 
 
 
@@ -39,7 +39,7 @@ $(GUEST_KERNEL_ELF):
 # $(GUEST_KERNEL_BIN): $(GUEST_KERNEL_ELF)
 # 	$(OBJCOPY) $(GUEST_KERNEL_ELF) --strip-all -O binary $@
 
-build: $(GUEST_KERNEL_ELF)
+build: $(GUEST_KERNEL_ELF) $(FS_IMG)
 	cargo build $(GUEST_KERNEL_FEATURE)
 
 $(KERNEL_BIN): build 
@@ -54,7 +54,7 @@ clean:
 	cargo clean
 	cd minikernel && cargo clean
 	cd minikernel/user && cargo clean
-	rm guest_kernel && rm guest.S && rm hyper.S
+	rm guest_kernel *.S $(FS_IMG)
 
 qemu-gdb: $(KERNEL_ELF)
 	$(QEMU) $(QEMUOPTS) -S -gdb tcp::1234

diff --git a/fs.img b/fs.img
diff --git a/src/guest/mod.rs b/src/guest/mod.rs
@@ -125,7 +125,9 @@ impl<P> GuestKernel<P> where P: PageDebug + PageTable {
         // 获取内核栈地址
         let (kernel_stack_bottom, kernel_stack_top) = kernel_stack_position(guest_id);
         // 将内核栈地址进行映射
-        HYPOCAUST.lock().hyper_space.exclusive_access().insert_framed_area(
+        let hypocaust = HYPOCAUST.lock();
+        let hypocaust = (&*hypocaust).as_ref().unwrap();
+        hypocaust.hyper_space.exclusive_access().insert_framed_area(
             kernel_stack_bottom.into(),
             kernel_stack_top.into(),
             MapPermission::R | MapPermission::W,
@@ -148,7 +150,7 @@ impl<P> GuestKernel<P> where P: PageDebug + PageTable {
         *trap_cx = TrapContext::app_init_context(
             GUEST_KERNEL_VIRT_START,
             0,
-            HYPOCAUST.lock().hyper_space.exclusive_access().token(),
+            hypocaust.hyper_space.exclusive_access().token(),
             kernel_stack_top,
             trap_handler as usize,
         );

diff --git a/src/guest/pmap.rs b/src/guest/pmap.rs
@@ -333,6 +333,8 @@ impl<P> GuestKernel<P> where P: PageDebug + PageTable {
         let root_gpa = (satp & 0xfff_ffff_ffff) << 12;
         let root_hppn = PhysPageNum::from(gpa2hpa(root_gpa, hart_id) >> 12);
         let gpt = P::from_ppn(root_hppn);
+        let hypocaust = HYPOCAUST.lock();
+        let hypocaust = (&*hypocaust).as_ref().unwrap();
         if self.shadow_state.shadow_page_tables.shadow_page_table(satp).is_none() {
             // 如果影子页表中没有发现，新建影子页表
             let mut spt;
@@ -358,11 +360,11 @@ impl<P> GuestKernel<P> where P: PageDebug + PageTable {
 
             // 为 `SPT` 映射跳板页
             // 无论是 guest spt 还是 user spt 都要映射跳板页与 Trap Context
-            let trampoline_hppn = HYPOCAUST.lock().hyper_space.exclusive_access().translate(VirtPageNum::from(TRAMPOLINE >> 12)).unwrap().ppn();
+            let trampoline_hppn = hypocaust.hyper_space.exclusive_access().translate(VirtPageNum::from(TRAMPOLINE >> 12)).unwrap().ppn();
             spt.map(VirtPageNum::from(TRAMPOLINE >> 12), trampoline_hppn, PTEFlags::R | PTEFlags::X);
 
             let trapctx_hvpn = VirtPageNum::from(self.translate_guest_paddr(TRAP_CONTEXT).unwrap() >> 12);
-            let trapctx_hppn = HYPOCAUST.lock().hyper_space.exclusive_access().translate(trapctx_hvpn).unwrap().ppn();
+            let trapctx_hppn = hypocaust.hyper_space.exclusive_access().translate(trapctx_hvpn).unwrap().ppn();
             spt.map(VirtPageNum::from(TRAP_CONTEXT >> 12), trapctx_hppn, PTEFlags::R | PTEFlags::W);
 
             // hdebug!("Make new SPT(satp -> {:#x}, spt -> {:#x}) ", satp, spt.token());
@@ -389,7 +391,7 @@ impl<P> GuestKernel<P> where P: PageDebug + PageTable {
                     synchronize_page_table::<P>(hart_id, satp);
                     let spt = &mut self.shadow_state.shadow_page_tables.shadow_page_table(satp).unwrap();
                     // 为 `SPT` 映射跳板页
-                    let trampoline_hppn = HYPOCAUST.lock().hyper_space.exclusive_access().translate(VirtPageNum::from(TRAMPOLINE >> 12)).unwrap().ppn();
+                    let trampoline_hppn = hypocaust.hyper_space.exclusive_access().translate(VirtPageNum::from(TRAMPOLINE >> 12)).unwrap().ppn();
                     if let Some(pte) = spt.translate(VirtPageNum::from(TRAMPOLINE >> 12)) {
                         if !pte.is_valid() {
                             htracking!("user remap trampoline");
@@ -401,7 +403,7 @@ impl<P> GuestKernel<P> where P: PageDebug + PageTable {
                     }
 
                     let trapctx_hvpn = VirtPageNum::from(self.translate_guest_paddr(TRAP_CONTEXT).unwrap() >> 12);
-                    let trapctx_hppn = HYPOCAUST.lock().hyper_space.exclusive_access().translate(trapctx_hvpn).unwrap().ppn();
+                    let trapctx_hppn = hypocaust.hyper_space.exclusive_access().translate(trapctx_hvpn).unwrap().ppn();
                     if let Some(pte) = spt.translate(VirtPageNum::from(TRAP_CONTEXT >> 12)) {
                         if !pte.is_valid() {
                             htracking!("user remap trap context");

diff --git a/src/hypervisor/device/block/virtio_blk.rs b/src/hypervisor/device/block/virtio_blk.rs
@@ -4,7 +4,7 @@ use alloc::vec;
 use virtio_drivers::transport::Transport;
 use virtio_drivers::transport::mmio::{VirtIOHeader, MmioTransport};
 use virtio_drivers::{Hal, device::blk::VirtIOBlk};
-use fdt::{ Fdt, node::FdtNode, standard_nodes::Compatible };
+use fdt::{ Fdt, node::FdtNode};
 
 use crate::hypervisor::HYPOCAUST;
 use crate::hypervisor::hyp_alloc::{frame_alloc, frame_dealloc};
@@ -37,7 +37,9 @@ pub fn initialize_virtio_blk(dtb: usize) -> Option<MmioTransport> {
     for node in fdt.all_nodes() {
         if let Some(compatible) = node.compatible() {
             if compatible.all().any(|s| s == "virtio,mmio") {
-                return virtio_probe(node)
+                if let Some(transport) = virtio_probe(node) {
+                    return Some(transport)
+                }
             }
         }
     }
@@ -48,18 +50,11 @@ pub fn initialize_virtio_blk(dtb: usize) -> Option<MmioTransport> {
 fn virtio_probe(node: FdtNode) -> Option<MmioTransport> {
     if let Some(reg) = node.reg().and_then(|mut reg| reg.next()) {
         let paddr = reg.starting_address as usize;
-        let size = reg.size.unwrap();
         let vaddr = paddr;
-        hdebug!("walk dt addr={:#x}, size={:#x}", paddr, size);
-        hdebug!(
-            "Device tree node {}: {:?}",
-            node.name,
-            node.compatible().map(Compatible::first)
-        );
         let header = NonNull::new(vaddr as *mut VirtIOHeader).unwrap();
         match unsafe{ MmioTransport::new(header) } {
-            Err(e) => { 
-                hwarning!("Error creating VirIO MMIO transport {}", e); 
+            Err(_) => { 
+                // hwarning!("Error creating VirIO MMIO transport {}", e); 
                 return None
             },
             Ok(transport) => {
@@ -101,7 +96,8 @@ impl Hal for VirtioHal {
     /// which the driver can use to access it.
     fn dma_alloc(pages: usize, _direction: virtio_drivers::BufferDirection) -> (virtio_drivers::PhysAddr, core::ptr::NonNull<u8>) {
         let hypocaust = HYPOCAUST.lock();
-        let mut queue = hypocaust.frame_queue.lock();
+        let hypocaust = (&*hypocaust).as_ref().unwrap();
+        let mut queue = hypocaust.frame_queue.exclusive_access();
         let mut ppn_base = PhysPageNum(0);
         for i in 0..pages {
             let frame = frame_alloc().unwrap();
@@ -153,8 +149,9 @@ impl Hal for VirtioHal {
 
 pub fn virtio_blk_test() {
     let mut hypocaust = HYPOCAUST.lock();
-    if let Some(virtio_blk) = &mut hypocaust.virtio_blk {
-        let mut blk = virtio_blk.0.exclusive_access();
+    let hypocaust = (&mut *hypocaust).as_mut().unwrap();
+    if let Some(blk) = hypocaust.virtio_blk.as_mut() {
+        let mut blk = blk.0.exclusive_access();
         let mut input = vec![0xffu8; 512];
         let mut output = vec![0; 512];
         for i in 0..32 {
@@ -167,7 +164,7 @@ pub fn virtio_blk_test() {
         }
         hdebug!("virtio-blk test finished");
     }else{
-        hwarning!("failed to find virtio-blk device");
+        hwarning!("failed to find virtio blk device");
     }
 
 }
diff --git a/src/hypervisor/fdt.rs b/src/hypervisor/fdt.rs
@@ -0,0 +1,51 @@
+///! ref: https://github.com/mit-pdos/RVirt/blob/HEAD/src/fdt.rs
+
+use arrayvec::ArrayVec;
+use fdt::Fdt;
+
+#[derive(Clone, Debug)]
+pub struct Device {
+    pub base_address: usize,
+    pub size: usize
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct MachineMeta{
+    pub physical_memory_offset: usize,
+    pub physical_memory_size: usize,
+
+    pub virtio: ArrayVec<Device, 16>
+}
+
+impl MachineMeta {
+    pub fn parse(dtb: usize) -> Self {
+        let fdt = unsafe{ Fdt::from_ptr(dtb as *const u8) }.unwrap();
+        let memory = fdt.memory();
+        let mut meta = MachineMeta::default();
+        for region in memory.regions() {
+            meta.physical_memory_offset = region.starting_address as usize;
+            meta.physical_memory_size = region.size.unwrap();
+        }
+        // 发现 virtio mmio 设备
+        for node in fdt.find_all_nodes("/soc/virtio_mmio") {
+            if let Some(reg) = node.reg().and_then(|mut reg| reg.next()) {
+                let paddr = reg.starting_address as usize;
+                let size = reg.size.unwrap();
+                let vaddr = paddr;
+                unsafe{
+                    let header = vaddr as *const u32;
+                    let device_id_addr = header.add(2);
+                    let device_id = core::ptr::read_volatile(device_id_addr);
+                    if device_id != 0 {
+                        hdebug!("virtio mmio addr: {:#x}, size: {:#x}", paddr, size);
+                        meta.virtio.push(
+                            Device { base_address: paddr, size }
+                        )
+                    }
+                }
+            }
+        }
+        meta.virtio.sort_unstable_by_key(|v| v.base_address);
+        meta
+    }
+}
diff --git a/src/hypervisor/mod.rs b/src/hypervisor/mod.rs
@@ -11,32 +11,43 @@ use crate::page_table::{PageTable, PageTableSv39};
 use crate::debug::PageDebug;
 use self::device::VirtIOBlock;
 pub use self::hyp_alloc::FrameTracker;
+pub use self::fdt::MachineMeta;
 
-use lazy_static::lazy_static;
 
 
 pub mod device;
 pub mod hyp_alloc;
 pub mod trap;
+pub mod fdt;
 
 pub struct Hypervisor<P: PageTable + PageDebug, T: Transport> {
     pub hyper_space: Arc<UPSafeCell<MemorySet<P>>>,
-    pub frame_queue: Mutex<Vec<FrameTracker>>,
-    pub virtio_blk: Option<VirtIOBlock<T>>
+    pub frame_queue: UPSafeCell<Vec<FrameTracker>>,
+    pub virtio_blk: Option<VirtIOBlock<T>>,
+    pub meta: MachineMeta
 }
 
-lazy_static! {
-    pub static ref HYPOCAUST: Mutex<Hypervisor<PageTableSv39, MmioTransport>> = Mutex::new(
-        Hypervisor { 
+
+pub static HYPOCAUST: Mutex<Option<Hypervisor<PageTableSv39, MmioTransport>>> = Mutex::new(None);
+
+
+
+pub fn initialize_vmm(meta: MachineMeta) {
+    unsafe{ HYPOCAUST.force_unlock(); }
+    let old = HYPOCAUST.lock().replace(
+        Hypervisor{
             hyper_space: Arc::new(unsafe { UPSafeCell::new(MemorySet::new_kernel()) }),
-            frame_queue: Mutex::new(Vec::new()),
-            virtio_blk: None
+            frame_queue: unsafe{ UPSafeCell::new(Vec::new()) },
+            virtio_blk: None,
+            meta
         }
     );
+    core::mem::forget(old);
 }
 
-impl<P: PageTable + PageDebug, T: Transport> Hypervisor<P, T> {
-    pub fn add_virtio_blk(&mut self, virtio_blk: VirtIOBlock<T>) {
-        self.virtio_blk.replace(virtio_blk);
-    }
+pub fn add_virtio_blk(virtio_blk: VirtIOBlock<MmioTransport>) {
+    let mut hypocaust = HYPOCAUST.lock();
+    let hypocaust = (&mut *hypocaust).as_mut().unwrap();
+    let old = hypocaust.virtio_blk.replace(virtio_blk);
+    core::mem::forget(old);
 }
diff --git a/src/main.rs b/src/main.rs
@@ -37,7 +37,6 @@ mod hypervisor;
 
 use crate::constants::layout::PAGE_SIZE;
 use crate::guest::{GuestKernel, GUEST_KERNEL_MANAGER, run_guest_kernel};
-use crate::hypervisor::HYPOCAUST;
 use crate::hypervisor::device::VirtIOBlock;
 use crate::mm::MemorySet;
 
@@ -95,16 +94,16 @@ pub fn hentry(hart_id: usize, device_tree_blob: usize) -> ! {
         clear_bss();
         hdebug!("Hello Hypocaust");
         hdebug!("hart_id: {}, device tree blob: {:#x}", hart_id, device_tree_blob);
+        let meta = hypervisor::fdt::MachineMeta::parse(device_tree_blob);
         // 初始化堆及帧分配器
         hypervisor::hyp_alloc::heap_init();
+        hypervisor::initialize_vmm(meta);
         // 获取 `transport`
         if let Some(transport) = hypervisor::device::initialize_virtio_blk(device_tree_blob) {
             let virtio_blk = VirtIOBlock::new(transport);
-            let mut hypocaust = HYPOCAUST.lock();
-            hdebug!("create virtio block");
-            // 添加 virtio block 设备
-            hypocaust.add_virtio_blk(virtio_blk);
-            drop(hypocaust);
+            hypervisor::add_virtio_blk(virtio_blk);
+            // 测试 virtio block
+            hypervisor::device::virtio_blk_test();
         }
         let guest_kernel_memory = MemorySet::new_guest_kernel(&GUEST_KERNEL);
         // 初始化虚拟内存
@@ -114,8 +113,6 @@ pub fn hentry(hart_id: usize, device_tree_blob: usize) -> ! {
         mm::remap_test();
         // 测试 guest kernel 内存映射
         mm::guest_kernel_test();
-        // 测试 virtio block
-        hypervisor::device::virtio_blk_test();
         // 开启时钟中断
         hypervisor::trap::enable_timer_interrupt();
         timer::set_default_next_trigger();

diff --git a/src/mm/memory_set.rs b/src/mm/memory_set.rs
@@ -462,6 +462,7 @@ bitflags! {
 #[allow(unused)]
 pub fn remap_test() {
     let hypocaust = HYPOCAUST.lock();
+    let hypocaust = (&*hypocaust).as_ref().unwrap();
     let mut kernel_space = hypocaust.hyper_space.exclusive_access();
     let mid_text: VirtAddr = ((stext as usize + etext as usize) / 2).into();
     let mid_rodata: VirtAddr = ((srodata as usize + erodata as usize) / 2).into();
@@ -483,13 +484,14 @@ pub fn remap_test() {
         .unwrap()
         .executable(),);
     // 测试 guest ketnel
-    println!("[hypervisor] remap test passed!");
+    hdebug!("remap test passed!");
 }
 
 #[allow(unused)]
 pub fn guest_kernel_test() {
     use crate::constants::layout::GUEST_KERNEL_PHY_START_1;
     let hypocaust = HYPOCAUST.lock();
+    let hypocaust = (&*hypocaust).as_ref().unwrap();
     let mut kernel_space = hypocaust.hyper_space.exclusive_access();
 
     let guest_kernel_text: VirtAddr = GUEST_KERNEL_PHY_START_1.into();
@@ -501,6 +503,6 @@ pub fn guest_kernel_test() {
         core::ptr::read(GUEST_KERNEL_PHY_START_1 as *const u32);
     }
     // 测试 guest ketnel
-    println!("[hypervisor] guest kernel test passed!");
+    hdebug!("guest kernel test passed!");
 }
 
diff --git a/src/mm/mod.rs b/src/mm/mod.rs
@@ -10,6 +10,7 @@ use crate::page_table::PageTableSv39;
 
 pub fn vm_init(guest_kernel_memory: &MemorySet<PageTableSv39>) {
     let hypocaust = HYPOCAUST.lock();
+    let hypocaust = (&*hypocaust).as_ref().unwrap();
     let mut hyper_space = hypocaust.hyper_space.exclusive_access();
     hyper_space.hyper_load_guest_kernel(guest_kernel_memory);
     hyper_space.activate();
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,4 +3,4 @@ hyper.S @@
     guest.S
     guest_kernel
     .gdb_history
-    fd.img
+    fs.img