kernel_x86_64: allow access of registers from interrupt handlers

This tidies up how we pass interrupt stack frames to Rust, and also saves
all the registers (instead of just caller-saved ones) and allows Rust to
access them. This will probably only be needed by the breakpoint handler,
but it's probably not that bad performance-wise (we can change it back
later if it proves to be slow), and is useful while we're trying to debug
this weird issue.

Interestingly, this changes the problem from presenting as a #PF into a #GP.

kernel/hal_x86_64/src/hw/idt.rs

@@ -156,9 +156,169 @@ impl IndexMut<u8> for Idt {
 #[derive(Clone, Copy, Debug)]
 #[repr(C)]
 pub struct InterruptStackFrame {
+    pub r15: u64,
+    pub r14: u64,
+    pub r13: u64,
+    pub r12: u64,
+    pub r11: u64,
+    pub r10: u64,
+    pub r9: u64,
+    pub r8: u64,
+    pub rbp: u64,
+    pub rdi: u64,
+    pub rsi: u64,
+    pub rdx: u64,
+    pub rcx: u64,
+    pub rbx: u64,
+    pub rax: u64,
+
     pub instruction_pointer: VirtualAddress,
     pub code_segment: u64,
     pub cpu_flags: CpuFlags,
     pub stack_pointer: VirtualAddress,
     pub stack_segment: u64,
 }
+
+#[derive(Clone, Copy, Debug)]
+#[repr(C)]
+pub struct ExceptionWithErrorStackFrame {
+    pub r15: u64,
+    pub r14: u64,
+    pub r13: u64,
+    pub r12: u64,
+    pub r11: u64,
+    pub r10: u64,
+    pub r9: u64,
+    pub r8: u64,
+    pub rbp: u64,
+    pub rdi: u64,
+    pub rsi: u64,
+    pub rdx: u64,
+    pub rcx: u64,
+    pub rbx: u64,
+    pub rax: u64,
+
+    pub error_code: u64,
+    pub instruction_pointer: VirtualAddress,
+    pub code_segment: u64,
+    pub cpu_flags: CpuFlags,
+    pub stack_pointer: VirtualAddress,
+    pub stack_segment: u64,
+}
+
+/// Macro to save the registers. We only need to save the scratch registers (`rax`, `rcx`, `rdx`, `rdi`, `rsi`,
+/// `r8`, `r9`, and `r10`), as callee-saved registers will be saved by the Rust handler, but we save all of them so
+/// we can access them in the handler if we want to.
+///
+/// The order we save them is important because we rely on their layout on the stack in `InterruptStackFrame` and
+/// `ExceptionWithErrorStackFrame`
+#[allow(unused_macros)] // `rustc` says this is unused for some reason
+macro save_regs() {
+    llvm_asm!("push rax
+               push rbx
+               push rcx
+               push rdx
+               push rsi
+               push rdi
+               push rbp
+               push r8
+               push r9
+               push r10
+               push r11
+               push r12
+               push r13
+               push r14
+               push r15"
+        :
+        :
+        :
+        : "intel"
+        );
+}
+
+/// Restore the saved registers. Must be in the opposite order to `save_regs`.
+#[allow(unused_macros)] // `rustc` says this is unused for some reason
+macro restore_regs() {
+    llvm_asm!("pop r15
+               pop r14
+               pop r13
+               pop r12
+               pop r11
+               pop r10
+               pop r9
+               pop r8
+               pop rbp
+               pop rdi
+               pop rsi
+               pop rdx
+               pop rcx
+               pop rbx
+               pop rax"
+    :
+    :
+    :
+    : "intel"
+    );
+}
+
+pub macro wrap_handler($name: path) {
+    {
+        #[naked]
+        extern "C" fn wrapper() -> ! {
+            unsafe {
+                /*
+                 * Without an error code, a total of `0xa0` bytes are pushed onto the stack by both the CPU and us.
+                 * Because `rsp+8` must be divisible by 0x10, we must align the stack.
+                 */
+                save_regs!();
+                llvm_asm!("mov rdi, rsp
+                           sub rsp, 8
+                           call $0
+                           add rsp, 8"
+                :
+                : "i"($name as extern "C" fn(&InterruptStackFrame))
+                : "rdi"
+                : "intel"
+                );
+                restore_regs!();
+                llvm_asm!("iretq" : : : : "intel");
+                unreachable!();
+            }
+        }
+
+        wrapper
+    }
+}
+
+pub macro wrap_handler_with_error_code($name: path) {
+    {
+        #[naked]
+        extern "C" fn wrapper() -> ! {
+            unsafe {
+                save_regs!();
+                /*
+                 * With an error code, a total of `0xa8` bytes are pushed onto the stack, and so `rsp+8` is already
+                 * divisible by 0x10, so no additional alignment is needed.
+                 */
+                llvm_asm!("mov rdi, rsp
+                           call $0"
+                :
+                : "i"($name as extern "C" fn(&ExceptionWithErrorStackFrame))
+                : "rdi"
+                : "intel"
+                );
+                restore_regs!();
+                llvm_asm!("add rsp, 8 // Pop the error code
+                           iretq"
+                :
+                :
+                :
+                : "intel"
+                );
+                unreachable!();
+            }
+        }
+
+        wrapper
+    }
+}

kernel/kernel_x86_64/src/interrupts/exception.rs

@@ -3,7 +3,10 @@
 //! panics.
 
 use bit_field::BitField;
-use hal_x86_64::hw::{idt::InterruptStackFrame, registers::read_control_reg};
+use hal_x86_64::hw::{
+    idt::{ExceptionWithErrorStackFrame, InterruptStackFrame},
+    registers::read_control_reg,
+};
 use log::{error, info};
 use pebble_util::BinaryPrettyPrint;
 
@@ -12,7 +15,7 @@ pub extern "C" fn nmi_handler(_: &InterruptStackFrame) {
 }
 
 pub extern "C" fn breakpoint_handler(stack_frame: &InterruptStackFrame) {
-    info!("BREAKPOINT: {:#?}", stack_frame);
+    info!("BREAKPOINT: {:#x?}", stack_frame);
 }
 
 pub extern "C" fn invalid_opcode_handler(stack_frame: &InterruptStackFrame) {
@@ -21,21 +24,21 @@ pub extern "C" fn invalid_opcode_handler(stack_frame: &InterruptStackFrame) {
     loop {}
 }
 
-pub extern "C" fn general_protection_fault_handler(stack_frame: &InterruptStackFrame, error_code: u64) {
-    error!("General protection fault (error code = {:#x}). Interrupt stack frame: ", error_code);
-    error!("{:#?}", stack_frame);
+pub extern "C" fn general_protection_fault_handler(stack_frame: &ExceptionWithErrorStackFrame) {
+    error!("General protection fault (error code = {:#x}). Interrupt stack frame: ", stack_frame.error_code);
+    error!("{:#x?}", stack_frame);
 
     loop {}
 }
 
-pub extern "C" fn page_fault_handler(stack_frame: &InterruptStackFrame, error_code: u64) {
+pub extern "C" fn page_fault_handler(stack_frame: &ExceptionWithErrorStackFrame) {
     error!(
         "PAGE_FAULT: {} ({:#x})",
         match (
-            error_code.get_bit(2), // User / Supervisor
-            error_code.get_bit(4), // Instruction / Data
-            error_code.get_bit(1), // Read / Write
-            error_code.get_bit(0)  // Present
+            stack_frame.error_code.get_bit(2), // User / Supervisor
+            stack_frame.error_code.get_bit(4), // Instruction / Data
+            stack_frame.error_code.get_bit(1), // Read / Write
+            stack_frame.error_code.get_bit(0)  // Present
         ) {
             // Page faults caused by the kernel
             (false, false, false, false) => "Kernel read non-present page",
@@ -58,17 +61,17 @@ pub extern "C" fn page_fault_handler(stack_frame: &InterruptStackFrame, error_co
         read_control_reg!(cr2) // CR2 holds the address of the page that caused the #PF
     );
 
-    error!("Error code: {}", BinaryPrettyPrint(error_code));
-    error!("{:#?}", stack_frame);
+    error!("Error code: {}", BinaryPrettyPrint(stack_frame.error_code));
+    error!("{:#x?}", stack_frame);
 
     /*
      * Page-faults can be recovered from and so are faults, but we never will so just give up.
      */
     loop {}
 }
 
-pub extern "C" fn double_fault_handler(stack_frame: &InterruptStackFrame, error_code: u64) {
-    error!("EXCEPTION: DOUBLE FAULT   (Error code: {})\n{:#?}", error_code, stack_frame);
+pub extern "C" fn double_fault_handler(stack_frame: &ExceptionWithErrorStackFrame) {
+    error!("EXCEPTION: DOUBLE FAULT   (Error code: {})\n{:#?}", stack_frame.error_code, stack_frame);
 
     loop {}
 }

kernel/kernel_x86_64/src/interrupts/mod.rs

@@ -11,13 +11,13 @@ use hal_x86_64::{
         cpu::CpuInfo,
         gdt::KERNEL_CODE_SELECTOR,
         i8259_pic::Pic,
-        idt::{Idt, InterruptStackFrame},
+        idt::{wrap_handler, wrap_handler_with_error_code, Idt, InterruptStackFrame},
         local_apic::LocalApic,
         registers::write_msr,
     },
     kernel_map,
 };
-use log::{info, warn};
+use log::warn;
 use pci::PciResolver;
 use pebble_util::InitGuard;
 
@@ -199,7 +199,7 @@ impl InterruptController {
             write_msr(IA32_LSTAR, syscall_handler as u64);
             // TODO: set this up properly (e.g. disable interrupts until we switch stacks, set up DF and such to be
             // as the kernel expects them)
-            write_msr(IA32_FMASK, 0);
+            write_msr(IA32_FMASK, 0x0);
         }
     }
 }
@@ -212,114 +212,3 @@ extern "C" fn local_apic_timer_handler(_: &InterruptStackFrame) {
 }
 
 extern "C" fn spurious_handler(_: &InterruptStackFrame) {}
-
-/// Macro to save the scratch registers. In System-V, `rbx`, `rbp`, `r12`, `r13`, `r14`, and `r15`
-/// must be restored by the callee, so Rust automatically generates code to restore them, but for
-/// the rest we have to manually preserve them. Use `restore_regs` to restore the scratch registers
-/// before returning from the handler.
-macro save_regs() {
-    llvm_asm!("push rax
-          push rcx
-          push rdx
-          push rsi
-          push rdi
-          push r8
-          push r9
-          push r10
-          push r11"
-        :
-        :
-        :
-        : "intel"
-        );
-}
-
-/// Restore the saved scratch registers.
-macro restore_regs() {
-    llvm_asm!("pop r11
-          pop r10
-          pop r9
-          pop r8
-          pop rdi
-          pop rsi
-          pop rdx
-          pop rcx
-          pop rax"
-        :
-        :
-        :
-        : "intel"
-        );
-}
-
-macro wrap_handler($name: path) {
-    {
-        #[naked]
-        extern "C" fn wrapper() -> ! {
-            unsafe {
-                /*
-                 * To calculate the address of the exception stack frame, we add 0x48 bytes (9
-                 * 64-bit registers). We don't need to manually align the stack, as it should
-                 * already be aligned correctly.
-                 */
-                save_regs!();
-                llvm_asm!("mov rdi, rsp
-                      add rdi, 0x48
-                      call $0"
-                    :
-                    : "i"($name as extern "C" fn(&InterruptStackFrame))
-                    : "rdi"
-                    : "intel"
-                    );
-                restore_regs!();
-                llvm_asm!("iretq"
-                     :
-                     :
-                     :
-                     : "intel"
-                     );
-                unreachable!();
-            }
-        }
-
-        wrapper
-    }
-}
-
-macro wrap_handler_with_error_code($name: path) {
-    {
-        #[naked]
-        extern "C" fn wrapper() -> ! {
-            unsafe {
-                /*
-                 * To calculate the address of the exception stack frame, we add 0x48 bytes (9
-                 * 64-bit registers), and then the two bytes of the error code. Because we skip
-                 * 0x50 bytes, we need to manually align the stack.
-                 */
-                save_regs!();
-                llvm_asm!("mov rsi, [rsp+0x48]   // Put the error code in RSI
-                      mov rdi, rsp
-                      add rdi, 0x50
-                      sub rsp, 8            // Align the stack pointer
-                      call $0
-                      add rsp, 8            // Restore the stack pointer"
-                     :
-                     : "i"($name as extern "C" fn(&InterruptStackFrame, _error_code: u64))
-                     : "rdi", "rsi"
-                     : "intel"
-                    );
-                restore_regs!();
-                llvm_asm!("add rsp, 8            // Pop the error code
-                      iretq"
-                     :
-                     :
-                     :
-                     : "intel"
-                    );
-                unreachable!();
-            }
-        }
-
-        wrapper
-    }
-}