Initialize trap handling earlier

include/asmmacros.h

@@ -28,6 +28,16 @@
     .previous; \
     91:
 
+#define OPTIONAL_OR_INSTR(option, instr) \
+    .section .hotpatch, "a";             \
+    .quad 0x6, 94f, 91f, 95f-94f;        \
+    .previous;                           \
+    .section .rodata;                    \
+94: instr;                               \
+95:                                      \
+    .previous;                           \
+91:
+
 /*
  * Taken from Linux: Fill the CPU return stack buffer.
  *

kernel/hotpatch.cc

@@ -15,7 +15,7 @@ struct patch {
   u64 opcode;
   u64 alternative;
   u64 end;
-  u64 padding;
+  u64 string_len;
 };
 
 #define PATCH_SEGMENT_KTEXT 0x1
@@ -169,9 +169,12 @@ void apply_hotpatches()
           remove_range(text_bases[i], p->start+5, p->end);
           break;
         case PATCH_OPCODE_OR_STRING:
+          assert(p->string_len > 0);
+          assert(p->string_len <= p->end - p->start);
           memcpy(&text_bases[i][p->start - KTEXT],
                  (char*)p->alternative,
-                 p->end - p->start);
+                 p->string_len);
+          remove_range(text_bases[i], p->start + p->string_len, p->end);
           break;
         default:
           panic("hotpatch: bad opcode");

kernel/main.cc

@@ -29,6 +29,7 @@ void cleanuppg(void);
 void inittls(struct cpu *);
 void initcodex(void);
 void inittrap(void);
+void initvectoredtrap(void);
 void initfpu(void);
 void initmsr(void);
 void initseg(struct cpu *);
@@ -180,43 +181,39 @@ bootothers(void)
 void
 cmain(u64 mbmagic, u64 mbaddr)
 {
-
   // Make cpus[0] work.  CPU 0's percpu data is pre-allocated directly
   // in the image.  *cpu and such won't work until we inittls.
   percpu_offsets[0] = __percpu_start;
 
-  extern u64 text;
-  writefs(UDSEG);
-  writemsr(MSR_FS_BASE, (u64)&text);
+  // Initialize trap handling
+  initseg(&cpus[0]);
+  inittls(&cpus[0]);       // Requires initseg
+  inittrap();
 
+  // Initialize output
   initmultiboot(mbmagic, mbaddr);
   inithz();
   inituart();              // Requires inithz
-
   debugmultiboot(mbmagic, mbaddr);
   initcmdline();           // Requires initmultiboot
   initvga();               // Requires initmultiboot, initcmdline
+
   initphysmem();           // Requires initmultiboot
   initpg(&cpus[0]);        // Requires initphysmem
-  initseg(&cpus[0]);
-  inittls(&cpus[0]);       // Requires initseg
+  initvectoredtrap();      // Requires initpg
   initdoublebuffer();      // Requires initpg
-
   initacpitables();        // Requires initpg, inittls
   initlapic();             // Requires initpg, inithz
   initnuma();              // Requires initacpitables, initlapic
   initpercpu();            // Requires initnuma
-  initcpus();              // Requires initnuma, initpercpu,
-                           // suggests initacpitables
-
-  initpic();       // interrupt controller
+  initcpus();              // Requires initnuma, initpercpu, suggests initacpitables
+  initpic();               // interrupt controller
   initiommu();             // Requires initlapic
   initextpic();            // Requires initpic
   // Interrupt routing is now configured
 
   inituartcons();          // Requires interrupt routing
   initcga();
-
   initpageinfo();          // Requires initnuma
 
   // Some global constructors require mycpu()->id (via myid()) which
@@ -232,17 +229,16 @@ cmain(u64 mbmagic, u64 mbaddr)
       (*__init_array_start[i])(0, nullptr, nullptr);
 
   inithotpatch();
-  inittrap();              // Requires inithotpatch
   inithpet();              // Requires initacpitables
   inittsc();               // Requires inithpet
   initfpu();               // Requires nothing
   initmsr();               // Requires nothing
   initkalloc();            // Requires initpageinfo
-  initproc();      // process table
-  initsched();     // scheduler run queues
+  initproc();
+  initsched();
   initidle();
-  initgc();        // gc epochs and threads
-  initrefcache();  // Requires initsched
+  initgc();
+  initrefcache();          // Requires initsched
   initconsole();
   initsamp();
   initlockstat();
@@ -254,11 +250,11 @@ cmain(u64 mbmagic, u64 mbaddr)
   initpci();               // Suggests initacpi
   initnet();
   initrtc();               // Requires inithpet
-  initdev();               // Misc /dev nodes
-  initide();               // IDE driver
+  initdev();
+  initide();
   initmemide();
   initpartition();
-  initinode();     // inode cache
+  initinode();
   initmfs();
 
   // XXX hack until mnodes can load from disk
@@ -269,13 +265,13 @@ cmain(u64 mbmagic, u64 mbaddr)
 #if CODEX
   initcodex();
 #endif
-  bootothers();    // start other processors
+  bootothers();
   cleanuppg();             // Requires bootothers
   initcpprt();
   initwd();                // Requires initnmi
   initattack(); // for spectre demo
 
-  inituser();      // first user process
+  inituser();
   idleloop();
 
   panic("Unreachable");

kernel/trap.cc

@@ -132,7 +132,7 @@ do_pagefault(struct trapframe *tf, bool had_secrets)
     if(r >= 0 || myproc()->deliver_signal(SIGSEGV)){
       return 0;
     }
-  } else if (myproc()->uaccess_) {
+  } else if (myproc() && myproc()->uaccess_) {
     // Normally __uaccess_* functions must be called with interrupts disabled so
     // that we can process page faults caused by unmapped pages. However, futex
     // critical sections need to hold a lock while checking user memory, so we
@@ -418,7 +418,11 @@ inittrap(void)
   // And reserve interrupt 255 (Intel SDM Vol. 3 suggests this can't
   // be used for MSI).
   irq_info[255 - T_IRQ0].in_use = true;
+}
 
+void
+initvectoredtrap(void)
+{
   // Configure double fault handling. Any double fault results in a kernel
   // panic, so it is harmless to just share double fault stacks globally.
   for (int c = 0; c < ncpu; c++) {
@@ -555,6 +559,10 @@ initseg(struct cpu *c)
   writemsr(MSR_STAR, star);
   writemsr(MSR_LSTAR, (u64)&sysentry);
   writemsr(MSR_SFMASK, FL_TF | FL_IF);
+
+  extern u64 text;
+  writefs(UDSEG);
+  writemsr(MSR_FS_BASE, (u64)&text);
 }
 
 // Pushcli/popcli are like cli/sti except that they are matched:

kernel/trapasm.S

@@ -241,14 +241,14 @@ OPTIONAL("lazy_barrier", "no")
 OPTIONAL_OR_NOPS("lazy_barrier")
         movq %rsp, %rdi  // first argument to trap
         movq (secrets_mapped), %rsi  // second argument to trap
-OPTIONAL("fsgsbase", "yes")
-        rdfsbase %rax
-OPTIONAL_OR_CALL("fsgsbase", emulate_rdfsbase)
+OPTIONAL("fsgsbase", "no")
+        call emulate_rdfsbase
+OPTIONAL_OR_INSTR("fsgsbase", rdfsbase %rax)
         pushq %rax
         movq $text, %rax
-OPTIONAL("fsgsbase", "yes")
-        wrfsbase %rax
-OPTIONAL_OR_CALL("fsgsbase", emulate_wrfsbase)
+OPTIONAL("fsgsbase", "no")
+        call emulate_wrfsbase
+OPTIONAL_OR_INSTR("fsgsbase", wrfsbase %rax)
         subq $8, %rsp
         call trap_c
         addq $8, %rsp
@@ -259,9 +259,9 @@ OPTIONAL_OR_CALL("fsgsbase", emulate_wrfsbase)
 .balign 8
 trapret:
         cli
-OPTIONAL("fsgsbase", "yes")
-        wrfsbase %rax
-OPTIONAL_OR_CALL("fsgsbase", emulate_wrfsbase)
+OPTIONAL("fsgsbase", "no")
+        call emulate_wrfsbase
+OPTIONAL_OR_INSTR("fsgsbase", wrfsbase %rax)
         addq $0x10, %rsp
         RESTORE_REGISTERS
         addq $0x10, %rsp  // trapno, err