Large page TLB flush

QEMU uses a fixed page size for the CPU TLB. If the guest uses large pages then we effectively split these into multiple smaller pages, and populate the corresponding TLB entries on demand. When the guest invalidates the TLB by virtual address we must invalidate all entries covered by the large page. However the address used to invalidate the entry may not be present in the QEMU TLB, so we do not know which regions to clear. Implementing a full vaiable size TLB is hard and slow, so just keep a simple address/mask pair to record which addresses may have been mapped by large pages. If the guest invalidates this region then flush the whole TLB. Signed-off-by: Paul Brook <[email protected]>
Kustsnpulten · Mar 17, 2010 · d4c430a · d4c430a
1 parent 409dbce
commit d4c430a
Show file tree

Hide file tree

Showing 14 changed files with 136 additions and 80 deletions.
diff --git a/cpu-defs.h b/cpu-defs.h
@@ -111,6 +111,8 @@ typedef struct CPUTLBEntry {
     /* The meaning of the MMU modes is defined in the target code. */   \
     CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
     target_phys_addr_t iotlb[NB_MMU_MODES][CPU_TLB_SIZE];               \
+    target_ulong tlb_flush_addr;                                        \
+    target_ulong tlb_flush_mask;
 
 #else
 

diff --git a/exec-all.h b/exec-all.h
@@ -96,17 +96,9 @@ void tb_invalidate_page_range(target_ulong start, target_ulong end);
 void tlb_flush_page(CPUState *env, target_ulong addr);
 void tlb_flush(CPUState *env, int flush_global);
 #if !defined(CONFIG_USER_ONLY)
-int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
-                      target_phys_addr_t paddr, int prot,
-                      int mmu_idx, int is_softmmu);
-static inline int tlb_set_page(CPUState *env1, target_ulong vaddr,
-                               target_phys_addr_t paddr, int prot,
-                               int mmu_idx, int is_softmmu)
-{
-    if (prot & PAGE_READ)
-        prot |= PAGE_EXEC;
-    return tlb_set_page_exec(env1, vaddr, paddr, prot, mmu_idx, is_softmmu);
-}
+void tlb_set_page(CPUState *env, target_ulong vaddr,
+                  target_phys_addr_t paddr, int prot,
+                  int mmu_idx, target_ulong size);
 #endif
 
 #define CODE_GEN_ALIGN           16 /* must be >= of the size of a icache line */

diff --git a/exec.c b/exec.c
@@ -1918,6 +1918,8 @@ void tlb_flush(CPUState *env, int flush_global)
 
     memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
 
+    env->tlb_flush_addr = -1;
+    env->tlb_flush_mask = 0;
     tlb_flush_count++;
 }
 
@@ -1941,6 +1943,16 @@ void tlb_flush_page(CPUState *env, target_ulong addr)
 #if defined(DEBUG_TLB)
     printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
 #endif
+    /* Check if we need to flush due to large pages.  */
+    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
+#if defined(DEBUG_TLB)
+        printf("tlb_flush_page: forced full flush ("
+               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
+               env->tlb_flush_addr, env->tlb_flush_mask);
+#endif
+        tlb_flush(env, 1);
+        return;
+    }
     /* must reset current TB so that interrupts cannot modify the
        links while we are modifying them */
     env->current_tb = NULL;
@@ -2090,25 +2102,50 @@ static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
         tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
 }
 
-/* add a new TLB entry. At most one entry for a given virtual address
-   is permitted. Return 0 if OK or 2 if the page could not be mapped
-   (can only happen in non SOFTMMU mode for I/O pages or pages
-   conflicting with the host address space). */
-int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
-                      target_phys_addr_t paddr, int prot,
-                      int mmu_idx, int is_softmmu)
+/* Our TLB does not support large pages, so remember the area covered by
+   large pages and trigger a full TLB flush if these are invalidated.  */
+static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
+                               target_ulong size)
+{
+    target_ulong mask = ~(size - 1);
+
+    if (env->tlb_flush_addr == (target_ulong)-1) {
+        env->tlb_flush_addr = vaddr & mask;
+        env->tlb_flush_mask = mask;
+        return;
+    }
+    /* Extend the existing region to include the new page.
+       This is a compromise between unnecessary flushes and the cost
+       of maintaining a full variable size TLB.  */
+    mask &= env->tlb_flush_mask;
+    while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
+        mask <<= 1;
+    }
+    env->tlb_flush_addr &= mask;
+    env->tlb_flush_mask = mask;
+}
+
+/* Add a new TLB entry. At most one entry for a given virtual address
+   is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
+   supplied size is only used by tlb_flush_page.  */
+void tlb_set_page(CPUState *env, target_ulong vaddr,
+                  target_phys_addr_t paddr, int prot,
+                  int mmu_idx, target_ulong size)
 {
     PhysPageDesc *p;
     unsigned long pd;
     unsigned int index;
     target_ulong address;
     target_ulong code_address;
     target_phys_addr_t addend;
-    int ret;
     CPUTLBEntry *te;
     CPUWatchpoint *wp;
     target_phys_addr_t iotlb;
 
+    assert(size >= TARGET_PAGE_SIZE);
+    if (size != TARGET_PAGE_SIZE) {
+        tlb_add_large_page(env, vaddr, size);
+    }
     p = phys_page_find(paddr >> TARGET_PAGE_BITS);
     if (!p) {
         pd = IO_MEM_UNASSIGNED;
@@ -2120,7 +2157,6 @@ int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
            vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
 #endif
 
-    ret = 0;
     address = vaddr;
     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
         /* IO memory case (romd handled later) */
@@ -2190,7 +2226,6 @@ int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
     } else {
         te->addr_write = -1;
     }
-    return ret;
 }
 
 #else

diff --git a/hw/alpha_palcode.c b/hw/alpha_palcode.c
@@ -1003,11 +1003,14 @@ int cpu_ppc_handle_mmu_fault (CPUState *env, uint32_t address, int rw,
         /* No fault */
         page_size = 1ULL << zbits;
         address &= ~(page_size - 1);
+        /* FIXME: page_size should probably be passed to tlb_set_page,
+           and this loop removed.   */
         for (end = physical + page_size; physical < end; physical += 0x1000) {
-            ret = tlb_set_page(env, address, physical, prot,
-                               mmu_idx, is_softmmu);
+            tlb_set_page(env, address, physical, prot, mmu_idx,
+                         TARGET_PAGE_SIZE);
             address += 0x1000;
         }
+        ret = 0;
         break;
 #if 0
     case 1:

diff --git a/target-arm/helper.c b/target-arm/helper.c
@@ -894,7 +894,8 @@ static uint32_t get_level1_table_address(CPUState *env, uint32_t address)
 }
 
 static int get_phys_addr_v5(CPUState *env, uint32_t address, int access_type,
-			    int is_user, uint32_t *phys_ptr, int *prot)
+			    int is_user, uint32_t *phys_ptr, int *prot,
+                            target_ulong *page_size)
 {
     int code;
     uint32_t table;
@@ -927,6 +928,7 @@ static int get_phys_addr_v5(CPUState *env, uint32_t address, int access_type,
         phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
         ap = (desc >> 10) & 3;
         code = 13;
+        *page_size = 1024 * 1024;
     } else {
         /* Lookup l2 entry.  */
 	if (type == 1) {
@@ -944,10 +946,12 @@ static int get_phys_addr_v5(CPUState *env, uint32_t address, int access_type,
         case 1: /* 64k page.  */
             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
             ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
+            *page_size = 0x10000;
             break;
         case 2: /* 4k page.  */
             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
             ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
+            *page_size = 0x1000;
             break;
         case 3: /* 1k page.  */
 	    if (type == 1) {
@@ -962,6 +966,7 @@ static int get_phys_addr_v5(CPUState *env, uint32_t address, int access_type,
 		phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
 	    }
             ap = (desc >> 4) & 3;
+            *page_size = 0x400;
             break;
         default:
             /* Never happens, but compiler isn't smart enough to tell.  */
@@ -981,7 +986,8 @@ static int get_phys_addr_v5(CPUState *env, uint32_t address, int access_type,
 }
 
 static int get_phys_addr_v6(CPUState *env, uint32_t address, int access_type,
-			    int is_user, uint32_t *phys_ptr, int *prot)
+			    int is_user, uint32_t *phys_ptr, int *prot,
+                            target_ulong *page_size)
 {
     int code;
     uint32_t table;
@@ -1021,9 +1027,11 @@ static int get_phys_addr_v6(CPUState *env, uint32_t address, int access_type,
         if (desc & (1 << 18)) {
             /* Supersection.  */
             phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
+            *page_size = 0x1000000;
         } else {
             /* Section.  */
             phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
+            *page_size = 0x100000;
         }
         ap = ((desc >> 10) & 3) | ((desc >> 13) & 4);
         xn = desc & (1 << 4);
@@ -1040,10 +1048,12 @@ static int get_phys_addr_v6(CPUState *env, uint32_t address, int access_type,
         case 1: /* 64k page.  */
             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
             xn = desc & (1 << 15);
+            *page_size = 0x10000;
             break;
         case 2: case 3: /* 4k page.  */
             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
             xn = desc & 1;
+            *page_size = 0x1000;
             break;
         default:
             /* Never happens, but compiler isn't smart enough to tell.  */
@@ -1132,7 +1142,8 @@ static int get_phys_addr_mpu(CPUState *env, uint32_t address, int access_type,
 
 static inline int get_phys_addr(CPUState *env, uint32_t address,
                                 int access_type, int is_user,
-                                uint32_t *phys_ptr, int *prot)
+                                uint32_t *phys_ptr, int *prot,
+                                target_ulong *page_size)
 {
     /* Fast Context Switch Extension.  */
     if (address < 0x02000000)
@@ -1142,34 +1153,39 @@ static inline int get_phys_addr(CPUState *env, uint32_t address,
         /* MMU/MPU disabled.  */
         *phys_ptr = address;
         *prot = PAGE_READ | PAGE_WRITE;
+        *page_size = TARGET_PAGE_SIZE;
         return 0;
     } else if (arm_feature(env, ARM_FEATURE_MPU)) {
+        *page_size = TARGET_PAGE_SIZE;
 	return get_phys_addr_mpu(env, address, access_type, is_user, phys_ptr,
 				 prot);
     } else if (env->cp15.c1_sys & (1 << 23)) {
         return get_phys_addr_v6(env, address, access_type, is_user, phys_ptr,
-                                prot);
+                                prot, page_size);
     } else {
         return get_phys_addr_v5(env, address, access_type, is_user, phys_ptr,
-                                prot);
+                                prot, page_size);
     }
 }
 
 int cpu_arm_handle_mmu_fault (CPUState *env, target_ulong address,
                               int access_type, int mmu_idx, int is_softmmu)
 {
     uint32_t phys_addr;
+    target_ulong page_size;
     int prot;
     int ret, is_user;
 
     is_user = mmu_idx == MMU_USER_IDX;
-    ret = get_phys_addr(env, address, access_type, is_user, &phys_addr, &prot);
+    ret = get_phys_addr(env, address, access_type, is_user, &phys_addr, &prot,
+                        &page_size);
     if (ret == 0) {
         /* Map a single [sub]page.  */
         phys_addr &= ~(uint32_t)0x3ff;
         address &= ~(uint32_t)0x3ff;
-        return tlb_set_page (env, address, phys_addr, prot, mmu_idx,
-                             is_softmmu);
+        tlb_set_page (env, address, phys_addr, prot | PAGE_EXEC, mmu_idx,
+                      page_size);
+        return 0;
     }
 
     if (access_type == 2) {
@@ -1189,10 +1205,11 @@ int cpu_arm_handle_mmu_fault (CPUState *env, target_ulong address,
 target_phys_addr_t cpu_get_phys_page_debug(CPUState *env, target_ulong addr)
 {
     uint32_t phys_addr;
+    target_ulong page_size;
     int prot;
     int ret;
 
-    ret = get_phys_addr(env, addr, 0, 0, &phys_addr, &prot);
+    ret = get_phys_addr(env, addr, 0, 0, &phys_addr, &prot, &page_size);
 
     if (ret != 0)
         return -1;
@@ -1406,18 +1423,7 @@ void HELPER(set_cp15)(CPUState *env, uint32_t insn, uint32_t val)
             tlb_flush(env, 0);
             break;
         case 1: /* Invalidate single TLB entry.  */
-#if 0
-            /* ??? This is wrong for large pages and sections.  */
-            /* As an ugly hack to make linux work we always flush a 4K
-               pages.  */
-            val &= 0xfffff000;
-            tlb_flush_page(env, val);
-            tlb_flush_page(env, val + 0x400);
-            tlb_flush_page(env, val + 0x800);
-            tlb_flush_page(env, val + 0xc00);
-#else
-            tlb_flush(env, 1);
-#endif
+            tlb_flush_page(env, val & TARGET_PAGE_MASK);
             break;
         case 2: /* Invalidate on ASID.  */
             tlb_flush(env, val == 0);

diff --git a/target-cris/helper.c b/target-cris/helper.c
@@ -100,8 +100,9 @@ int cpu_cris_handle_mmu_fault (CPUState *env, target_ulong address, int rw,
 		 */
 		phy = res.phy & ~0x80000000;
 		prot = res.prot;
-		r = tlb_set_page(env, address & TARGET_PAGE_MASK,
-				 phy, prot, mmu_idx, is_softmmu);
+		tlb_set_page(env, address & TARGET_PAGE_MASK, phy,
+                             prot | PAGE_EXEC, mmu_idx, TARGET_PAGE_SIZE);
+                r = 0;
 	}
 	if (r > 0)
 		D_LOG("%s returns %d irqreq=%x addr=%x"

diff --git a/target-i386/helper.c b/target-i386/helper.c
@@ -531,14 +531,13 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
    -1 = cannot handle fault
    0  = nothing more to do
    1  = generate PF fault
-   2  = soft MMU activation required for this block
 */
 int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
                              int is_write1, int mmu_idx, int is_softmmu)
 {
     uint64_t ptep, pte;
     target_ulong pde_addr, pte_addr;
-    int error_code, is_dirty, prot, page_size, ret, is_write, is_user;
+    int error_code, is_dirty, prot, page_size, is_write, is_user;
     target_phys_addr_t paddr;
     uint32_t page_offset;
     target_ulong vaddr, virt_addr;
@@ -799,8 +798,8 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
     paddr = (pte & TARGET_PAGE_MASK) + page_offset;
     vaddr = virt_addr + page_offset;
 
-    ret = tlb_set_page_exec(env, vaddr, paddr, prot, mmu_idx, is_softmmu);
-    return ret;
+    tlb_set_page(env, vaddr, paddr, prot, mmu_idx, page_size);
+    return 0;
  do_fault_protect:
     error_code = PG_ERROR_P_MASK;
  do_fault:

diff --git a/target-m68k/helper.c b/target-m68k/helper.c
@@ -368,8 +368,9 @@ int cpu_m68k_handle_mmu_fault (CPUState *env, target_ulong address, int rw,
     int prot;
 
     address &= TARGET_PAGE_MASK;
-    prot = PAGE_READ | PAGE_WRITE;
-    return tlb_set_page(env, address, address, prot, mmu_idx, is_softmmu);
+    prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
+    tlb_set_page(env, address, address, prot, mmu_idx, TARGET_PAGE_SIZE);
+    return 0;
 }
 
 /* Notify CPU of a pending interrupt.  Prioritization and vectoring should

diff --git a/target-microblaze/helper.c b/target-microblaze/helper.c
@@ -76,8 +76,8 @@ int cpu_mb_handle_mmu_fault (CPUState *env, target_ulong address, int rw,
 
             DMMU(qemu_log("MMU map mmu=%d v=%x p=%x prot=%x\n",
                      mmu_idx, vaddr, paddr, lu.prot));
-            r = tlb_set_page(env, vaddr,
-                             paddr, lu.prot, mmu_idx, is_softmmu);
+            tlb_set_page(env, vaddr, paddr, lu.prot, mmu_idx, TARGET_PAGE_SIZE);
+            r = 0;
         } else {
             env->sregs[SR_EAR] = address;
             DMMU(qemu_log("mmu=%d miss v=%x\n", mmu_idx, address));
@@ -107,7 +107,8 @@ int cpu_mb_handle_mmu_fault (CPUState *env, target_ulong address, int rw,
         /* MMU disabled or not available.  */
         address &= TARGET_PAGE_MASK;
         prot = PAGE_BITS;
-        r = tlb_set_page(env, address, address, prot, mmu_idx, is_softmmu);
+        tlb_set_page(env, address, address, prot, mmu_idx, TARGET_PAGE_SIZE);
+        r = 0;
     }
     return r;
 }

diff --git a/target-mips/helper.c b/target-mips/helper.c
@@ -296,9 +296,10 @@ int cpu_mips_handle_mmu_fault (CPUState *env, target_ulong address, int rw,
     qemu_log("%s address=" TARGET_FMT_lx " ret %d physical " TARGET_FMT_plx " prot %d\n",
               __func__, address, ret, physical, prot);
     if (ret == TLBRET_MATCH) {
-       ret = tlb_set_page(env, address & TARGET_PAGE_MASK,
-                          physical & TARGET_PAGE_MASK, prot,
-                          mmu_idx, is_softmmu);
+       tlb_set_page(env, address & TARGET_PAGE_MASK,
+                    physical & TARGET_PAGE_MASK, prot | PAGE_EXEC,
+                    mmu_idx, TARGET_PAGE_SIZE);
+       ret = 0;
     } else if (ret < 0)
 #endif
     {