Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/rmind-uvmplock]: src/sys/arch Apply renovated patch to significantly red...
details: https://anonhg.NetBSD.org/src/rev/69d9112ddaaa
branches: rmind-uvmplock
changeset: 753056:69d9112ddaaa
user: rmind <rmind%NetBSD.org@localhost>
date: Mon Apr 26 02:43:34 2010 +0000
description:
Apply renovated patch to significantly reduce TLB shootdowns in x86 pmap,
also provide TLBSTATS option to measure and track TLB shootdowns. Details:
http://mail-index.netbsd.org/port-i386/2009/01/11/msg001018.html
Patch from Andrew Doran, proposed on tech-x86 [sic], in January 2009.
XXX: amd64 and xen are not yet; work in progress.
diffstat:
sys/arch/i386/i386/genassym.cf | 15 +-
sys/arch/i386/i386/vector.S | 139 +----
sys/arch/x86/include/cpu.h | 4 +-
sys/arch/x86/include/i82489var.h | 9 +-
sys/arch/x86/include/pmap.h | 56 +-
sys/arch/x86/x86/cpu.c | 12 +-
sys/arch/x86/x86/lapic.c | 10 +-
sys/arch/x86/x86/pmap.c | 856 ++++++++++++++++++++++----------------
8 files changed, 611 insertions(+), 490 deletions(-)
diffs (truncated from 1813 to 300 lines):
diff -r f82b8da03b38 -r 69d9112ddaaa sys/arch/i386/i386/genassym.cf
--- a/sys/arch/i386/i386/genassym.cf Mon Apr 26 02:20:59 2010 +0000
+++ b/sys/arch/i386/i386/genassym.cf Mon Apr 26 02:43:34 2010 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.85 2010/02/22 23:52:17 jym Exp $
+# $NetBSD: genassym.cf,v 1.85.2.1 2010/04/26 02:43:34 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -160,6 +160,7 @@
define PG_V PG_V
define PG_KW PG_KW
define PG_KR PG_KR
+define PG_G PG_G
define PGEX_U PGEX_U
define L2_SLOT_KERNBASE pl2_pi(KERNBASE)
@@ -288,7 +289,6 @@
define CPU_INFO_SELF offsetof(struct cpu_info, ci_self)
define CPU_INFO_RESCHED offsetof(struct cpu_info, ci_want_resched)
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
-define CPU_INFO_PMAP_CPU offsetof(struct cpu_info, ci_pmap_cpu)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
define TLBSTATE_LAZY TLBSTATE_LAZY
@@ -426,12 +426,11 @@
define RW_READER RW_READER
define RW_WRITER RW_WRITER
-define MB_POINTER offsetof(struct pmap_mbox, mb_pointer)
-define MB_GLOBAL offsetof(struct pmap_mbox, mb_global)
-define MB_ADDR1 offsetof(struct pmap_mbox, mb_addr1)
-define MB_ADDR2 offsetof(struct pmap_mbox, mb_addr2)
-define MB_HEAD offsetof(struct pmap_mbox, mb_head)
-define MB_TAIL offsetof(struct pmap_mbox, mb_tail)
+define TM_PENDING offsetof(struct pmap_tlb_mailbox, tm_pending)
+define TP_COUNT offsetof(struct pmap_tlb_packet, tp_count)
+define TP_VA offsetof(struct pmap_tlb_packet, tp_va)
+define TP_USERMASK offsetof(struct pmap_tlb_packet, tp_usermask)
+define TP_PTE offsetof(struct pmap_tlb_packet, tp_pte)
define PM_CPUS offsetof(struct pmap, pm_cpus)
diff -r f82b8da03b38 -r 69d9112ddaaa sys/arch/i386/i386/vector.S
--- a/sys/arch/i386/i386/vector.S Mon Apr 26 02:20:59 2010 +0000
+++ b/sys/arch/i386/i386/vector.S Mon Apr 26 02:43:34 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vector.S,v 1.53 2010/02/22 06:42:14 darran Exp $ */
+/* $NetBSD: vector.S,v 1.53.2.1 2010/04/26 02:43:34 rmind Exp $ */
/*
* Copyright 2002 (c) Wasabi Systems, Inc.
@@ -65,7 +65,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.53 2010/02/22 06:42:14 darran Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.53.2.1 2010/04/26 02:43:34 rmind Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -186,10 +186,10 @@
IDTVEC_END(resume_lapic_ipi)
/*
- * Multicast TLB shootdown handler for !kernel_pmap.
+ * TLB shootdown handler.
*/
-IDTVEC(intr_lapic_tlb_mcast)
- /* Save state. */
+IDTVEC(intr_lapic_tlb)
+ /* Save state and ack the interrupt. */
pushl %eax
pushl %ebx
pushl %ecx
@@ -198,45 +198,44 @@
pushl %fs
movl $GSEL(GDATA_SEL, SEL_KPL), %eax
movl $GSEL(GCPU_SEL, SEL_KPL), %edx
- movl %eax, %ds
- movl %edx, %fs
- /* Count it. */
- addl $1, CPUVAR(TLB_EVCNT)+EV_COUNT
- adcl $0, CPUVAR(TLB_EVCNT)+EV_COUNT+4
+ mov %ax, %ds
+ mov %dx, %fs
+ movl $0, _C_LABEL(local_apic)+LAPIC_EOI
+
/* Find out what we need to invalidate. */
- movl CPUVAR(PMAP_CPU), %ecx
- movl MB_ADDR1(%ecx), %eax
- movl MB_ADDR2(%ecx), %edx
- xorl %ebx, %ebx
- xchgl MB_POINTER(%ecx), %ebx
- movl $0, _C_LABEL(local_apic)+LAPIC_EOI
- cmpl $-1, %eax
+ leal _C_LABEL(pmap_tlb_packet), %ebx
+ movswl TP_COUNT(%ebx), %ecx
+ cmpl $-1, %ecx
je 4f
+ leal TP_VA(%ebx), %edx
1:
/* Invalidate a single page or a range of pages. */
+ movl (%edx), %eax
invlpg (%eax)
- addl $PAGE_SIZE, %eax
- cmpl %edx, %eax
- jb 1b
+ addl $4, %edx
+ decl %ecx
+ jg 1b
2:
- /* Ack the request. */
- lock
- incl (%ebx)
/*
- * Check the current TLB state. If we don't want further
+ * Check the current TLB state. If we do not want further
* invalidations for this pmap, then take the CPU out of
* the pmap's bitmask.
*/
+ movl CPUVAR(CPUMASK), %eax
cmpl $TLBSTATE_LAZY, CPUVAR(TLBSTATE)
jne 3f
+ testl %eax, TP_USERMASK(%ebx)
+ jz 3f
movl CPUVAR(PMAP), %edx
- movl CPUVAR(CPUMASK), %ecx
+ movl %eax, %ecx
notl %ecx
lock
andl %ecx, PM_CPUS(%edx)
movl $TLBSTATE_STALE, CPUVAR(TLBSTATE)
3:
- /* Restore state and return. */
+ /* Ack the request, restore state & return. */
+ lock
+ xorl %eax, _C_LABEL(pmap_tlb_mailbox)+TM_PENDING
popl %fs
popl %ds
popl %edx
@@ -245,11 +244,18 @@
popl %eax
iret
4:
+ /* Invalidate whole address space: */
+ testw $PG_G, TP_PTE(%ebx)
+ jnz 5f
/*
- * Get the emap generation number. Invalidate user TLB entries.
- * Perform emap update, pass the generation number. Note that
- * caller-save registers might be modified (all saved in the
- * beginning). Only %ebx value is used by 2b context.
+ * a) Invalidating user TLB entries only.
+ *
+ * - Get the emap generation number.
+ * - Invalidate TLB entries.
+ * - Perform emap update, pass the generation number.
+ *
+ * Note that caller-save registers might be modified (all saved in the
+ * beginning). Only %ebx value must be preserved for the 2b context.
*/
call _C_LABEL(uvm_emap_gen_return)
movl %eax, %edx
@@ -259,83 +265,24 @@
call _C_LABEL(uvm_emap_update)
addl $4, %esp
jmp 2b
-IDTVEC_END(intr_lapic_tlb_mcast)
-
-/*
- * Broadcast TLB shootdown handler for kernel_pmap.
- */
-IDTVEC(intr_lapic_tlb_bcast)
- /* Save state and ack the interrupt. */
- pushl %eax
- pushl %ebx
- pushl %ecx
- pushl %edx
- pushl %ds
- pushl %fs
- movl $GSEL(GDATA_SEL, SEL_KPL), %eax
- movl $GSEL(GCPU_SEL, SEL_KPL), %edx
- movl %eax, %ds
- movl %edx, %fs
- /* Find out what we need to invalidate. */
- movl %ss:_C_LABEL(pmap_mbox)+MB_ADDR1, %eax
- movl %ss:_C_LABEL(pmap_mbox)+MB_ADDR2, %edx
- movl %ss:_C_LABEL(pmap_mbox)+MB_GLOBAL, %ebx
- movl $0, %ss:_C_LABEL(local_apic)+LAPIC_EOI
- cmpl $-1, %eax
- je,pn 3f
-1:
- /* Invalidate a single page or a range of pages. */
- invlpg %ss:(%eax)
- addl $PAGE_SIZE, %eax
- cmpl %edx, %eax
- jb 1b
-2:
- /* Ack the request, restore state & return. */
- lock
- incl %ss:_C_LABEL(pmap_mbox)+MB_TAIL
- popl %fs
- popl %ds
- popl %edx
- popl %ecx
- popl %ebx
- popl %eax
- iret
-3:
- testl %ebx, %ebx
- jz 4f
+5:
/*
- * If we have been asked to invalidate the entire TLB we arrive here.
- * Get the emap generation before flush, and use it after for update.
- * Note that caller-save registers might be modified, though no
- * registers need to be preserved for 2b context.
+ * b) Invalidating user and kernel TLB entries.
+ *
+ * See notes above.
*/
call _C_LABEL(uvm_emap_gen_return)
- movl %eax, %ebx
+ movl %eax, %ecx
movl %cr4, %eax
movl %eax, %edx
andl $~CR4_PGE, %edx
movl %edx, %cr4
movl %eax, %cr4
- pushl %ebx
+ pushl %ecx
call _C_LABEL(uvm_emap_update)
addl $4, %esp
jmp 2b
-4:
- /*
- * Get the emap generation number. Invalidate user TLB entries.
- * Perform emap update, pass the generation number. Note that
- * caller-save registers might be modified, though no registers
- * need to be preserved for 2b context.
- */
- call _C_LABEL(uvm_emap_gen_return)
- movl %eax, %ebx
- movl %cr3, %eax
- movl %eax, %cr3
- pushl %ebx
- call _C_LABEL(uvm_emap_update)
- addl $4, %esp
- jmp 2b
-IDTVEC_END(intr_lapic_tlb_bcast)
+IDTVEC_END(intr_lapic_tlb)
#if defined(DDB)
IDTVEC(intrddbipi)
diff -r f82b8da03b38 -r 69d9112ddaaa sys/arch/x86/include/cpu.h
--- a/sys/arch/x86/include/cpu.h Mon Apr 26 02:20:59 2010 +0000
+++ b/sys/arch/x86/include/cpu.h Mon Apr 26 02:43:34 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.20 2010/01/18 16:40:17 rmind Exp $ */
+/* $NetBSD: cpu.h,v 1.20.4.1 2010/04/26 02:43:34 rmind Exp $ */
/*-
* Copyright (c) 1990 The Regents of the University of California.
@@ -88,7 +88,6 @@
*/
struct cpu_info *ci_next; /* next cpu */
struct lwp *ci_curlwp; /* current owner of the processor */
- struct pmap_cpu *ci_pmap_cpu; /* per-CPU pmap data */
struct lwp *ci_fpcurlwp; /* current owner of the FPU */
int ci_fpsaving; /* save in progress */
int ci_fpused; /* XEN: FPU was used by curlwp */
@@ -114,6 +113,7 @@
int ci_curldt; /* current LDT descriptor */
int ci_nintrhand; /* number of H/W interrupt handlers */
uint64_t ci_scratch;
+ uintptr_t ci_pmap_data[128 / sizeof(uintptr_t)];
#ifdef XEN
struct iplsource *ci_isources[NIPL];
diff -r f82b8da03b38 -r 69d9112ddaaa sys/arch/x86/include/i82489var.h
--- a/sys/arch/x86/include/i82489var.h Mon Apr 26 02:20:59 2010 +0000
+++ b/sys/arch/x86/include/i82489var.h Mon Apr 26 02:43:34 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: i82489var.h,v 1.12 2008/04/28 20:23:40 martin Exp $ */
+/* $NetBSD: i82489var.h,v 1.12.22.1 2010/04/26 02:43:34 rmind Exp $ */
/*-
* Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -76,11 +76,8 @@
extern void Xresume_lapic_ipi(void);
#define LAPIC_IPI_VECTOR 0xe0
-extern void Xintr_lapic_tlb_bcast(void);
-#define LAPIC_TLB_BCAST_VECTOR 0xe1
-
-extern void Xintr_lapic_tlb_mcast(void);
-#define LAPIC_TLB_MCAST_VECTOR 0xe2
+extern void Xintr_lapic_tlb(void);
+#define LAPIC_TLB_VECTOR 0xe1
Home |
Main Index |
Thread Index |
Old Index