Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/rmind-uvmplock]: src/sys/arch Partly rewrite amd64 TLB shutdown handler ...
details: https://anonhg.NetBSD.org/src/rev/d14c96527287
branches: rmind-uvmplock
changeset: 753057:d14c96527287
user: rmind <rmind%NetBSD.org@localhost>
date: Mon Apr 26 04:48:49 2010 +0000
description:
Partly rewrite amd64 TLB shutdown handler for the changes in x86 pmap.
At this point, branch seems to pass preliminar stress tests on amd64.
diffstat:
sys/arch/amd64/amd64/genassym.cf | 16 ++--
sys/arch/amd64/amd64/vector.S | 135 +++++++++++---------------------------
sys/arch/x86/include/pmap.h | 10 +-
sys/arch/x86/x86/pmap.c | 12 +-
4 files changed, 58 insertions(+), 115 deletions(-)
diffs (truncated from 315 to 300 lines):
diff -r 69d9112ddaaa -r d14c96527287 sys/arch/amd64/amd64/genassym.cf
--- a/sys/arch/amd64/amd64/genassym.cf Mon Apr 26 02:43:34 2010 +0000
+++ b/sys/arch/amd64/amd64/genassym.cf Mon Apr 26 04:48:49 2010 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.43 2009/11/27 03:23:04 rmind Exp $
+# $NetBSD: genassym.cf,v 1.43.4.1 2010/04/26 04:48:49 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -136,6 +136,8 @@
define KERNTEXTOFF_LO KERNTEXTOFF_LO
define KERNTEXTOFF KERNTEXTOFF
+define PG_G PG_G
+
define NBPG NBPG
define L4_SLOT_KERNBASE L4_SLOT_KERNBASE
@@ -225,7 +227,6 @@
define CPU_INFO_SELF offsetof(struct cpu_info, ci_self)
define CPU_INFO_RESCHED offsetof(struct cpu_info, ci_want_resched)
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
-define CPU_INFO_PMAP_CPU offsetof(struct cpu_info, ci_pmap_cpu)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
define TLBSTATE_LAZY TLBSTATE_LAZY
@@ -349,12 +350,11 @@
define RW_READER RW_READER
define RW_WRITER RW_WRITER
-define MB_POINTER offsetof(struct pmap_mbox, mb_pointer)
-define MB_GLOBAL offsetof(struct pmap_mbox, mb_global)
-define MB_ADDR1 offsetof(struct pmap_mbox, mb_addr1)
-define MB_ADDR2 offsetof(struct pmap_mbox, mb_addr2)
-define MB_HEAD offsetof(struct pmap_mbox, mb_head)
-define MB_TAIL offsetof(struct pmap_mbox, mb_tail)
+define TM_PENDING offsetof(struct pmap_tlb_mailbox, tm_pending)
+define TP_COUNT offsetof(struct pmap_tlb_packet, tp_count)
+define TP_VA offsetof(struct pmap_tlb_packet, tp_va)
+define TP_USERMASK offsetof(struct pmap_tlb_packet, tp_usermask)
+define TP_PTE offsetof(struct pmap_tlb_packet, tp_pte)
define PM_CPUS offsetof(struct pmap, pm_cpus)
diff -r 69d9112ddaaa -r d14c96527287 sys/arch/amd64/amd64/vector.S
--- a/sys/arch/amd64/amd64/vector.S Mon Apr 26 02:43:34 2010 +0000
+++ b/sys/arch/amd64/amd64/vector.S Mon Apr 26 04:48:49 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vector.S,v 1.33 2010/02/23 06:27:40 cegger Exp $ */
+/* $NetBSD: vector.S,v 1.33.2.1 2010/04/26 04:48:49 rmind Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -480,13 +480,10 @@
#ifndef XEN
/*
- * Multicast TLB shootdown handler for !kernel_pmap.
+ * TLB shootdown handler.
*/
-IDTVEC(intr_lapic_tlb_mcast)
- /*
- * Save state, ack interrupt and count it. Count locally
- * so other CPUs don't have to touch the counter's line.
- */
+IDTVEC(intr_lapic_tlb)
+ /* Save state and ack the interrupt. */
testq $SEL_UPL,8(%rsp)
jz 0f
swapgs
@@ -499,41 +496,42 @@
pushq %rcx
pushq %r8
pushq %r9
- incq CPUVAR(TLB_EVCNT)+EV_COUNT
- /* Find out what needs to be invalidated and unlock the mailbox. */
- movq CPUVAR(PMAP_CPU),%rcx
- movq MB_ADDR1(%rcx), %rax
- movq MB_ADDR2(%rcx), %rdx
- xorq %rbx, %rbx
- xchgq MB_POINTER(%rcx), %rbx
movl $0, _C_LABEL(local_apic)+LAPIC_EOI
- cmpq $-1, %rax
+
+ /* Find out what we need to invalidate. */
+ leaq _C_LABEL(pmap_tlb_packet)(%rip), %rbx
+ movswq TP_COUNT(%rbx), %rcx
+ cmpq $-1, %rcx
je 5f
+ leaq TP_VA(%rbx), %rdx
1:
/* Invalidate a single page or a range of pages. */
+ movq (%rdx), %rax
invlpg (%rax)
- addq $PAGE_SIZE, %rax
- cmpq %rdx, %rax
- jb 1b
+ addq $8, %rdx
+ decq %rcx
+ jg 1b
2:
- /* Ack the request. */
- lock
- incq (%rbx)
/*
- * Check the current TLB state. If we don't want further
+ * Check the current TLB state. If we do not want further
* invalidations for this pmap, then take the CPU out of
* the pmap's bitmask.
*/
+ movl CPUVAR(CPUMASK), %eax
cmpl $TLBSTATE_LAZY, CPUVAR(TLBSTATE)
jne 3f
- movq CPUVAR(PMAP), %rdx
- movl CPUVAR(CPUMASK), %ecx
+ testl %eax, TP_USERMASK(%rbx)
+ jz 3f
+ movl CPUVAR(PMAP), %edx
+ movl %eax, %ecx
notl %ecx
lock
- andl %ecx, PM_CPUS(%rdx)
+ andl %ecx, PM_CPUS(%edx)
movl $TLBSTATE_STALE, CPUVAR(TLBSTATE)
3:
- /* Restore state and return. */
+ /* Ack the request, restore state & return. */
+ lock
+ xorl %eax, _C_LABEL(pmap_tlb_mailbox)+TM_PENDING
popq %r9
popq %r8
popq %rcx
@@ -548,11 +546,18 @@
4:
iretq
5:
+ /* Invalidate whole address space: */
+ testw $PG_G, TP_PTE(%rbx)
+ jnz 6f
/*
- * Get the emap generation number. Invalidate user TLB entries.
- * Perform emap update, pass the generation number. Note that
- * caller-save registers might be modified (all saved in the
- * beginning). Only %rbx value is used by 2b context.
+ * a) Invalidating user TLB entries only.
+ *
+ * - Get the emap generation number.
+ * - Invalidate TLB entries.
+ * - Perform emap update, pass the generation number.
+ *
+ * Note that caller-save registers might be modified (all saved in the
+ * beginning). Only %rbx value must be preserved for the 2b context.
*/
callq _C_LABEL(uvm_emap_gen_return)
movq %rax, %rdi
@@ -560,60 +565,11 @@
movq %rax, %cr3
callq _C_LABEL(uvm_emap_update)
jmp 2b
-
-/*
- * Broadcast TLB shootdown handler for kernel_pmap.
- */
-IDTVEC(intr_lapic_tlb_bcast)
- testq $SEL_UPL,8(%rsp)
- jz 0f
- swapgs
-0:
- /* Save state. */
- pushq %rax
- pushq %rdi
- pushq %rsi
- pushq %rdx
- pushq %rcx
- pushq %r8
- pushq %r9
- /* Find out what needs to be invalidated. */
- movq _C_LABEL(pmap_mbox)+MB_ADDR1, %rax
- movq _C_LABEL(pmap_mbox)+MB_ADDR2, %rdx
- movq _C_LABEL(pmap_mbox)+MB_GLOBAL, %rdi
- movl $0, _C_LABEL(local_apic)+LAPIC_EOI
- cmpq $-1, %rax
- je,pn 3f
-1:
- /* Invalidate a single page or a range of pages. */
- invlpg (%rax)
- addq $PAGE_SIZE, %rax
- cmpq %rdx, %rax
- jb 1b
-2:
- /* Notify waiter of completion, restore state & return */
- lock
- incq _C_LABEL(pmap_mbox)+MB_TAIL
- popq %r9
- popq %r8
- popq %rcx
- popq %rdx
- popq %rsi
- popq %rdi
- popq %rax
- testq $SEL_UPL, 8(%rsp)
- jz 5f
- swapgs
-5:
- iretq
-3:
- testq %rdi, %rdi
- jz 4f
+6:
/*
- * If we have been asked to invalidate the entire TLB we arrive here.
- * Get the emap generation before flush, and use it after for update.
- * Note that caller-save registers might be modified, though no
- * registers need to be preserved for 2b context.
+ * b) Invalidating user and kernel TLB entries.
+ *
+ * See notes above.
*/
callq _C_LABEL(uvm_emap_gen_return)
movq %rax, %rdi
@@ -624,19 +580,6 @@
movq %rax, %cr4
callq _C_LABEL(uvm_emap_update)
jmp 2b
-4:
- /*
- * Get the emap generation number. Invalidate user TLB entries.
- * Perform emap update, pass the generation number. Note that
- * caller-save registers might be modified, though no registers
- * need to be preserved for 2b context.
- */
- callq _C_LABEL(uvm_emap_gen_return)
- movq %rax, %rdi
- movq %cr3, %rax
- movq %rax, %cr3
- callq _C_LABEL(uvm_emap_update)
- jmp 2b
#endif /* !XEN */
diff -r 69d9112ddaaa -r d14c96527287 sys/arch/x86/include/pmap.h
--- a/sys/arch/x86/include/pmap.h Mon Apr 26 02:43:34 2010 +0000
+++ b/sys/arch/x86/include/pmap.h Mon Apr 26 04:48:49 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.29.2.2 2010/04/26 02:43:34 rmind Exp $ */
+/* $NetBSD: pmap.h,v 1.29.2.3 2010/04/26 04:48:49 rmind Exp $ */
/*
*
@@ -454,10 +454,10 @@
#define TP_MAXVA 6 /* no more than N seperate invlpg */
struct pmap_tlb_mailbox {
- uintptr_t tm_pending;
- uintptr_t tm_gen;
- uintptr_t tm_usergen;
- uintptr_t tm_globalgen;
+ uint32_t tm_pending;
+ uint32_t tm_gen;
+ uint32_t tm_usergen;
+ uint32_t tm_globalgen;
char tm_pad[64 - sizeof(uintptr_t) * 4];
};
diff -r 69d9112ddaaa -r d14c96527287 sys/arch/x86/x86/pmap.c
--- a/sys/arch/x86/x86/pmap.c Mon Apr 26 02:43:34 2010 +0000
+++ b/sys/arch/x86/x86/pmap.c Mon Apr 26 04:48:49 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.105.2.4 2010/04/26 02:43:35 rmind Exp $ */
+/* $NetBSD: pmap.c,v 1.105.2.5 2010/04/26 04:48:49 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -177,7 +177,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.4 2010/04/26 02:43:35 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.5 2010/04/26 04:48:49 rmind Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -1779,14 +1779,14 @@
evcnt_attach_dynamic(&tlbstat_single_issue, EVCNT_TYPE_MISC,
NULL, "tlbshoot single page", "issues");
#endif
-
+#if 0 /* XXXrmind */
evcnt_attach_dynamic(&pmap_tlb_evcnt, EVCNT_TYPE_INTR,
NULL, "TLB", "shootdown");
evcnt_attach_dynamic(&pmap_iobmp_evcnt, EVCNT_TYPE_MISC,
NULL, "x86", "io bitmap copy");
evcnt_attach_dynamic(&pmap_ldt_evcnt, EVCNT_TYPE_MISC,
NULL, "x86", "ldt sync");
-
+#endif
/*
* done: pmap module is up (and ready for business)
*/
@@ -4785,13 +4785,13 @@
Home |
Main Index |
Thread Index |
Old Index