Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/rmind-uvmplock]: src/sys/arch - Fix tlbflushg() to behave like tlbflush(...
details: https://anonhg.NetBSD.org/src/rev/e722a99eed9e
branches: rmind-uvmplock
changeset: 753080:e722a99eed9e
user: rmind <rmind%NetBSD.org@localhost>
date: Thu Mar 17 04:46:28 2011 +0000
description:
- Fix tlbflushg() to behave like tlbflush(), if page global extension (PGE)
is not (yet) enabled. This fixes the issue of stale TLB entry, experienced
early on boot, when PGE is not yet set on primary CPU.
- Rewrite i386/amd64 TLB interrupt handlers in C (only stubs are in assembly),
which simplifies and unifies (under x86) code, plus fixes few bugs.
- cpu_attach: remove assignment to cpus_running, as primary CPU might not be
attached first, which causes reset (and thus missed secondary CPUs).
diffstat:
sys/arch/amd64/amd64/cpufunc.S | 7 +-
sys/arch/amd64/amd64/genassym.cf | 14 +---
sys/arch/amd64/amd64/vector.S | 97 +------------------------
sys/arch/i386/i386/genassym.cf | 13 +---
sys/arch/i386/i386/i386func.S | 11 +-
sys/arch/i386/i386/vector.S | 93 +----------------------
sys/arch/x86/include/pmap.h | 30 +-------
sys/arch/x86/x86/cpu.c | 5 +-
sys/arch/x86/x86/pmap.c | 8 +-
sys/arch/x86/x86/pmap_tlb.c | 146 ++++++++++++++++++++++++++++++--------
10 files changed, 146 insertions(+), 278 deletions(-)
diffs (truncated from 731 to 300 lines):
diff -r e3a32f986c73 -r e722a99eed9e sys/arch/amd64/amd64/cpufunc.S
--- a/sys/arch/amd64/amd64/cpufunc.S Thu Mar 10 08:45:34 2011 +0000
+++ b/sys/arch/amd64/amd64/cpufunc.S Thu Mar 17 04:46:28 2011 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.16.4.1 2011/03/05 20:49:14 rmind Exp $ */
+/* $NetBSD: cpufunc.S,v 1.16.4.2 2011/03/17 04:46:29 rmind Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -136,10 +136,14 @@
* modify the PSE, PGE, or PAE flag."
*
* (the alternatives not quoted above are not an option here.)
+ *
+ * If PGE is not in use, we reload CR3.
*/
#ifndef XEN
ENTRY(tlbflushg)
movq %cr4, %rax
+ testq $CR4_PGE, %rax
+ jz 1f
movq %rax, %rdx
andq $~CR4_PGE, %rdx
movq %rdx, %cr4
@@ -147,6 +151,7 @@
ret
ENTRY(tlbflush)
+1:
movq %cr3, %rax
movq %rax, %cr3
ret
diff -r e3a32f986c73 -r e722a99eed9e sys/arch/amd64/amd64/genassym.cf
--- a/sys/arch/amd64/amd64/genassym.cf Thu Mar 10 08:45:34 2011 +0000
+++ b/sys/arch/amd64/amd64/genassym.cf Thu Mar 17 04:46:28 2011 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.43.4.3 2011/03/05 20:49:15 rmind Exp $
+# $NetBSD: genassym.cf,v 1.43.4.4 2011/03/17 04:46:29 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -136,8 +136,6 @@
define KERNTEXTOFF_LO KERNTEXTOFF_LO
define KERNTEXTOFF KERNTEXTOFF
-define PG_G PG_G
-
define NBPG NBPG
define L4_SLOT_KERNBASE L4_SLOT_KERNBASE
@@ -230,8 +228,6 @@
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
-define TLBSTATE_LAZY TLBSTATE_LAZY
-define TLBSTATE_STALE TLBSTATE_STALE
define CPU_INFO_TLB_EVCNT offsetof(struct cpu_info, ci_tlb_evcnt)
define CPU_INFO_CURLWP offsetof(struct cpu_info, ci_curlwp)
define CPU_INFO_CURLDT offsetof(struct cpu_info, ci_curldt)
@@ -352,14 +348,6 @@
define RW_READER RW_READER
define RW_WRITER RW_WRITER
-define TM_PENDING offsetof(struct pmap_tlb_mailbox, tm_pending)
-define TP_COUNT offsetof(struct pmap_tlb_packet, tp_count)
-define TP_VA offsetof(struct pmap_tlb_packet, tp_va)
-define TP_USERMASK offsetof(struct pmap_tlb_packet, tp_usermask)
-define TP_PTE offsetof(struct pmap_tlb_packet, tp_pte)
-
-define PM_CPUS offsetof(struct pmap, pm_cpus)
-
define EV_COUNT offsetof(struct evcnt, ev_count)
define OPTERON_MSR_PASSCODE OPTERON_MSR_PASSCODE
diff -r e3a32f986c73 -r e722a99eed9e sys/arch/amd64/amd64/vector.S
--- a/sys/arch/amd64/amd64/vector.S Thu Mar 10 08:45:34 2011 +0000
+++ b/sys/arch/amd64/amd64/vector.S Thu Mar 17 04:46:28 2011 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vector.S,v 1.33.2.3 2011/03/05 20:49:15 rmind Exp $ */
+/* $NetBSD: vector.S,v 1.33.2.4 2011/03/17 04:46:29 rmind Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -479,97 +479,12 @@
* TLB shootdown handler.
*/
IDTVEC(intr_lapic_tlb)
- /* Save state and ack the interrupt. */
- testq $SEL_UPL,8(%rsp)
- jz 0f
- swapgs
-0:
- pushq %rax
- pushq %rbx
- pushq %rdi
- pushq %rsi
- pushq %rdx
- pushq %rcx
- pushq %r8
- pushq %r9
+ pushq $0
+ pushq $T_ASTFLT
+ INTRENTRY
movl $0, _C_LABEL(local_apic)+LAPIC_EOI
-
- /* Find out what we need to invalidate. */
- leaq _C_LABEL(pmap_tlb_packet)(%rip), %rbx
- movswq TP_COUNT(%rbx), %rcx
- cmpq $-1, %rcx
- je 5f
- leaq TP_VA(%rbx), %rdx
-1:
- /* Invalidate a single page or a range of pages. */
- movq (%rdx), %rax
- invlpg (%rax)
- addq $8, %rdx
- decq %rcx
- jg 1b
-2:
- /*
- * Check the current TLB state. If we do not want further
- * invalidations for this pmap, then take the CPU out of
- * the pmap's bitmask.
- */
- movl CPUVAR(CPUMASK), %eax
- cmpl $TLBSTATE_LAZY, CPUVAR(TLBSTATE)
- jne 3f
- testl %eax, TP_USERMASK(%rbx)
- jz 3f
- movl CPUVAR(PMAP), %edx
- movl %eax, %ecx
- notl %ecx
- lock
- andl %ecx, PM_CPUS(%edx)
- movl $TLBSTATE_STALE, CPUVAR(TLBSTATE)
-3:
- /* Ack the request, restore state & return. */
- lock
- xorl %eax, _C_LABEL(pmap_tlb_mailbox)+TM_PENDING
- popq %r9
- popq %r8
- popq %rcx
- popq %rdx
- popq %rsi
- popq %rdi
- popq %rbx
- popq %rax
- testq $SEL_UPL, 8(%rsp)
- jz 4f
- swapgs
-4:
- iretq
-5:
- /*
- * Note that caller-save registers might be modified (all saved in the
- * beginning). Only %rbx value must be preserved for the 2f context.
- */
-
- /* Get the emap generation number. */
- callq _C_LABEL(uvm_emap_gen_return)
- movq %rax, %rdi
-
- /* Which entries we are invalidating? */
- testw $PG_G, TP_PTE(%rbx)
- jnz 6f
-
- /* a) Invalidating user TLB entries only. */
- movq %cr3, %rax
- movq %rax, %cr3
- jmp 7f
-6:
- /* b) Invalidating user and kernel TLB entries. */
- movq %cr4, %rax
- movq %rax, %rdx
- andq $~CR4_PGE, %rdx
- movq %rdx, %cr4
- movq %rax, %cr4
-7:
- /* Perform emap update, pass the generation number. */
- callq _C_LABEL(uvm_emap_update)
- jmp 2b
+ callq _C_LABEL(pmap_tlb_intr)
+ INTRFASTEXIT
#endif /* !XEN */
diff -r e3a32f986c73 -r e722a99eed9e sys/arch/i386/i386/genassym.cf
--- a/sys/arch/i386/i386/genassym.cf Thu Mar 10 08:45:34 2011 +0000
+++ b/sys/arch/i386/i386/genassym.cf Thu Mar 17 04:46:28 2011 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.85.2.3 2011/03/05 20:50:39 rmind Exp $
+# $NetBSD: genassym.cf,v 1.85.2.4 2011/03/17 04:46:29 rmind Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -160,7 +160,6 @@
define PG_V PG_V
define PG_KW PG_KW
define PG_KR PG_KR
-define PG_G PG_G
define PGEX_U PGEX_U
define L2_SLOT_KERNBASE pl2_pi(KERNBASE)
@@ -288,8 +287,6 @@
define CPU_INFO_WANT_PMAPLOAD offsetof(struct cpu_info, ci_want_pmapload)
define CPU_INFO_TLBSTATE offsetof(struct cpu_info, ci_tlbstate)
define TLBSTATE_VALID TLBSTATE_VALID
-define TLBSTATE_LAZY TLBSTATE_LAZY
-define TLBSTATE_STALE TLBSTATE_STALE
define CPU_INFO_TLB_EVCNT offsetof(struct cpu_info, ci_tlb_evcnt)
define CPU_INFO_CURLWP offsetof(struct cpu_info, ci_curlwp)
define CPU_INFO_FPCURLWP offsetof(struct cpu_info, ci_fpcurlwp)
@@ -424,14 +421,6 @@
define RW_READER RW_READER
define RW_WRITER RW_WRITER
-define TM_PENDING offsetof(struct pmap_tlb_mailbox, tm_pending)
-define TP_COUNT offsetof(struct pmap_tlb_packet, tp_count)
-define TP_VA offsetof(struct pmap_tlb_packet, tp_va)
-define TP_USERMASK offsetof(struct pmap_tlb_packet, tp_usermask)
-define TP_PTE offsetof(struct pmap_tlb_packet, tp_pte)
-
-define PM_CPUS offsetof(struct pmap, pm_cpus)
-
define EV_COUNT offsetof(struct evcnt, ev_count)
define OPTERON_MSR_PASSCODE OPTERON_MSR_PASSCODE
diff -r e3a32f986c73 -r e722a99eed9e sys/arch/i386/i386/i386func.S
--- a/sys/arch/i386/i386/i386func.S Thu Mar 10 08:45:34 2011 +0000
+++ b/sys/arch/i386/i386/i386func.S Thu Mar 17 04:46:28 2011 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: i386func.S,v 1.15 2008/05/25 15:56:12 chs Exp $ */
+/* $NetBSD: i386func.S,v 1.15.20.1 2011/03/17 04:46:29 rmind Exp $ */
/*-
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: i386func.S,v 1.15 2008/05/25 15:56:12 chs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: i386func.S,v 1.15.20.1 2011/03/17 04:46:29 rmind Exp $");
#include <machine/specialreg.h>
#include <machine/segments.h>
@@ -99,13 +99,12 @@
*
* (the alternatives not quoted above are not an option here.)
*
- * If PGE is not in use, we reload CR3 for the benefit of
- * pre-P6-family processors.
+ * If PGE is not in use, we reload CR3.
*/
ENTRY(tlbflushg)
- testl $CPUID_PGE, _C_LABEL(cpu_feature)
+ movl %cr4, %eax
+ testl $CR4_PGE, %eax
jz 1f
- movl %cr4, %eax
movl %eax, %edx
andl $~CR4_PGE, %edx
movl %edx, %cr4
diff -r e3a32f986c73 -r e722a99eed9e sys/arch/i386/i386/vector.S
--- a/sys/arch/i386/i386/vector.S Thu Mar 10 08:45:34 2011 +0000
+++ b/sys/arch/i386/i386/vector.S Thu Mar 17 04:46:28 2011 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: vector.S,v 1.53.2.3 2011/03/05 20:50:41 rmind Exp $ */
+/* $NetBSD: vector.S,v 1.53.2.4 2011/03/17 04:46:29 rmind Exp $ */
/*
* Copyright 2002 (c) Wasabi Systems, Inc.
@@ -65,7 +65,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.53.2.3 2011/03/05 20:50:41 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.53.2.4 2011/03/17 04:46:29 rmind Exp $");
#include "opt_ddb.h"
#include "opt_multiprocessor.h"
@@ -187,91 +187,12 @@
* TLB shootdown handler.
*/
IDTVEC(intr_lapic_tlb)
- /* Save state and ack the interrupt. */
- pushl %eax
- pushl %ebx
- pushl %ecx
- pushl %edx
- pushl %ds
- pushl %fs
- movl $GSEL(GDATA_SEL, SEL_KPL), %eax
- movl $GSEL(GCPU_SEL, SEL_KPL), %edx
- mov %ax, %ds
- mov %dx, %fs
+ pushl $0
+ pushl $T_ASTFLT
+ INTRENTRY
movl $0, _C_LABEL(local_apic)+LAPIC_EOI
-
Home |
Main Index |
Thread Index |
Old Index