Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/rmind-uvmplock]: src/sys/arch/x86 Split x86 TLB shootdown code into a se...
details: https://anonhg.NetBSD.org/src/rev/2cfcdae19d1e
branches: rmind-uvmplock
changeset: 753059:2cfcdae19d1e
user: rmind <rmind%NetBSD.org@localhost>
date: Wed May 26 04:55:23 2010 +0000
description:
Split x86 TLB shootdown code into a separate file.
Code part is under TNF license, as per pmap.c 1.105.2.4 revision.
diffstat:
sys/arch/x86/conf/files.x86 | 3 +-
sys/arch/x86/include/pmap.h | 7 +-
sys/arch/x86/x86/pmap.c | 282 +-------------------------------------
sys/arch/x86/x86/pmap_tlb.c | 323 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 335 insertions(+), 280 deletions(-)
diffs (truncated from 693 to 300 lines):
diff -r be9a0e9a8fb6 -r 2cfcdae19d1e sys/arch/x86/conf/files.x86
--- a/sys/arch/x86/conf/files.x86 Mon Apr 26 17:06:21 2010 +0000
+++ b/sys/arch/x86/conf/files.x86 Wed May 26 04:55:23 2010 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: files.x86,v 1.54 2009/10/05 23:59:31 rmind Exp $
+# $NetBSD: files.x86,v 1.54.4.1 2010/05/26 04:55:23 rmind Exp $
# options for MP configuration through the MP spec
defflag opt_mpbios.h MPBIOS MPVERBOSE MPDEBUG MPBIOS_SCANPCI
@@ -65,6 +65,7 @@
file arch/x86/x86/patch.c
file arch/x86/x86/platform.c
file arch/x86/x86/pmap.c
+file arch/x86/x86/pmap_tlb.c
file arch/x86/x86/sys_machdep.c
file arch/x86/x86/syscall.c
file arch/x86/x86/vm_machdep.c
diff -r be9a0e9a8fb6 -r 2cfcdae19d1e sys/arch/x86/include/pmap.h
--- a/sys/arch/x86/include/pmap.h Mon Apr 26 17:06:21 2010 +0000
+++ b/sys/arch/x86/include/pmap.h Wed May 26 04:55:23 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.29.2.3 2010/04/26 04:48:49 rmind Exp $ */
+/* $NetBSD: pmap.h,v 1.29.2.4 2010/05/26 04:55:23 rmind Exp $ */
/*
*
@@ -251,8 +251,9 @@
TLBSHOOT__MAX,
} tlbwhy_t;
-void pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, tlbwhy_t);
-void pmap_tlb_shootnow(void);
+void pmap_tlb_init(void);
+void pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, tlbwhy_t);
+void pmap_tlb_shootnow(void);
#define __HAVE_PMAP_EMAP
diff -r be9a0e9a8fb6 -r 2cfcdae19d1e sys/arch/x86/x86/pmap.c
--- a/sys/arch/x86/x86/pmap.c Mon Apr 26 17:06:21 2010 +0000
+++ b/sys/arch/x86/x86/pmap.c Wed May 26 04:55:23 2010 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.105.2.5 2010/04/26 04:48:49 rmind Exp $ */
+/* $NetBSD: pmap.c,v 1.105.2.6 2010/05/26 04:55:23 rmind Exp $ */
/*-
* Copyright (c) 2008, 2010 The NetBSD Foundation, Inc.
@@ -177,7 +177,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.5 2010/04/26 04:48:49 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105.2.6 2010/05/26 04:55:23 rmind Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -321,15 +321,6 @@
* - pmaps_lock
* this lock protects the list of active pmaps (headed by "pmaps").
* we lock it when adding or removing pmaps from this list.
- *
- * tlb shootdown
- *
- * tlb shootdowns are hard interrupts that operate outside the spl
- * framework: they don't need to be blocked provided that the pmap module
- * gets the order of events correct. the calls are made by poking the
- * lapic directly. the stub to handle the interrupts is short and does
- * one of the following: invalidate a set of pages, all user tlb entries
- * or the entire tlb.
*/
const vaddr_t ptp_masks[] = PTP_MASK_INITIALIZER;
@@ -372,41 +363,6 @@
struct evcnt pmap_ldt_evcnt;
/*
- * TLB shootdown state.
- */
-struct evcnt pmap_tlb_evcnt __aligned(64);
-struct pmap_tlb_packet pmap_tlb_packet __aligned(64);
-struct pmap_tlb_mailbox pmap_tlb_mailbox __aligned(64);
-
-/*
- * TLB shootdown statistics.
- */
-
-#ifdef TLBSTATS
-static struct evcnt tlbstat_local[TLBSHOOT__MAX];
-static struct evcnt tlbstat_remote[TLBSHOOT__MAX];
-static struct evcnt tlbstat_kernel[TLBSHOOT__MAX];
-static struct evcnt tlbstat_single_req;
-static struct evcnt tlbstat_single_issue;
-static const char *tlbstat_name[] = {
- "APTE",
- "KENTER",
- "KREMOVE",
- "FREE_PTP1",
- "FREE_PTP2",
- "REMOVE_PTE",
- "REMOVE_PTES",
- "SYNC_PV1",
- "SYNC_PV2",
- "WRITE_PROTECT",
- "ENTER",
- "UPDATE",
- "BUS_DMA",
- "BUS_SPACE",
-};
-#endif
-
-/*
* global data structures
*/
@@ -1761,32 +1717,14 @@
mutex_init(&pv_hash_locks[i].lock, MUTEX_NODEBUG, IPL_VM);
}
-#ifdef TLBSTATS
- for (i = 0; i < TLBSHOOT__MAX; i++) {
- evcnt_attach_dynamic(&tlbstat_local[i], EVCNT_TYPE_MISC,
- NULL, "tlbshoot local", tlbstat_name[i]);
- }
- for (i = 0; i < TLBSHOOT__MAX; i++) {
- evcnt_attach_dynamic(&tlbstat_remote[i], EVCNT_TYPE_MISC,
- NULL, "tlbshoot remote", tlbstat_name[i]);
- }
- for (i = 0; i < TLBSHOOT__MAX; i++) {
- evcnt_attach_dynamic(&tlbstat_kernel[i], EVCNT_TYPE_MISC,
- NULL, "tlbshoot kernel", tlbstat_name[i]);
- }
- evcnt_attach_dynamic(&tlbstat_single_req, EVCNT_TYPE_MISC,
- NULL, "tlbshoot single page", "requests");
- evcnt_attach_dynamic(&tlbstat_single_issue, EVCNT_TYPE_MISC,
- NULL, "tlbshoot single page", "issues");
-#endif
-#if 0 /* XXXrmind */
- evcnt_attach_dynamic(&pmap_tlb_evcnt, EVCNT_TYPE_INTR,
- NULL, "TLB", "shootdown");
+ pmap_tlb_init();
+
evcnt_attach_dynamic(&pmap_iobmp_evcnt, EVCNT_TYPE_MISC,
NULL, "x86", "io bitmap copy");
evcnt_attach_dynamic(&pmap_ldt_evcnt, EVCNT_TYPE_MISC,
NULL, "x86", "ldt sync");
-#endif
+
+
/*
* done: pmap module is up (and ready for business)
*/
@@ -4617,214 +4555,6 @@
}
#endif
-static inline void
-pmap_tlbstat_count(struct pmap *pm, vaddr_t va, tlbwhy_t why)
-{
-#ifdef TLBSTATS
- uint32_t mask;
-
- if (va != (vaddr_t)-1LL) {
- atomic_inc_64(&tlbstat_single_req.ev_count);
- }
- if (pm == pmap_kernel()) {
- atomic_inc_64(&tlbstat_kernel[why].ev_count);
- return;
-
- }
- if (va >= VM_MAXUSER_ADDRESS) {
- mask = pm->pm_cpus | pm->pm_kernel_cpus;
- } else {
- mask = pm->pm_cpus;
- }
- if ((mask & curcpu()->ci_cpumask) != 0) {
- atomic_inc_64(&tlbstat_local[why].ev_count);
- }
- if ((mask & ~curcpu()->ci_cpumask) != 0) {
- atomic_inc_64(&tlbstat_remote[why].ev_count);
- }
-#endif
-}
-
-/*
- * pmap_tlb_shootdown: invalidate a page on all CPUs using pmap 'pm'
- */
-
-__noinline void
-pmap_tlb_shootdown(struct pmap *pm, vaddr_t va, pt_entry_t pte, tlbwhy_t why)
-{
- struct pmap_tlb_packet *tp;
- int s;
-
- KASSERT((pte & PG_G) == 0 || pm == pmap_kernel());
-
- /*
- * If tearing down the pmap, do nothing. We will flush later
- * when we are ready to recycle/destroy it.
- */
- if (__predict_false(curlwp->l_md.md_gc_pmap == pm)) {
- return;
- }
-
- if ((pte & PG_PS) != 0) {
- va &= PG_LGFRAME;
- }
-
- /*
- * Add the shootdown operation to our pending set.
- */
- s = splvm();
- tp = (struct pmap_tlb_packet *)curcpu()->ci_pmap_data;
- tp->tp_pte |= (uint16_t)pte;
- if (tp->tp_count < TP_MAXVA && va != (vaddr_t)-1LL) {
- /* Flush a single page. */
- tp->tp_va[tp->tp_count++] = va;
- } else {
- /* Flush everything. */
- tp->tp_count = (uint16_t)-1;
- }
-
- if (pm == pmap_kernel()) {
- tp->tp_cpumask = cpus_running;
- } else if (va >= VM_MAXUSER_ADDRESS) {
- tp->tp_cpumask |= (pm->pm_cpus | pm->pm_kernel_cpus);
- tp->tp_usermask |= (pm->pm_cpus | pm->pm_kernel_cpus);;
- } else {
- tp->tp_cpumask |= pm->pm_cpus;
- tp->tp_usermask |= pm->pm_cpus;
- }
- pmap_tlbstat_count(pm, va, why);
- splx(s);
-}
-
-/*
- * pmap_tlb_shootnow: process pending TLB shootdowns queued on current CPU.
- *
- * => Must be called with preemption disabled.
- */
-
-__noinline void
-pmap_tlb_shootnow(void)
-{
- struct pmap_tlb_packet *tp;
- struct pmap_tlb_mailbox *tm;
- struct cpu_info *ci, *lci;
- CPU_INFO_ITERATOR cii;
- uint32_t remote;
- uintptr_t gen;
- int s, err, i, count;
-
- KASSERT(kpreempt_disabled());
-
- s = splvm();
- ci = curcpu();
- tp = (struct pmap_tlb_packet *)ci->ci_pmap_data;
- if (tp->tp_count == 0) {
- splx(s);
- return;
- }
- tm = &pmap_tlb_mailbox;
- remote = tp->tp_cpumask & ~ci->ci_cpumask;
- gen = 0; /* XXXgcc */
- if (remote != 0) {
- /*
- * Gain ownership of the shootdown mailbox. We must stay
- * at splvm once we own it or could deadlock against an
- * interrupt on this cpu trying to do the same.
- */
- while (atomic_cas_32(&tm->tm_pending, 0, remote) != 0) {
- splx(s);
- count = SPINLOCK_BACKOFF_MIN;
- while (tm->tm_pending != 0) {
- SPINLOCK_BACKOFF(count);
- }
- s = splvm();
- /* An interrupt might have done it for us. */
- if (tp->tp_count == 0) {
- splx(s);
- return;
- }
- }
-
- /*
- * Start a new generation of updates. Copy our shootdown
- * requests into the global buffer.
- */
- gen = ++tm->tm_gen;
- memcpy(&pmap_tlb_packet, tp, sizeof(*tp));
- pmap_tlb_evcnt.ev_count++;
-
- /*
- * Initiate shootdowns on remote CPUs.
- */
- if (tp->tp_cpumask == cpus_running) {
- err = x86_ipi(LAPIC_TLB_VECTOR, LAPIC_DEST_ALLEXCL,
- LAPIC_DLMODE_FIXED);
- } else {
- err = 0;
- for (CPU_INFO_FOREACH(cii, lci)) {
Home |
Main Index |
Thread Index |
Old Index