Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/uvm/pmap Allow this to handle H/W tlbs. Some ARM allow ...
details: https://anonhg.NetBSD.org/src/rev/bfb39c0c966f
branches: trunk
changeset: 328300:bfb39c0c966f
user: matt <matt%NetBSD.org@localhost>
date: Sun Mar 30 15:26:15 2014 +0000
description:
Allow this to handle H/W tlbs. Some ARM allow for a cheap way to flush all
entries using an ASID from the TLB. Add support for taking advantage of it.
Most ARMs don't have an easy way to find out what's in the TLB so make
record_asids can just say all ASIDs are in use. Fix some off by 1 errors.
diffstat:
sys/uvm/pmap/pmap_tlb.c | 202 ++++++++++++++++++++++++++++++++++-------------
sys/uvm/pmap/pmap_tlb.h | 6 +-
2 files changed, 148 insertions(+), 60 deletions(-)
diffs (truncated from 457 to 300 lines):
diff -r 7c73be66ca4a -r bfb39c0c966f sys/uvm/pmap/pmap_tlb.c
--- a/sys/uvm/pmap/pmap_tlb.c Sun Mar 30 15:20:54 2014 +0000
+++ b/sys/uvm/pmap/pmap_tlb.c Sun Mar 30 15:26:15 2014 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_tlb.c,v 1.4 2014/03/18 18:20:44 riastradh Exp $ */
+/* $NetBSD: pmap_tlb.c,v 1.5 2014/03/30 15:26:15 matt Exp $ */
/*-
* Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.4 2014/03/18 18:20:44 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.5 2014/03/30 15:26:15 matt Exp $");
/*
* Manages address spaces in a TLB.
@@ -136,7 +136,7 @@
#include <uvm/uvm.h>
-static kmutex_t pmap_tlb0_mutex __cacheline_aligned;
+static kmutex_t pmap_tlb0_lock __cacheline_aligned;
#define IFCONSTANT(x) (__builtin_constant_p((x)) ? (x) : 0)
@@ -145,13 +145,13 @@
.ti_asid_hint = KERNEL_PID + 1,
#ifdef PMAP_TLB_NUM_PIDS
.ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1),
- .ti_asids_free = IFCONSTANT(PMAP_TLB_NUM_PIDS - (KERNEL_PID + 1)),
+ .ti_asids_free = IFCONSTANT(PMAP_TLB_NUM_PIDS - (1 + KERNEL_PID)),
#endif
.ti_asid_bitmap[0] = (2 << KERNEL_PID) - 1,
#ifdef PMAP_TLB_WIRED_UPAGES
.ti_wired = PMAP_TLB_WIRED_UPAGES,
#endif
- .ti_lock = &pmap_tlb0_mutex,
+ .ti_lock = &pmap_tlb0_lock,
.ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais),
#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
.ti_tlbinvop = TLBINV_NOBODY,
@@ -174,6 +174,8 @@
#define __BITMAP_ISSET_P(bm, n) \
(((bm)[(n) / (8*sizeof(bm[0]))] & (1LU << ((n) % (8*sizeof(bm[0]))))) != 0)
+#define TLBINFO_ASID_MARK_UNUSED(ti, asid) \
+ __BITMAP_CLR((ti)->ti_asid_bitmap, (asid))
#define TLBINFO_ASID_MARK_USED(ti, asid) \
__BITMAP_SET((ti)->ti_asid_bitmap, (asid))
#define TLBINFO_ASID_INUSE_P(ti, asid) \
@@ -231,6 +233,25 @@
pai->pai_link.le_prev = NULL; /* tagged as unlinked */
#endif
/*
+ * If the platform has a cheap way to flush ASIDs then free the ASID
+ * back into the pool. On multiprocessor systems, we will flush the
+ * ASID from the TLB when it's allocated. That way we know the flush
+ * was always done in the correct TLB space. On uniprocessor systems,
+ * just do the flush now since we know that it has been used. This has
+ * a bit less overhead. Either way, this will mean that we will only
+ * need to flush all ASIDs if all ASIDs are in use and we need to
+ * allocate a new one.
+ */
+ if (PMAP_TLB_FLUSH_ASID_ON_RESET) {
+#ifndef MULTIPROCESSOR
+ tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
+#endif
+ if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
+ TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid);
+ ti->ti_asids_free++;
+ }
+ }
+ /*
* Note that we don't mark the ASID as not in use in the TLB's ASID
* bitmap (thus it can't be allocated until the ASID space is exhausted
* and therefore reinitialized). We don't want to flush the TLB for
@@ -312,14 +333,17 @@
#endif
#endif /* MULTIPROCESSOR */
KASSERT(ti == &pmap_tlb0_info);
+ KASSERT(ti->ti_lock == &pmap_tlb0_lock);
+ //printf("ti_lock %p ", ti->ti_lock);
mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED);
#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
kcpuset_create(&ti->ti_kcpuset, true);
- kcpuset_set(&ti->ti_kcpuset, cpu_index(curcpu()));
+ kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu()));
#endif
+ //printf("asid ");
if (ti->ti_asid_max == 0) {
ti->ti_asid_max = pmap_md_tlb_asid_max();
- ti->ti_asids_free = ti->ti_asid_max - (KERNEL_PID + 1);
+ ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID;
}
KASSERT(ti->ti_asid_max < sizeof(ti->ti_asid_bitmap)*8);
@@ -336,8 +360,8 @@
TLBINFO_LOCK(ti);
#if PMAP_TLB_MAX > 1
kcpuset_set(ti->ti_kcpuset, cpu_index(ci));
+ cpu_set_tlb_info(ci, ti);
#endif
- cpu_set_tlb_info(ci, ti);
/*
* Do any MD tlb info init.
@@ -372,6 +396,8 @@
pmap_pai_check(ti);
+ ti->ti_evcnt_asid_reinits.ev_count++;
+
/*
* First, clear the ASID bitmap (except for ASID 0 which belongs
* to the kernel).
@@ -427,11 +453,14 @@
word++) {
ti->ti_asid_bitmap[word] = 0;
}
+ ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID;
#if !defined(MULTIPROCESSOR) || defined(PMAP_NEED_TLB_SHOOTDOWN)
} else {
ti->ti_asids_free -= asids_found;
}
#endif /* !MULTIPROCESSOR || PMAP_NEED_TLB_SHOOTDOWN */
+ KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u",
+ ti->ti_asids_free);
break;
}
default:
@@ -465,10 +494,9 @@
}
}
#ifdef DIAGNOSTIC
- size_t free_count = ti->ti_asid_max - pmap_tlb_asid_count(ti);
- if (free_count != ti->ti_asids_free)
- panic("%s: bitmap error: %zu != %u",
- __func__, free_count, ti->ti_asids_free);
+ size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti);
+ KASSERTMSG(free_count == ti->ti_asids_free,
+ "bitmap error: %zu != %u", free_count, ti->ti_asids_free);
#endif
}
@@ -697,11 +725,18 @@
struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+
KASSERT(kpreempt_disabled());
+ UVMHIST_LOG(maphist, " (pm=%#x va=%#x) ti=%#x asid=%#x",
+ pm, va, ti, pai->pai_asid);
+
TLBINFO_LOCK(ti);
if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
pmap_tlb_asid_check();
+ UVMHIST_LOG(maphist, " invalidating %#x asid %#x",
+ va, pai->pai_asid, 0, 0);
tlb_invalidate_addr(va, pai->pai_asid);
pmap_tlb_asid_check();
}
@@ -709,6 +744,7 @@
pm->pm_shootdown_pending = 1;
#endif
TLBINFO_UNLOCK(ti);
+ UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
}
static inline void
@@ -723,55 +759,69 @@
KASSERT(pai->pai_asid == 0);
KASSERT(pai->pai_link.le_prev == NULL);
#if defined(MULTIPROCESSOR)
- KASSERT(pmap_tlb_intersecting_onproc_p(pm, ti));
- KASSERT(pmap_tlb_intersecting_active_p(pm, ti));
+ KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
+ KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
#endif
KASSERT(ti->ti_asids_free > 0);
- KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
+ KASSERT(ti->ti_asid_hint > KERNEL_PID);
+
+ /*
+ * If the last ASID allocated was the maximum ASID, then the
+ * hint will be out of range. Reset the hint to first
+ * available ASID.
+ */
+ if (PMAP_TLB_FLUSH_ASID_ON_RESET
+ && ti->ti_asid_hint > ti->ti_asid_max) {
+ ti->ti_asid_hint = KERNEL_PID + 1;
+ }
+ KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u",
+ ti->ti_asid_hint);
/*
* Let's see if the hinted ASID is free. If not search for
* a new one.
*/
- if (__predict_false(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {
-#ifdef DIAGNOSTIC
- const size_t words = __arraycount(ti->ti_asid_bitmap);
-#endif
- const size_t nbpw = 8 * sizeof(ti->ti_asid_bitmap[0]);
- for (size_t i = 0; i < ti->ti_asid_hint / nbpw; i++) {
- KASSERT(~ti->ti_asid_bitmap[i] == 0);
+ if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {
+ const size_t nbpw __diagused = 8*sizeof(ti->ti_asid_bitmap[0]);
+ size_t i;
+ u_long bits;
+ for (i = 0; (bits = ~ti->ti_asid_bitmap[i]) == 0; i++) {
+ KASSERT(i < __arraycount(ti->ti_asid_bitmap) - 1);
}
- for (size_t i = ti->ti_asid_hint / nbpw;; i++) {
- KASSERT(i < words);
- /*
- * ffs wants to find the first bit set while we want
- * to find the first bit cleared.
- */
- u_long bits = ~ti->ti_asid_bitmap[i];
- if (__predict_true(bits)) {
- u_int n = 0;
- if ((bits & 0xffffffff) == 0) {
- bits = (bits >> 31) >> 1;
- KASSERT(bits);
- n += 32;
- }
- n += ffs(bits) - 1;
- KASSERT(n < nbpw);
- ti->ti_asid_hint = n + i * nbpw;
- break;
- }
- }
- KASSERT(ti->ti_asid_hint > KERNEL_PID);
- KASSERT(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint-1));
- KASSERT(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint));
+ /*
+ * ffs wants to find the first bit set while we want
+ * to find the first bit cleared.
+ */
+ const u_int n = __builtin_ffsl(bits) - 1;
+ KASSERTMSG((bits << (nbpw - (n+1))) == (1ul << (nbpw-1)),
+ "n %u bits %#lx", n, bits);
+ KASSERT(n < nbpw);
+ ti->ti_asid_hint = n + i * nbpw;
}
+ KASSERT(ti->ti_asid_hint > KERNEL_PID);
+ KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
+ KASSERTMSG(PMAP_TLB_FLUSH_ASID_ON_RESET
+ || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1),
+ "hint %u bitmap %p", ti->ti_asid_hint, ti->ti_asid_bitmap);
+ KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint),
+ "hint %u bitmap %p", ti->ti_asid_hint, ti->ti_asid_bitmap);
+
/*
* The hint contains our next ASID so take it and advance the hint.
* Mark it as used and insert the pai into the list of active asids.
* There is also one less asid free in this TLB.
*/
+ KASSERT(ti->ti_asid_hint > KERNEL_PID);
pai->pai_asid = ti->ti_asid_hint++;
+#ifdef MULTIPROCESSOR
+ if (PMAP_TLB_FLUSH_ASID_ON_RESET) {
+ /*
+ * Clean the new ASID from the TLB.
+ */
+ tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
+ }
+#endif
TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link);
ti->ti_asids_free--;
@@ -803,12 +853,17 @@
KASSERT(kpreempt_disabled());
+ UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
+
/*
* Kernels use a fixed ASID and thus doesn't need to acquire one.
*/
- if (pm == pmap_kernel())
+ if (pm == pmap_kernel()) {
+ UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0);
return;
+ }
+ UVMHIST_LOG(maphist, " (pm=%#x, l=%#x, ti=%#x)", pm, l, ti, 0);
TLBINFO_LOCK(ti);
KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL);
KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL);
@@ -819,13 +874,16 @@
*/
if (__predict_false(tlbinfo_noasids_p(ti))) {
KASSERT(l == curlwp);
+ UVMHIST_LOG(maphist, " asid reinit", 0, 0, 0, 0);
Home |
Main Index |
Thread Index |
Old Index