Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/uvm/pmap Make this kcpuset_t instead of the private __cp...
details: https://anonhg.NetBSD.org/src/rev/0f2289534a18
branches: trunk
changeset: 788670:0f2289534a18
user: matt <matt%NetBSD.org@localhost>
date: Wed Jul 17 23:15:20 2013 +0000
description:
Make this kcpuset_t instead of the private __cpuset_t
Add improvements for single TLB implementation (PPC, ARM).
diffstat:
sys/uvm/pmap/pmap.c | 23 ++++---
sys/uvm/pmap/pmap.h | 13 +++-
sys/uvm/pmap/pmap_tlb.c | 127 +++++++++++++++++++++++++++++++++--------------
sys/uvm/pmap/pmap_tlb.h | 16 ++++-
4 files changed, 122 insertions(+), 57 deletions(-)
diffs (truncated from 480 to 300 lines):
diff -r 3e89943e8ae4 -r 0f2289534a18 sys/uvm/pmap/pmap.c
--- a/sys/uvm/pmap/pmap.c Wed Jul 17 23:09:26 2013 +0000
+++ b/sys/uvm/pmap/pmap.c Wed Jul 17 23:15:20 2013 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.1 2012/10/03 00:51:45 christos Exp $ */
+/* $NetBSD: pmap.c,v 1.2 2013/07/17 23:15:20 matt Exp $ */
/*-
* Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.1 2012/10/03 00:51:45 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.2 2013/07/17 23:15:20 matt Exp $");
/*
* Manages physical address maps.
@@ -209,10 +209,6 @@
.pm_segtab = PMAP_INVALID_SEGTAB_ADDRESS,
.pm_minaddr = VM_MIN_KERNEL_ADDRESS,
.pm_maxaddr = VM_MAX_KERNEL_ADDRESS,
-#ifdef MULTIPROCESSOR
- .pm_active = 1,
- .pm_onproc = 1,
-#endif
},
};
@@ -309,18 +305,22 @@
#endif
struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
pv_entry_t pv = &mdpg->mdpg_first;
- __cpuset_t onproc = CPUSET_NULLSET;
+ kcpuset_t *onproc;
+#ifdef MULTIPROCESSOR
+ kcpuset_create(&onproc, true);
+#endif
(void)VM_PAGEMD_PVLIST_LOCK(mdpg, false);
+
if (pv->pv_pmap != NULL) {
for (; pv != NULL; pv = pv->pv_next) {
#ifdef MULTIPROCESSOR
- CPUSET_MERGE(onproc, pv->pv_pmap->pm_onproc);
- if (CPUSET_EQUAL_P(onproc, cpuset_info.cpus_running)) {
+ kcpuset_merge(onproc, pv->pv_pmap->pm_onproc);
+ if (kcpuset_match(onproc, kcpuset_running)) {
break;
}
#else
if (pv->pv_pmap == curpmap) {
- onproc = CPUSET_SINGLE(0);
+ onproc = curcpu()->ci_data.cpu_kcpuset;
break;
}
#endif
@@ -329,6 +329,9 @@
VM_PAGEMD_PVLIST_UNLOCK(mdpg);
kpreempt_disable();
pmap_md_page_syncicache(pg, onproc);
+#ifdef MULTIPROCESSOR
+ kcpuset_destroy(onproc);
+#endif
kpreempt_enable();
}
diff -r 3e89943e8ae4 -r 0f2289534a18 sys/uvm/pmap/pmap.h
--- a/sys/uvm/pmap/pmap.h Wed Jul 17 23:09:26 2013 +0000
+++ b/sys/uvm/pmap/pmap.h Wed Jul 17 23:15:20 2013 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.2 2013/07/02 09:35:48 matt Exp $ */
+/* $NetBSD: pmap.h,v 1.3 2013/07/17 23:15:20 matt Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -113,6 +113,9 @@
extern kmutex_t pmap_segtab_lock;
#endif /* _KERNEL */
+#ifdef MULTIPROCESSOR
+#include <sys/kcpuset.h>
+#endif
#include <uvm/pmap/pmap_tlb.h>
/*
@@ -120,8 +123,8 @@
*/
struct pmap {
#ifdef MULTIPROCESSOR
- __cpuset_t pm_active; /* pmap was active on ... */
- __cpuset_t pm_onproc; /* pmap is active on ... */
+ kcpuset_t *pm_active; /* pmap was active on ... */
+ kcpuset_t *pm_onproc; /* pmap is active on ... */
volatile u_int pm_shootdown_pending;
#endif
pmap_segtab_t * pm_segtab; /* pointers to pages of PTEs */
@@ -137,8 +140,8 @@
#ifdef _KERNEL
struct pmap_kernel {
struct pmap kernel_pmap;
-#ifdef MULTIPROCESSOR
- struct pmap_asid_info kernel_pai[MAXCPUS-1];
+#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
+ struct pmap_asid_info kernel_pai[PMAP_TLB_MAX-1];
#endif
};
diff -r 3e89943e8ae4 -r 0f2289534a18 sys/uvm/pmap/pmap_tlb.c
--- a/sys/uvm/pmap/pmap_tlb.c Wed Jul 17 23:09:26 2013 +0000
+++ b/sys/uvm/pmap/pmap_tlb.c Wed Jul 17 23:15:20 2013 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_tlb.c,v 1.2 2013/07/02 09:35:48 matt Exp $ */
+/* $NetBSD: pmap_tlb.c,v 1.3 2013/07/17 23:15:20 matt Exp $ */
/*-
* Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.2 2013/07/02 09:35:48 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.3 2013/07/17 23:15:20 matt Exp $");
/*
* Manages address spaces in a TLB.
@@ -153,16 +153,15 @@
#endif
.ti_lock = &pmap_tlb0_mutex,
.ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais),
-#if defined(MULTIPROCESSOR)
- .ti_cpu_mask = 1,
+#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
.ti_tlbinvop = TLBINV_NOBODY,
#endif
};
#undef IFCONSTANT
-#if defined(MULTIPROCESSOR)
-struct pmap_tlb_info *pmap_tlbs[MAXCPUS] = {
+#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
+struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = {
[0] = &pmap_tlb0_info,
};
u_int pmap_ntlbs = 1;
@@ -194,6 +193,28 @@
#endif
}
+#ifdef MULTIPROCESSOR
+static inline bool
+pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)
+{
+#if PMAP_TLB_MAX == 1
+ return !kcpuset_iszero(pm->pm_active);
+#else
+ return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset);
+#endif
+}
+
+static inline bool
+pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti)
+{
+#if PMAP_TLB_MAX == 1
+ return !kcpuset_iszero(pm->pm_onproc);
+#else
+ return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset);
+#endif
+}
+#endif
+
static inline void
pmap_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,
struct pmap *pm)
@@ -203,7 +224,7 @@
*/
KASSERT(pai->pai_asid > KERNEL_PID);
#if defined(MULTIPROCESSOR)
- KASSERT((pm->pm_onproc & ti->ti_cpu_mask) == 0);
+ KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
#endif
LIST_REMOVE(pai, pai_link);
#ifdef DIAGNOSTIC
@@ -224,7 +245,11 @@
* The bits in pm_active belonging to this TLB can only be changed
* while this TLB's lock is held.
*/
- CPUSET_DELSET(pm->pm_active, ti->ti_cpu_mask);
+#if PMAP_TLB_MAX == 1
+ kcpuset_zero(pm->pm_active);
+#else
+ kcpuset_atomicly_remove(pm->pm_active, ti->ti_kcpuset);
+#endif
#endif /* MULTIPROCESSOR */
}
@@ -260,7 +285,11 @@
pmap_tlb_info_init(struct pmap_tlb_info *ti)
{
#if defined(MULTIPROCESSOR)
+#if PMAP_TLB_MAX == 1
+ KASSERT(ti == &pmap_tlb0_info);
+#else
if (ti != &pmap_tlb0_info) {
+ KASSERT(pmap_ntlbs < PMAP_TLB_MAX);
KASSERT(pmap_tlbs[pmap_ntlbs] == NULL);
@@ -271,7 +300,7 @@
ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID;
ti->ti_tlbinvop = TLBINV_NOBODY,
ti->ti_victim = NULL;
- ti->ti_cpu_mask = 0;
+ kcpuset_create(&ti->ti_kcpuset, true);
ti->ti_index = pmap_ntlbs++;
ti->ti_wired = 0;
pmap_tlbs[ti->ti_index] = ti;
@@ -280,9 +309,14 @@
pmap_tlb_info_evcnt_attach(ti);
return;
}
+#endif
#endif /* MULTIPROCESSOR */
KASSERT(ti == &pmap_tlb0_info);
mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED);
+#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
+ kcpuset_create(&ti->ti_kcpuset, true);
+ kcpuset_set(&ti->ti_kcpuset, cpu_index(curcpu()));
+#endif
if (ti->ti_asid_max == 0) {
ti->ti_asid_max = pmap_md_tlb_asid_max();
ti->ti_asids_free = ti->ti_asid_max - (KERNEL_PID + 1);
@@ -300,8 +334,9 @@
KASSERT(cold);
TLBINFO_LOCK(ti);
- const __cpuset_t cpu_mask = CPUSET_SINGLE(cpu_index(ci));
- CPUSET_ADDSET(ti->ti_cpu_mask, cpu_mask);
+#if PMAP_TLB_MAX > 1
+ kcpuset_set(ti->ti_kcpuset, cpu_index(ci));
+#endif
cpu_set_tlb_info(ci, ti);
/*
@@ -310,11 +345,9 @@
pmap_md_tlb_info_attach(ti, ci);
/*
- * Mark the kernel as active and "onproc" for this cpu. We assume
- * we are the only CPU running so atomic ops are not needed.
+ * The kernel pmap uses the kcpuset_running set so it's always
+ * up-to-date.
*/
- CPUSET_ADDSET(pmap_kernel()->pm_active, cpu_mask);
- CPUSET_ADDSET(pmap_kernel()->pm_onproc, cpu_mask);
TLBINFO_UNLOCK(ti);
}
#endif /* MULTIPROCESSOR */
@@ -417,7 +450,7 @@
KASSERT(pm != pmap_kernel());
KASSERT(pai->pai_asid > KERNEL_PID);
#if defined(MULTIPROCESSOR)
- if (!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask))) {
+ if (pmap_tlb_intersecting_onproc_p(pm, ti)) {
if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
ti->ti_asids_free--;
@@ -440,6 +473,9 @@
}
#if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN)
+#if PMAP_MAX_TLB == 1
+#error shootdown not required for single TLB systems
+#endif
void
pmap_tlb_shootdown_process(void)
{
@@ -463,7 +499,7 @@
*/
struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
KASSERT(ti->ti_victim != pmap_kernel());
- if (!CPUSET_EMPTY_P(CPUSET_SUBSET(ti->ti_victim->pm_onproc, ti->ti_cpu_mask))) {
+ if (!pmap_tlb_intersecting_onproc_p(ti_victim->pm_onproc, ti)) {
/*
* The victim is an active pmap so we will just
* invalidate its TLB entries.
@@ -479,7 +515,7 @@
* next called for this pmap, it will allocate a new
* ASID.
*/
- KASSERT(!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask)));
+ KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
pmap_pai_reset(ti, pai, PAI_PMAP(pai, ti));
}
break;
@@ -546,8 +582,11 @@
/*
* We don't need to deal our own TLB.
*/
- __cpuset_t pm_active =
Home |
Main Index |
Thread Index |
Old Index