Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/arch/i386/i386 fixes and optimizations from Stephan Uphoff.
details: https://anonhg.NetBSD.org/src/rev/b46171af5509
branches: trunk
changeset: 556909:b46171af5509
user: yamt <yamt%NetBSD.org@localhost>
date: Tue Dec 30 03:55:01 2003 +0000
description:
fixes and optimizations from Stephan Uphoff.
- pmap_enter: zap PTE and read attributes atomically to
eliminate a race window which could cause lost of attributes.
- reduce number of TLB shootdown by using some assumptions
about PTE handling.
for more details, see "SMP improvements for pmap" thread on port-i386@
around May 2003.
diffstat:
sys/arch/i386/i386/pmap.c | 400 ++++++++++++++++++++++++++++-----------------
1 files changed, 247 insertions(+), 153 deletions(-)
diffs (truncated from 598 to 300 lines):
diff -r 93ef5660da83 -r b46171af5509 sys/arch/i386/i386/pmap.c
--- a/sys/arch/i386/i386/pmap.c Tue Dec 30 03:54:35 2003 +0000
+++ b/sys/arch/i386/i386/pmap.c Tue Dec 30 03:55:01 2003 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.167 2003/12/26 11:50:51 yamt Exp $ */
+/* $NetBSD: pmap.c,v 1.168 2003/12/30 03:55:01 yamt Exp $ */
/*
*
@@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.167 2003/12/26 11:50:51 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.168 2003/12/30 03:55:01 yamt Exp $");
#include "opt_cputype.h"
#include "opt_user_ldt.h"
@@ -819,7 +819,7 @@
if (opte & PG_PS)
panic("pmap_kenter_pa: PG_PS");
#endif
- if (pmap_valid_entry(opte)) {
+ if ((opte & (PG_V | PG_U)) == (PG_V | PG_U)) {
#if defined(MULTIPROCESSOR)
int32_t cpumask = 0;
@@ -868,7 +868,8 @@
panic("pmap_kremove: PG_PVLIST mapping for 0x%lx",
va);
#endif
- pmap_tlb_shootdown(pmap_kernel(), va, opte, &cpumask);
+ if ((opte & (PG_V | PG_U)) == (PG_V | PG_U))
+ pmap_tlb_shootdown(pmap_kernel(), va, opte, &cpumask);
}
pmap_tlb_shootnow(cpumask);
}
@@ -1496,12 +1497,36 @@
}
/*
+ * pmap_lock_pvhs: Lock pvh1 and optional pvh2
+ * Observe locking order when locking both pvhs
+ */
+
+__inline static void
+pmap_lock_pvhs(struct pv_head *pvh1, struct pv_head *pvh2)
+{
+
+ if (pvh2 == NULL) {
+ simple_lock(&pvh1->pvh_lock);
+ return;
+ }
+
+ if (pvh1 < pvh2) {
+ simple_lock(&pvh1->pvh_lock);
+ simple_lock(&pvh2->pvh_lock);
+ } else {
+ simple_lock(&pvh2->pvh_lock);
+ simple_lock(&pvh1->pvh_lock);
+ }
+}
+
+
+/*
* main pv_entry manipulation functions:
* pmap_enter_pv: enter a mapping onto a pv_head list
* pmap_remove_pv: remove a mappiing from a pv_head list
*
- * NOTE: pmap_enter_pv expects to lock the pvh itself
- * pmap_remove_pv expects te caller to lock the pvh before calling
+ * NOTE: Both pmap_enter_pv and pmap_remove_pv expect the caller to lock
+ * the pvh before calling
*/
/*
@@ -1509,7 +1534,7 @@
*
* => caller should hold the proper lock on pmap_main_lock
* => caller should have pmap locked
- * => we will gain the lock on the pv_head and allocate the new pv_entry
+ * => caller should have the pv_head locked
* => caller should adjust ptp's wire_count before calling
*/
@@ -1524,9 +1549,7 @@
pve->pv_pmap = pmap;
pve->pv_va = va;
pve->pv_ptp = ptp; /* NULL for kernel pmap */
- simple_lock(&pvh->pvh_lock); /* lock pv_head */
SPLAY_INSERT(pvtree, &pvh->pvh_root, pve); /* add to locked list */
- simple_unlock(&pvh->pvh_lock); /* unlock, done! */
}
/*
@@ -2191,10 +2214,16 @@
pmap->pm_stats.wired_count--;
pmap->pm_stats.resident_count--;
- pmap_tlb_shootdown(pmap, startva, opte, cpumaskp);
-
- if (ptp)
+ if (opte & PG_U)
+ pmap_tlb_shootdown(pmap, startva, opte, cpumaskp);
+
+ if (ptp) {
ptp->wire_count--; /* dropping a PTE */
+ /* Make sure that the PDE is flushed */
+ if ((ptp->wire_count <= 1) && !(opte & PG_U))
+ pmap_tlb_shootdown(pmap, startva, opte,
+ cpumaskp);
+ }
/*
* if we are not on a pv_head list we are done.
@@ -2274,11 +2303,16 @@
pmap->pm_stats.wired_count--;
pmap->pm_stats.resident_count--;
- if (ptp)
+ if (opte & PG_U)
+ pmap_tlb_shootdown(pmap, va, opte, cpumaskp);
+
+ if (ptp) {
ptp->wire_count--; /* dropping a PTE */
-
- pmap_tlb_shootdown(pmap, va, opte, cpumaskp);
-
+ /* Make sure that the PDE is flushed */
+ if ((ptp->wire_count <= 1) && !(opte & PG_U))
+ pmap_tlb_shootdown(pmap, va, opte, cpumaskp);
+
+ }
/*
* if we are not on a pv_head list we are done.
*/
@@ -2424,7 +2458,6 @@
TAILQ_FIRST(&pmap->pm_obj.memq);
ptp->wire_count = 0;
ptp->flags |= PG_ZERO;
- /* Postpone free to shootdown */
uvm_pagerealloc(ptp, NULL, 0);
TAILQ_INSERT_TAIL(&empty_ptps, ptp, listq);
}
@@ -2599,7 +2632,10 @@
pve->pv_pmap->pm_stats.wired_count--;
pve->pv_pmap->pm_stats.resident_count--;
- pmap_tlb_shootdown(pve->pv_pmap, pve->pv_va, opte, &cpumask);
+ /* Shootdown only if referenced */
+ if (opte & PG_U)
+ pmap_tlb_shootdown(pve->pv_pmap, pve->pv_va, opte,
+ &cpumask);
/* sync R/M bits */
pg->mdpage.mp_attrs |= (opte & (PG_U|PG_M));
@@ -2608,6 +2644,14 @@
if (pve->pv_ptp) {
pve->pv_ptp->wire_count--;
if (pve->pv_ptp->wire_count <= 1) {
+ /*
+ * Do we have to shootdown the page just to
+ * get the pte out of the TLB ?
+ */
+ if(!(opte & PG_U))
+ pmap_tlb_shootdown(pve->pv_pmap,
+ pve->pv_va, opte, &cpumask);
+
/* zap! */
opte = x86_atomic_testset_ul(
&pve->pv_pmap->pm_pdir[pdei(pve->pv_va)],
@@ -2674,7 +2718,8 @@
int *myattrs;
struct pv_head *pvh;
struct pv_entry *pve;
- pt_entry_t *ptes, pte;
+ volatile pt_entry_t *ptes;
+ pt_entry_t pte;
#if DIAGNOSTIC
int bank, off;
@@ -2772,12 +2817,44 @@
ptes = pmap_map_ptes(pve->pv_pmap); /* locks pmap */
opte = ptes[x86_btop(pve->pv_va)];
if (opte & clearbits) {
+ /* We need to do something */
+ if (clearbits == PG_RW) {
+ result |= PG_RW;
+
+ /*
+ * On write protect we might not need to flush
+ * the TLB
+ */
+
+ /* First zap the RW bit! */
+ x86_atomic_clearbits_l(
+ &ptes[x86_btop(pve->pv_va)], PG_RW);
+ opte = ptes[x86_btop(pve->pv_va)];
+
+ /*
+ * Then test if it is not cached as RW the TLB
+ */
+ if (!(opte & PG_M))
+ goto no_tlb_shootdown;
+ }
+
+ /*
+ * Since we need a shootdown me might as well
+ * always clear PG_U AND PG_M.
+ */
+
+ /* zap! */
+ opte = x86_atomic_testset_ul(
+ &ptes[x86_btop(pve->pv_va)],
+ (opte & ~(PG_U | PG_M)));
+
result |= (opte & clearbits);
- x86_atomic_clearbits_l(&ptes[x86_btop(pve->pv_va)],
- (opte & clearbits)); /* zap! */
+ *myattrs |= (opte & ~(clearbits));
+
pmap_tlb_shootdown(pve->pv_pmap, pve->pv_va, opte,
- &cpumask);
+ &cpumask);
}
+no_tlb_shootdown:
pmap_unmap_ptes(pve->pv_pmap); /* unlocks pmap */
}
@@ -2788,6 +2865,7 @@
return(result != 0);
}
+
/*
* p m a p p r o t e c t i o n f u n c t i o n s
*/
@@ -2819,7 +2897,8 @@
vaddr_t sva, eva;
vm_prot_t prot;
{
- pt_entry_t *ptes, *spte, *epte;
+ pt_entry_t *ptes, *epte;
+ volatile pt_entry_t *spte;
vaddr_t blockend;
int32_t cpumask = 0;
@@ -2864,8 +2943,10 @@
for (/*null */; spte < epte ; spte++) {
if ((*spte & (PG_RW|PG_V)) == (PG_RW|PG_V)) {
x86_atomic_clearbits_l(spte, PG_RW); /* zap! */
- pmap_tlb_shootdown(pmap,
- x86_ptob(spte - ptes), *spte, &cpumask);
+ if (*spte & PG_M)
+ pmap_tlb_shootdown(pmap,
+ x86_ptob(spte - ptes),
+ *spte, &cpumask);
}
}
}
@@ -2903,7 +2984,7 @@
panic("pmap_unwire: invalid (unmapped) va 0x%lx", va);
#endif
if ((ptes[x86_btop(va)] & PG_W) != 0) {
- ptes[x86_btop(va)] &= ~PG_W;
+ x86_atomic_clearbits_l(&ptes[x86_btop(va)], PG_W);
pmap->pm_stats.wired_count--;
}
#ifdef DIAGNOSTIC
@@ -2970,10 +3051,9 @@
pt_entry_t *ptes, opte, npte;
struct vm_page *ptp, *pg;
struct vm_page_md *mdpg;
- struct pv_head *pvh;
- struct pv_entry *pve;
+ struct pv_head *old_pvh, *new_pvh;
+ struct pv_entry *pve = NULL; /* XXX gcc */
int error;
- int ptpdelta, wireddelta, resdelta;
boolean_t wired = (flags & PMAP_WIRED) != 0;
#ifdef DIAGNOSTIC
@@ -2990,13 +3070,21 @@
panic("pmap_enter: missing kernel PTP!");
#endif
+ npte = pa | protection_codes[prot] | PG_V;
+
+ if (wired)
+ npte |= PG_W;
+
+ if (va < VM_MAXUSER_ADDRESS)
+ npte |= PG_u;
+ else if (va < VM_MAX_ADDRESS)
+ npte |= (PG_u | PG_RW); /* XXXCDC: no longer needed? */
+ if (pmap == pmap_kernel())
+ npte |= pmap_pg_g;
+
/* get lock */
PMAP_MAP_TO_HEAD_LOCK();
Home |
Main Index |
Thread Index |
Old Index