[src/trunk]: src/sys/arch/sparc64/sparc64 blast_dcache() SMP friendly:

[mrg <mrg%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/203a75bdde7e
branches:  trunk
changeset: 752842:203a75bdde7e
user:      mrg <mrg%NetBSD.org@localhost>
date:      Mon Mar 08 08:59:06 2010 +0000

description:
blast_dcache() SMP friendly:

- blast_dcache() becomes sp_blast_dcache(dcache_size, dcache_line_size)
- new smp_blast_dcache(sparc64_cpuset_t) that blasts the D$ on this cpuset
- sparc64_ipi_blast_dcache() to support the above
- in pmap_remove_all(), when freeing mmu contexts for this pmap, mark
  the set of cpus to blast the d$ on as well and convert the
  blast_dcache() call into smp_blast_dcache() on the cpus who ran this
  pmap, or, sp_blast_dcache(dcache_size, dcache_line_size)
- convert the remaining blast_dcache() in machdep.c to sp_blast_dcache()
- in pmap_destroy()/pmap_remove_all() take the pmap_lock() always since
  we assert it is held always.

with these changes, NFS builds on the U60 seem to be stable now, and
the USIII machines also can often complete a single build.sh run now,
diskful or diskless.


reviewed by mlelstv and partially by martin, tested by martin and myself,
with some ideas from chuq as well.

diffstat:

 sys/arch/sparc64/sparc64/cache.h    |  26 +++++++++++--------
 sys/arch/sparc64/sparc64/ipifuncs.c |  17 +++++++++++-
 sys/arch/sparc64/sparc64/locore.s   |  48 ++++++++++++++++++++++++++----------
 sys/arch/sparc64/sparc64/pmap.c     |  27 ++++++++++++++------
 4 files changed, 84 insertions(+), 34 deletions(-)

diffs (261 lines):

diff -r b6bce5d93e9f -r 203a75bdde7e sys/arch/sparc64/sparc64/cache.h
--- a/sys/arch/sparc64/sparc64/cache.h  Mon Mar 08 08:56:03 2010 +0000
+++ b/sys/arch/sparc64/sparc64/cache.h  Mon Mar 08 08:59:06 2010 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cache.h,v 1.16 2010/03/06 08:08:29 mrg Exp $ */
+/*     $NetBSD: cache.h,v 1.17 2010/03/08 08:59:06 mrg Exp $ */
 
 /*
  * Copyright (c) 1996
@@ -72,10 +72,17 @@
  * set-associative -- each bank is 8K.  No conflict there.)
  */
 
+/* Various cache size/line sizes */
+extern int     ecache_min_line_size;
+extern int     dcache_line_size;
+extern int     dcache_size;
+extern int     icache_line_size;
+extern int     icache_size;
+
 /* The following are for I$ and D$ flushes and are in locore.s */
 void   dcache_flush_page_us(paddr_t);  /* flush page from D$ */
 void   dcache_flush_page_usiii(paddr_t); /* flush page from D$ */
-void   blast_dcache(void);             /* Clear entire D$ */
+void   sp_blast_dcache(int, int);      /* Clear entire D$ */
 void   blast_icache_us(void);          /* Clear entire I$ */
 void   blast_icache_usiii(void);       /* Clear entire I$ */
 
@@ -137,17 +144,14 @@
 
 #ifdef MULTIPROCESSOR
 void smp_tlb_flush_pte(vaddr_t, struct pmap *);
-#define        tlb_flush_pte(va,pm)    smp_tlb_flush_pte(va, pm)
 void smp_dcache_flush_page_all(paddr_t pa);
+void smp_blast_dcache(sparc64_cpuset_t);
+#define        tlb_flush_pte(va,pm     )       smp_tlb_flush_pte(va, pm)
 #define        dcache_flush_page_all(pa)       smp_dcache_flush_page_all(pa)
+#define        blast_dcache()                  smp_blast_dcache(cpus_active)
 #else
-#define        tlb_flush_pte(va,pm)    sp_tlb_flush_pte(va, (pm)->pm_ctx[0])
+#define        tlb_flush_pte(va,pm)            sp_tlb_flush_pte(va, (pm)->pm_ctx[0])
 #define        dcache_flush_page_all(pa)       dcache_flush_page(pa)
+#define        blast_dcache()                  sp_blast_dcache(dcache_size, \
+                                                       dcache_line_size)
 #endif
-
-/* Various cache size/line sizes */
-extern int     ecache_min_line_size;
-extern int     dcache_line_size;
-extern int     dcache_size;
-extern int     icache_line_size;
-extern int     icache_size;
diff -r b6bce5d93e9f -r 203a75bdde7e sys/arch/sparc64/sparc64/ipifuncs.c
--- a/sys/arch/sparc64/sparc64/ipifuncs.c       Mon Mar 08 08:56:03 2010 +0000
+++ b/sys/arch/sparc64/sparc64/ipifuncs.c       Mon Mar 08 08:59:06 2010 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: ipifuncs.c,v 1.34 2010/02/24 09:49:36 mrg Exp $ */
+/*     $NetBSD: ipifuncs.c,v 1.35 2010/03/08 08:59:06 mrg Exp $ */
 
 /*-
  * Copyright (c) 2004 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.34 2010/02/24 09:49:36 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.35 2010/03/08 08:59:06 mrg Exp $");
 
 #include "opt_ddb.h"
 
@@ -73,6 +73,7 @@
 void   sparc64_ipi_flush_pte_usiii(void *);
 void   sparc64_ipi_dcache_flush_page_us(void *);
 void   sparc64_ipi_dcache_flush_page_usiii(void *);
+void   sparc64_ipi_blast_dcache(void *);
 
 /*
  * Process cpu stop-self event.
@@ -428,6 +429,18 @@
 }
 
 /*
+ * Flush the D$ on this set of CPUs.
+ */
+void
+smp_blast_dcache(sparc64_cpuset_t activecpus)
+{
+
+       sparc64_multicast_ipi(activecpus, sparc64_ipi_blast_dcache,
+                             dcache_size, dcache_line_size);
+       sp_blast_dcache(dcache_size, dcache_line_size);
+}
+
+/*
  * Print an error message.
  */
 void
diff -r b6bce5d93e9f -r 203a75bdde7e sys/arch/sparc64/sparc64/locore.s
--- a/sys/arch/sparc64/sparc64/locore.s Mon Mar 08 08:56:03 2010 +0000
+++ b/sys/arch/sparc64/sparc64/locore.s Mon Mar 08 08:59:06 2010 +0000
@@ -1,6 +1,7 @@
-/*     $NetBSD: locore.s,v 1.325 2010/03/07 01:52:44 mrg Exp $ */
+/*     $NetBSD: locore.s,v 1.326 2010/03/08 08:59:06 mrg Exp $ */
 
 /*
+ * Copyright (c) 2006-2010 Matthew R. Green
  * Copyright (c) 1996-2002 Eduardo Horvath
  * Copyright (c) 1996 Paul Kranenburg
  * Copyright (c) 1996
@@ -4918,14 +4919,12 @@
         wrpr   %o3, %pstate
 
 /*
- * blast_dcache()
+ * sp_blast_dcache(int dcache_size, int dcache_line_size)
  *
  * Clear out all of D$ regardless of contents
- * Does not modify %o0
- *
  */
        .align 8
-ENTRY(blast_dcache)
+ENTRY(sp_blast_dcache)
 /*
  * We turn off interrupts for the duration to prevent RED exceptions.
  */
@@ -4934,18 +4933,14 @@
 #endif
 
        rdpr    %pstate, %o3
-       sethi   %hi(dcache_size), %o1
-       ld      [%o1 + %lo(dcache_size)], %o1
-       sethi   %hi(dcache_line_size), %o5
-       ld      [%o5 + %lo(dcache_line_size)], %o5
-       sub     %o1, %o5, %o1
+       sub     %o0, %o1, %o0
        andn    %o3, PSTATE_IE, %o4                     ! Turn off PSTATE_IE bit
        wrpr    %o4, 0, %pstate
 1:
-       stxa    %g0, [%o1] ASI_DCACHE_TAG
+       stxa    %g0, [%o0] ASI_DCACHE_TAG
        membar  #Sync
-       brnz,pt %o1, 1b
-        sub    %o1, %o5, %o1
+       brnz,pt %o0, 1b
+        sub    %o0, %o1, %o0
 
        sethi   %hi(KERNBASE), %o2
        flush   %o2
@@ -4959,6 +4954,33 @@
         wrpr   %o3, %pstate
 #endif
 
+#ifdef MULTIPROCESSOR
+/*
+ * void sparc64_ipi_blast_dcache(int dcache_size, int dcache_line_size)
+ *
+ * Clear out all of D$ regardless of contents
+ *
+ * On entry:
+ *     %g2 = dcache_size
+ *     %g3 = dcache_line_size
+ */
+       .align 8
+ENTRY(sparc64_ipi_blast_dcache)
+       sub     %g2, %g3, %g2
+1:
+       stxa    %g0, [%g2] ASI_DCACHE_TAG
+       membar  #Sync
+       brnz,pt %g2, 1b
+        sub    %g2, %g3, %g2
+
+       sethi   %hi(KERNBASE), %g5
+       flush   %g5
+       membar  #Sync
+
+       ba,a    ret_from_intr_vector
+        nop
+#endif /* MULTIPROCESSOR */
+
 /*
  * blast_icache_us()
  * blast_icache_usiii()
diff -r b6bce5d93e9f -r 203a75bdde7e sys/arch/sparc64/sparc64/pmap.c
--- a/sys/arch/sparc64/sparc64/pmap.c   Mon Mar 08 08:56:03 2010 +0000
+++ b/sys/arch/sparc64/sparc64/pmap.c   Mon Mar 08 08:59:06 2010 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.c,v 1.257 2010/03/06 08:08:30 mrg Exp $   */
+/*     $NetBSD: pmap.c,v 1.258 2010/03/08 08:59:06 mrg Exp $   */
 /*
  *
  * Copyright (C) 1996-1999 Eduardo Horvath.
@@ -26,7 +26,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.257 2010/03/06 08:08:30 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.258 2010/03/08 08:59:06 mrg Exp $");
 
 #undef NO_VCACHE /* Don't forget the locked TLB in dostart */
 #define        HWREF
@@ -1365,16 +1365,16 @@
                return;
        }
        DPRINTF(PDB_DESTROY, ("pmap_destroy: freeing pmap %p\n", pm));
+       mutex_enter(&pmap_lock);
 #ifdef MULTIPROCESSOR
-       mutex_enter(&pmap_lock);
        for (ci = cpus; ci != NULL; ci = ci->ci_next) {
                if (CPUSET_HAS(cpus_active, ci->ci_index))
                        ctx_free(pm, ci);
        }
-       mutex_exit(&pmap_lock);
 #else
        ctx_free(pm, curcpu());
 #endif
+       mutex_exit(&pmap_lock);
 
        /* we could be a little smarter and leave pages zeroed */
        for (pg = TAILQ_FIRST(&pm->pm_obj.memq); pg != NULL; pg = nextpg) {
@@ -1863,6 +1863,7 @@
 {
 #ifdef MULTIPROCESSOR
        struct cpu_info *ci;
+       sparc64_cpuset_t pmap_cpus_active;
 #endif
 
        if (pm == pmap_kernel()) {
@@ -1870,18 +1871,28 @@
        }
        write_user_windows();
        pm->pm_refs = 0;
+
+       mutex_enter(&pmap_lock);
 #ifdef MULTIPROCESSOR
-       mutex_enter(&pmap_lock);
+       CPUSET_CLEAR(pmap_cpus_active);
        for (ci = cpus; ci != NULL; ci = ci->ci_next) {
-               if (CPUSET_HAS(cpus_active, ci->ci_index))
+               if (CPUSET_HAS(cpus_active, ci->ci_index)) {
+                       if (pm->pm_ctx[ci->ci_index] > 0)
+                               CPUSET_ADD(pmap_cpus_active, ci->ci_index);
                        ctx_free(pm, ci);
+               }
        }
-       mutex_exit(&pmap_lock);
 #else
        ctx_free(pm, curcpu());
 #endif
+       mutex_exit(&pmap_lock);
+
        REMOVE_STAT(flushes);
-       blast_dcache();
+#ifdef MULTIPROCESSOR
+       smp_blast_dcache(pmap_cpus_active);
+#else
+       sp_blast_dcache(dcache_size, dcache_line_size);
+#endif
 }
 
 /*