[src/trunk]: src/sys/arch/aarch64 * fix icache invalidations.

[ryo <ryo%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/ce963dafd831
branches:  trunk
changeset: 324831:ce963dafd831
user:      ryo <ryo%NetBSD.org@localhost>
date:      Mon Jul 23 22:51:39 2018 +0000

description:
* fix icache invalidations.
* "ic ivau" (aarch64_icache_sync_range) with VA generates permission fault in some situations, therefore use KSEG address for now.

diffstat:

 sys/arch/aarch64/aarch64/cpufunc_asm_armv8.S |   8 ++--
 sys/arch/aarch64/aarch64/pmap.c              |  44 ++++++++++++++++++++-------
 sys/arch/aarch64/include/cpufunc.h           |   4 +-
 3 files changed, 39 insertions(+), 17 deletions(-)

diffs (161 lines):

diff -r 92613b9fabf9 -r ce963dafd831 sys/arch/aarch64/aarch64/cpufunc_asm_armv8.S
--- a/sys/arch/aarch64/aarch64/cpufunc_asm_armv8.S      Mon Jul 23 22:32:22 2018 +0000
+++ b/sys/arch/aarch64/aarch64/cpufunc_asm_armv8.S      Mon Jul 23 22:51:39 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc_asm_armv8.S,v 1.1 2018/04/01 04:35:03 ryo Exp $        */
+/*     $NetBSD: cpufunc_asm_armv8.S,v 1.2 2018/07/23 22:51:39 ryo Exp $        */
 
 /*-
  * Copyright (c) 2014 Robin Randhawa
@@ -133,14 +133,14 @@
 ENTRY(aarch64_icache_inv_all)
        dsb     ish
 #ifdef MULTIPROCESSOR
-       ic      iallu
+       ic      ialluis
 #else
-       ic      ialluis
+       ic      iallu
 #endif
        dsb     ish
        isb
        ret
-END(aarch64_icache_sync_range)
+END(aarch64_icache_inv_all)
 
 
 
diff -r 92613b9fabf9 -r ce963dafd831 sys/arch/aarch64/aarch64/pmap.c
--- a/sys/arch/aarch64/aarch64/pmap.c   Mon Jul 23 22:32:22 2018 +0000
+++ b/sys/arch/aarch64/aarch64/pmap.c   Mon Jul 23 22:51:39 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pmap.c,v 1.12 2018/07/23 22:32:22 ryo Exp $    */
+/*     $NetBSD: pmap.c,v 1.13 2018/07/23 22:51:39 ryo Exp $    */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.12 2018/07/23 22:32:22 ryo Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.13 2018/07/23 22:51:39 ryo Exp $");
 
 #include "opt_arm_debug.h"
 #include "opt_ddb.h"
@@ -1120,6 +1120,9 @@
 
        for (va = sva; va < eva; va += PAGE_SIZE) {
                pt_entry_t *ptep, pte;
+#ifdef UVMHIST
+               pt_entry_t opte;
+#endif
                struct vm_page *pg;
                paddr_t pa;
                uint32_t mdattr;
@@ -1153,8 +1156,20 @@
                }
 
                pte = *ptep;
+#ifdef UVMHIST
+               opte = pte;
+#endif
                executable = l3pte_executable(pte);
                pte = _pmap_pte_adjust_prot(pte, prot, mdattr);
+
+               if (!executable && (prot & VM_PROT_EXECUTE)) {
+                       /* non-exec -> exec */
+                       UVMHIST_LOG(pmaphist, "icache_sync: pm=%p, va=%016lx, pte: %016lx -> %016lx",
+                           pm, va, opte, pte);
+                       cpu_icache_sync_range(AARCH64_PA_TO_KVA(pa), PAGE_SIZE);
+                       cpu_icache_inv_all();   /* for VIPT/VIVT */
+               }
+
                atomic_swap_64(ptep, pte);
 
 #if 0
@@ -1162,11 +1177,6 @@
 #else
                aarch64_tlbi_by_va(va);
 #endif
-
-               if (!executable && (prot & VM_PROT_EXECUTE)) {
-                       /* non-exec -> exec */
-                       cpu_icache_sync_range(va, PAGE_SIZE);
-               }
        }
 
        pm_unlock(pm);
@@ -1285,6 +1295,9 @@
        struct pv_entry *spv;
        pd_entry_t pde;
        pt_entry_t attr, pte, *ptep;
+#ifdef UVMHIST
+       pt_entry_t opte;
+#endif
        pd_entry_t *l0, *l1, *l2, *l3;
        paddr_t pdppa;
        uint32_t mdattr;
@@ -1386,6 +1399,9 @@
        ptep = &l3[idx];        /* as PTE */
 
        pte = *ptep;
+#ifdef UVMHIST
+       opte = pte;
+#endif
        executable = l3pte_executable(pte);
 
        if (l3pte_valid(pte)) {
@@ -1464,6 +1480,15 @@
 #endif
 
        pte = pa | attr;
+
+       if (!executable && (prot & VM_PROT_EXECUTE)) {
+               UVMHIST_LOG(pmaphist, "icache_sync: pm=%p, va=%016lx, pte: %016lx -> %016lx",
+                   pm, va, opte, pte);
+               /* non-exec -> exec */
+               cpu_icache_sync_range(AARCH64_PA_TO_KVA(pa), PAGE_SIZE);
+               cpu_icache_inv_all();   /* for VIPT/VIVT */
+       }
+
        atomic_swap_64(ptep, pte);
 
 #if 0
@@ -1473,11 +1498,6 @@
        aarch64_tlbi_by_va(va);
 #endif
 
-       if (!executable && (prot & VM_PROT_EXECUTE)) {
-               /* non-exec -> exec */
-               cpu_icache_sync_range(va, PAGE_SIZE);
-       }
-
        if (pte & LX_BLKPAG_OS_WIRED)
                pm->pm_stats.wired_count++;
        pm->pm_stats.resident_count++;
diff -r 92613b9fabf9 -r ce963dafd831 sys/arch/aarch64/include/cpufunc.h
--- a/sys/arch/aarch64/include/cpufunc.h        Mon Jul 23 22:32:22 2018 +0000
+++ b/sys/arch/aarch64/include/cpufunc.h        Mon Jul 23 22:51:39 2018 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: cpufunc.h,v 1.1 2018/04/01 04:35:03 ryo Exp $  */
+/*     $NetBSD: cpufunc.h,v 1.2 2018/07/23 22:51:39 ryo Exp $  */
 
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo%nerv.org@localhost>
@@ -89,6 +89,7 @@
 void aarch64_dcache_wbinv_range(vaddr_t, vsize_t);
 void aarch64_dcache_inv_range(vaddr_t, vsize_t);
 void aarch64_dcache_wb_range(vaddr_t, vsize_t);
+void aarch64_icache_inv_all(void);
 void aarch64_drain_writebuf(void);
 
 /* tlb op in cpufunc_asm_armv8.S */
@@ -113,6 +114,7 @@
        (aarch64_dcache_wbinv_all(), aarch64_icache_inv_all())
 #define cpu_icache_sync_all()          \
        (aarch64_dcache_wb_all(), aarch64_icache_inv_all())
+#define cpu_icache_inv_all()           aarch64_icache_inv_all()
 
 #define cpu_dcache_wbinv_range(v,s)    aarch64_dcache_wbinv_range((v),(s))
 #define cpu_dcache_inv_range(v,s)      aarch64_dcache_inv_range((v),(s))