Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys Introduce per-cpu IDTs



details:   https://anonhg.NetBSD.org/src/rev/25279a73c466
branches:  trunk
changeset: 973823:25279a73c466
user:      yamaguchi <yamaguchi%NetBSD.org@localhost>
date:      Tue Jul 14 00:45:52 2020 +0000

description:
Introduce per-cpu IDTs

This is realized by following modifications:
- Add IDT pages and its allocation maps for each cpu in "struct cpu_info"
- Load per-cpu IDTs at cpu_init_idt(struct cpu_info*)
- Copy the IDT entries for cpu0 to other CPUs at attach
   - These are, for example, exceptions, db, system calls, etc.

And, added a kernel option named PCPU_IDT to enable the feature.

diffstat:

 sys/arch/amd64/amd64/db_interface.c |    8 +-
 sys/arch/amd64/amd64/machdep.c      |  100 +++++++++++++++++++++++++++++++----
 sys/arch/amd64/conf/ALL             |    5 +-
 sys/arch/amd64/include/segments.h   |   17 +++--
 sys/arch/i386/conf/ALL              |    5 +-
 sys/arch/i386/i386/db_interface.c   |   11 ++-
 sys/arch/i386/i386/machdep.c        |   78 ++++++++++++++++++++++++---
 sys/arch/i386/i386/trap.c           |   10 ++-
 sys/arch/i386/include/segments.h    |   18 +++---
 sys/arch/x86/conf/files.x86         |    4 +-
 sys/arch/x86/include/cpu.h          |    9 ++-
 sys/arch/x86/include/cpuvar.h       |    3 +-
 sys/arch/x86/include/pmap.h         |    4 +-
 sys/arch/x86/x86/cpu.c              |    7 +-
 sys/arch/x86/x86/hyperv.c           |  100 +++++++++++++++++++++++++++++++----
 sys/arch/x86/x86/idt.c              |   45 ++++++++++++---
 sys/arch/x86/x86/intr.c             |   65 +++++++++++++++++------
 sys/arch/x86/x86/lapic.c            |   23 ++++---
 sys/arch/x86/x86/pmap.c             |    8 +--
 sys/arch/x86/x86/svs.c              |    8 +-
 sys/arch/xen/x86/cpu.c              |    9 ++-
 sys/arch/xen/xen/hypervisor.c       |   10 ++-
 sys/dev/hyperv/vmbusvar.h           |    4 +-
 sys/dev/nvmm/x86/nvmm_x86_vmx.c     |   12 ++-
 24 files changed, 433 insertions(+), 130 deletions(-)

diffs (truncated from 1510 to 300 lines):

diff -r 42cb0e232211 -r 25279a73c466 sys/arch/amd64/amd64/db_interface.c
--- a/sys/arch/amd64/amd64/db_interface.c       Mon Jul 13 23:56:41 2020 +0000
+++ b/sys/arch/amd64/amd64/db_interface.c       Tue Jul 14 00:45:52 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: db_interface.c,v 1.37 2020/02/29 15:00:28 christos Exp $       */
+/*     $NetBSD: db_interface.c,v 1.38 2020/07/14 00:45:52 yamaguchi Exp $      */
 
 /*
  * Mach Operating System
@@ -33,7 +33,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.37 2020/02/29 15:00:28 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: db_interface.c,v 1.38 2020/07/14 00:45:52 yamaguchi Exp $");
 
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
@@ -111,12 +111,14 @@
 
 #ifdef MULTIPROCESSOR
 #ifndef XENPV
+       struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
        vector *handler = &Xintr_ddbipi;
+       idt_descriptor_t *idt = iv->iv_idt;
 #if NLAPIC > 0
        if (lapic_is_x2apic())
                handler = &Xintr_x2apic_ddbipi;
 #endif
-       ddb_vec = idt_vec_alloc(0xf0, 0xff);
+       ddb_vec = idt_vec_alloc(iv, 0xf0, 0xff);
        set_idtgate(&idt[ddb_vec], handler, 1, SDT_SYS386IGT, SEL_KPL,
            GSEL(GCODE_SEL, SEL_KPL));
 #else
diff -r 42cb0e232211 -r 25279a73c466 sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c    Mon Jul 13 23:56:41 2020 +0000
+++ b/sys/arch/amd64/amd64/machdep.c    Tue Jul 14 00:45:52 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: machdep.c,v 1.355 2020/05/10 06:30:57 maxv Exp $       */
+/*     $NetBSD: machdep.c,v 1.356 2020/07/14 00:45:52 yamaguchi Exp $  */
 
 /*
  * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.355 2020/05/10 06:30:57 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.356 2020/07/14 00:45:52 yamaguchi Exp $");
 
 #include "opt_modular.h"
 #include "opt_user_ldt.h"
@@ -1392,11 +1392,15 @@
 char *gdtstore;
 
 void
-setgate(struct gate_descriptor *gd, void *func, int ist, int type, int dpl, int sel)
+setgate(struct gate_descriptor *gd, void *func,
+    int ist, int type, int dpl, int sel)
 {
+       vaddr_t vaddr;
+
+       vaddr = ((vaddr_t)gd) & ~PAGE_MASK;
 
        kpreempt_disable();
-       pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+       pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
 
        gd->gd_looffset = (uint64_t)func & 0xffff;
        gd->gd_selector = sel;
@@ -1410,20 +1414,23 @@
        gd->gd_xx2 = 0;
        gd->gd_xx3 = 0;
 
-       pmap_changeprot_local(idt_vaddr, VM_PROT_READ);
+       pmap_changeprot_local(vaddr, VM_PROT_READ);
        kpreempt_enable();
 }
 
 void
 unsetgate(struct gate_descriptor *gd)
 {
+       vaddr_t vaddr;
+
+       vaddr = ((vaddr_t)gd) & ~PAGE_MASK;
 
        kpreempt_disable();
-       pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+       pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
 
        memset(gd, 0, sizeof (*gd));
 
-       pmap_changeprot_local(idt_vaddr, VM_PROT_READ);
+       pmap_changeprot_local(vaddr, VM_PROT_READ);
        kpreempt_enable();
 }
 
@@ -1470,10 +1477,12 @@
 }
 
 void
-cpu_init_idt(void)
+cpu_init_idt(struct cpu_info *ci)
 {
        struct region_descriptor region;
+       idt_descriptor_t *idt;
 
+       idt = ci->ci_idtvec.iv_idt;
        setregion(&region, idt, NIDT * sizeof(idt[0]) - 1);
        lidt(&region);
 }
@@ -1677,6 +1686,8 @@
        extern void consinit(void);
        struct region_descriptor region;
        struct mem_segment_descriptor *ldt_segp;
+       struct idt_vec *iv;
+       idt_descriptor_t *idt;
        int x;
        struct pcb *pcb;
        extern vaddr_t lwp0uarea;
@@ -1807,7 +1818,9 @@
 
        pmap_update(pmap_kernel());
 
-       idt = (idt_descriptor_t *)idt_vaddr;
+       iv = &(cpu_info_primary.ci_idtvec);
+       idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary));
+       idt = iv->iv_idt;
        gdtstore = (char *)gdt_vaddr;
        ldtstore = (char *)ldt_vaddr;
 
@@ -1872,7 +1885,7 @@
                sel = SEL_KPL;
                ist = 0;
 
-               idt_vec_reserve(x);
+               idt_vec_reserve(iv, x);
 
                switch (x) {
                case 1: /* DB */
@@ -1902,7 +1915,7 @@
        }
 
        /* new-style interrupt gate for syscalls */
-       idt_vec_reserve(128);
+       idt_vec_reserve(iv, 128);
        set_idtgate(&idt[128], &IDTVEC(osyscall), 0, SDT_SYS386IGT, SEL_UPL,
            GSEL(GCODE_SEL, SEL_KPL));
 
@@ -1920,7 +1933,7 @@
                panic("HYPERVISOR_set_callbacks() failed");
 #endif /* XENPV */
 
-       cpu_init_idt();
+       cpu_init_idt(&cpu_info_primary);
 
 #ifdef XENPV
        xen_init_ksyms();
@@ -1961,6 +1974,14 @@
 void
 cpu_reset(void)
 {
+#ifndef XENPV
+       idt_descriptor_t *idt;
+       vaddr_t vaddr;
+
+       idt = cpu_info_primary.ci_idtvec.iv_idt;
+       vaddr = (vaddr_t)idt;
+#endif
+
        x86_disable_intr();
 
 #ifdef XENPV
@@ -1974,7 +1995,7 @@
         * invalid and causing a fault.
         */
        kpreempt_disable();
-       pmap_changeprot_local(idt_vaddr, VM_PROT_READ|VM_PROT_WRITE);
+       pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
        memset((void *)idt, 0, NIDT * sizeof(idt[0]));
        kpreempt_enable();
        breakpoint();
@@ -2321,3 +2342,56 @@
        return false;
 #endif
 }
+
+static void
+idt_vec_copy(struct idt_vec *dst, struct idt_vec *src)
+{
+       idt_descriptor_t *idt_dst;
+
+       idt_dst = dst->iv_idt;
+
+       kpreempt_disable();
+       pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ|VM_PROT_WRITE);
+
+       memcpy(idt_dst, src->iv_idt, PAGE_SIZE);
+       memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap));
+
+       pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ);
+       kpreempt_enable();
+}
+
+void
+idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid)
+{
+       vaddr_t va;
+
+       if (cid != cpu_index(&cpu_info_primary) &&
+           idt_vec_is_pcpu()) {
+#ifdef __HAVE_PCPU_AREA
+               va = (vaddr_t)&pcpuarea->ent[cid].idt;
+#else
+               struct vm_page *pg;
+
+               va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
+                   UVM_KMF_VAONLY);
+               pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
+               if (pg == NULL) {
+                       panic("failed to allocate a page for IDT");
+               }
+               pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
+                   VM_PROT_READ|VM_PROT_WRITE, 0);
+               pmap_update(pmap_kernel());
+#endif
+
+               memset((void *)va, 0, PAGE_SIZE);
+#ifndef XENPV
+               pmap_changeprot_local(va, VM_PROT_READ);
+#endif
+               pmap_update(pmap_kernel());
+
+               iv->iv_idt = (void *)va;
+               idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec));
+       } else {
+               iv->iv_idt = (void *)idt_vaddr;
+       }
+}
diff -r 42cb0e232211 -r 25279a73c466 sys/arch/amd64/conf/ALL
--- a/sys/arch/amd64/conf/ALL   Mon Jul 13 23:56:41 2020 +0000
+++ b/sys/arch/amd64/conf/ALL   Tue Jul 14 00:45:52 2020 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: ALL,v 1.155 2020/06/29 23:58:44 riastradh Exp $
+# $NetBSD: ALL,v 1.156 2020/07/14 00:45:52 yamaguchi Exp $
 # From NetBSD: GENERIC,v 1.787 2006/10/01 18:37:54 bouyer Exp
 #
 # ALL machine description file
@@ -17,7 +17,7 @@
 
 options        INCLUDE_CONFIG_FILE     # embed config file in kernel binary
 
-#ident         "ALL-$Revision: 1.155 $"
+#ident         "ALL-$Revision: 1.156 $"
 
 maxusers       64              # estimated number of users
 
@@ -29,6 +29,7 @@
 #options       PAE             # PAE mode (36 bits physical addressing)
 makeoptions    SPECTRE_V2_GCC_MITIGATION=1     # GCC Spectre variant 2
                                                # migitation
+options        PCPU_IDT        # Per CPU IDT
 
 # CPU features
 acpicpu*       at cpu?         # ACPI CPU (including frequency scaling)
diff -r 42cb0e232211 -r 25279a73c466 sys/arch/amd64/include/segments.h
--- a/sys/arch/amd64/include/segments.h Mon Jul 13 23:56:41 2020 +0000
+++ b/sys/arch/amd64/include/segments.h Tue Jul 14 00:45:52 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: segments.h,v 1.36 2019/02/11 14:59:32 cherry Exp $     */
+/*     $NetBSD: segments.h,v 1.37 2020/07/14 00:45:52 yamaguchi Exp $  */
 
 /*
  * Copyright (c) 1990 The Regents of the University of California.
@@ -238,7 +238,6 @@
 #else
 typedef struct gate_descriptor idt_descriptor_t; 
 #endif /* XENPV */
-extern idt_descriptor_t *idt;
 extern char *gdtstore;
 extern char *ldtstore;
 
@@ -251,14 +250,16 @@
     int, int, int);
 void set_mem_segment(struct mem_segment_descriptor *, void *, size_t,
     int, int, int, int, int);
-void cpu_init_idt(void);
 void update_descriptor(void *, void *);
 
-
-void idt_vec_reserve(int);
-int idt_vec_alloc(int, int);
-void idt_vec_set(int, void (*)(void));
-void idt_vec_free(int);
+struct idt_vec;
+void idt_vec_reserve(struct idt_vec *, int);
+int idt_vec_alloc(struct idt_vec *, int, int);
+void idt_vec_set(struct idt_vec *, int, void (*)(void));
+void idt_vec_free(struct idt_vec *, int);
+void idt_vec_init_cpu_md(struct idt_vec *, cpuid_t);



Home | Main Index | Thread Index | Old Index