Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/arch Add a __HAVE_PCPU_AREA option, enabled by default o...
details: https://anonhg.NetBSD.org/src/rev/fc160fffcda9
branches: trunk
changeset: 828815:fc160fffcda9
user: maxv <maxv%NetBSD.org@localhost>
date: Fri Jan 05 08:04:20 2018 +0000
description:
Add a __HAVE_PCPU_AREA option, enabled by default on native amd64 but not
Xen.
With this option, the CPU structures that must always be present in the
CPU's page tables are moved on L4 slot 384, which means address
0xffffc00000000000.
A new pcpu_area structure is defined. It contains shared structures (IDT,
LDT), and then an array of pcpu_entry structures, indexed by cpu_index(ci).
Theoretically the LDT should be in the array, but this will be done later.
During the boot procedure, cpu0 calls pmap_init_pcpu, which creates a
page tree that is able to map the pcpu_area structure entirely. cpu0 then
immediately maps the shared structures. Later, every CPU goes through
cpu_pcpuarea_init, which allocates physical pages and kenters the relevant
pcpu_entry to them. Finally, each pointer is replaced to point to pcpuarea.
The point of this change is to make sure that the structures that must
always be present in the page tables have their own L4 slot. Until now
their L4 slot was that of pmap_kernel, and making a distinction between
what must be mapped and what does not need to be was complicated.
Even in the non-speculative-bug case this change makes some sense: there
are several x86 instructions that leak the addresses of the CPU structures,
and putting these structures inside pmap_kernel actually offered a way to
compute the address of the kernel heap - which would have made ASLR on it
plainly useless, had we implemented that.
Note that, for now, pcpuarea does not contain rsp0.
Unfortunately this change adds many #ifdefs, and makes the code harder to
understand. There is also some duplication, but that will be solved later.
diffstat:
sys/arch/amd64/amd64/gdt.c | 18 ++++-
sys/arch/amd64/amd64/machdep.c | 22 ++++++-
sys/arch/amd64/include/types.h | 3 +-
sys/arch/x86/include/cpu.h | 4 +-
sys/arch/x86/include/pmap.h | 27 ++++++++-
sys/arch/x86/x86/cpu.c | 37 +++++++++++-
sys/arch/x86/x86/pmap.c | 126 ++++++++++++++++++++++++++++++++++++++--
7 files changed, 218 insertions(+), 19 deletions(-)
diffs (truncated from 497 to 300 lines):
diff -r 9a8ca10ea143 -r fc160fffcda9 sys/arch/amd64/amd64/gdt.c
--- a/sys/arch/amd64/amd64/gdt.c Fri Jan 05 03:07:15 2018 +0000
+++ b/sys/arch/amd64/amd64/gdt.c Fri Jan 05 08:04:20 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: gdt.c,v 1.44 2018/01/04 20:38:30 maxv Exp $ */
+/* $NetBSD: gdt.c,v 1.45 2018/01/05 08:04:20 maxv Exp $ */
/*
* Copyright (c) 1996, 1997, 2009 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: gdt.c,v 1.44 2018/01/04 20:38:30 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: gdt.c,v 1.45 2018/01/05 08:04:20 maxv Exp $");
#include "opt_multiprocessor.h"
#include "opt_xen.h"
@@ -132,8 +132,6 @@
gdt_init(void)
{
char *old_gdt;
- struct vm_page *pg;
- vaddr_t va;
struct cpu_info *ci = &cpu_info_primary;
/* Initialize the global values */
@@ -142,6 +140,13 @@
old_gdt = gdtstore;
+#ifdef __HAVE_PCPU_AREA
+ /* The GDT is part of the pcpuarea */
+ gdtstore = (char *)&pcpuarea->ent[cpu_index(ci)].gdt;
+#else
+ struct vm_page *pg;
+ vaddr_t va;
+
/* Allocate gdt_size bytes of memory. */
gdtstore = (char *)uvm_km_alloc(kernel_map, gdt_size, 0,
UVM_KMF_VAONLY);
@@ -155,6 +160,7 @@
VM_PROT_READ | VM_PROT_WRITE, 0);
}
pmap_update(pmap_kernel());
+#endif
/* Copy the initial bootstrap GDT into the new area. */
memcpy(gdtstore, old_gdt, DYNSEL_START);
@@ -174,6 +180,9 @@
void
gdt_alloc_cpu(struct cpu_info *ci)
{
+#ifdef __HAVE_PCPU_AREA
+ ci->ci_gdt = (union descriptor *)&pcpuarea->ent[cpu_index(ci)].gdt;
+#else
struct vm_page *pg;
vaddr_t va;
@@ -189,6 +198,7 @@
VM_PROT_READ | VM_PROT_WRITE, 0);
}
pmap_update(pmap_kernel());
+#endif
memcpy(ci->ci_gdt, gdtstore, gdt_size);
}
diff -r 9a8ca10ea143 -r fc160fffcda9 sys/arch/amd64/amd64/machdep.c
--- a/sys/arch/amd64/amd64/machdep.c Fri Jan 05 03:07:15 2018 +0000
+++ b/sys/arch/amd64/amd64/machdep.c Fri Jan 05 08:04:20 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.283 2018/01/04 13:36:30 maxv Exp $ */
+/* $NetBSD: machdep.c,v 1.284 2018/01/05 08:04:20 maxv Exp $ */
/*
* Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
@@ -110,7 +110,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.283 2018/01/04 13:36:30 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.284 2018/01/05 08:04:20 maxv Exp $");
/* #define XENDEBUG_LOW */
@@ -393,6 +393,9 @@
x86_bus_space_mallocok();
#endif
+#ifdef __HAVE_PCPU_AREA
+ cpu_pcpuarea_init(&cpu_info_primary);
+#endif
gdt_init();
x86_64_proc0_pcb_ldt_init();
@@ -502,21 +505,36 @@
void
cpu_init_tss(struct cpu_info *ci)
{
+#ifdef __HAVE_PCPU_AREA
+ const cpuid_t cid = cpu_index(ci);
+#endif
struct cpu_tss *cputss;
uintptr_t p;
+#ifdef __HAVE_PCPU_AREA
+ cputss = (struct cpu_tss *)&pcpuarea->ent[cid].tss;
+#else
cputss = (struct cpu_tss *)uvm_km_alloc(kernel_map,
sizeof(struct cpu_tss), 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
+#endif
cputss->tss.tss_iobase = IOMAP_INVALOFF << 16;
/* cputss->tss.tss_ist[0] is filled by cpu_intr_init */
/* double fault */
+#ifdef __HAVE_PCPU_AREA
+ p = (vaddr_t)&pcpuarea->ent[cid].ist1;
+#else
p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED);
+#endif
cputss->tss.tss_ist[1] = p + PAGE_SIZE - 16;
/* NMI */
+#ifdef __HAVE_PCPU_AREA
+ p = (vaddr_t)&pcpuarea->ent[cid].ist2;
+#else
p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED);
+#endif
cputss->tss.tss_ist[2] = p + PAGE_SIZE - 16;
ci->ci_tss = cputss;
diff -r 9a8ca10ea143 -r fc160fffcda9 sys/arch/amd64/include/types.h
--- a/sys/arch/amd64/include/types.h Fri Jan 05 03:07:15 2018 +0000
+++ b/sys/arch/amd64/include/types.h Fri Jan 05 08:04:20 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: types.h,v 1.52 2017/01/26 15:55:09 christos Exp $ */
+/* $NetBSD: types.h,v 1.53 2018/01/05 08:04:21 maxv Exp $ */
/*-
* Copyright (c) 1990 The Regents of the University of California.
@@ -102,6 +102,7 @@
#include "opt_xen.h"
#if defined(__x86_64__) && !defined(XEN)
+#define __HAVE_PCPU_AREA 1
#define __HAVE_DIRECT_MAP 1
#define __HAVE_MM_MD_DIRECT_MAPPED_IO
#define __HAVE_MM_MD_DIRECT_MAPPED_PHYS
diff -r 9a8ca10ea143 -r fc160fffcda9 sys/arch/x86/include/cpu.h
--- a/sys/arch/x86/include/cpu.h Fri Jan 05 03:07:15 2018 +0000
+++ b/sys/arch/x86/include/cpu.h Fri Jan 05 08:04:20 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.86 2018/01/04 13:36:30 maxv Exp $ */
+/* $NetBSD: cpu.h,v 1.87 2018/01/05 08:04:21 maxv Exp $ */
/*
* Copyright (c) 1990 The Regents of the University of California.
@@ -332,6 +332,8 @@
void cpu_broadcast_halt(void);
void cpu_kick(struct cpu_info *);
+void cpu_pcpuarea_init(struct cpu_info *);
+
#define curcpu() x86_curcpu()
#define curlwp x86_curlwp()
#define curpcb ((struct pcb *)lwp_getpcb(curlwp))
diff -r 9a8ca10ea143 -r fc160fffcda9 sys/arch/x86/include/pmap.h
--- a/sys/arch/x86/include/pmap.h Fri Jan 05 03:07:15 2018 +0000
+++ b/sys/arch/x86/include/pmap.h Fri Jan 05 08:04:20 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.72 2017/12/28 14:34:39 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.73 2018/01/05 08:04:21 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -153,6 +153,25 @@
vaddr_t emodule;
};
+#ifndef MAXGDTSIZ
+#define MAXGDTSIZ 65536 /* XXX */
+#endif
+
+struct pcpu_entry {
+ uint8_t gdt[MAXGDTSIZ];
+ uint8_t tss[PAGE_SIZE];
+ uint8_t ist1[PAGE_SIZE];
+ uint8_t ist2[PAGE_SIZE];
+} __packed;
+
+struct pcpu_area {
+ uint8_t idt[PAGE_SIZE];
+ uint8_t ldt[PAGE_SIZE];
+ struct pcpu_entry ent[MAXCPUS];
+} __packed;
+
+extern struct pcpu_area *pcpuarea;
+
/*
* pmap data structures: see pmap.c for details of locking.
*/
@@ -526,6 +545,12 @@
*/
#define POOL_VTOPHYS(va) vtophys((vaddr_t) (va))
+#ifdef __HAVE_PCPU_AREA
+extern struct pcpu_area *pcpuarea;
+#define PDIR_SLOT_PCPU 384
+#define PMAP_PCPU_BASE (VA_SIGN_NEG((PDIR_SLOT_PCPU * NBPD_L4)))
+#endif
+
#ifdef __HAVE_DIRECT_MAP
extern vaddr_t pmap_direct_base;
diff -r 9a8ca10ea143 -r fc160fffcda9 sys/arch/x86/x86/cpu.c
--- a/sys/arch/x86/x86/cpu.c Fri Jan 05 03:07:15 2018 +0000
+++ b/sys/arch/x86/x86/cpu.c Fri Jan 05 08:04:20 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.141 2017/11/11 11:00:46 maxv Exp $ */
+/* $NetBSD: cpu.c,v 1.142 2018/01/05 08:04:21 maxv Exp $ */
/*
* Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.141 2017/11/11 11:00:46 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.142 2018/01/05 08:04:21 maxv Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -221,6 +221,36 @@
return 1;
}
+#ifdef __HAVE_PCPU_AREA
+void
+cpu_pcpuarea_init(struct cpu_info *ci)
+{
+ struct vm_page *pg;
+ size_t i, npages;
+ vaddr_t base, va;
+ paddr_t pa;
+
+ CTASSERT(sizeof(struct pcpu_entry) % PAGE_SIZE == 0);
+
+ npages = sizeof(struct pcpu_entry) / PAGE_SIZE;
+ base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)];
+
+ for (i = 0; i < npages; i++) {
+ pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
+ if (pg == NULL) {
+ panic("failed to allocate pcpu PA");
+ }
+
+ va = base + i * PAGE_SIZE;
+ pa = VM_PAGE_TO_PHYS(pg);
+
+ pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, 0);
+ }
+
+ pmap_update(pmap_kernel());
+}
+#endif
+
static void
cpu_vm_init(struct cpu_info *ci)
{
@@ -358,6 +388,9 @@
"mi_cpu_attach failed with %d\n", error);
return;
}
+#ifdef __HAVE_PCPU_AREA
+ cpu_pcpuarea_init(ci);
+#endif
cpu_init_tss(ci);
} else {
KASSERT(ci->ci_data.cpu_idlelwp != NULL);
diff -r 9a8ca10ea143 -r fc160fffcda9 sys/arch/x86/x86/pmap.c
--- a/sys/arch/x86/x86/pmap.c Fri Jan 05 03:07:15 2018 +0000
+++ b/sys/arch/x86/x86/pmap.c Fri Jan 05 08:04:20 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.275 2018/01/04 13:36:30 maxv Exp $ */
+/* $NetBSD: pmap.c,v 1.276 2018/01/05 08:04:21 maxv Exp $ */
/*
* Copyright (c) 2008, 2010, 2016, 2017 The NetBSD Foundation, Inc.
@@ -170,7 +170,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.275 2018/01/04 13:36:30 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.276 2018/01/05 08:04:21 maxv Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -322,6 +322,8 @@
struct pmap_head pmaps;
Home |
Main Index |
Thread Index |
Old Index