tech-kern archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
Re: kernel memory allocators
> Do you have your changes available for review?
The kmem patch it includes:
- enhanced vmk caching in the uvm_km module not only for page sized
allocation but low integer multiplies.
(changed for rump as well)
- a changed kmem(9) implementation (using these new caches) (it's not
using vmem see note below)
- removed the malloc(9) bucket system and made malloc(9) a thin
wrapper around kmem, just like in the yamt-kmem branch.
(changed vmstat to deal with non more existing symbol for the malloc buckets)
- pool_subsystem_init is split into pool_subsystem_bootstrap and
pool_subsystem_init,
after bootstrap static allocated pools can be initialized and after
init allocation is allowed.
the only instances (as fas as I found them) that do static pool
initialization earlier are some pmaps those are changed accordingly.
(Tested i386 and amd64 so far)
vmem:
Status quo:
The kmem(9) implementation used vmem for its backing, with an
pool_allocator for each size this is unusual for caches.
The vmem(9) backing kmem(9) uses a quatum size of the machine
alignment so 4 or 8 bytes, therefore the quantum caches of the vmem
are very small and kmem extends these to larger ones.
The import functions for vmem do this on a page sized basis and the
uvm_map subsystem is in charge of controlling the virtual address
layout and vmem is just an extra layer.
Questions:
Shouldn't vmem provide the pool caches with pages for import into the
pools and the quantum caches of vmem should provide these pages for
the low integer multiplied sizes? That's the way I understand the idea
of vmem and it's implementation in solaris.
But this makes only sense if vmem(9) is in charge of controlling lets
say the kmem map and not the uvm_map system, slices of this submap
would be described by vmem entries and not by map entries.
With the extended vmk caching for the kernel_map and kmem_map I
implemented the quatum caching idea.
Results on an amd64 four-core 8gb machine:
sizes after: building a kernel with make -j200, du /, ./build.sh -j8
distribution
current
changed kmem
pool size: 915mb / 950mb 942mb/956mb
pmap -R0 | wc 2700 1915
sizes after pushing the memory system with several instances of the
Sieve of Eratosthenes each one consuming about 540mb to shrink the
pools.
current
changed kmem
pool size: 657mb / 760mb 620mb/740mb
pmap -R0 | wc 4280 3327
those numbers are not precise (especially the later ones) at all but
they do hint in an direction.
Keep in mind that allocations that go to malloc in the current
implementation go to the pool in the changed one.
Runtime of the build process was the same within a few seconds difference.
kind rgards,
Lars
? .git
? external/mit/xorg/lib/xkeyboard-config/xkeyboard-config.pc
? sys/arch/amd64/compile/MNEMONIC
? sys/arch/amd64/conf/MNEMONIC
? sys/arch/i386/compile/BOX
? sys/arch/i386/conf/BOX
? usr.bin/vmstat/.gdbinit
Index: sys/arch/alpha/alpha/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/arch/alpha/alpha/pmap.c,v
retrieving revision 1.255
diff -u -r1.255 pmap.c
--- sys/arch/alpha/alpha/pmap.c 12 Nov 2010 07:59:25 -0000 1.255
+++ sys/arch/alpha/alpha/pmap.c 21 Jan 2011 09:09:31 -0000
@@ -882,20 +882,6 @@
/* Nothing to do; it's already zero'd */
/*
- * Initialize the pmap pools and list.
- */
- pmap_ncpuids = ncpuids;
- pool_cache_bootstrap(&pmap_pmap_cache, PMAP_SIZEOF(pmap_ncpuids), 0,
- 0, 0, "pmap", NULL, IPL_NONE, NULL, NULL, NULL);
- pool_cache_bootstrap(&pmap_l1pt_cache, PAGE_SIZE, 0, 0, 0, "pmapl1pt",
- &pmap_l1pt_allocator, IPL_NONE, pmap_l1pt_ctor, NULL, NULL);
- pool_cache_bootstrap(&pmap_pv_cache, sizeof(struct pv_entry), 0, 0,
- PR_LARGECACHE, "pmappv", &pmap_pv_page_allocator, IPL_NONE, NULL,
- NULL, NULL);
-
- TAILQ_INIT(&pmap_all_pmaps);
-
- /*
* Initialize the ASN logic.
*/
pmap_max_asn = maxasn;
@@ -907,6 +893,9 @@
/*
* Initialize the locks.
*/
+
+ TAILQ_INIT(&pmap_all_pmaps);
+
rw_init(&pmap_main_lock);
mutex_init(&pmap_all_pmaps_lock, MUTEX_DEFAULT, IPL_NONE);
for (i = 0; i < __arraycount(pmap_pvh_locks); i++) {
@@ -931,19 +920,6 @@
mutex_init(&pmap_kernel()->pm_lock, MUTEX_DEFAULT, IPL_NONE);
TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap_kernel(), pm_list);
-#if defined(MULTIPROCESSOR)
- /*
- * Initialize the TLB shootdown queues.
- */
- pool_cache_bootstrap(&pmap_tlb_shootdown_job_cache,
- sizeof(struct pmap_tlb_shootdown_job), CACHE_LINE_SIZE,
- 0, PR_LARGECACHE, "pmaptlb", NULL, IPL_VM, NULL, NULL, NULL);
- for (i = 0; i < ALPHA_MAXPROCS; i++) {
- TAILQ_INIT(&pmap_tlb_shootdown_q[i].pq_head);
- mutex_init(&pmap_tlb_shootdown_q[i].pq_lock, MUTEX_DEFAULT,
- IPL_SCHED);
- }
-#endif
/*
* Set up lwp0's PCB such that the ptbr points to the right place
@@ -1094,6 +1070,33 @@
printf("pmap_init()\n");
#endif
+ /*
+ * Initialize the pmap pools and list.
+ */
+ pmap_ncpuids = ncpuids;
+ pool_cache_bootstrap(&pmap_pmap_cache, PMAP_SIZEOF(pmap_ncpuids), 0,
+ 0, 0, "pmap", NULL, IPL_NONE, NULL, NULL, NULL);
+ pool_cache_bootstrap(&pmap_l1pt_cache, PAGE_SIZE, 0, 0, 0, "pmapl1pt",
+ &pmap_l1pt_allocator, IPL_NONE, pmap_l1pt_ctor, NULL, NULL);
+ pool_cache_bootstrap(&pmap_pv_cache, sizeof(struct pv_entry), 0, 0,
+ PR_LARGECACHE, "pmappv", &pmap_pv_page_allocator, IPL_NONE,
NULL,
+ NULL, NULL);
+
+
+#if defined(MULTIPROCESSOR)
+ /*
+ * Initialize the TLB shootdown queues.
+ */
+ pool_cache_bootstrap(&pmap_tlb_shootdown_job_cache,
+ sizeof(struct pmap_tlb_shootdown_job), CACHE_LINE_SIZE,
+ 0, PR_LARGECACHE, "pmaptlb", NULL, IPL_VM, NULL, NULL, NULL);
+ for (i = 0; i < ALPHA_MAXPROCS; i++) {
+ TAILQ_INIT(&pmap_tlb_shootdown_q[i].pq_head);
+ mutex_init(&pmap_tlb_shootdown_q[i].pq_lock, MUTEX_DEFAULT,
+ IPL_SCHED);
+ }
+#endif
+
/* initialize protection array */
alpha_protection_init();
Index: sys/arch/arm/arm32/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/arch/arm/arm32/pmap.c,v
retrieving revision 1.219
diff -u -r1.219 pmap.c
--- sys/arch/arm/arm32/pmap.c 12 Nov 2010 07:59:25 -0000 1.219
+++ sys/arch/arm/arm32/pmap.c 21 Jan 2011 09:09:33 -0000
@@ -281,11 +281,6 @@
* Pool of PV structures
*/
static struct pool pmap_pv_pool;
-static void *pmap_bootstrap_pv_page_alloc(struct pool *, int);
-static void pmap_bootstrap_pv_page_free(struct pool *, void *);
-static struct pool_allocator pmap_bootstrap_pv_allocator = {
- pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free
-};
/*
* Pool and cache of l2_dtable structures.
@@ -5285,32 +5280,11 @@
pm->pm_pl1vec = NULL;
/*
- * Initialize the pmap cache
+ * Initialize the pmap list
*/
- pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0,
- "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL);
LIST_INIT(&pmap_pmaps);
LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list);
- /*
- * Initialize the pv pool.
- */
- pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl",
- &pmap_bootstrap_pv_allocator, IPL_NONE);
-
- /*
- * Initialize the L2 dtable pool and cache.
- */
- pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0,
- 0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL, NULL);
-
- /*
- * Initialise the L2 descriptor table pool and cache
- */
- pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 0,
- L2_TABLE_SIZE_REAL, 0, "l2ptppl", NULL, IPL_NONE,
- pmap_l2ptp_ctor, NULL, NULL);
-
cpu_dcache_wbinv_all();
}
@@ -5378,6 +5352,30 @@
void
pmap_init(void)
{
+ /*
+ * Initialize the pmap cache
+ */
+ pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0,
+ "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL);
+
+ /*
+ * Initialize the pv pool.
+ */
+ pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl",
+ &pool_allocator_kmem, IPL_NONE);
+
+ /*
+ * Initialize the L2 dtable pool and cache.
+ */
+ pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0,
+ 0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL,
NULL);
+
+ /*
+ * Initialise the L2 descriptor table pool and cache
+ */
+ pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 0,
+ L2_TABLE_SIZE_REAL, 0, "l2ptppl", NULL, IPL_NONE,
+ pmap_l2ptp_ctor, NULL, NULL);
/*
* Set the available memory vars - These do not map to real memory
@@ -5406,50 +5404,6 @@
pmap_initialized = true;
}
-static vaddr_t last_bootstrap_page = 0;
-static void *free_bootstrap_pages = NULL;
-
-static void *
-pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags)
-{
- extern void *pool_page_alloc(struct pool *, int);
- vaddr_t new_page;
- void *rv;
-
- if (pmap_initialized)
- return (pool_page_alloc(pp, flags));
-
- if (free_bootstrap_pages) {
- rv = free_bootstrap_pages;
- free_bootstrap_pages = *((void **)rv);
- return (rv);
- }
-
- new_page = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
- UVM_KMF_WIRED | ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT));
-
- KASSERT(new_page > last_bootstrap_page);
- last_bootstrap_page = new_page;
- return ((void *)new_page);
-}
-
-static void
-pmap_bootstrap_pv_page_free(struct pool *pp, void *v)
-{
- extern void pool_page_free(struct pool *, void *);
-
- if ((vaddr_t)v <= last_bootstrap_page) {
- *((void **)v) = free_bootstrap_pages;
- free_bootstrap_pages = v;
- return;
- }
-
- if (pmap_initialized) {
- pool_page_free(pp, v);
- return;
- }
-}
-
/*
* pmap_postinit()
*
Index: sys/arch/ia64/ia64/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/arch/ia64/ia64/pmap.c,v
retrieving revision 1.27
diff -u -r1.27 pmap.c
--- sys/arch/ia64/ia64/pmap.c 12 Nov 2010 07:59:26 -0000 1.27
+++ sys/arch/ia64/ia64/pmap.c 21 Jan 2011 09:09:36 -0000
@@ -638,24 +638,9 @@
VM_GATEWAY_SIZE;
/*
- * Initialize the pmap pools and list.
+ * Initialize the pmap rids and list.
*/
pmap_ncpuids = pmap_ridmax;
- pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
- &pool_allocator_nointr, IPL_NONE); /* This may block. */
-
- /* XXX: Need to convert ia64_kptdir[][] to a pool. ????*/
-
- /* The default pool allocator uses uvm_km_alloc & friends.
- * XXX: We should be using regular vm_alloced mem for regular,
non-kernel ptesl
- */
-
- pool_init(&pmap_ia64_lpte_pool, sizeof (struct ia64_lpte),
- sizeof(void *), 0, 0, "ptpl", NULL, IPL_NONE);
-
- pool_init(&pmap_pv_pool, sizeof (struct pv_entry), sizeof(void *),
- 0, 0, "pvpl", &pmap_pv_page_allocator, IPL_NONE);
-
TAILQ_INIT(&pmap_all_pmaps);
@@ -794,7 +779,23 @@
void
pmap_init(void)
{
+ /*
+ * Initialize the pmap pools and list.
+ */
+ pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
+ &pool_allocator_nointr, IPL_NONE); /* This may block. */
+
+ /* XXX: Need to convert ia64_kptdir[][] to a pool. ????*/
+ /* The default pool allocator uses uvm_km_alloc & friends.
+ * XXX: We should be using regular vm_alloced mem for regular,
non-kernel ptesl
+ */
+
+ pool_init(&pmap_ia64_lpte_pool, sizeof (struct ia64_lpte),
+ sizeof(void *), 0, 0, "ptpl", NULL, IPL_NONE);
+
+ pool_init(&pmap_pv_pool, sizeof (struct pv_entry), sizeof(void *),
+ 0, 0, "pvpl", &pmap_pv_page_allocator, IPL_NONE);
/*
* Set a low water mark on the pv_entry pool, so that we are
Index: sys/arch/mips/mips/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/arch/mips/mips/pmap.c,v
retrieving revision 1.192
diff -u -r1.192 pmap.c
--- sys/arch/mips/mips/pmap.c 12 Nov 2010 07:59:26 -0000 1.192
+++ sys/arch/mips/mips/pmap.c 21 Jan 2011 09:09:37 -0000
@@ -365,14 +365,6 @@
uvm_pageboot_alloc(sizeof(struct pv_entry) * pv_table_npages);
/*
- * Initialize the pools.
- */
- pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
- &pool_allocator_nointr, IPL_NONE);
- pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl",
- &pmap_pv_page_allocator, IPL_NONE);
-
- /*
* Initialize the kernel pmap.
*/
pmap_kernel()->pm_count = 1;
@@ -516,6 +508,14 @@
#endif
/*
+ * Initialize the pools.
+ */
+ pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
+ &pool_allocator_nointr, IPL_NONE);
+ pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl",
+ &pmap_pv_page_allocator, IPL_NONE);
+
+ /*
* Memory for the pv entry heads has
* already been allocated. Initialize the physical memory
* segments.
Index: sys/arch/x86/x86/intr.c
===================================================================
RCS file: /cvsroot/src/sys/arch/x86/x86/intr.c,v
retrieving revision 1.69
diff -u -r1.69 intr.c
--- sys/arch/x86/x86/intr.c 24 Nov 2010 14:56:18 -0000 1.69
+++ sys/arch/x86/x86/intr.c 21 Jan 2011 09:09:40 -0000
@@ -987,7 +987,6 @@
struct intrsource *isp;
#if NLAPIC > 0 && defined(MULTIPROCESSOR)
int i;
- static int first = 1;
#endif
#ifdef INTRSTACKSIZE
vaddr_t istack;
@@ -1002,10 +1001,8 @@
isp->is_handlers = &fake_timer_intrhand;
isp->is_pic = &local_pic;
ci->ci_isources[LIR_TIMER] = isp;
- evcnt_attach_dynamic(&isp->is_evcnt,
- first ? EVCNT_TYPE_INTR : EVCNT_TYPE_MISC, NULL,
+ evcnt_attach_dynamic(&isp->is_evcnt, EVCNT_TYPE_INTR, NULL,
device_xname(ci->ci_dev), "timer");
- first = 0;
#ifdef MULTIPROCESSOR
isp = kmem_zalloc(sizeof(*isp), KM_SLEEP);
Index: sys/arch/x86/x86/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/arch/x86/x86/pmap.c,v
retrieving revision 1.113
diff -u -r1.113 pmap.c
--- sys/arch/x86/x86/pmap.c 24 Jul 2010 00:45:56 -0000 1.113
+++ sys/arch/x86/x86/pmap.c 21 Jan 2011 09:09:41 -0000
@@ -1559,24 +1559,6 @@
pmap_cpu_init_early(curcpu());
/*
- * initialize caches.
- */
-
- pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0,
- "pmappl", NULL, IPL_NONE, NULL, NULL, NULL);
-#ifdef PAE
- pool_cache_bootstrap(&pmap_pdp_cache, PAGE_SIZE * PDP_SIZE, 0, 0, 0,
- "pdppl", &pmap_pdp_allocator, IPL_NONE,
- pmap_pdp_ctor, pmap_pdp_dtor, NULL);
-#else /* PAE */
- pool_cache_bootstrap(&pmap_pdp_cache, PAGE_SIZE, 0, 0, 0,
- "pdppl", NULL, IPL_NONE, pmap_pdp_ctor, pmap_pdp_dtor, NULL);
-#endif /* PAE */
- pool_cache_bootstrap(&pmap_pv_cache, sizeof(struct pv_entry), 0, 0,
- PR_LARGECACHE, "pvpl", &pool_allocator_meta, IPL_NONE, NULL,
- NULL, NULL);
-
- /*
* ensure the TLB is sync'd with reality by flushing it...
*/
@@ -1662,6 +1644,24 @@
{
int i;
+ /*
+ * initialize caches.
+ */
+
+ pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0,
+ "pmappl", NULL, IPL_NONE, NULL, NULL, NULL);
+#ifdef PAE
+ pool_cache_bootstrap(&pmap_pdp_cache, PAGE_SIZE * PDP_SIZE, 0, 0, 0,
+ "pdppl", &pmap_pdp_allocator, IPL_NONE,
+ pmap_pdp_ctor, pmap_pdp_dtor, NULL);
+#else /* PAE */
+ pool_cache_bootstrap(&pmap_pdp_cache, PAGE_SIZE, 0, 0, 0,
+ "pdppl", NULL, IPL_NONE, pmap_pdp_ctor, pmap_pdp_dtor, NULL);
+#endif /* PAE */
+ pool_cache_bootstrap(&pmap_pv_cache, sizeof(struct pv_entry), 0, 0,
+ PR_LARGECACHE, "pvpl", &pool_allocator_meta, IPL_NONE, NULL,
+ NULL, NULL);
+
for (i = 0; i < PV_HASH_SIZE; i++) {
SLIST_INIT(&pv_hash_heads[i].hh_list);
}
@@ -1707,7 +1707,7 @@
NULL, "x86", "ldt sync");
}
- evcnt_attach_dynamic(&ci->ci_tlb_evcnt, EVCNT_TYPE_MISC,
+ evcnt_attach_dynamic(&ci->ci_tlb_evcnt, EVCNT_TYPE_INTR,
NULL, device_xname(ci->ci_dev), "TLB IPI");
#ifdef PAE
Index: sys/kern/kern_malloc.c
===================================================================
RCS file: /cvsroot/src/sys/kern/kern_malloc.c,v
retrieving revision 1.131
diff -u -r1.131 kern_malloc.c
--- sys/kern/kern_malloc.c 5 May 2010 02:20:42 -0000 1.131
+++ sys/kern/kern_malloc.c 21 Jan 2011 09:09:44 -0000
@@ -72,6 +72,7 @@
#include <sys/proc.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
+#include <sys/kmem.h>
#include <sys/systm.h>
#include <sys/debug.h>
#include <sys/mutex.h>
@@ -79,247 +80,23 @@
#include <uvm/uvm_extern.h>
-static struct vm_map_kernel kmem_map_store;
-struct vm_map *kmem_map = NULL;
-
#include "opt_kmempages.h"
-#ifdef NKMEMCLUSTERS
-#error NKMEMCLUSTERS is obsolete; remove it from your kernel config file and
use NKMEMPAGES instead or let the kernel auto-size
-#endif
-
-/*
- * Default number of pages in kmem_map. We attempt to calculate this
- * at run-time, but allow it to be either patched or set in the kernel
- * config file.
- */
-#ifndef NKMEMPAGES
-#define NKMEMPAGES 0
-#endif
-int nkmempages = NKMEMPAGES;
-
-/*
- * Defaults for lower- and upper-bounds for the kmem_map page count.
- * Can be overridden by kernel config options.
- */
-#ifndef NKMEMPAGES_MIN
-#define NKMEMPAGES_MIN NKMEMPAGES_MIN_DEFAULT
-#endif
-
-#ifndef NKMEMPAGES_MAX
-#define NKMEMPAGES_MAX NKMEMPAGES_MAX_DEFAULT
-#endif
-
#include "opt_kmemstats.h"
#include "opt_malloclog.h"
#include "opt_malloc_debug.h"
-#define MINALLOCSIZE (1 << MINBUCKET)
-#define BUCKETINDX(size) \
- ((size) <= (MINALLOCSIZE * 128) \
- ? (size) <= (MINALLOCSIZE * 8) \
- ? (size) <= (MINALLOCSIZE * 2) \
- ? (size) <= (MINALLOCSIZE * 1) \
- ? (MINBUCKET + 0) \
- : (MINBUCKET + 1) \
- : (size) <= (MINALLOCSIZE * 4) \
- ? (MINBUCKET + 2) \
- : (MINBUCKET + 3) \
- : (size) <= (MINALLOCSIZE* 32) \
- ? (size) <= (MINALLOCSIZE * 16) \
- ? (MINBUCKET + 4) \
- : (MINBUCKET + 5) \
- : (size) <= (MINALLOCSIZE * 64) \
- ? (MINBUCKET + 6) \
- : (MINBUCKET + 7) \
- : (size) <= (MINALLOCSIZE * 2048) \
- ? (size) <= (MINALLOCSIZE * 512) \
- ? (size) <= (MINALLOCSIZE * 256) \
- ? (MINBUCKET + 8) \
- : (MINBUCKET + 9) \
- : (size) <= (MINALLOCSIZE * 1024) \
- ? (MINBUCKET + 10) \
- : (MINBUCKET + 11) \
- : (size) <= (MINALLOCSIZE * 8192) \
- ? (size) <= (MINALLOCSIZE * 4096) \
- ? (MINBUCKET + 12) \
- : (MINBUCKET + 13) \
- : (size) <= (MINALLOCSIZE * 16384) \
- ? (MINBUCKET + 14) \
- : (MINBUCKET + 15))
-
-/*
- * Array of descriptors that describe the contents of each page
- */
-struct kmemusage {
- short ku_indx; /* bucket index */
- union {
- u_short freecnt;/* for small allocations, free pieces in page */
- u_short pagecnt;/* for large allocations, pages alloced */
- } ku_un;
-};
-#define ku_freecnt ku_un.freecnt
-#define ku_pagecnt ku_un.pagecnt
-
-struct kmembuckets kmembuckets[MINBUCKET + 16];
-struct kmemusage *kmemusage;
-char *kmembase, *kmemlimit;
-
-#ifdef DEBUG
-static void *malloc_freecheck;
-#endif
-
-/*
- * Turn virtual addresses into kmem map indicies
- */
-#define btokup(addr) (&kmemusage[((char *)(addr) - kmembase) >>
PGSHIFT])
-
struct malloc_type *kmemstatistics;
-#ifdef MALLOCLOG
-#ifndef MALLOCLOGSIZE
-#define MALLOCLOGSIZE 100000
-#endif
-
-struct malloclog {
- void *addr;
- long size;
- struct malloc_type *type;
- int action;
- const char *file;
- long line;
-} malloclog[MALLOCLOGSIZE];
-
-long malloclogptr;
-
/*
* Fuzz factor for neighbour address match this must be a mask of the lower
* bits we wish to ignore when comparing addresses
*/
__uintptr_t malloclog_fuzz = 0x7FL;
-
-static void
-domlog(void *a, long size, struct malloc_type *type, int action,
- const char *file, long line)
-{
-
- malloclog[malloclogptr].addr = a;
- malloclog[malloclogptr].size = size;
- malloclog[malloclogptr].type = type;
- malloclog[malloclogptr].action = action;
- malloclog[malloclogptr].file = file;
- malloclog[malloclogptr].line = line;
- malloclogptr++;
- if (malloclogptr >= MALLOCLOGSIZE)
- malloclogptr = 0;
-}
-
-#ifdef DIAGNOSTIC
-static void
-hitmlog(void *a)
-{
- struct malloclog *lp;
- long l;
-
-#define PRT do { \
- lp = &malloclog[l]; \
- if (lp->addr == a && lp->action) { \
- printf("malloc log entry %ld:\n", l); \
- printf("\taddr = %p\n", lp->addr); \
- printf("\tsize = %ld\n", lp->size); \
- printf("\ttype = %s\n", lp->type->ks_shortdesc); \
- printf("\taction = %s\n", lp->action == 1 ? "alloc" : "free"); \
- printf("\tfile = %s\n", lp->file); \
- printf("\tline = %ld\n", lp->line); \
- } \
-} while (/* CONSTCOND */0)
-
-/*
- * Print fuzzy matched "neighbour" - look for the memory block that has
- * been allocated below the address we are interested in. We look for a
- * base address + size that is within malloclog_fuzz of our target
- * address. If the base address and target address are the same then it is
- * likely we have found a free (size is 0 in this case) so we won't report
- * those, they will get reported by PRT anyway.
- */
-#define NPRT do { \
- __uintptr_t fuzz_mask = ~(malloclog_fuzz); \
- lp = &malloclog[l]; \
- if ((__uintptr_t)lp->addr != (__uintptr_t)a && \
- (((__uintptr_t)lp->addr + lp->size + malloclog_fuzz) & fuzz_mask) \
- == ((__uintptr_t)a & fuzz_mask) && lp->action) { \
- printf("neighbour malloc log entry %ld:\n", l); \
- printf("\taddr = %p\n", lp->addr); \
- printf("\tsize = %ld\n", lp->size); \
- printf("\ttype = %s\n", lp->type->ks_shortdesc); \
- printf("\taction = %s\n", lp->action == 1 ? "alloc" : "free"); \
- printf("\tfile = %s\n", lp->file); \
- printf("\tline = %ld\n", lp->line); \
- } \
-} while (/* CONSTCOND */0)
-
- for (l = malloclogptr; l < MALLOCLOGSIZE; l++) {
- PRT;
- NPRT;
- }
-
-
- for (l = 0; l < malloclogptr; l++) {
- PRT;
- NPRT;
- }
-
-#undef PRT
-}
-#endif /* DIAGNOSTIC */
-#endif /* MALLOCLOG */
-
-#ifdef DIAGNOSTIC
-/*
- * This structure provides a set of masks to catch unaligned frees.
- */
-const long addrmask[] = { 0,
- 0x00000001, 0x00000003, 0x00000007, 0x0000000f,
- 0x0000001f, 0x0000003f, 0x0000007f, 0x000000ff,
- 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff,
- 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff,
-};
-
-/*
- * The WEIRD_ADDR is used as known text to copy into free objects so
- * that modifications after frees can be detected.
- */
-#define WEIRD_ADDR ((uint32_t) 0xdeadbeef)
-#ifdef DEBUG
-#define MAX_COPY PAGE_SIZE
-#else
-#define MAX_COPY 32
-#endif
-
-/*
- * Normally the freelist structure is used only to hold the list pointer
- * for free objects. However, when running with diagnostics, the first
- * 8/16 bytes of the structure is unused except for diagnostic information,
- * and the free list pointer is at offset 8/16 in the structure. Since the
- * first 8 bytes is the portion of the structure most often modified, this
- * helps to detect memory reuse problems and avoid free list corruption.
- */
-struct freelist {
- uint32_t spare0;
-#ifdef _LP64
- uint32_t spare1; /* explicit padding */
-#endif
- struct malloc_type *type;
- void * next;
-};
-#else /* !DIAGNOSTIC */
-struct freelist {
- void * next;
+struct malloc_header {
+ size_t mh_size;
};
-#endif /* DIAGNOSTIC */
-
-kmutex_t malloc_lock;
/*
* Allocate a block of memory
@@ -333,212 +110,23 @@
kern_malloc(unsigned long size, struct malloc_type *ksp, int flags)
#endif /* MALLOCLOG */
{
- struct kmembuckets *kbp;
- struct kmemusage *kup;
- struct freelist *freep;
- long indx, npg, allocsize;
- char *va, *cp, *savedlist;
-#ifdef DIAGNOSTIC
- uint32_t *end, *lp;
- int copysize;
-#endif
-
-#ifdef LOCKDEBUG
- if ((flags & M_NOWAIT) == 0) {
- ASSERT_SLEEPABLE();
- }
-#endif
-#ifdef MALLOC_DEBUG
- if (debug_malloc(size, ksp, flags, (void *) &va)) {
- if (va != 0) {
- FREECHECK_OUT(&malloc_freecheck, (void *)va);
- }
- return ((void *) va);
+ struct malloc_header *mh;
+ int kmflags = ((flags & M_NOWAIT) != 0) ? KM_NOSLEEP : KM_SLEEP;
+ size_t allocsize = sizeof(struct malloc_header) + size;
+ void *p;
+
+ if ((flags & M_ZERO) != 0) {
+ p = kmem_zalloc(allocsize, kmflags);
+ } else {
+ p = kmem_alloc(allocsize, kmflags);
}
-#endif
- indx = BUCKETINDX(size);
- kbp = &kmembuckets[indx];
- mutex_spin_enter(&malloc_lock);
-#ifdef KMEMSTATS
- while (ksp->ks_memuse >= ksp->ks_limit) {
- if (flags & M_NOWAIT) {
- mutex_spin_exit(&malloc_lock);
- return ((void *) NULL);
- }
- if (ksp->ks_limblocks < 65535)
- ksp->ks_limblocks++;
- mtsleep((void *)ksp, PSWP+2, ksp->ks_shortdesc, 0,
- &malloc_lock);
- }
- ksp->ks_size |= 1 << indx;
-#ifdef DIAGNOSTIC
- if (ksp->ks_active[indx - MINBUCKET] == USHRT_MAX)
- panic("too many allocations in bucket");
-#endif
- ksp->ks_active[indx - MINBUCKET]++;
-#endif
-#ifdef DIAGNOSTIC
- copysize = 1 << indx < MAX_COPY ? 1 << indx : MAX_COPY;
-#endif
- if (kbp->kb_next == NULL) {
- int s;
- kbp->kb_last = NULL;
- if (size > MAXALLOCSAVE)
- allocsize = round_page(size);
- else
- allocsize = 1 << indx;
- npg = btoc(allocsize);
- mutex_spin_exit(&malloc_lock);
- s = splvm();
- va = (void *) uvm_km_alloc(kmem_map,
- (vsize_t)ctob(npg), 0,
- ((flags & M_NOWAIT) ? UVM_KMF_NOWAIT : 0) |
- ((flags & M_CANFAIL) ? UVM_KMF_CANFAIL : 0) |
- UVM_KMF_WIRED);
- splx(s);
- if (__predict_false(va == NULL)) {
- /*
- * Kmem_malloc() can return NULL, even if it can
- * wait, if there is no map space available, because
- * it can't fix that problem. Neither can we,
- * right now. (We should release pages which
- * are completely free and which are in kmembuckets
- * with too many free elements.)
- */
- if ((flags & (M_NOWAIT|M_CANFAIL)) == 0)
- panic("malloc: out of space in kmem_map");
- return (NULL);
- }
- mutex_spin_enter(&malloc_lock);
-#ifdef KMEMSTATS
- kbp->kb_total += kbp->kb_elmpercl;
-#endif
- kup = btokup(va);
- kup->ku_indx = indx;
- if (allocsize > MAXALLOCSAVE) {
- if (npg > 65535)
- panic("malloc: allocation too large");
- kup->ku_pagecnt = npg;
-#ifdef KMEMSTATS
- ksp->ks_memuse += allocsize;
-#endif
- goto out;
- }
-#ifdef KMEMSTATS
- kup->ku_freecnt = kbp->kb_elmpercl;
- kbp->kb_totalfree += kbp->kb_elmpercl;
-#endif
- /*
- * Just in case we blocked while allocating memory,
- * and someone else also allocated memory for this
- * kmembucket, don't assume the list is still empty.
- */
- savedlist = kbp->kb_next;
- kbp->kb_next = cp = va + (npg << PAGE_SHIFT) - allocsize;
- for (;;) {
- freep = (struct freelist *)cp;
-#ifdef DIAGNOSTIC
- /*
- * Copy in known text to detect modification
- * after freeing.
- */
- end = (uint32_t *)&cp[copysize];
- for (lp = (uint32_t *)cp; lp < end; lp++)
- *lp = WEIRD_ADDR;
- freep->type = M_FREE;
-#endif /* DIAGNOSTIC */
- if (cp <= va)
- break;
- cp -= allocsize;
- freep->next = cp;
- }
- freep->next = savedlist;
- if (savedlist == NULL)
- kbp->kb_last = (void *)freep;
- }
- va = kbp->kb_next;
- kbp->kb_next = ((struct freelist *)va)->next;
-#ifdef DIAGNOSTIC
- freep = (struct freelist *)va;
- /* XXX potential to get garbage pointer here. */
- if (kbp->kb_next) {
- int rv;
- vaddr_t addr = (vaddr_t)kbp->kb_next;
-
- vm_map_lock(kmem_map);
- rv = uvm_map_checkprot(kmem_map, addr,
- addr + sizeof(struct freelist), VM_PROT_WRITE);
- vm_map_unlock(kmem_map);
-
- if (__predict_false(rv == 0)) {
- printf("Data modified on freelist: "
- "word %ld of object %p size %ld previous type %s "
- "(invalid addr %p)\n",
- (long)((int32_t *)&kbp->kb_next - (int32_t *)kbp),
- va, size, "foo", kbp->kb_next);
-#ifdef MALLOCLOG
- hitmlog(va);
-#endif
- kbp->kb_next = NULL;
- }
+ if (p == NULL) {
+ return NULL;
}
+ mh = (void *)p;
+ mh->mh_size = allocsize;
- /* Fill the fields that we've used with WEIRD_ADDR */
-#ifdef _LP64
- freep->type = (struct malloc_type *)
- (WEIRD_ADDR | (((u_long) WEIRD_ADDR) << 32));
-#else
- freep->type = (struct malloc_type *) WEIRD_ADDR;
-#endif
- end = (uint32_t *)&freep->next +
- (sizeof(freep->next) / sizeof(int32_t));
- for (lp = (uint32_t *)&freep->next; lp < end; lp++)
- *lp = WEIRD_ADDR;
-
- /* and check that the data hasn't been modified. */
- end = (uint32_t *)&va[copysize];
- for (lp = (uint32_t *)va; lp < end; lp++) {
- if (__predict_true(*lp == WEIRD_ADDR))
- continue;
- printf("Data modified on freelist: "
- "word %ld of object %p size %ld previous type %s "
- "(0x%x != 0x%x)\n",
- (long)(lp - (uint32_t *)va), va, size,
- "bar", *lp, WEIRD_ADDR);
-#ifdef MALLOCLOG
- hitmlog(va);
-#endif
- break;
- }
-
- freep->spare0 = 0;
-#endif /* DIAGNOSTIC */
-#ifdef KMEMSTATS
- kup = btokup(va);
- if (kup->ku_indx != indx)
- panic("malloc: wrong bucket");
- if (kup->ku_freecnt == 0)
- panic("malloc: lost data");
- kup->ku_freecnt--;
- kbp->kb_totalfree--;
- ksp->ks_memuse += 1 << indx;
-out:
- kbp->kb_calls++;
- ksp->ks_inuse++;
- ksp->ks_calls++;
- if (ksp->ks_memuse > ksp->ks_maxused)
- ksp->ks_maxused = ksp->ks_memuse;
-#else
-out:
-#endif
-#ifdef MALLOCLOG
- domlog(va, size, ksp, 1, file, line);
-#endif
- mutex_spin_exit(&malloc_lock);
- if ((flags & M_ZERO) != 0)
- memset(va, 0, size);
- FREECHECK_OUT(&malloc_freecheck, (void *)va);
- return ((void *) va);
+ return mh + 1;
}
/*
@@ -552,145 +140,11 @@
kern_free(void *addr, struct malloc_type *ksp)
#endif /* MALLOCLOG */
{
- struct kmembuckets *kbp;
- struct kmemusage *kup;
- struct freelist *freep;
- long size;
-#ifdef DIAGNOSTIC
- void *cp;
- int32_t *end, *lp;
- long alloc, copysize;
-#endif
-
- FREECHECK_IN(&malloc_freecheck, addr);
-#ifdef MALLOC_DEBUG
- if (debug_free(addr, ksp))
- return;
-#endif
-
-#ifdef DIAGNOSTIC
- /*
- * Ensure that we're free'ing something that we could
- * have allocated in the first place. That is, check
- * to see that the address is within kmem_map.
- */
- if (__predict_false((vaddr_t)addr < vm_map_min(kmem_map) ||
- (vaddr_t)addr >= vm_map_max(kmem_map)))
- panic("free: addr %p not within kmem_map", addr);
-#endif
-
- kup = btokup(addr);
- size = 1 << kup->ku_indx;
- kbp = &kmembuckets[kup->ku_indx];
-
- LOCKDEBUG_MEM_CHECK(addr,
- size <= MAXALLOCSAVE ? size : ctob(kup->ku_pagecnt));
-
- mutex_spin_enter(&malloc_lock);
-#ifdef MALLOCLOG
- domlog(addr, 0, ksp, 2, file, line);
-#endif
-#ifdef DIAGNOSTIC
- /*
- * Check for returns of data that do not point to the
- * beginning of the allocation.
- */
- if (size > PAGE_SIZE)
- alloc = addrmask[BUCKETINDX(PAGE_SIZE)];
- else
- alloc = addrmask[kup->ku_indx];
- if (((u_long)addr & alloc) != 0)
- panic("free: unaligned addr %p, size %ld, type %s, mask %ld",
- addr, size, ksp->ks_shortdesc, alloc);
-#endif /* DIAGNOSTIC */
- if (size > MAXALLOCSAVE) {
- uvm_km_free(kmem_map, (vaddr_t)addr, ctob(kup->ku_pagecnt),
- UVM_KMF_WIRED);
-#ifdef KMEMSTATS
- size = kup->ku_pagecnt << PGSHIFT;
- ksp->ks_memuse -= size;
-#ifdef DIAGNOSTIC
- if (ksp->ks_active[kup->ku_indx - MINBUCKET] == 0)
- panic("no active allocation(1), probably double free");
-#endif
- ksp->ks_active[kup->ku_indx - MINBUCKET]--;
- kup->ku_indx = 0;
- kup->ku_pagecnt = 0;
- if (ksp->ks_memuse + size >= ksp->ks_limit &&
- ksp->ks_memuse < ksp->ks_limit)
- wakeup((void *)ksp);
-#ifdef DIAGNOSTIC
- if (ksp->ks_inuse == 0)
- panic("free 1: inuse 0, probable double free");
-#endif
- ksp->ks_inuse--;
- kbp->kb_total -= 1;
-#endif
- mutex_spin_exit(&malloc_lock);
- return;
- }
- freep = (struct freelist *)addr;
-#ifdef DIAGNOSTIC
- /*
- * Check for multiple frees. Use a quick check to see if
- * it looks free before laboriously searching the freelist.
- */
- if (__predict_false(freep->spare0 == WEIRD_ADDR)) {
- for (cp = kbp->kb_next; cp;
- cp = ((struct freelist *)cp)->next) {
- if (addr != cp)
- continue;
- printf("multiply freed item %p\n", addr);
-#ifdef MALLOCLOG
- hitmlog(addr);
-#endif
- panic("free: duplicated free");
- }
- }
+ struct malloc_header *mh;
- /*
- * Copy in known text to detect modification after freeing
- * and to make it look free. Also, save the type being freed
- * so we can list likely culprit if modification is detected
- * when the object is reallocated.
- */
- copysize = size < MAX_COPY ? size : MAX_COPY;
- end = (int32_t *)&((char *)addr)[copysize];
- for (lp = (int32_t *)addr; lp < end; lp++)
- *lp = WEIRD_ADDR;
- freep->type = ksp;
-#endif /* DIAGNOSTIC */
-#ifdef KMEMSTATS
- kup->ku_freecnt++;
- if (kup->ku_freecnt >= kbp->kb_elmpercl) {
- if (kup->ku_freecnt > kbp->kb_elmpercl)
- panic("free: multiple frees");
- else if (kbp->kb_totalfree > kbp->kb_highwat)
- kbp->kb_couldfree++;
- }
- kbp->kb_totalfree++;
- ksp->ks_memuse -= size;
-#ifdef DIAGNOSTIC
- if (ksp->ks_active[kup->ku_indx - MINBUCKET] == 0)
- panic("no active allocation(2), probably double free");
-#endif
- ksp->ks_active[kup->ku_indx - MINBUCKET]--;
- if (ksp->ks_memuse + size >= ksp->ks_limit &&
- ksp->ks_memuse < ksp->ks_limit)
- wakeup((void *)ksp);
-#ifdef DIAGNOSTIC
- if (ksp->ks_inuse == 0)
- panic("free 2: inuse 0, probable double free");
-#endif
- ksp->ks_inuse--;
-#endif
- if (kbp->kb_next == NULL)
- kbp->kb_next = addr;
- else
- ((struct freelist *)kbp->kb_last)->next = addr;
- freep->next = NULL;
- kbp->kb_last = addr;
- mutex_spin_exit(&malloc_lock);
+ mh = addr;
+ mh--;
+ kmem_free(mh, mh->mh_size);
}
/*
@@ -700,12 +154,13 @@
kern_realloc(void *curaddr, unsigned long newsize, struct malloc_type *ksp,
int flags)
{
- struct kmemusage *kup;
unsigned long cursize;
void *newaddr;
-#ifdef DIAGNOSTIC
- long alloc;
-#endif
+
+ struct malloc_header *mh;
+ mh = curaddr;
+ mh--;
+ cursize = mh->mh_size;
/*
* realloc() with a NULL pointer is the same as malloc().
@@ -721,37 +176,6 @@
return (NULL);
}
-#ifdef LOCKDEBUG
- if ((flags & M_NOWAIT) == 0) {
- ASSERT_SLEEPABLE();
- }
-#endif
-
- /*
- * Find out how large the old allocation was (and do some
- * sanity checking).
- */
- kup = btokup(curaddr);
- cursize = 1 << kup->ku_indx;
-
-#ifdef DIAGNOSTIC
- /*
- * Check for returns of data that do not point to the
- * beginning of the allocation.
- */
- if (cursize > PAGE_SIZE)
- alloc = addrmask[BUCKETINDX(PAGE_SIZE)];
- else
- alloc = addrmask[kup->ku_indx];
- if (((u_long)curaddr & alloc) != 0)
- panic("realloc: "
- "unaligned addr %p, size %ld, type %s, mask %ld\n",
- curaddr, cursize, ksp->ks_shortdesc, alloc);
-#endif /* DIAGNOSTIC */
-
- if (cursize > MAXALLOCSAVE)
- cursize = ctob(kup->ku_pagecnt);
-
/*
* If we already actually have as much as they want, we're done.
*/
@@ -782,51 +206,11 @@
}
/*
- * Roundup size to the actual allocation size.
- */
-unsigned long
-malloc_roundup(unsigned long size)
-{
-
- if (size > MAXALLOCSAVE)
- return (roundup(size, PAGE_SIZE));
- else
- return (1 << BUCKETINDX(size));
-}
-
-/*
* Add a malloc type to the system.
*/
void
malloc_type_attach(struct malloc_type *type)
{
-
- if (nkmempages == 0)
- panic("malloc_type_attach: nkmempages == 0");
-
- if (type->ks_magic != M_MAGIC)
- panic("malloc_type_attach: bad magic");
-
-#ifdef DIAGNOSTIC
- {
- struct malloc_type *ksp;
- for (ksp = kmemstatistics; ksp != NULL; ksp = ksp->ks_next) {
- if (ksp == type)
- panic("%s: `%s' already on list", __func__,
- type->ks_shortdesc);
- }
- }
-#endif
-
-#ifdef KMEMSTATS
- if (type->ks_limit == 0)
- type->ks_limit = ((u_long)nkmempages << PAGE_SHIFT) * 6U / 10U;
-#else
- type->ks_limit = 0;
-#endif
-
- type->ks_next = kmemstatistics;
- kmemstatistics = type;
}
/*
@@ -835,29 +219,6 @@
void
malloc_type_detach(struct malloc_type *type)
{
- struct malloc_type *ksp;
-
-#ifdef DIAGNOSTIC
- if (type->ks_magic != M_MAGIC)
- panic("malloc_type_detach: bad magic");
-#endif
-
- if (type == kmemstatistics)
- kmemstatistics = type->ks_next;
- else {
- for (ksp = kmemstatistics; ksp->ks_next != NULL;
- ksp = ksp->ks_next) {
- if (ksp->ks_next == type) {
- ksp->ks_next = type->ks_next;
- break;
- }
- }
-#ifdef DIAGNOSTIC
- if (ksp->ks_next == NULL)
- panic("malloc_type_detach: not on list");
-#endif
- }
- type->ks_next = NULL;
}
/*
@@ -866,39 +227,6 @@
void
malloc_type_setlimit(struct malloc_type *type, u_long limit)
{
-#ifdef KMEMSTATS
- mutex_spin_enter(&malloc_lock);
- type->ks_limit = limit;
- mutex_spin_exit(&malloc_lock);
-#endif
-}
-
-/*
- * Compute the number of pages that kmem_map will map, that is,
- * the size of the kernel malloc arena.
- */
-void
-kmeminit_nkmempages(void)
-{
- int npages;
-
- if (nkmempages != 0) {
- /*
- * It's already been set (by us being here before, or
- * by patching or kernel config options), bail out now.
- */
- return;
- }
-
- npages = physmem;
-
- if (npages > NKMEMPAGES_MAX)
- npages = NKMEMPAGES_MAX;
-
- if (npages < NKMEMPAGES_MIN)
- npages = NKMEMPAGES_MIN;
-
- nkmempages = npages;
}
/*
@@ -909,130 +237,9 @@
{
__link_set_decl(malloc_types, struct malloc_type);
struct malloc_type * const *ksp;
- vaddr_t kmb, kml;
-#ifdef KMEMSTATS
- long indx;
-#endif
-
-#if ((MAXALLOCSAVE & (MAXALLOCSAVE - 1)) != 0)
- ERROR!_kmeminit:_MAXALLOCSAVE_not_power_of_2
-#endif
-#if (MAXALLOCSAVE > MINALLOCSIZE * 32768)
- ERROR!_kmeminit:_MAXALLOCSAVE_too_big
-#endif
-#if (MAXALLOCSAVE < NBPG)
- ERROR!_kmeminit:_MAXALLOCSAVE_too_small
-#endif
-
- if (sizeof(struct freelist) > (1 << MINBUCKET))
- panic("minbucket too small/struct freelist too big");
-
- mutex_init(&malloc_lock, MUTEX_DEFAULT, IPL_VM);
-
- /*
- * Compute the number of kmem_map pages, if we have not
- * done so already.
- */
- kmeminit_nkmempages();
-
- kmemusage = (struct kmemusage *) uvm_km_alloc(kernel_map,
- (vsize_t)(nkmempages * sizeof(struct kmemusage)), 0,
- UVM_KMF_WIRED|UVM_KMF_ZERO);
- kmb = 0;
- kmem_map = uvm_km_suballoc(kernel_map, &kmb,
- &kml, ((vsize_t)nkmempages << PAGE_SHIFT),
- VM_MAP_INTRSAFE, false, &kmem_map_store);
- uvm_km_vacache_init(kmem_map, "kvakmem", 0);
- kmembase = (char *)kmb;
- kmemlimit = (char *)kml;
-#ifdef KMEMSTATS
- for (indx = 0; indx < MINBUCKET + 16; indx++) {
- if (1 << indx >= PAGE_SIZE)
- kmembuckets[indx].kb_elmpercl = 1;
- else
- kmembuckets[indx].kb_elmpercl = PAGE_SIZE / (1 << indx);
- kmembuckets[indx].kb_highwat =
- 5 * kmembuckets[indx].kb_elmpercl;
- }
-#endif
/* Attach all of the statically-linked malloc types. */
__link_set_foreach(ksp, malloc_types)
malloc_type_attach(*ksp);
-
-#ifdef MALLOC_DEBUG
- debug_malloc_init();
-#endif
}
-#ifdef DDB
-#include <ddb/db_output.h>
-
-/*
- * Dump kmem statistics from ddb.
- *
- * usage: call dump_kmemstats
- */
-void dump_kmemstats(void);
-
-void
-dump_kmemstats(void)
-{
-#ifdef KMEMSTATS
- struct malloc_type *ksp;
-
- for (ksp = kmemstatistics; ksp != NULL; ksp = ksp->ks_next) {
- if (ksp->ks_memuse == 0)
- continue;
- db_printf("%s%.*s %ld\n", ksp->ks_shortdesc,
- (int)(20 - strlen(ksp->ks_shortdesc)),
- " ",
- ksp->ks_memuse);
- }
-#else
- db_printf("Kmem stats are not being collected.\n");
-#endif /* KMEMSTATS */
-}
-#endif /* DDB */
-
-
-#if 0
-/*
- * Diagnostic messages about "Data modified on
- * freelist" indicate a memory corruption, but
- * they do not help tracking it down.
- * This function can be called at various places
- * to sanity check malloc's freelist and discover
- * where does the corruption take place.
- */
-int
-freelist_sanitycheck(void) {
- int i,j;
- struct kmembuckets *kbp;
- struct freelist *freep;
- int rv = 0;
-
- for (i = MINBUCKET; i <= MINBUCKET + 15; i++) {
- kbp = &kmembuckets[i];
- freep = (struct freelist *)kbp->kb_next;
- j = 0;
- while(freep) {
- vm_map_lock(kmem_map);
- rv = uvm_map_checkprot(kmem_map, (vaddr_t)freep,
- (vaddr_t)freep + sizeof(struct freelist),
- VM_PROT_WRITE);
- vm_map_unlock(kmem_map);
-
- if ((rv == 0) || (*(int *)freep != WEIRD_ADDR)) {
- printf("bucket %i, chunck %d at %p modified\n",
- i, j, freep);
- return 1;
- }
- freep = (struct freelist *)freep->next;
- j++;
- }
- }
-
- return 0;
-}
-#endif
Index: sys/kern/subr_kmem.c
===================================================================
RCS file: /cvsroot/src/sys/kern/subr_kmem.c,v
retrieving revision 1.33
diff -u -r1.33 subr_kmem.c
--- sys/kern/subr_kmem.c 11 Feb 2010 23:13:46 -0000 1.33
+++ sys/kern/subr_kmem.c 21 Jan 2011 09:09:44 -0000
@@ -58,8 +58,6 @@
/*
* allocator of kernel wired memory.
*
- * TODO:
- * - worth to have "intrsafe" version? maybe..
*/
#include <sys/cdefs.h>
@@ -68,57 +66,74 @@
#include <sys/param.h>
#include <sys/callback.h>
#include <sys/kmem.h>
-#include <sys/vmem.h>
+#include <sys/pool.h>
#include <sys/debug.h>
#include <sys/lockdebug.h>
#include <sys/cpu.h>
#include <uvm/uvm_extern.h>
#include <uvm/uvm_map.h>
-#include <uvm/uvm_kmguard.h>
#include <lib/libkern/libkern.h>
#include <machine/stdarg.h>
-#define KMEM_QUANTUM_SIZE (ALIGNBYTES + 1)
-#define KMEM_QCACHE_MAX (KMEM_QUANTUM_SIZE * 32)
-#define KMEM_CACHE_COUNT 16
-
-typedef struct kmem_cache {
- pool_cache_t kc_cache;
- struct pool_allocator kc_pa;
- char kc_name[12];
-} kmem_cache_t;
-
-static vmem_t *kmem_arena;
-static struct callback_entry kmem_kva_reclaim_entry;
-
-static kmem_cache_t kmem_cache[KMEM_CACHE_COUNT + 1];
-static size_t kmem_cache_max;
-static size_t kmem_cache_min;
-static size_t kmem_cache_mask;
-static int kmem_cache_shift;
+
+struct kmem_cache_info {
+ int kc_size;
+ const char *kc_name;
+ struct pool_allocator *kc_allocator;
+};
+
+static const struct kmem_cache_info kmem_cache_tiny_sizes[] = {
+ { 16, "kmem-16", NULL },
+ { 32, "kmem-32", NULL },
+ { 48, "kmem-48", NULL },
+ { 64, "kmem-64", NULL },
+ { 96, "kmem-96", NULL },
+ { 128, "kmem-128", NULL },
+ { 192, "kmem-192", NULL },
+ { 256, "kmem-256", NULL },
+ { 384, "kmem-384", NULL },
+ { 512, "kmem-512", NULL },
+ { 768, "kmem-768", NULL },
+ { 1024, "kmem-1024", NULL },
+ /* { 1536, "kmem-1536", NULL }, */
+ { 2048, "kmem-2048", NULL },
+ /* { 3072, "kmem-3072", NULL }, */
+ { 4096, "kmem-4096", NULL },
+ { 0, NULL, NULL }
+};
+
+static struct pool_allocator kmem_big_8k_pa;
+static struct pool_allocator kmem_big_12k_pa;
+static struct pool_allocator kmem_big_16k_pa;
+
+static const struct kmem_cache_info kmem_cache_big_sizes[] = {
+ { 8192, "kmem-8192", &kmem_big_8k_pa },
+ { 12288, "kmem-12288", &kmem_big_12k_pa },
+ { 16384, "kmem-16384", &kmem_big_16k_pa },
+ { 0, NULL, NULL }
+};
+
+#define KMEM_TINY_MAXSIZE 4096
+#define KMEM_TINY_ALIGN 16
+#define KMEM_TINY_SHIFT 4
+
+#define KMEM_BIG_MAXSIZE (4*4096)
+#define KMEM_BIG_SHIFT 12
+
+static pool_cache_t kmem_cache_tiny[KMEM_TINY_MAXSIZE >> KMEM_TINY_SHIFT];
+static pool_cache_t kmem_cache_big[KMEM_BIG_MAXSIZE >> KMEM_BIG_SHIFT];
+
+#define DBUG
#if defined(DEBUG)
-int kmem_guard_depth;
-size_t kmem_guard_size;
-static struct uvm_kmguard kmem_guard;
static void *kmem_freecheck;
-#define KMEM_POISON
#define KMEM_REDZONE
#define KMEM_SIZE
-#define KMEM_GUARD
#endif /* defined(DEBUG) */
-#if defined(KMEM_POISON)
-static void kmem_poison_fill(void *, size_t);
-static void kmem_poison_check(void *, size_t);
-#else /* defined(KMEM_POISON) */
-#define kmem_poison_fill(p, sz) /* nothing */
-#define kmem_poison_check(p, sz) /* nothing */
-#endif /* defined(KMEM_POISON) */
-
#if defined(KMEM_REDZONE)
#define REDZONE_SIZE 1
#else /* defined(KMEM_REDZONE) */
@@ -126,7 +141,7 @@
#endif /* defined(KMEM_REDZONE) */
#if defined(KMEM_SIZE)
-#define SIZE_SIZE (max(KMEM_QUANTUM_SIZE, sizeof(size_t)))
+#define SIZE_SIZE (max(KMEM_TINY_ALIGN, sizeof(size_t)))
static void kmem_size_set(void *, size_t);
static void kmem_size_check(const void *, size_t);
#else
@@ -135,47 +150,24 @@
#define kmem_size_check(p, sz) /* nothing */
#endif
-static vmem_addr_t kmem_backend_alloc(vmem_t *, vmem_size_t, vmem_size_t *,
- vm_flag_t);
-static void kmem_backend_free(vmem_t *, vmem_addr_t, vmem_size_t);
-static int kmem_kva_reclaim_callback(struct callback_entry *, void *, void *);
CTASSERT(KM_SLEEP == PR_WAITOK);
CTASSERT(KM_NOSLEEP == PR_NOWAIT);
-static inline vm_flag_t
-kmf_to_vmf(km_flag_t kmflags)
-{
- vm_flag_t vmflags;
-
- KASSERT((kmflags & (KM_SLEEP|KM_NOSLEEP)) != 0);
- KASSERT((~kmflags & (KM_SLEEP|KM_NOSLEEP)) != 0);
-
- vmflags = 0;
- if ((kmflags & KM_SLEEP) != 0) {
- vmflags |= VM_SLEEP;
- }
- if ((kmflags & KM_NOSLEEP) != 0) {
- vmflags |= VM_NOSLEEP;
- }
-
- return vmflags;
-}
-
static void *
-kmem_poolpage_alloc(struct pool *pool, int prflags)
+kmem_bigpoolpage_alloc(struct pool *pool, int prflags)
{
+ bool waitok = (prflags & PR_WAITOK) ? true : false;
- return (void *)vmem_alloc(kmem_arena, pool->pr_alloc->pa_pagesz,
- kmf_to_vmf(prflags) | VM_INSTANTFIT);
-
+ return ((void *) uvm_km_alloc_poolpage_cache(kmem_map,
+ pool->pr_alloc->pa_pagesz, waitok));
}
static void
-kmem_poolpage_free(struct pool *pool, void *addr)
+kmem_bigpoolpage_free(struct pool *pool, void *addr)
{
-
- vmem_free(kmem_arena, (vmem_addr_t)addr, pool->pr_alloc->pa_pagesz);
+ uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) addr,
+ pool->pr_alloc->pa_pagesz);
}
/* ---- kmem API */
@@ -189,32 +181,27 @@
void *
kmem_alloc(size_t size, km_flag_t kmflags)
{
- kmem_cache_t *kc;
+ pool_cache_t pc;
+ size_t index;
uint8_t *p;
- KASSERT(!cpu_intr_p());
- KASSERT(!cpu_softintr_p());
KASSERT(size > 0);
-#ifdef KMEM_GUARD
- if (size <= kmem_guard_size) {
- return uvm_kmguard_alloc(&kmem_guard, size,
- (kmflags & KM_SLEEP) != 0);
- }
-#endif
-
size += REDZONE_SIZE + SIZE_SIZE;
- if (size >= kmem_cache_min && size <= kmem_cache_max) {
- kc = &kmem_cache[(size + kmem_cache_mask) >> kmem_cache_shift];
- KASSERT(size <= kc->kc_pa.pa_pagesz);
- kmflags &= (KM_SLEEP | KM_NOSLEEP);
- p = pool_cache_get(kc->kc_cache, kmflags);
+
+ if ((index = ((size - 1) >> KMEM_TINY_SHIFT)) < (KMEM_TINY_MAXSIZE >>
KMEM_TINY_SHIFT)) {
+ pc = kmem_cache_tiny[index];
+ } else if ((index = ((size - 1) >> KMEM_BIG_SHIFT)) < (KMEM_BIG_MAXSIZE
>> KMEM_BIG_SHIFT)) {
+ pc = kmem_cache_big[index];
} else {
- p = (void *)vmem_alloc(kmem_arena, size,
- kmf_to_vmf(kmflags) | VM_INSTANTFIT);
+ return (void *) uvm_km_alloc(kmem_map,
+ (vsize_t)round_page(size), 0,
+ ((kmflags & KM_SLEEP) ? 0 : UVM_KMF_NOWAIT |
UVM_KMF_TRYLOCK) | UVM_KMF_WIRED);
}
+
+ p = pool_cache_get(pc, kmflags);
+
if (__predict_true(p != NULL)) {
- kmem_poison_check(p, kmem_roundup_size(size));
FREECHECK_OUT(&kmem_freecheck, p);
kmem_size_set(p, size);
p = (uint8_t *)p + SIZE_SIZE;
@@ -249,195 +236,100 @@
void
kmem_free(void *p, size_t size)
{
- kmem_cache_t *kc;
+ pool_cache_t pc;
+ size_t index;
- KASSERT(!cpu_intr_p());
- KASSERT(!cpu_softintr_p());
KASSERT(p != NULL);
KASSERT(size > 0);
-#ifdef KMEM_GUARD
- if (size <= kmem_guard_size) {
- uvm_kmguard_free(&kmem_guard, size, p);
+ size += REDZONE_SIZE + SIZE_SIZE;
+
+ if ((index = ((size - 1) >> KMEM_TINY_SHIFT)) < (KMEM_TINY_MAXSIZE >>
KMEM_TINY_SHIFT)) {
+ pc = kmem_cache_tiny[index];
+ } else if ((index = ((size - 1) >> KMEM_BIG_SHIFT)) <
(KMEM_BIG_MAXSIZE >> KMEM_BIG_SHIFT)) {
+ pc = kmem_cache_big[index];
+ } else {
+ uvm_km_free(kmem_map, (vaddr_t)p, round_page(size),
UVM_KMF_WIRED);
return;
- }
-#endif
- size += SIZE_SIZE;
+ }
+
p = (uint8_t *)p - SIZE_SIZE;
- kmem_size_check(p, size + REDZONE_SIZE);
+ kmem_size_check(p, size);
FREECHECK_IN(&kmem_freecheck, p);
- LOCKDEBUG_MEM_CHECK(p, size);
- kmem_poison_check((char *)p + size,
- kmem_roundup_size(size + REDZONE_SIZE) - size);
- kmem_poison_fill(p, size);
- size += REDZONE_SIZE;
- if (size >= kmem_cache_min && size <= kmem_cache_max) {
- kc = &kmem_cache[(size + kmem_cache_mask) >> kmem_cache_shift];
- KASSERT(size <= kc->kc_pa.pa_pagesz);
- pool_cache_put(kc->kc_cache, p);
- } else {
- vmem_free(kmem_arena, (vmem_addr_t)p, size);
- }
-}
+ LOCKDEBUG_MEM_CHECK(p, size - REDZONE_SIZE);
+ pool_cache_put(pc, p);
+}
-void
-kmem_init(void)
+static void
+kmem_create_caches(const struct kmem_cache_info *array, pool_cache_t
alloc_table[],
+ size_t maxbuf, int shift)
{
- kmem_cache_t *kc;
- size_t sz;
+ size_t table_unit = (1 << shift);
+ size_t size = table_unit;
int i;
-#ifdef KMEM_GUARD
- uvm_kmguard_init(&kmem_guard, &kmem_guard_depth, &kmem_guard_size,
- kernel_map);
-#endif
-
- kmem_arena = vmem_create("kmem", 0, 0, KMEM_QUANTUM_SIZE,
- kmem_backend_alloc, kmem_backend_free, NULL, KMEM_QCACHE_MAX,
- VM_SLEEP, IPL_NONE);
- callback_register(&vm_map_to_kernel(kernel_map)->vmk_reclaim_callback,
- &kmem_kva_reclaim_entry, kmem_arena, kmem_kva_reclaim_callback);
+ for (i = 0; array[i].kc_size != 0 ; i++) {
+ size_t cache_size = array[i].kc_size;
+ size_t align = (cache_size < PAGE_SIZE) ?
+ KMEM_TINY_ALIGN : PAGE_SIZE;
+ const char *name = array[i].kc_name;
+ pool_cache_t pc;
- /*
- * kmem caches start at twice the size of the largest vmem qcache
- * and end at PAGE_SIZE or earlier. assert that KMEM_QCACHE_MAX
- * is a power of two.
- */
- KASSERT(ffs(KMEM_QCACHE_MAX) != 0);
- KASSERT(KMEM_QCACHE_MAX - (1 << (ffs(KMEM_QCACHE_MAX) - 1)) == 0);
- kmem_cache_shift = ffs(KMEM_QCACHE_MAX);
- kmem_cache_min = 1 << kmem_cache_shift;
- kmem_cache_mask = kmem_cache_min - 1;
- for (i = 1; i <= KMEM_CACHE_COUNT; i++) {
- sz = i << kmem_cache_shift;
- if (sz > PAGE_SIZE) {
+ /* check if we reached the requested size */
+ if (size > maxbuf)
break;
+
+ pc = pool_cache_init(cache_size,
+ align, 0, PR_NOALIGN,
+ name, array[i].kc_allocator, IPL_VM,
+ NULL, NULL, NULL);
+
+ while (size <= cache_size) {
+ alloc_table[(size - 1) >> shift] = pc;
+ size += table_unit;
}
- kmem_cache_max = sz;
- kc = &kmem_cache[i];
- kc->kc_pa.pa_pagesz = sz;
- kc->kc_pa.pa_alloc = kmem_poolpage_alloc;
- kc->kc_pa.pa_free = kmem_poolpage_free;
- sprintf(kc->kc_name, "kmem-%zu", sz);
- kc->kc_cache = pool_cache_init(sz,
- KMEM_QUANTUM_SIZE, 0, PR_NOALIGN | PR_NOTOUCH,
- kc->kc_name, &kc->kc_pa, IPL_NONE,
- NULL, NULL, NULL);
- KASSERT(kc->kc_cache != NULL);
}
}
-size_t
-kmem_roundup_size(size_t size)
+void
+kmem_init(void)
{
+ /*
+ * Make allocation size the same size as the va cache size in uvm_km to
lower fragmentation.
+ */
+ kmem_big_8k_pa.pa_pagesz = 2*4096;
+ kmem_big_8k_pa.pa_alloc = kmem_bigpoolpage_alloc;
+ kmem_big_8k_pa.pa_free = kmem_bigpoolpage_free;
+ kmem_big_8k_pa.pa_backingmap = NULL;
+
+ kmem_big_12k_pa.pa_pagesz = 3*4096;
+ kmem_big_12k_pa.pa_alloc = kmem_bigpoolpage_alloc;
+ kmem_big_12k_pa.pa_free = kmem_bigpoolpage_free;
+ kmem_big_12k_pa.pa_backingmap = NULL;
+
+ kmem_big_16k_pa.pa_pagesz = 4*4096;
+ kmem_big_16k_pa.pa_alloc = kmem_bigpoolpage_alloc;
+ kmem_big_16k_pa.pa_free = kmem_bigpoolpage_free;
+ kmem_big_16k_pa.pa_backingmap = NULL;
- return vmem_roundup_size(kmem_arena, size);
-}
-
-/* ---- uvm glue */
-
-static vmem_addr_t
-kmem_backend_alloc(vmem_t *dummy, vmem_size_t size, vmem_size_t *resultsize,
- vm_flag_t vmflags)
-{
- uvm_flag_t uflags;
- vaddr_t va;
+ kmem_create_caches(kmem_cache_tiny_sizes, kmem_cache_tiny,
+ KMEM_TINY_MAXSIZE, KMEM_TINY_SHIFT);
- KASSERT(dummy == NULL);
- KASSERT(size != 0);
- KASSERT((vmflags & (VM_SLEEP|VM_NOSLEEP)) != 0);
- KASSERT((~vmflags & (VM_SLEEP|VM_NOSLEEP)) != 0);
+ kmem_create_caches(kmem_cache_big_sizes, kmem_cache_big,
+ KMEM_BIG_MAXSIZE, KMEM_BIG_SHIFT);
- if ((vmflags & VM_NOSLEEP) != 0) {
- uflags = UVM_KMF_TRYLOCK | UVM_KMF_NOWAIT;
- } else {
- uflags = UVM_KMF_WAITVA;
- }
- *resultsize = size = round_page(size);
- va = uvm_km_alloc(kernel_map, size, 0,
- uflags | UVM_KMF_WIRED | UVM_KMF_CANFAIL);
- if (va != 0) {
- kmem_poison_fill((void *)va, size);
- }
- return (vmem_addr_t)va;
}
-static void
-kmem_backend_free(vmem_t *dummy, vmem_addr_t addr, vmem_size_t size)
-{
-
- KASSERT(dummy == NULL);
- KASSERT(addr != 0);
- KASSERT(size != 0);
- KASSERT(size == round_page(size));
-
- kmem_poison_check((void *)addr, size);
- uvm_km_free(kernel_map, (vaddr_t)addr, size, UVM_KMF_WIRED);
-}
-static int
-kmem_kva_reclaim_callback(struct callback_entry *ce, void *obj, void *arg)
+size_t
+kmem_roundup_size(size_t size)
{
- vmem_t *vm = obj;
-
- vmem_reap(vm);
- return CALLBACK_CHAIN_CONTINUE;
+ return (size + (KMEM_TINY_ALIGN - 1)) & ~(KMEM_TINY_ALIGN - 1);
}
/* ---- debug */
-#if defined(KMEM_POISON)
-
-#if defined(_LP64)
-#define PRIME 0x9e37fffffffc0001UL
-#else /* defined(_LP64) */
-#define PRIME 0x9e3779b1
-#endif /* defined(_LP64) */
-
-static inline uint8_t
-kmem_poison_pattern(const void *p)
-{
-
- return (uint8_t)((((uintptr_t)p) * PRIME)
- >> ((sizeof(uintptr_t) - sizeof(uint8_t))) * CHAR_BIT);
-}
-
-static void
-kmem_poison_fill(void *p, size_t sz)
-{
- uint8_t *cp;
- const uint8_t *ep;
-
- cp = p;
- ep = cp + sz;
- while (cp < ep) {
- *cp = kmem_poison_pattern(cp);
- cp++;
- }
-}
-
-static void
-kmem_poison_check(void *p, size_t sz)
-{
- uint8_t *cp;
- const uint8_t *ep;
-
- cp = p;
- ep = cp + sz;
- while (cp < ep) {
- const uint8_t expected = kmem_poison_pattern(cp);
-
- if (*cp != expected) {
- panic("%s: %p: 0x%02x != 0x%02x\n",
- __func__, cp, *cp, expected);
- }
- cp++;
- }
-}
-
-#endif /* defined(KMEM_POISON) */
-
#if defined(KMEM_SIZE)
static void
kmem_size_set(void *p, size_t sz)
Index: sys/kern/subr_pool.c
===================================================================
RCS file: /cvsroot/src/sys/kern/subr_pool.c,v
retrieving revision 1.188
diff -u -r1.188 subr_pool.c
--- sys/kern/subr_pool.c 17 Jan 2011 07:36:58 -0000 1.188
+++ sys/kern/subr_pool.c 21 Jan 2011 09:09:45 -0000
@@ -72,6 +72,12 @@
* an internal pool of page headers (`phpool').
*/
+/* Flag for beeing bootstrapped */
+static int bootstrapped = 0;
+
+/* Flag for beeing initialized */
+static int initialized = 0;
+
/* List of all pools */
static TAILQ_HEAD(, pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head);
@@ -86,9 +92,6 @@
static struct pool psppool;
#endif
-static SLIST_HEAD(, pool_allocator) pa_deferinitq =
- SLIST_HEAD_INITIALIZER(pa_deferinitq);
-
static void *pool_page_alloc_meta(struct pool *, int);
static void pool_page_free_meta(struct pool *, void *);
@@ -595,10 +598,6 @@
struct pool *pp;
KASSERT(pa->pa_backingmap == NULL);
- if (map == NULL) {
- SLIST_INSERT_HEAD(&pa_deferinitq, pa, pa_q);
- return;
- }
pa->pa_backingmap = map;
TAILQ_FOREACH(pp, &pa->pa_list, pr_alloc_list) {
pool_reclaim_register(pp);
@@ -606,23 +605,51 @@
}
/*
- * Initialize all the pools listed in the "pools" link set.
+ * Initialize all the internal pools.
*/
void
-pool_subsystem_init(void)
+pool_subsystem_bootstrap(void)
{
- struct pool_allocator *pa;
+ int idx;
+ size_t size;
mutex_init(&pool_head_lock, MUTEX_DEFAULT, IPL_NONE);
mutex_init(&pool_allocator_lock, MUTEX_DEFAULT, IPL_NONE);
cv_init(&pool_busy, "poolbusy");
- while ((pa = SLIST_FIRST(&pa_deferinitq)) != NULL) {
- KASSERT(pa->pa_backingmapptr != NULL);
- KASSERT(*pa->pa_backingmapptr != NULL);
- SLIST_REMOVE_HEAD(&pa_deferinitq, pa_q);
- pa_reclaim_register(pa);
+ /* setup flag to stop pool_init from panicing, when initializing our
private pools */
+ bootstrapped = 1;
+
+ for (idx = 0; idx < PHPOOL_MAX; idx++) {
+ static char phpool_names[PHPOOL_MAX][6+1+6+1];
+ int nelem;
+ size_t sz;
+
+ nelem = PHPOOL_FREELIST_NELEM(idx);
+ snprintf(phpool_names[idx], sizeof(phpool_names[idx]),
+ "phpool-%d", nelem);
+ sz = sizeof(struct pool_item_header);
+ if (nelem) {
+ sz = offsetof(struct pool_item_header,
+ ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
+ }
+ pool_init(&phpool[idx], sz, 0, 0, 0,
+ phpool_names[idx], &pool_allocator_meta, IPL_VM);
}
+#ifdef POOL_SUBPAGE
+ pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0,
+ PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM);
+#endif
+
+ size = sizeof(pcg_t) +
+ (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t);
+ pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0,
+ "pcgnormal", &pool_allocator_meta, IPL_VM);
+
+ size = sizeof(pcg_t) +
+ (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t);
+ pool_init(&pcg_large_pool, size, coherency_unit, 0, 0,
+ "pcglarge", &pool_allocator_meta, IPL_VM);
pool_init(&cache_pool, sizeof(struct pool_cache), coherency_unit,
0, 0, "pcache", &pool_allocator_nointr, IPL_NONE);
@@ -631,6 +658,12 @@
0, 0, "pcachecpu", &pool_allocator_nointr, IPL_NONE);
}
+void
+pool_subsystem_init(void)
+{
+ initialized = 1;
+}
+
/*
* Initialize the given pool resource structure.
*
@@ -645,6 +678,9 @@
size_t trysize, phsize;
int off, slack;
+ if (!bootstrapped)
+ panic("pool_init: pool-subsystem not bootstrapped");
+
#ifdef DEBUG
/*
* Check that the pool hasn't already been initialised and
@@ -675,8 +711,7 @@
palloc = &pool_allocator_nointr_fullpage;
}
#endif /* POOL_SUBPAGE */
- if (!cold)
- mutex_enter(&pool_allocator_lock);
+ mutex_enter(&pool_allocator_lock);
if (palloc->pa_refcnt++ == 0) {
if (palloc->pa_pagesz == 0)
palloc->pa_pagesz = PAGE_SIZE;
@@ -691,8 +726,7 @@
pa_reclaim_register(palloc);
}
}
- if (!cold)
- mutex_exit(&pool_allocator_lock);
+ mutex_exit(&pool_allocator_lock);
if (align == 0)
align = ALIGN(1);
@@ -824,48 +858,8 @@
cv_init(&pp->pr_cv, wchan);
pp->pr_ipl = ipl;
- /*
- * Initialize private page header pool and cache magazine pool if we
- * haven't done so yet.
- * XXX LOCKING.
- */
- if (phpool[0].pr_size == 0) {
- int idx;
- for (idx = 0; idx < PHPOOL_MAX; idx++) {
- static char phpool_names[PHPOOL_MAX][6+1+6+1];
- int nelem;
- size_t sz;
-
- nelem = PHPOOL_FREELIST_NELEM(idx);
- snprintf(phpool_names[idx], sizeof(phpool_names[idx]),
- "phpool-%d", nelem);
- sz = sizeof(struct pool_item_header);
- if (nelem) {
- sz = offsetof(struct pool_item_header,
- ph_bitmap[howmany(nelem, BITMAP_SIZE)]);
- }
- pool_init(&phpool[idx], sz, 0, 0, 0,
- phpool_names[idx], &pool_allocator_meta, IPL_VM);
- }
-#ifdef POOL_SUBPAGE
- pool_init(&psppool, POOL_SUBPAGE, POOL_SUBPAGE, 0,
- PR_RECURSIVE, "psppool", &pool_allocator_meta, IPL_VM);
-#endif
-
- size = sizeof(pcg_t) +
- (PCG_NOBJECTS_NORMAL - 1) * sizeof(pcgpair_t);
- pool_init(&pcg_normal_pool, size, coherency_unit, 0, 0,
- "pcgnormal", &pool_allocator_meta, IPL_VM);
-
- size = sizeof(pcg_t) +
- (PCG_NOBJECTS_LARGE - 1) * sizeof(pcgpair_t);
- pool_init(&pcg_large_pool, size, coherency_unit, 0, 0,
- "pcglarge", &pool_allocator_meta, IPL_VM);
- }
-
/* Insert into the list of all pools. */
- if (!cold)
- mutex_enter(&pool_head_lock);
+ mutex_enter(&pool_head_lock);
TAILQ_FOREACH(pp1, &pool_head, pr_poollist) {
if (strcmp(pp1->pr_wchan, pp->pr_wchan) > 0)
break;
@@ -874,15 +868,12 @@
TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);
else
TAILQ_INSERT_BEFORE(pp1, pp, pr_poollist);
- if (!cold)
- mutex_exit(&pool_head_lock);
+ mutex_exit(&pool_head_lock);
/* Insert this into the list of pools using this allocator. */
- if (!cold)
- mutex_enter(&palloc->pa_lock);
+ mutex_enter(&palloc->pa_lock);
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);
- if (!cold)
- mutex_exit(&palloc->pa_lock);
+ mutex_exit(&palloc->pa_lock);
pool_reclaim_register(pp);
}
@@ -991,6 +982,9 @@
struct pool_item_header *ph;
void *v;
+ if (!initialized)
+ panic("pool_get: called prior to pool_subsystem
initialization");
+
#ifdef DIAGNOSTIC
if (pp->pr_itemsperpage == 0)
panic("pool_get: pool '%s': pr_itemsperpage is zero, "
@@ -2130,8 +2124,7 @@
}
/* Add to list of all pools. */
- if (__predict_true(!cold))
- mutex_enter(&pool_head_lock);
+ mutex_enter(&pool_head_lock);
TAILQ_FOREACH(pc1, &pool_cache_head, pc_cachelist) {
if (strcmp(pc1->pc_pool.pr_wchan, pc->pc_pool.pr_wchan) > 0)
break;
@@ -2140,8 +2133,7 @@
TAILQ_INSERT_TAIL(&pool_cache_head, pc, pc_cachelist);
else
TAILQ_INSERT_BEFORE(pc1, pc, pc_cachelist);
- if (__predict_true(!cold))
- mutex_exit(&pool_head_lock);
+ mutex_exit(&pool_head_lock);
membar_sync();
pp->pr_cache = pc;
@@ -2881,14 +2873,14 @@
{
bool waitok = (flags & PR_WAITOK) ? true : false;
- return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, waitok));
+ return ((void *) uvm_km_alloc_poolpage_cache(kmem_map, PAGE_SIZE,
waitok));
}
void
pool_page_free(struct pool *pp, void *v)
{
- uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) v);
+ uvm_km_free_poolpage_cache(kmem_map, (vaddr_t) v, PAGE_SIZE);
}
static void *
@@ -2896,14 +2888,14 @@
{
bool waitok = (flags & PR_WAITOK) ? true : false;
- return ((void *) uvm_km_alloc_poolpage(kmem_map, waitok));
+ return ((void *) uvm_km_alloc_poolpage(kmem_map, PAGE_SIZE, waitok));
}
static void
pool_page_free_meta(struct pool *pp, void *v)
{
- uvm_km_free_poolpage(kmem_map, (vaddr_t) v);
+ uvm_km_free_poolpage(kmem_map, (vaddr_t) v, PAGE_SIZE);
}
#ifdef POOL_SUBPAGE
@@ -2940,14 +2932,14 @@
{
bool waitok = (flags & PR_WAITOK) ? true : false;
- return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, waitok));
+ return ((void *) uvm_km_alloc_poolpage_cache(kernel_map, PAGE_SIZE,
waitok));
}
void
pool_page_free_nointr(struct pool *pp, void *v)
{
- uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v);
+ uvm_km_free_poolpage_cache(kernel_map, (vaddr_t) v, PAGE_SIZE);
}
#if defined(DDB)
Index: sys/rump/librump/rumpkern/vm.c
===================================================================
RCS file: /cvsroot/src/sys/rump/librump/rumpkern/vm.c,v
retrieving revision 1.107
diff -u -r1.107 vm.c
--- sys/rump/librump/rumpkern/vm.c 18 Jan 2011 22:21:23 -0000 1.107
+++ sys/rump/librump/rumpkern/vm.c 21 Jan 2011 09:09:46 -0000
@@ -713,32 +713,32 @@
}
vaddr_t
-uvm_km_alloc_poolpage(struct vm_map *map, bool waitok)
+uvm_km_alloc_poolpage(struct vm_map *map, size_t size, bool waitok)
{
- return (vaddr_t)rump_hypermalloc(PAGE_SIZE, PAGE_SIZE,
+ return (vaddr_t)rump_hypermalloc(size, PAGE_SIZE,
waitok, "kmalloc");
}
void
-uvm_km_free_poolpage(struct vm_map *map, vaddr_t addr)
+uvm_km_free_poolpage(struct vm_map *map, vaddr_t addr, size_t size)
{
- rump_hyperfree((void *)addr, PAGE_SIZE);
+ rump_hyperfree((void *)addr, size);
}
vaddr_t
-uvm_km_alloc_poolpage_cache(struct vm_map *map, bool waitok)
+uvm_km_alloc_poolpage_cache(struct vm_map *map, size_t size, bool waitok)
{
- return uvm_km_alloc_poolpage(map, waitok);
+ return uvm_km_alloc_poolpage(map, size, waitok);
}
void
-uvm_km_free_poolpage_cache(struct vm_map *map, vaddr_t vaddr)
+uvm_km_free_poolpage_cache(struct vm_map *map, vaddr_t vaddr, size_t size)
{
- uvm_km_free_poolpage(map, vaddr);
+ uvm_km_free_poolpage(map, vaddr, size);
}
void
Index: sys/sys/pool.h
===================================================================
RCS file: /cvsroot/src/sys/sys/pool.h,v
retrieving revision 1.70
diff -u -r1.70 pool.h
--- sys/sys/pool.h 3 Jun 2010 10:40:17 -0000 1.70
+++ sys/sys/pool.h 21 Jan 2011 09:09:46 -0000
@@ -265,6 +265,7 @@
extern struct pool_allocator pool_allocator_nointr_fullpage;
#endif
+void pool_subsystem_bootstrap(void);
void pool_subsystem_init(void);
void pool_init(struct pool *, size_t, u_int, u_int,
Index: sys/uvm/uvm_extern.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_extern.h,v
retrieving revision 1.168
diff -u -r1.168 uvm_extern.h
--- sys/uvm/uvm_extern.h 4 Jan 2011 08:26:33 -0000 1.168
+++ sys/uvm/uvm_extern.h 21 Jan 2011 09:09:46 -0000
@@ -652,12 +652,11 @@
struct vm_map *uvm_km_suballoc(struct vm_map *, vaddr_t *,
vaddr_t *, vsize_t, int, bool,
struct vm_map_kernel *);
-vaddr_t uvm_km_alloc_poolpage(struct vm_map *, bool);
-void uvm_km_free_poolpage(struct vm_map *, vaddr_t);
-vaddr_t uvm_km_alloc_poolpage_cache(struct vm_map *,
bool);
-void uvm_km_free_poolpage_cache(struct vm_map *, vaddr_t);
-void uvm_km_vacache_init(struct vm_map *,
- const char *, size_t);
+vaddr_t uvm_km_alloc_poolpage(struct vm_map *, size_t,
bool);
+void uvm_km_free_poolpage(struct vm_map *, vaddr_t, size_t);
+vaddr_t uvm_km_alloc_poolpage_cache(struct vm_map *,
size_t, bool);
+void uvm_km_free_poolpage_cache(struct vm_map *, vaddr_t,
size_t);
+void uvm_km_vacache_init(struct vm_map *, const char *);
/* uvm_map.c */
int uvm_map(struct vm_map *, vaddr_t *, vsize_t,
Index: sys/uvm/uvm_init.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_init.c,v
retrieving revision 1.38
diff -u -r1.38 uvm_init.c
--- sys/uvm/uvm_init.c 14 Nov 2010 04:31:02 -0000 1.38
+++ sys/uvm/uvm_init.c 21 Jan 2011 09:09:46 -0000
@@ -100,7 +100,6 @@
memset(&uvm, 0, sizeof(uvm));
averunnable.fscale = FSCALE;
- uvm_amap_init();
/*
* step 2: init the page sub-system. this includes allocating the
@@ -111,6 +110,7 @@
*/
uvm_page_init(&kvm_start, &kvm_end);
+ pool_subsystem_bootstrap();
/*
* step 3: init the map sub-system. allocates the static pool of
@@ -119,59 +119,50 @@
*/
uvm_map_init();
+ uvm_amap_init();
/*
* step 4: setup the kernel's virtual memory data structures. this
* includes setting up the kernel_map/kernel_object.
+ * Bootstrap all kernel memory allocators
*/
uvm_km_init(kvm_start, kvm_end);
/*
- * step 5: init the pmap module. the pmap module is free to allocate
- * memory for its private use (e.g. pvlists).
+ * step 5: init the kernel memory allocator. after this call the
+ * kernel memory allocator (malloc) can be used.
*/
- pmap_init();
+ kmeminit(); /* kill this */
+ pool_subsystem_init();
+ kmem_init();
+
/*
- * step 6: init the kernel memory allocator. after this call the
- * kernel memory allocator (malloc) can be used. this includes
- * setting up the kmem_map.
+ * step 6: init the pmap module. the pmap module is free to allocate
+ * memory for its private use (e.g. pvlists).
*/
- kmeminit();
+ pmap_init();
#ifdef DEBUG
debug_init();
#endif
/*
- * step 7: init all pagers and the pager_map.
+ * step 6: init all pagers and the pager_map.
*/
uvm_pager_init();
/*
- * step 8: init the uvm_loan() facility.
+ * step 7: init the uvm_loan() facility.
*/
uvm_loan_init();
/*
- * Initialize pools. This must be done before anyone manipulates
- * any vm_maps because we use a pool for some map entry structures.
- */
-
- pool_subsystem_init();
-
- /*
- * init slab memory allocator kmem(9).
- */
-
- kmem_init();
-
- /*
* init emap subsystem.
*/
Index: sys/uvm/uvm_km.c
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_km.c,v
retrieving revision 1.107
diff -u -r1.107 uvm_km.c
--- sys/uvm/uvm_km.c 4 Jan 2011 08:26:33 -0000 1.107
+++ sys/uvm/uvm_km.c 21 Jan 2011 09:09:47 -0000
@@ -136,6 +136,7 @@
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/pool.h>
+#include <sys/kmem.h>
#include <uvm/uvm.h>
@@ -144,6 +145,7 @@
*/
struct vm_map *kernel_map = NULL;
+struct vm_map *kmem_map = NULL;
/*
* local data structues
@@ -152,35 +154,52 @@
static struct vm_map_kernel kernel_map_store;
static struct vm_map_entry kernel_first_mapent_store;
-#if !defined(PMAP_MAP_POOLPAGE)
+/*
+ * Default number of pages in kmem_map. We attempt to calculate this
+ * at run-time, but allow it to be either patched or set in the kernel
+ * config file.
+ */
+#ifndef NKMEMPAGES
+#define NKMEMPAGES 0
+#endif
+int nkmempages = NKMEMPAGES;
+
+/*
+ * Defaults for lower- and upper-bounds for the kmem_map page count.
+ * Can be overridden by kernel config options.
+ */
+#ifndef NKMEMPAGES_MIN
+#define NKMEMPAGES_MIN NKMEMPAGES_MIN_DEFAULT
+#endif
+
+#ifndef NKMEMPAGES_MAX
+#define NKMEMPAGES_MAX NKMEMPAGES_MAX_DEFAULT
+#endif
+
+static struct vm_map_kernel kmem_map_store;
+static char *kmembase, *kmemlimit;
/*
* kva cache
*
- * XXX maybe it's better to do this at the uvm_map layer.
+ * caches are up to PAGE_SIZE * 4.
+ * KM_VACACHE_SIZE should be multiply of largest cache.
+ * if PMAP_MAP_POOLPAGE is defined the PAGE_SIZE cache is skipped.
*/
-
-#define KM_VACACHE_SIZE (32 * PAGE_SIZE) /* XXX tune */
+#define KM_VACACHE_SIZE (32 * PAGE_SIZE)
static void *km_vacache_alloc(struct pool *, int);
static void km_vacache_free(struct pool *, void *);
static void km_vacache_init(struct vm_map *, const char *, size_t);
-/* XXX */
-#define KM_VACACHE_POOL_TO_MAP(pp) \
- ((struct vm_map *)((char *)(pp) - \
- offsetof(struct vm_map_kernel, vmk_vacache)))
-
static void *
km_vacache_alloc(struct pool *pp, int flags)
{
vaddr_t va;
size_t size;
- struct vm_map *map;
+ struct vm_map *map = pp->pr_alloc->pa_backingmap;
size = pp->pr_alloc->pa_pagesz;
- map = KM_VACACHE_POOL_TO_MAP(pp);
-
va = vm_map_min(map); /* hint */
if (uvm_map(map, &va, size, NULL, UVM_UNKNOWN_OFFSET, size,
UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE,
@@ -197,9 +216,8 @@
{
vaddr_t va = (vaddr_t)v;
size_t size = pp->pr_alloc->pa_pagesz;
- struct vm_map *map;
+ struct vm_map *map = pp->pr_alloc->pa_backingmap;
- map = KM_VACACHE_POOL_TO_MAP(pp);
uvm_unmap1(map, va, va + size, UVM_FLAG_QUANTUM|UVM_FLAG_VAONLY);
}
@@ -213,52 +231,52 @@
struct vm_map_kernel *vmk;
struct pool *pp;
struct pool_allocator *pa;
+ char *cn;
int ipl;
+ int i;
KASSERT(VM_MAP_IS_KERNEL(map));
KASSERT(size < (vm_map_max(map) - vm_map_min(map)) / 2); /* sanity */
vmk = vm_map_to_kernel(map);
- pp = &vmk->vmk_vacache;
- pa = &vmk->vmk_vacache_allocator;
- memset(pa, 0, sizeof(*pa));
- pa->pa_alloc = km_vacache_alloc;
- pa->pa_free = km_vacache_free;
- pa->pa_pagesz = (unsigned int)size;
- pa->pa_backingmap = map;
- pa->pa_backingmapptr = NULL;
-
- if ((map->flags & VM_MAP_INTRSAFE) != 0)
- ipl = IPL_VM;
- else
- ipl = IPL_NONE;
- pool_init(pp, PAGE_SIZE, 0, 0, PR_NOTOUCH | PR_RECURSIVE, name, pa,
- ipl);
+#if defined(PMAP_MAP_POOLPAGE)
+ i = 1;
+#else
+ i = 0;
+#endif
+ for (; i < VMK_VACACHE_COUNT; i++) {
+ size_t cachesize = (i+1) * PAGE_SIZE;
+ pp = &vmk->vmk_vacache[i];
+ pa = &vmk->vmk_vacache_allocator[i];
+ cn = vmk->vmk_vacache_name[i];
+ memset(pa, 0, sizeof(*pa));
+ pa->pa_alloc = km_vacache_alloc;
+ pa->pa_free = km_vacache_free;
+ pa->pa_pagesz = (unsigned int)size;
+ pa->pa_backingmap = map;
+ pa->pa_backingmapptr = NULL;
+
+ if ((map->flags & VM_MAP_INTRSAFE) != 0)
+ ipl = IPL_VM;
+ else
+ ipl = IPL_NONE;
+
+ sprintf(cn, "%s-%i", name, i+1);
+ pool_init(pp, cachesize, 0, 0, PR_NOTOUCH | PR_RECURSIVE, cn,
pa,
+ ipl);
+ }
}
void
-uvm_km_vacache_init(struct vm_map *map, const char *name, size_t size)
+uvm_km_vacache_init(struct vm_map *map, const char *name)
{
map->flags |= VM_MAP_VACACHE;
- if (size == 0)
- size = KM_VACACHE_SIZE;
- km_vacache_init(map, name, size);
+ km_vacache_init(map, name, KM_VACACHE_SIZE);
}
-#else /* !defined(PMAP_MAP_POOLPAGE) */
-
-void
-uvm_km_vacache_init(struct vm_map *map, const char *name, size_t size)
-{
-
- /* nothing */
-}
-
-#endif /* !defined(PMAP_MAP_POOLPAGE) */
-
void
uvm_km_va_drain(struct vm_map *map, uvm_flag_t flags)
{
@@ -280,6 +298,8 @@
uvm_km_init(vaddr_t start, vaddr_t end)
{
vaddr_t base = VM_MIN_KERNEL_ADDRESS;
+ vaddr_t kmb = 0;
+ vaddr_t kml;
/*
* next, init kernel memory objects.
@@ -288,7 +308,8 @@
/* kernel_object: for pageable anonymous kernel memory */
uao_init();
uvm_kernel_object = uao_create(VM_MAX_KERNEL_ADDRESS -
- VM_MIN_KERNEL_ADDRESS, UAO_FLAG_KERNOBJ);
+ VM_MIN_KERNEL_ADDRESS, UAO_FLAG_KERNOBJ);
+
/*
* init the map and reserve any space that might already
@@ -323,7 +344,25 @@
*/
kernel_map = &kernel_map_store.vmk_map;
- uvm_km_vacache_init(kernel_map, "kvakernel", 0);
+ uvm_km_vacache_init(kernel_map, "kvakernel");
+
+ /*
+ * Boostrap kmem submap
+ */
+ nkmempages = physmem;
+
+ if (nkmempages > NKMEMPAGES_MAX)
+ nkmempages = NKMEMPAGES_MAX;
+
+ if (nkmempages < NKMEMPAGES_MIN)
+ nkmempages = NKMEMPAGES_MIN;
+
+ kmem_map = uvm_km_suballoc(kernel_map, &kmb,
+ &kml, ((vsize_t)nkmempages << PAGE_SHIFT),
+ VM_MAP_INTRSAFE, false, &kmem_map_store);
+ uvm_km_vacache_init(kmem_map, "kvakmem");
+ kmembase = (char *)kmb;
+ kmemlimit = (char *)kml;
}
/*
@@ -689,70 +728,88 @@
/* ARGSUSED */
vaddr_t
-uvm_km_alloc_poolpage_cache(struct vm_map *map, bool waitok)
+uvm_km_alloc_poolpage_cache(struct vm_map *map, size_t size, bool waitok)
{
-#if defined(PMAP_MAP_POOLPAGE)
- return uvm_km_alloc_poolpage(map, waitok);
-#else
struct vm_page *pg;
- struct pool *pp = &vm_map_to_kernel(map)->vmk_vacache;
+ int index;
+ struct pool *pp;
vaddr_t va;
+ vaddr_t loopva;
+ vsize_t loopsize;
+
+
+#if defined(PMAP_MAP_POOLPAGE)
+ if (size == PAGE_SIZE)
+ return uvm_km_alloc_poolpage(map, size, waitok);
+#endif /* PMAP_MAP_POOLPAGE */
if ((map->flags & VM_MAP_VACACHE) == 0)
- return uvm_km_alloc_poolpage(map, waitok);
+ return uvm_km_alloc_poolpage(map, size, waitok);
+
+ index = (size >> PAGE_SHIFT) - 1;
+ pp = &vm_map_to_kernel(map)->vmk_vacache[index];
va = (vaddr_t)pool_get(pp, waitok ? PR_WAITOK : PR_NOWAIT);
if (va == 0)
return 0;
- KASSERT(!pmap_extract(pmap_kernel(), va, NULL));
-again:
- pg = uvm_pagealloc(NULL, 0, NULL, waitok ? 0 : UVM_PGA_USERESERVE);
- if (__predict_false(pg == NULL)) {
- if (waitok) {
- uvm_wait("plpg");
- goto again;
- } else {
- pool_put(pp, (void *)va);
- return 0;
+
+ loopva = va;
+ loopsize = size;
+
+ while (loopsize) {
+ KASSERT(!pmap_extract(pmap_kernel(), loopva, NULL));
+
+ pg = uvm_pagealloc(NULL, 0, NULL, waitok ? 0 :
UVM_PGA_USERESERVE);
+ if (__predict_false(pg == NULL)) {
+ if (waitok) {
+ uvm_wait("plpg");
+ continue;
+ } else {
+ pool_put(pp, (void *)va);
+ return 0;
+ }
}
+
+ pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg),
+ VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE);
+
+ loopva += PAGE_SIZE;
+ loopsize -= PAGE_SIZE;
}
- pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
- VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE);
pmap_update(pmap_kernel());
return va;
-#endif /* PMAP_MAP_POOLPAGE */
}
vaddr_t
-uvm_km_alloc_poolpage(struct vm_map *map, bool waitok)
+uvm_km_alloc_poolpage(struct vm_map *map, size_t size, bool waitok)
{
+ vaddr_t va;
#if defined(PMAP_MAP_POOLPAGE)
struct vm_page *pg;
- vaddr_t va;
-
+ if (size == PAGE_SIZE) {
again:
#ifdef PMAP_ALLOC_POOLPAGE
- pg = PMAP_ALLOC_POOLPAGE(waitok ? 0 : UVM_PGA_USERESERVE);
+ pg = PMAP_ALLOC_POOLPAGE(waitok ? 0 : UVM_PGA_USERESERVE);
#else
- pg = uvm_pagealloc(NULL, 0, NULL, waitok ? 0 : UVM_PGA_USERESERVE);
+ pg = uvm_pagealloc(NULL, 0, NULL, waitok ? 0 :
UVM_PGA_USERESERVE);
#endif
- if (__predict_false(pg == NULL)) {
- if (waitok) {
- uvm_wait("plpg");
- goto again;
- } else
- return (0);
+ if (__predict_false(pg == NULL)) {
+ if (waitok) {
+ uvm_wait("plpg");
+ goto again;
+ } else
+ return (0);
+ }
+ va = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg));
+ if (__predict_false(va == 0))
+ uvm_pagefree(pg);
+ return (va);
}
- va = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg));
- if (__predict_false(va == 0))
- uvm_pagefree(pg);
- return (va);
#else
- vaddr_t va;
- va = uvm_km_alloc(map, PAGE_SIZE, 0,
+ va = uvm_km_alloc(map, size, 0,
(waitok ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK) | UVM_KMF_WIRED);
return (va);
#endif /* PMAP_MAP_POOLPAGE */
@@ -766,40 +823,41 @@
/* ARGSUSED */
void
-uvm_km_free_poolpage_cache(struct vm_map *map, vaddr_t addr)
+uvm_km_free_poolpage_cache(struct vm_map *map, vaddr_t addr, size_t size)
{
-#if defined(PMAP_UNMAP_POOLPAGE)
- uvm_km_free_poolpage(map, addr);
-#else
struct pool *pp;
+ int index;
+
+#if defined(PMAP_UNMAP_POOLPAGE)
+ if (size == PAGE_SIZE)
+ uvm_km_free_poolpage(map, addr, size);
+#endif /* PMAP_UNMAP_POOLPAGE */
if ((map->flags & VM_MAP_VACACHE) == 0) {
- uvm_km_free_poolpage(map, addr);
+ uvm_km_free_poolpage(map, addr, size);
return;
}
- KASSERT(pmap_extract(pmap_kernel(), addr, NULL));
- uvm_km_pgremove_intrsafe(map, addr, addr + PAGE_SIZE);
- pmap_kremove(addr, PAGE_SIZE);
-#if defined(DEBUG)
- pmap_update(pmap_kernel());
-#endif
- KASSERT(!pmap_extract(pmap_kernel(), addr, NULL));
- pp = &vm_map_to_kernel(map)->vmk_vacache;
+ uvm_km_pgremove_intrsafe(map, addr, addr + size);
+ pmap_kremove(addr, size);
+
+ index = (size >> PAGE_SHIFT) - 1;
+ pp = &vm_map_to_kernel(map)->vmk_vacache[index];
pool_put(pp, (void *)addr);
-#endif
}
/* ARGSUSED */
void
-uvm_km_free_poolpage(struct vm_map *map, vaddr_t addr)
+uvm_km_free_poolpage(struct vm_map *map, vaddr_t addr, size_t size)
{
#if defined(PMAP_UNMAP_POOLPAGE)
paddr_t pa;
- pa = PMAP_UNMAP_POOLPAGE(addr);
- uvm_pagefree(PHYS_TO_VM_PAGE(pa));
+ if (size == PAGE_SIZE) {
+ pa = PMAP_UNMAP_POOLPAGE(addr);
+ uvm_pagefree(PHYS_TO_VM_PAGE(pa));
+ } else
#else
- uvm_km_free(map, addr, PAGE_SIZE, UVM_KMF_WIRED);
+ uvm_km_free(map, addr, size, UVM_KMF_WIRED);
#endif /* PMAP_UNMAP_POOLPAGE */
}
Index: sys/uvm/uvm_map.h
===================================================================
RCS file: /cvsroot/src/sys/uvm/uvm_map.h,v
retrieving revision 1.65
diff -u -r1.65 uvm_map.h
--- sys/uvm/uvm_map.h 25 Sep 2010 01:42:40 -0000 1.65
+++ sys/uvm/uvm_map.h 21 Jan 2011 09:09:47 -0000
@@ -230,6 +230,8 @@
#include <sys/callback.h>
+/* we cache up to 4 times PAGE_SIZE va ranges */
+#define VMK_VACACHE_COUNT 4
struct vm_map_kernel {
struct vm_map vmk_map;
LIST_HEAD(, uvm_kmapent_hdr) vmk_kentry_free;
@@ -238,10 +240,9 @@
/* Merged entries, kept for later splitting */
struct callback_head vmk_reclaim_callback;
-#if !defined(PMAP_MAP_POOLPAGE)
- struct pool vmk_vacache; /* kva cache */
- struct pool_allocator vmk_vacache_allocator; /* ... and its allocator */
-#endif
+ struct pool vmk_vacache[VMK_VACACHE_COUNT]; /* kva cache */
+ struct pool_allocator vmk_vacache_allocator[VMK_VACACHE_COUNT]; /* ...
and its allocator */
+ char vmk_vacache_name[VMK_VACACHE_COUNT][28];
};
#endif /* defined(_KERNEL) */
Index: usr.bin/vmstat/vmstat.c
===================================================================
RCS file: /cvsroot/src/usr.bin/vmstat/vmstat.c,v
retrieving revision 1.175
diff -u -r1.175 vmstat.c
--- usr.bin/vmstat/vmstat.c 25 Dec 2010 23:36:59 -0000 1.175
+++ usr.bin/vmstat/vmstat.c 21 Jan 2011 09:09:49 -0000
@@ -160,23 +160,19 @@
{ .n_name = "_stathz" },
#define X_NCHSTATS 3
{ .n_name = "_nchstats" },
-#define X_KMEMSTAT 4
- { .n_name = "_kmemstatistics" },
-#define X_KMEMBUCKETS 5
- { .n_name = "_kmembuckets" },
-#define X_ALLEVENTS 6
+#define X_ALLEVENTS 4
{ .n_name = "_allevents" },
-#define X_POOLHEAD 7
+#define X_POOLHEAD 5
{ .n_name = "_pool_head" },
-#define X_UVMEXP 8
+#define X_UVMEXP 6
{ .n_name = "_uvmexp" },
-#define X_TIME_SECOND 9
+#define X_TIME_SECOND 7
{ .n_name = "_time_second" },
-#define X_TIME 10
+#define X_TIME 8
{ .n_name = "_time" },
-#define X_CPU_QUEUE 11
+#define X_CPU_QUEUE 9
{ .n_name = "_cpu_queue" },
-#define X_NL_SIZE 12
+#define X_NL_SIZE 10
{ .n_name = NULL },
};
@@ -301,7 +297,6 @@
void doevcnt(int verbose);
void dohashstat(int, int, const char *);
void dointr(int verbose);
-void domem(void);
void dopool(int, int);
void dopoolcache(void);
void dosum(void);
@@ -514,7 +509,6 @@
(void)putchar('\n');
}
if (todo & MEMSTAT) {
- domem();
dopool(verbose, wide);
(void)putchar('\n');
}
@@ -1142,119 +1136,6 @@
}
}
-static char memname[64];
-
-void
-domem(void)
-{
- struct kmembuckets *kp;
- struct malloc_type ks, *ksp;
- int i, j;
- int len, size, first;
- long totuse = 0, totfree = 0, totreq = 0;
- struct kmembuckets buckets[MINBUCKET + 16];
-
- kread(namelist, X_KMEMBUCKETS, buckets, sizeof(buckets));
- for (first = 1, i = MINBUCKET, kp = &buckets[i]; i < MINBUCKET + 16;
- i++, kp++) {
- if (kp->kb_calls == 0)
- continue;
- if (first) {
- (void)printf("Memory statistics by bucket size\n");
- (void)printf(
- " Size In Use Free Requests HighWater Couldfree\n");
- first = 0;
- }
- size = 1 << i;
- (void)printf("%8d %8ld %6ld %10ld %7ld %10ld\n", size,
- kp->kb_total - kp->kb_totalfree,
- kp->kb_totalfree, kp->kb_calls,
- kp->kb_highwat, kp->kb_couldfree);
- totfree += size * kp->kb_totalfree;
- }
-
- /*
- * If kmem statistics are not being gathered by the kernel,
- * first will still be 1.
- */
- if (first) {
- warnx("Kmem statistics are not being gathered by the kernel.");
- return;
- }
-
- (void)printf("\nMemory usage type by bucket size\n");
- (void)printf(" Size Type(s)\n");
- kp = &buckets[MINBUCKET];
- for (j = 1 << MINBUCKET; j < 1 << (MINBUCKET + 16); j <<= 1, kp++) {
- if (kp->kb_calls == 0)
- continue;
- first = 1;
- len = 8;
- for (kread(namelist, X_KMEMSTAT, &ksp, sizeof(ksp));
- ksp != NULL; ksp = ks.ks_next) {
- deref_kptr(ksp, &ks, sizeof(ks), "malloc type");
- if (ks.ks_calls == 0)
- continue;
- if ((ks.ks_size & j) == 0)
- continue;
- deref_kptr(ks.ks_shortdesc, memname,
- sizeof(memname), "malloc type name");
- len += 2 + strlen(memname);
- if (first)
- (void)printf("%8d %s", j, memname);
- else
- (void)printf(",");
- if (len >= 80) {
- (void)printf("\n\t ");
- len = 10 + strlen(memname);
- }
- if (!first)
- (void)printf(" %s", memname);
- first = 0;
- }
- (void)putchar('\n');
- }
-
- (void)printf(
- "\nMemory statistics by type Type
Kern\n");
- (void)printf(
-" Type InUse MemUse HighUse Limit Requests Limit Limit
Size(s)\n");
- for (kread(namelist, X_KMEMSTAT, &ksp, sizeof(ksp));
- ksp != NULL; ksp = ks.ks_next) {
- deref_kptr(ksp, &ks, sizeof(ks), "malloc type");
- if (ks.ks_calls == 0)
- continue;
- deref_kptr(ks.ks_shortdesc, memname,
- sizeof(memname), "malloc type name");
- (void)printf("%15s %5ld %6ldK %6ldK %6ldK %10ld %5u %5u",
- memname,
- ks.ks_inuse, howmany(ks.ks_memuse, KILO),
- howmany(ks.ks_maxused, KILO),
- howmany(ks.ks_limit, KILO), ks.ks_calls,
- ks.ks_limblocks, ks.ks_mapblocks);
- first = 1;
- for (j = 1 << MINBUCKET, i = MINBUCKET;
- j < 1 << (MINBUCKET + 16);
- j <<= 1, i++)
- {
- if ((ks.ks_size & j) == 0)
- continue;
- if (first)
- (void)printf(" %d", j);
- else
- (void)printf(",%d", j);
- first = 0;
- (void)printf(":%u", ks.ks_active[i - MINBUCKET]);
- }
- (void)printf("\n");
- totuse += ks.ks_memuse;
- totreq += ks.ks_calls;
- }
- (void)printf("\nMemory totals: In Use Free Requests\n");
- (void)printf(" %7ldK %6ldK %8ld\n\n",
- howmany(totuse, KILO), howmany(totfree, KILO), totreq);
-}
-
void
dopool(int verbose, int wide)
{
Home |
Main Index |
Thread Index |
Old Index