Re: Patch: optimize kmem_alloc for frequent mid-sized allocations

[enami tsugutomo <enami%sm.sony.co.jp@localhost>]

> > Your point is to keep memory in pool cache and not in pool itself,
> > right?  If so, we want to bypass pool layer to avoid page header
> > allocation, don't we?
> 
> I think that's a great idea. It could be done without an ABI change if an
> additional flag was introduced and the 'palloc' argument to pool_cache_init
> was made a 'void *'.

I initially thought it requires a bit more complicated change but
actually we can just call pool_allocator_{alloc,free} instead of
pool_{get,put} for minimum modification.

enami.

Index: sys/pool.h
===================================================================
RCS file: /cvsroot/src/sys/sys/pool.h,v
retrieving revision 1.64
diff -u -r1.64 pool.h
--- sys/pool.h  4 Jul 2008 16:38:59 -0000       1.64
+++ sys/pool.h  12 Feb 2009 14:22:37 -0000
@@ -122,6 +122,7 @@
 #define PR_NOTOUCH     0x400   /* don't use free items to keep internal state*/
 #define PR_NOALIGN     0x800   /* don't assume backend alignment */
 #define        PR_LARGECACHE   0x1000  /* use large cache groups */
+#define        PR_NOPOOL       0x2000  /* don't use pool layer */
 
        /*
         * `pr_lock' protects the pool's data structures when removing
Index: kern/subr_kmem.c
===================================================================
RCS file: /cvsroot/src/sys/kern/subr_kmem.c,v
retrieving revision 1.24
diff -u -r1.24 subr_kmem.c
--- kern/subr_kmem.c    6 Feb 2009 22:58:49 -0000       1.24
+++ kern/subr_kmem.c    12 Feb 2009 14:22:30 -0000
@@ -296,7 +296,8 @@
                kc->kc_pa.pa_free = kmem_poolpage_free;
                sprintf(kc->kc_name, "kmem-%zd", sz);
                kc->kc_cache = pool_cache_init(sz,
-                   KMEM_QUANTUM_SIZE, 0, PR_NOALIGN | PR_NOTOUCH,
+                   KMEM_QUANTUM_SIZE, 0,
+                   PR_NOALIGN | PR_NOTOUCH | PR_NOPOOL,
                    kc->kc_name, &kc->kc_pa, IPL_NONE,
                    NULL, NULL, NULL);
                KASSERT(kc->kc_cache != NULL);
Index: kern/subr_pool.c
===================================================================
RCS file: /cvsroot/src/sys/kern/subr_pool.c,v
retrieving revision 1.171
diff -u -r1.171 subr_pool.c
--- kern/subr_pool.c    11 Nov 2008 16:13:03 -0000      1.171
+++ kern/subr_pool.c    12 Feb 2009 14:22:37 -0000
@@ -2255,9 +2255,13 @@
 static void
 pool_cache_destruct_object1(pool_cache_t pc, void *object)
 {
+       struct pool *pp = &pc->pc_pool;
 
        (*pc->pc_dtor)(pc->pc_arg, object);
-       pool_put(&pc->pc_pool, object);
+       if (pp->pr_roflags & PR_NOPOOL)
+               pool_allocator_free(pp, object);
+       else
+               pool_put(pp, object);
 }
 
 /*
@@ -2364,6 +2368,7 @@
 pool_cache_get_slow(pool_cache_cpu_t *cc, int s, void **objectp,
                    paddr_t *pap, int flags)
 {
+       struct pool *pp;
        pcg_t *pcg, *cur;
        uint64_t ncsw;
        pool_cache_t pc;
@@ -2425,7 +2430,11 @@
        mutex_exit(&pc->pc_lock);
        splx(s);
 
-       object = pool_get(&pc->pc_pool, flags);
+       pp = &pc->pc_pool;
+       if (pp->pr_roflags & PR_NOPOOL)
+               object = pool_allocator_alloc(pp, flags);
+       else
+               object = pool_get(pp, flags);
        *objectp = object;
        if (__predict_false(object == NULL))
                return false;