Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/kern kern: Eliminate most __HAVE_ATOMIC_AS_MEMBAR condit...
details: https://anonhg.NetBSD.org/src/rev/43a765c88c88
branches: trunk
changeset: 373679:43a765c88c88
user: riastradh <riastradh%NetBSD.org@localhost>
date: Fri Feb 24 11:02:27 2023 +0000
description:
kern: Eliminate most __HAVE_ATOMIC_AS_MEMBAR conditionals.
I'm leaving in the conditional around the legacy membar_enters
(store-before-load, store-before-store) in kern_mutex.c and in
kern_lock.c because they may still matter: store-before-load barriers
tend to be the most expensive kind, so eliding them is probably
worthwhile on x86. (It also may not matter; I just don't care to do
measurements right now, and it's a single valid and potentially
justifiable use case in the whole tree.)
However, membar_release/acquire can be mere instruction barriers on
all TSO platforms including x86, so there's no need to go out of our
way with a bad API to conditionalize them. If the procedure call
overhead is measurable we just could change them to be macros on x86
that expand into __insn_barrier.
Discussed on tech-kern:
https://mail-index.netbsd.org/tech-kern/2023/02/23/msg028729.html
diffstat:
sys/kern/kern_auth.c | 8 ++------
sys/kern/kern_descrip.c | 18 ++----------------
sys/kern/kern_mutex_obj.c | 8 ++------
sys/kern/kern_rwlock.c | 9 ++-------
sys/kern/kern_rwlock_obj.c | 8 ++------
sys/kern/subr_copy.c | 6 ++----
sys/kern/subr_ipi.c | 16 ++--------------
sys/kern/subr_pcq.c | 6 ++----
sys/kern/subr_pool.c | 10 ++--------
sys/kern/sys_futex.c | 8 ++------
sys/kern/uipc_mbuf.c | 8 ++------
sys/kern/vfs_mount.c | 8 ++------
sys/kern/vfs_vnode.c | 16 ++--------------
13 files changed, 26 insertions(+), 103 deletions(-)
diffs (truncated from 579 to 300 lines):
diff -r 73f353e1c9bc -r 43a765c88c88 sys/kern/kern_auth.c
--- a/sys/kern/kern_auth.c Fri Feb 24 11:02:05 2023 +0000
+++ b/sys/kern/kern_auth.c Fri Feb 24 11:02:27 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_auth.c,v 1.81 2022/04/09 23:38:33 riastradh Exp $ */
+/* $NetBSD: kern_auth.c,v 1.82 2023/02/24 11:02:27 riastradh Exp $ */
/*-
* Copyright (c) 2005, 2006 Elad Efrat <elad%NetBSD.org@localhost>
@@ -28,7 +28,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_auth.c,v 1.81 2022/04/09 23:38:33 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_auth.c,v 1.82 2023/02/24 11:02:27 riastradh Exp $");
#include <sys/types.h>
#include <sys/param.h>
@@ -144,14 +144,10 @@
KASSERT(cred->cr_refcnt > 0);
ASSERT_SLEEPABLE();
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
if (atomic_dec_uint_nv(&cred->cr_refcnt) > 0)
return;
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_acquire();
-#endif
kauth_cred_hook(cred, KAUTH_CRED_FREE, NULL, NULL);
specificdata_fini(kauth_domain, &cred->cr_sd);
diff -r 73f353e1c9bc -r 43a765c88c88 sys/kern/kern_descrip.c
--- a/sys/kern/kern_descrip.c Fri Feb 24 11:02:05 2023 +0000
+++ b/sys/kern/kern_descrip.c Fri Feb 24 11:02:27 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_descrip.c,v 1.254 2023/02/23 03:00:15 riastradh Exp $ */
+/* $NetBSD: kern_descrip.c,v 1.255 2023/02/24 11:02:27 riastradh Exp $ */
/*-
* Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
@@ -70,7 +70,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_descrip.c,v 1.254 2023/02/23 03:00:15 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_descrip.c,v 1.255 2023/02/24 11:02:27 riastradh Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -429,9 +429,7 @@
* will safely wait for references to drain.
*/
atomic_inc_uint(&ff->ff_refcnt);
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_acquire();
-#endif
}
/*
@@ -485,9 +483,7 @@
* the file after it has been freed or recycled by another
* CPU.
*/
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
/*
* Be optimistic and start out with the assumption that no other
@@ -637,9 +633,7 @@
* waiting for other users of the file to drain. Release
* our reference, and wake up the closer.
*/
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
atomic_dec_uint(&ff->ff_refcnt);
cv_broadcast(&ff->ff_closing);
mutex_exit(&fdp->fd_lock);
@@ -674,13 +668,9 @@
refcnt = --(ff->ff_refcnt);
} else {
/* Multi threaded. */
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
refcnt = atomic_dec_uint_nv(&ff->ff_refcnt);
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_acquire();
-#endif
}
if (__predict_false(refcnt != 0)) {
/*
@@ -1566,14 +1556,10 @@
KASSERT(fdp->fd_dtbuiltin.dt_nfiles == NDFILE);
KASSERT(fdp->fd_dtbuiltin.dt_link == NULL);
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
if (atomic_dec_uint_nv(&fdp->fd_refcnt) > 0)
return;
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_acquire();
-#endif
/*
* Close any files that the process holds open.
diff -r 73f353e1c9bc -r 43a765c88c88 sys/kern/kern_mutex_obj.c
--- a/sys/kern/kern_mutex_obj.c Fri Feb 24 11:02:05 2023 +0000
+++ b/sys/kern/kern_mutex_obj.c Fri Feb 24 11:02:27 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_mutex_obj.c,v 1.10 2022/10/26 23:21:19 riastradh Exp $ */
+/* $NetBSD: kern_mutex_obj.c,v 1.11 2023/02/24 11:02:27 riastradh Exp $ */
/*-
* Copyright (c) 2008, 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_mutex_obj.c,v 1.10 2022/10/26 23:21:19 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_mutex_obj.c,v 1.11 2023/02/24 11:02:27 riastradh Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -155,15 +155,11 @@
"%s: lock %p: mo->mo_refcnt (%#x) == 0",
__func__, mo, mo->mo_refcnt);
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
if (atomic_dec_uint_nv(&mo->mo_refcnt) > 0) {
return false;
}
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_acquire();
-#endif
mutex_destroy(&mo->mo_lock);
pool_cache_put(mutex_obj_cache, mo);
return true;
diff -r 73f353e1c9bc -r 43a765c88c88 sys/kern/kern_rwlock.c
--- a/sys/kern/kern_rwlock.c Fri Feb 24 11:02:05 2023 +0000
+++ b/sys/kern/kern_rwlock.c Fri Feb 24 11:02:27 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_rwlock.c,v 1.68 2023/02/24 11:01:43 riastradh Exp $ */
+/* $NetBSD: kern_rwlock.c,v 1.69 2023/02/24 11:02:27 riastradh Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007, 2008, 2009, 2019, 2020
@@ -45,7 +45,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_rwlock.c,v 1.68 2023/02/24 11:01:43 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_rwlock.c,v 1.69 2023/02/24 11:02:27 riastradh Exp $");
#include "opt_lockdebug.h"
@@ -100,13 +100,8 @@
/*
* Memory barriers.
*/
-#ifdef __HAVE_ATOMIC_AS_MEMBAR
-#define RW_MEMBAR_ACQUIRE()
-#define RW_MEMBAR_RELEASE()
-#else
#define RW_MEMBAR_ACQUIRE() membar_acquire()
#define RW_MEMBAR_RELEASE() membar_release()
-#endif
/*
* For platforms that do not provide stubs, or for the LOCKDEBUG case.
diff -r 73f353e1c9bc -r 43a765c88c88 sys/kern/kern_rwlock_obj.c
--- a/sys/kern/kern_rwlock_obj.c Fri Feb 24 11:02:05 2023 +0000
+++ b/sys/kern/kern_rwlock_obj.c Fri Feb 24 11:02:27 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_rwlock_obj.c,v 1.8 2022/10/26 23:22:22 riastradh Exp $ */
+/* $NetBSD: kern_rwlock_obj.c,v 1.9 2023/02/24 11:02:27 riastradh Exp $ */
/*-
* Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_rwlock_obj.c,v 1.8 2022/10/26 23:22:22 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_rwlock_obj.c,v 1.9 2023/02/24 11:02:27 riastradh Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -145,15 +145,11 @@
KASSERT(ro->ro_magic == RW_OBJ_MAGIC);
KASSERT(ro->ro_refcnt > 0);
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
if (atomic_dec_uint_nv(&ro->ro_refcnt) > 0) {
return false;
}
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_acquire();
-#endif
rw_destroy(&ro->ro_lock);
pool_cache_put(rw_obj_cache, ro);
return true;
diff -r 73f353e1c9bc -r 43a765c88c88 sys/kern/subr_copy.c
--- a/sys/kern/subr_copy.c Fri Feb 24 11:02:05 2023 +0000
+++ b/sys/kern/subr_copy.c Fri Feb 24 11:02:27 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_copy.c,v 1.16 2022/04/09 23:51:09 riastradh Exp $ */
+/* $NetBSD: subr_copy.c,v 1.17 2023/02/24 11:02:27 riastradh Exp $ */
/*-
* Copyright (c) 1997, 1998, 1999, 2002, 2007, 2008, 2019
@@ -80,7 +80,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_copy.c,v 1.16 2022/04/09 23:51:09 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_copy.c,v 1.17 2023/02/24 11:02:27 riastradh Exp $");
#define __UFETCHSTORE_PRIVATE
#define __UCAS_PRIVATE
@@ -411,9 +411,7 @@
* Matches atomic_load_acquire in ucas_critical_wait -- turns
* the following atomic_dec_uint into a store-release.
*/
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
atomic_dec_uint(&ucas_critical_pausing_cpus);
/*
diff -r 73f353e1c9bc -r 43a765c88c88 sys/kern/subr_ipi.c
--- a/sys/kern/subr_ipi.c Fri Feb 24 11:02:05 2023 +0000
+++ b/sys/kern/subr_ipi.c Fri Feb 24 11:02:27 2023 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_ipi.c,v 1.10 2022/04/09 23:51:22 riastradh Exp $ */
+/* $NetBSD: subr_ipi.c,v 1.11 2023/02/24 11:02:27 riastradh Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.10 2022/04/09 23:51:22 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.11 2023/02/24 11:02:27 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -189,9 +189,7 @@
/* Mark as pending and return true if not previously marked. */
if ((atomic_load_acquire(&ci->ci_ipipend[i]) & bitm) == 0) {
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
atomic_or_32(&ci->ci_ipipend[i], bitm);
return true;
}
@@ -303,9 +301,7 @@
continue;
}
pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_acquire();
-#endif
while ((bit = ffs(pending)) != 0) {
const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id];
@@ -341,9 +337,7 @@
msg->func(msg->arg);
/* Ack the request. */
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
atomic_dec_uint(&msg->_pending);
}
}
@@ -364,9 +358,7 @@
KASSERT(curcpu() != ci);
msg->_pending = 1;
-#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_release();
-#endif
put_msg(&ipi_mboxes[id], msg);
ipi_trigger(IPI_SYNCH_ID, ci);
@@ -390,9 +382,7 @@
local = !!kcpuset_isset(target, cpu_index(self));
msg->_pending = kcpuset_countset(target) - local;
Home |
Main Index |
Thread Index |
Old Index