Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys Make insertion of data into socket buffers O(C):
details: https://anonhg.NetBSD.org/src/rev/d873b17c4835
branches: trunk
changeset: 533539:d873b17c4835
user: thorpej <thorpej%NetBSD.org@localhost>
date: Wed Jul 03 19:06:47 2002 +0000
description:
Make insertion of data into socket buffers O(C):
* Keep pointers to the first and last mbufs of the last record in the
socket buffer.
* Use the sb_lastrecord pointer in the sbappend*() family of functions
to avoid traversing the packet chain to find the last record.
* Add a new sbappend_stream() function for stream protocols which
guarantee that there will never be more than one record in the
socket buffer. This function uses the sb_mbtail pointer to perform
the data insertion. Make TCP use sbappend_stream().
On a profiling run, this makes sbappend of a TCP transmission using
a 1M socket buffer go from 50% of the time to .02% of the time.
Thanks to Bill Sommerfeld and YAMAMOTO Takashi for their debugging
assistance!
diffstat:
sys/dev/kttcp.c | 78 +++++++++++++++++-
sys/kern/uipc_socket.c | 76 +++++++++++++++++-
sys/kern/uipc_socket2.c | 190 +++++++++++++++++++++++++++++++++++++--------
sys/netccitt/if_x25subr.c | 12 ++-
sys/netccitt/pk_output.c | 5 +-
sys/netccitt/pk_usrreq.c | 5 +-
sys/netinet/tcp_input.c | 10 +-
sys/netinet/tcp_usrreq.c | 8 +-
sys/sys/socketvar.h | 25 +++++-
9 files changed, 349 insertions(+), 60 deletions(-)
diffs (truncated from 862 to 300 lines):
diff -r 40d688edc15d -r d873b17c4835 sys/dev/kttcp.c
--- a/sys/dev/kttcp.c Wed Jul 03 17:18:09 2002 +0000
+++ b/sys/dev/kttcp.c Wed Jul 03 19:06:47 2002 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: kttcp.c,v 1.1 2002/06/28 23:27:14 thorpej Exp $ */
+/* $NetBSD: kttcp.c,v 1.2 2002/07/03 19:06:47 thorpej Exp $ */
/*
* Copyright (c) 2002 Wasabi Systems, Inc.
@@ -268,6 +268,10 @@
if (space < resid && (atomic || space < so->so_snd.sb_lowat)) {
if (so->so_state & SS_NBIO)
snderr(EWOULDBLOCK);
+ SBLASTRECORDCHK(&so->so_rcv,
+ "kttcp_soreceive sbwait 1");
+ SBLASTMBUFCHK(&so->so_rcv,
+ "kttcp_soreceive sbwait 1");
sbunlock(&so->so_snd);
error = sbwait(&so->so_snd);
splx(s);
@@ -470,10 +474,18 @@
goto restart;
}
dontblock:
+ /*
+ * On entry here, m points to the first record of the socket buffer.
+ * While we process the initial mbufs containing address and control
+ * info, we save a copy of m->m_nextpkt into nextrecord.
+ */
#ifdef notyet /* XXXX */
if (uio->uio_procp)
uio->uio_procp->p_stats->p_ru.ru_msgrcv++;
#endif
+ KASSERT(m == so->so_rcv.sb_mb);
+ SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 1");
+ SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 1");
nextrecord = m->m_nextpkt;
if (pr->pr_flags & PR_ADDR) {
#ifdef DIAGNOSTIC
@@ -498,13 +510,39 @@
m = so->so_rcv.sb_mb;
}
}
+
+ /*
+ * If m is non-NULL, we have some data to read. From now on,
+ * make sure to keep sb_lastrecord consistent when working on
+ * the last packet on the chain (nextrecord == NULL) and we
+ * change m->m_nextpkt.
+ */
if (m) {
- if ((flags & MSG_PEEK) == 0)
+ if ((flags & MSG_PEEK) == 0) {
m->m_nextpkt = nextrecord;
+ /*
+ * If nextrecord == NULL (this is a single chain),
+ * then sb_lastrecord may not be valid here if m
+ * was changed earlier.
+ */
+ if (nextrecord == NULL) {
+ KASSERT(so->so_rcv.sb_mb == m);
+ so->so_rcv.sb_lastrecord = m;
+ }
+ }
type = m->m_type;
if (type == MT_OOBDATA)
flags |= MSG_OOB;
+ } else {
+ if ((flags & MSG_PEEK) == 0) {
+ KASSERT(so->so_rcv.sb_mb == m);
+ so->so_rcv.sb_mb = nextrecord;
+ SB_UPDATE_TAIL(&so->so_rcv);
+ }
}
+ SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 2");
+ SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 2");
+
moff = 0;
offset = 0;
while (m && resid > 0 && error == 0) {
@@ -550,8 +588,23 @@
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
- if (m)
+ /*
+ * If m != NULL, we also know that
+ * so->so_rcv.sb_mb != NULL.
+ */
+ KASSERT(so->so_rcv.sb_mb == m);
+ if (m) {
m->m_nextpkt = nextrecord;
+ if (nextrecord == NULL)
+ so->so_rcv.sb_lastrecord = m;
+ } else {
+ so->so_rcv.sb_mb = nextrecord;
+ SB_UPDATE_TAIL(&so->so_rcv);
+ }
+ SBLASTRECORDCHK(&so->so_rcv,
+ "kttcp_soreceive 3");
+ SBLASTMBUFCHK(&so->so_rcv,
+ "kttcp_soreceive 3");
}
} else {
if (flags & MSG_PEEK)
@@ -590,6 +643,10 @@
!sosendallatonce(so) && !nextrecord) {
if (so->so_error || so->so_state & SS_CANTRCVMORE)
break;
+ SBLASTRECORDCHK(&so->so_rcv,
+ "kttcp_soreceive sbwait 2");
+ SBLASTMBUFCHK(&so->so_rcv,
+ "kttcp_soreceive sbwait 2");
error = sbwait(&so->so_rcv);
if (error) {
sbunlock(&so->so_rcv);
@@ -607,8 +664,21 @@
(void) sbdroprecord(&so->so_rcv);
}
if ((flags & MSG_PEEK) == 0) {
- if (m == 0)
+ if (m == 0) {
+ /*
+ * First part is an SB_UPDATE_TAIL(). Second part
+ * makes sure sb_lastrecord is up-to-date if
+ * there is still data in the socket buffer.
+ */
so->so_rcv.sb_mb = nextrecord;
+ if (so->so_rcv.sb_mb == NULL) {
+ so->so_rcv.sb_mbtail = NULL;
+ so->so_rcv.sb_lastrecord = NULL;
+ } else if (nextrecord->m_nextpkt == NULL)
+ so->so_rcv.sb_lastrecord = nextrecord;
+ }
+ SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 4");
+ SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 4");
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb)
(*pr->pr_usrreq)(so, PRU_RCVD, (struct mbuf *)0,
(struct mbuf *)(long)flags, (struct mbuf *)0,
diff -r 40d688edc15d -r d873b17c4835 sys/kern/uipc_socket.c
--- a/sys/kern/uipc_socket.c Wed Jul 03 17:18:09 2002 +0000
+++ b/sys/kern/uipc_socket.c Wed Jul 03 19:06:47 2002 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uipc_socket.c,v 1.68 2002/06/11 00:21:33 matt Exp $ */
+/* $NetBSD: uipc_socket.c,v 1.69 2002/07/03 19:06:48 thorpej Exp $ */
/*-
* Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -72,7 +72,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.68 2002/06/11 00:21:33 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.69 2002/07/03 19:06:48 thorpej Exp $");
#include "opt_sock_counters.h"
#include "opt_sosend_loan.h"
@@ -894,6 +894,8 @@
error = EWOULDBLOCK;
goto release;
}
+ SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1");
+ SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1");
sbunlock(&so->so_rcv);
error = sbwait(&so->so_rcv);
splx(s);
@@ -902,10 +904,18 @@
goto restart;
}
dontblock:
+ /*
+ * On entry here, m points to the first record of the socket buffer.
+ * While we process the initial mbufs containing address and control
+ * info, we save a copy of m->m_nextpkt into nextrecord.
+ */
#ifdef notyet /* XXXX */
if (uio->uio_procp)
uio->uio_procp->p_stats->p_ru.ru_msgrcv++;
#endif
+ KASSERT(m == so->so_rcv.sb_mb);
+ SBLASTRECORDCHK(&so->so_rcv, "soreceive 1");
+ SBLASTMBUFCHK(&so->so_rcv, "soreceive 1");
nextrecord = m->m_nextpkt;
if (pr->pr_flags & PR_ADDR) {
#ifdef DIAGNOSTIC
@@ -958,13 +968,39 @@
controlp = &(*controlp)->m_next;
}
}
+
+ /*
+ * If m is non-NULL, we have some data to read. From now on,
+ * make sure to keep sb_lastrecord consistent when working on
+ * the last packet on the chain (nextrecord == NULL) and we
+ * change m->m_nextpkt.
+ */
if (m) {
- if ((flags & MSG_PEEK) == 0)
+ if ((flags & MSG_PEEK) == 0) {
m->m_nextpkt = nextrecord;
+ /*
+ * If nextrecord == NULL (this is a single chain),
+ * then sb_lastrecord may not be valid here if m
+ * was changed earlier.
+ */
+ if (nextrecord == NULL) {
+ KASSERT(so->so_rcv.sb_mb == m);
+ so->so_rcv.sb_lastrecord = m;
+ }
+ }
type = m->m_type;
if (type == MT_OOBDATA)
flags |= MSG_OOB;
+ } else {
+ if ((flags & MSG_PEEK) == 0) {
+ KASSERT(so->so_rcv.sb_mb == m);
+ so->so_rcv.sb_mb = nextrecord;
+ SB_UPDATE_TAIL(&so->so_rcv);
+ }
}
+ SBLASTRECORDCHK(&so->so_rcv, "soreceive 2");
+ SBLASTMBUFCHK(&so->so_rcv, "soreceive 2");
+
moff = 0;
offset = 0;
while (m && uio->uio_resid > 0 && error == 0) {
@@ -992,6 +1028,8 @@
* block interrupts again.
*/
if (mp == 0) {
+ SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
+ SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
splx(s);
error = uiomove(mtod(m, caddr_t) + moff, (int)len, uio);
s = splsoftnet();
@@ -1033,8 +1071,21 @@
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
- if (m)
+ /*
+ * If m != NULL, we also know that
+ * so->so_rcv.sb_mb != NULL.
+ */
+ KASSERT(so->so_rcv.sb_mb == m);
+ if (m) {
m->m_nextpkt = nextrecord;
+ if (nextrecord == NULL)
+ so->so_rcv.sb_lastrecord = m;
+ } else {
+ so->so_rcv.sb_mb = nextrecord;
+ SB_UPDATE_TAIL(&so->so_rcv);
+ }
+ SBLASTRECORDCHK(&so->so_rcv, "soreceive 3");
+ SBLASTMBUFCHK(&so->so_rcv, "soreceive 3");
}
} else {
if (flags & MSG_PEEK)
@@ -1090,6 +1141,8 @@
(struct mbuf *)(long)flags,
(struct mbuf *)0,
(struct proc *)0);
+ SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2");
+ SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2");
error = sbwait(&so->so_rcv);
if (error) {
sbunlock(&so->so_rcv);
@@ -1107,8 +1160,21 @@
(void) sbdroprecord(&so->so_rcv);
}
if ((flags & MSG_PEEK) == 0) {
- if (m == 0)
+ if (m == 0) {
+ /*
+ * First part is an inline SB_UPDATE_TAIL(). Second
+ * part makes sure sb_lastrecord is up-to-date if
+ * there is still data in the socket buffer.
+ */
so->so_rcv.sb_mb = nextrecord;
+ if (so->so_rcv.sb_mb == NULL) {
+ so->so_rcv.sb_mbtail = NULL;
+ so->so_rcv.sb_lastrecord = NULL;
+ } else if (nextrecord->m_nextpkt == NULL)
+ so->so_rcv.sb_lastrecord = nextrecord;
+ }
+ SBLASTRECORDCHK(&so->so_rcv, "soreceive 4");
+ SBLASTMBUFCHK(&so->so_rcv, "soreceive 4");
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb)
(*pr->pr_usrreq)(so, PRU_RCVD, (struct mbuf *)0,
(struct mbuf *)(long)flags, (struct mbuf *)0,
diff -r 40d688edc15d -r d873b17c4835 sys/kern/uipc_socket2.c
--- a/sys/kern/uipc_socket2.c Wed Jul 03 17:18:09 2002 +0000
+++ b/sys/kern/uipc_socket2.c Wed Jul 03 19:06:47 2002 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: uipc_socket2.c,v 1.42 2001/11/12 15:25:33 lukem Exp $ */
+/* $NetBSD: uipc_socket2.c,v 1.43 2002/07/03 19:06:49 thorpej Exp $ */
/*
* Copyright (c) 1982, 1986, 1988, 1990, 1993
@@ -36,7 +36,7 @@
Home |
Main Index |
Thread Index |
Old Index