Re: kern/54435: reading TCP urgent data with MSG_OOB doesn't clear poll(2) status

[Christos Zoulas <christos%zoulas.com@localhost>]

The following reply was made to PR kern/54435; it has been noted by GNATS.

From: Christos Zoulas <christos%zoulas.com@localhost>
To: gnats-bugs%netbsd.org@localhost
Cc: kern-bug-people%netbsd.org@localhost,
 gnats-admin%netbsd.org@localhost,
 netbsd-bugs%netbsd.org@localhost
Subject: Re: kern/54435: reading TCP urgent data with MSG_OOB doesn't clear
 poll(2) status
Date: Thu, 8 Aug 2019 19:07:54 +0300

 Darwin vpn1-1.astron.com 18.7.0 Darwin Kernel Version 18.7.0: Thu Jun 20 =
 18:42:21 PDT 2019; root:xnu-4903.270.47~4/RELEASE_X86_64 x86_64
 reading urgent data with MSG_OOB
 poll: revents =3D 0x93: HUP IN PRI RDBAND
 recv(MSG_OOB) =3D 1
 b
 poll: revents =3D 0x93: HUP IN PRI RDBAND
 recv(MSG_OOB) failed, retrying
 recv() =3D 1
 a
 poll: revents =3D 0x93: HUP IN PRI RDBAND
 recv(MSG_OOB) failed, retrying
 recv() =3D 1
 c
 poll: revents =3D 0x93: HUP IN PRI RDBAND
 recv(MSG_OOB) failed, retrying
 recv() =3D 0
 
 FreeBSD mb1.astron.com 12.0-CURRENT FreeBSD 12.0-CURRENT #0 r318137: Wed =
 May 10 15:09:31 UTC 2017     =
 root%releng3.nyi.freebsd.org@localhost:/usr/obj/usr/src/sys/GENERIC  amd64
 reading urgent data with MSG_OOB
 poll: revents =3D 0x1: IN
 recv() =3D 1
 a
 poll: revents =3D 0x83: IN PRI RDBAND
 recv(MSG_OOB) =3D 1
 b
 poll: revents =3D 0x83: IN PRI RDBAND
 recv(MSG_OOB) failed, retrying
 recv() =3D 1
 c
 poll: revents =3D 0x1: IN
 recv() =3D 0
 
 Linux mb1 4.9.0-8-amd64 #1 SMP Debian 4.9.144-3.1 (2019-02-19) x86_64 =
 GNU/Linux
 reading urgent data with MSG_OOB
 poll: revents =3D 0x3: IN PRI
 recv(MSG_OOB) =3D 1
 b
 poll: revents =3D 0x1: IN
 recv() =3D 1
 a
 poll: revents =3D 0x1: IN
 recv() =3D 1
 c
 poll: revents =3D 0x1: IN
 recv() =3D 0
 
 SunOS openindiana 5.11 illumos-6ccda740e0 i86pc i386 i86pc
 christos@openindiana:~/oob$ ./oobrecv=20
 reading urgent data with MSG_OOB
 poll: revents =3D 0x83: IN PRI RDBAND
 recv(MSG_OOB) =3D 1
 b
 poll: revents =3D 0x83: IN PRI RDBAND
 recv(MSG_OOB) failed, retrying
 recv() =3D 1
 a
 poll: revents =3D 0x83: IN PRI RDBAND
 recv(MSG_OOB) failed, retrying
 recv() =3D 1
 c
 poll: revents =3D 0x1: IN
 recv() =3D 0
 
 ------------------------------------------
 for portability on Solaris:
 #include <sys/param.h>
 #ifdef BSD4_4
 for socklen
 #endif
 
 #include <sys/sockio.h> for SIOCATMARK.
 
 I support the change to make us behave like Linux.
 
 christos
 
 > On Aug 3, 2019, at 5:50 PM, uwe%stderr.spb.ru@localhost wrote:
 >=20
 >> Number:         54435
 >> Category:       kern
 >> Synopsis:       reading TCP urgent data with MSG_OOB doesn't clear =
 poll(2) status
 >> Confidential:   no
 >> Severity:       non-critical
 >> Priority:       low
 >> Responsible:    kern-bug-people
 >> State:          open
 >> Class:          sw-bug
 >> Submitter-Id:   net
 >> Arrival-Date:   Sat Aug 03 14:50:00 +0000 2019
 >> Originator:     Valery Ushakov
 >> Release:        NetBSD-8
 >> Organization:
 >> Environment:
 > NetBSD pony 8.1_STABLE NetBSD 8.1_STABLE (GENERIC) #0: Sat Jun 15 =
 07:30:17 MSK 2019  =
 uwe@sampo:/home/uwe/work/netbsd/build8/obj/macppc/sys/arch/macppc/compile/=
 GENERIC macppc
 >=20
 >> Description:
 > It looks like POLLPRI | POLLRDBAND status in not cleared from the
 > socket when TCP urgent data is read with MSG_OOB
 >=20
 >> How-To-Repeat:
 > cat >> oobserv.c <<'__EOF__'
 > #include <sys/types.h>
 > #include <sys/socket.h>
 >=20
 > #include <netinet/in.h>
 > #include <arpa/inet.h>
 >=20
 > #include <err.h>
 > #include <errno.h>
 > #include <stdio.h>
 > #include <stdlib.h>
 > #include <string.h>
 > #include <unistd.h>
 >=20
 >=20
 > int
 > main(void)
 > {
 >    int status;
 >=20
 >    int server =3D socket(PF_INET, SOCK_STREAM, 0);
 >    if (server < 0)
 > 	err(EXIT_FAILURE, "socket");
 >=20
 >    struct sockaddr_in sin;
 >    memset(&sin, 0, sizeof(sin));
 > #if !defined(__linux__) && !defined(__sun__)
 >    sin.sin_len =3D sizeof(sin);
 > #endif
 >    sin.sin_family =3D AF_INET;
 >    sin.sin_addr.s_addr =3D htonl(INADDR_LOOPBACK);
 >    sin.sin_port =3D htons(12345);
 >=20
 >    status =3D bind(server, (struct sockaddr *)&sin, sizeof(sin));
 >    if (status < 0)
 > 	err(EXIT_FAILURE, "bind");
 >=20
 >    status =3D listen(server, 1);
 >    if (status < 0)
 > 	err(EXIT_FAILURE, "listen");
 >=20
 >    for (;;) {
 > 	int client =3D accept(server, NULL, 0);
 > 	if (client < 0)
 > 	    err(EXIT_FAILURE, "accept");
 >=20
 > 	send(client, "a", 1, 0);
 > 	send(client, "b", 1, MSG_OOB);
 > 	send(client, "c", 1, 0);
 >=20
 > 	close(client);
 >    }
 >=20
 >    return 0;
 > }
 > __EOF__
 >=20
 > cat >> oobrecv.c <<'__EOF__'
 > #include <sys/types.h>
 > #include <sys/socket.h>
 >=20
 > #include <err.h>
 > #include <errno.h>
 > #include <fcntl.h>
 > #include <poll.h>
 > #include <stdbool.h>
 > #include <stdio.h>
 > #include <stdlib.h>
 > #include <string.h>
 > #include <unistd.h>
 >=20
 > #include <netinet/in.h>
 > #include <arpa/inet.h>
 >=20
 > int usage(void);
 > int getsocket(void);
 > void oobtest(int);
 > int pollsock(int);
 >=20
 > bool oobinline =3D false;
 >=20
 > int
 > usage(void)
 > {
 >    fprintf(stderr, "usage: oobrecv [oob | inline]\n");
 >    fprintf(stderr, "default is \"%s\"\n",
 > 	    oobinline ? "inline" : "oob");
 >=20
 >    return EXIT_FAILURE;
 > }
 >=20
 >=20
 > int
 > main(int argc, char **argv)
 > {
 >    if (argc > 1) {
 > 	if (strcasecmp(argv[1], "oob") =3D=3D 0)
 > 	    oobinline =3D false;
 > 	else if (strcasecmp(argv[1], "inline") =3D=3D 0)
 > 	    oobinline =3D true;
 > 	else
 > 	    return usage();
 >    }
 >=20
 >    oobtest(getsocket());
 >    return 0;
 > }
 >=20
 >=20
 > int
 > getsocket(void)
 > {
 >    int s;
 >    int status;
 >=20
 >    s =3D socket(PF_INET, SOCK_STREAM, 0);
 >    if (s < 0)
 > 	err(EXIT_FAILURE, "socket");
 >=20
 >    struct sockaddr_in sin;
 >    memset(&sin, 0, sizeof(sin));
 > #if !defined(__linux__) && !defined(__sun__)
 >    sin.sin_len =3D sizeof(sin);
 > #endif
 >    sin.sin_family =3D AF_INET;
 >    sin.sin_addr.s_addr =3D htonl(INADDR_LOOPBACK);
 >    sin.sin_port =3D htons(12345);
 >=20
 >    if (oobinline) {
 > 	int one =3D 1;
 > 	status =3D setsockopt(s, SOL_SOCKET, SO_OOBINLINE,
 > 			    (char *)&one, sizeof(one));
 > 	if (status < 0)
 > 	    err(EXIT_FAILURE, "SO_OOBINLINE");
 >    }
 >=20
 >    status =3D connect(s, (struct sockaddr *)&sin, sizeof(sin));
 >    if (status < 0)
 > 	err(EXIT_FAILURE, "connect");
 >=20
 >    return s;
 > }
 >=20
 >=20
 > void
 > oobtest(int s)
 > {
 >    char buf[128];
 >    int status;
 >=20
 >    printf("reading urgent data with %s\n",
 > 	   oobinline ? "SO_OOBINLINE" : "MSG_OOB");
 >=20
 >    for (;;) {
 > 	int revents =3D pollsock(s);
 > 	if (revents =3D=3D 0)
 > 	    continue;
 >=20
 > 	if (revents & POLLNVAL) {
 > 	    errno =3D EBADF;
 > 	    err(EXIT_FAILURE, "poll");
 > 	}
 >=20
 > 	if (revents & POLLERR) {
 > 	    int sockerr;
 > 	    socklen_t optlen =3D (socklen_t)sizeof(sockerr);
 >=20
 >            status =3D getsockopt(s, SOL_SOCKET, SO_ERROR,
 > 				(char *)&sockerr, &optlen);
 > 	    if (status < 0)
 > 		err(EXIT_FAILURE, "SO_ERROR");
 >=20
 > 	    errno =3D sockerr;
 > 	    err(EXIT_FAILURE, NULL);
 > 	}
 >=20
 > 	int flags =3D 0;
 > 	if (!oobinline && (revents & (POLLPRI | POLLRDBAND)))
 > 	    flags =3D MSG_OOB;
 >=20
 > 	ssize_t nread =3D recv(s, buf, sizeof(buf), flags);
 > 	if (nread < 0)
 > 	    err(EXIT_FAILURE, "recv%s",
 > 		flags & MSG_OOB ? "(MSG_OOB)" : "");
 >=20
 > 	printf("recv(%s) =3D %zd\n",
 > 	       flags & MSG_OOB ? "MSG_OOB" : "",
 > 	       nread);
 >=20
 > 	if (nread =3D=3D 0)
 > 	    return;
 >=20
 > 	fwrite(buf, 1, nread, stdout);
 > 	printf("\n");
 >    }
 > }
 >=20
 >=20
 > int
 > pollsock(int s)
 > {
 >    struct pollfd fds[1];
 >=20
 >    fds[0].fd =3D s;
 >    fds[0].events =3D POLLIN | POLLPRI | POLLRDBAND;
 >    fds[0].revents =3D 0;
 >=20
 >    int nready =3D poll(fds, 1, -1);
 >    if (nready < 0)
 > 	err(EXIT_FAILURE, "poll");
 >=20
 >    if (nready =3D=3D 0)
 > 	return 0;
 >=20
 >    int revents =3D fds[0].revents;
 >=20
 >    printf("poll: revents =3D 0x%x", revents);
 >    if (fds[0].revents !=3D 0) {
 > 	printf(":");
 > 	if (revents & POLLNVAL)   printf(" NVAL");
 > 	if (revents & POLLERR) 	  printf(" ERR");
 > 	if (revents & POLLHUP) 	  printf(" HUP");
 > 	if (revents & POLLIN)  	  printf(" IN");
 > 	if (revents & POLLPRI) 	  printf(" PRI");
 > 	if (revents & POLLRDNORM) printf(" RDNORM");
 > 	if (revents & POLLRDBAND) printf(" RDBAND");
 >    }
 >    printf("\n");
 >=20
 >    return revents;
 > }
 > __EOF__
 >=20
 > cc -o oobserv oobserv.c
 > cc -o oobrecv oobrecv.c
 >=20
 > ./oobserv &
 > ./oobrecv oob
 >=20
 >=20
 > The server sends "a", then send "b" as urgent/MSG_OOB, then "c".
 >=20
 > The test invocation fails with:
 >=20
 > reading urgent data with MSG_OOB
 > poll: revents =3D 0x1: IN
 > recv() =3D 1
 > a
 > poll: revents =3D 0x82: PRI RDBAND
 > recv(MSG_OOB) =3D 1
 > b
 > poll: revents =3D 0x82: PRI RDBAND
 > oobrecv: recv(MSG_OOB): Invalid argument
 >=20
 > So after "b" was read (with MSG_OOB), the poll still reports POLLPRI.
 >=20
 > This doesn't happen with SO_OOBINLINE:
 >=20
 > $ ./oobrecv inline
 > reading urgent data with SO_OOBINLINE
 > poll: revents =3D 0x1: IN
 > recv() =3D 1
 > a
 > poll: revents =3D 0x83: IN PRI RDBAND
 > recv() =3D 1
 > b
 > poll: revents =3D 0x1: IN
 > recv() =3D 1
 > c
 > poll: revents =3D 0x1: IN
 > recv() =3D 0
 >=20
 > here POLLPRI | POLLRDBAND are cleared after "b" was read.
 >=20
 >> Fix: