Re: kern/54435: reading TCP urgent data with MSG_OOB doesn't clear poll(2) status

[Christos Zoulas <christos%zoulas.com@localhost>]

Darwin vpn1-1.astron.com 18.7.0 Darwin Kernel Version 18.7.0: Thu Jun 20 18:42:21 PDT 2019; root:xnu-4903.270.47~4/RELEASE_X86_64 x86_64
reading urgent data with MSG_OOB
poll: revents = 0x93: HUP IN PRI RDBAND
recv(MSG_OOB) = 1
b
poll: revents = 0x93: HUP IN PRI RDBAND
recv(MSG_OOB) failed, retrying
recv() = 1
a
poll: revents = 0x93: HUP IN PRI RDBAND
recv(MSG_OOB) failed, retrying
recv() = 1
c
poll: revents = 0x93: HUP IN PRI RDBAND
recv(MSG_OOB) failed, retrying
recv() = 0

FreeBSD mb1.astron.com 12.0-CURRENT FreeBSD 12.0-CURRENT #0 r318137: Wed May 10 15:09:31 UTC 2017     root%releng3.nyi.freebsd.org@localhost:/usr/obj/usr/src/sys/GENERIC  amd64
reading urgent data with MSG_OOB
poll: revents = 0x1: IN
recv() = 1
a
poll: revents = 0x83: IN PRI RDBAND
recv(MSG_OOB) = 1
b
poll: revents = 0x83: IN PRI RDBAND
recv(MSG_OOB) failed, retrying
recv() = 1
c
poll: revents = 0x1: IN
recv() = 0

Linux mb1 4.9.0-8-amd64 #1 SMP Debian 4.9.144-3.1 (2019-02-19) x86_64 GNU/Linux
reading urgent data with MSG_OOB
poll: revents = 0x3: IN PRI
recv(MSG_OOB) = 1
b
poll: revents = 0x1: IN
recv() = 1
a
poll: revents = 0x1: IN
recv() = 1
c
poll: revents = 0x1: IN
recv() = 0

SunOS openindiana 5.11 illumos-6ccda740e0 i86pc i386 i86pc
christos@openindiana:~/oob$ ./oobrecv 
reading urgent data with MSG_OOB
poll: revents = 0x83: IN PRI RDBAND
recv(MSG_OOB) = 1
b
poll: revents = 0x83: IN PRI RDBAND
recv(MSG_OOB) failed, retrying
recv() = 1
a
poll: revents = 0x83: IN PRI RDBAND
recv(MSG_OOB) failed, retrying
recv() = 1
c
poll: revents = 0x1: IN
recv() = 0

------------------------------------------
for portability on Solaris:
#include <sys/param.h>
#ifdef BSD4_4
for socklen
#endif

#include <sys/sockio.h> for SIOCATMARK.

I support the change to make us behave like Linux.

christos

> On Aug 3, 2019, at 5:50 PM, uwe%stderr.spb.ru@localhost wrote:
> 
>> Number:         54435
>> Category:       kern
>> Synopsis:       reading TCP urgent data with MSG_OOB doesn't clear poll(2) status
>> Confidential:   no
>> Severity:       non-critical
>> Priority:       low
>> Responsible:    kern-bug-people
>> State:          open
>> Class:          sw-bug
>> Submitter-Id:   net
>> Arrival-Date:   Sat Aug 03 14:50:00 +0000 2019
>> Originator:     Valery Ushakov
>> Release:        NetBSD-8
>> Organization:
>> Environment:
> NetBSD pony 8.1_STABLE NetBSD 8.1_STABLE (GENERIC) #0: Sat Jun 15 07:30:17 MSK 2019  uwe@sampo:/home/uwe/work/netbsd/build8/obj/macppc/sys/arch/macppc/compile/GENERIC macppc
> 
>> Description:
> It looks like POLLPRI | POLLRDBAND status in not cleared from the
> socket when TCP urgent data is read with MSG_OOB
> 
>> How-To-Repeat:
> cat >> oobserv.c <<'__EOF__'
> #include <sys/types.h>
> #include <sys/socket.h>
> 
> #include <netinet/in.h>
> #include <arpa/inet.h>
> 
> #include <err.h>
> #include <errno.h>
> #include <stdio.h>
> #include <stdlib.h>
> #include <string.h>
> #include <unistd.h>
> 
> 
> int
> main(void)
> {
>    int status;
> 
>    int server = socket(PF_INET, SOCK_STREAM, 0);
>    if (server < 0)
> 	err(EXIT_FAILURE, "socket");
> 
>    struct sockaddr_in sin;
>    memset(&sin, 0, sizeof(sin));
> #if !defined(__linux__) && !defined(__sun__)
>    sin.sin_len = sizeof(sin);
> #endif
>    sin.sin_family = AF_INET;
>    sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
>    sin.sin_port = htons(12345);
> 
>    status = bind(server, (struct sockaddr *)&sin, sizeof(sin));
>    if (status < 0)
> 	err(EXIT_FAILURE, "bind");
> 
>    status = listen(server, 1);
>    if (status < 0)
> 	err(EXIT_FAILURE, "listen");
> 
>    for (;;) {
> 	int client = accept(server, NULL, 0);
> 	if (client < 0)
> 	    err(EXIT_FAILURE, "accept");
> 
> 	send(client, "a", 1, 0);
> 	send(client, "b", 1, MSG_OOB);
> 	send(client, "c", 1, 0);
> 
> 	close(client);
>    }
> 
>    return 0;
> }
> __EOF__
> 
> cat >> oobrecv.c <<'__EOF__'
> #include <sys/types.h>
> #include <sys/socket.h>
> 
> #include <err.h>
> #include <errno.h>
> #include <fcntl.h>
> #include <poll.h>
> #include <stdbool.h>
> #include <stdio.h>
> #include <stdlib.h>
> #include <string.h>
> #include <unistd.h>
> 
> #include <netinet/in.h>
> #include <arpa/inet.h>
> 
> int usage(void);
> int getsocket(void);
> void oobtest(int);
> int pollsock(int);
> 
> bool oobinline = false;
> 
> int
> usage(void)
> {
>    fprintf(stderr, "usage: oobrecv [oob | inline]\n");
>    fprintf(stderr, "default is \"%s\"\n",
> 	    oobinline ? "inline" : "oob");
> 
>    return EXIT_FAILURE;
> }
> 
> 
> int
> main(int argc, char **argv)
> {
>    if (argc > 1) {
> 	if (strcasecmp(argv[1], "oob") == 0)
> 	    oobinline = false;
> 	else if (strcasecmp(argv[1], "inline") == 0)
> 	    oobinline = true;
> 	else
> 	    return usage();
>    }
> 
>    oobtest(getsocket());
>    return 0;
> }
> 
> 
> int
> getsocket(void)
> {
>    int s;
>    int status;
> 
>    s = socket(PF_INET, SOCK_STREAM, 0);
>    if (s < 0)
> 	err(EXIT_FAILURE, "socket");
> 
>    struct sockaddr_in sin;
>    memset(&sin, 0, sizeof(sin));
> #if !defined(__linux__) && !defined(__sun__)
>    sin.sin_len = sizeof(sin);
> #endif
>    sin.sin_family = AF_INET;
>    sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
>    sin.sin_port = htons(12345);
> 
>    if (oobinline) {
> 	int one = 1;
> 	status = setsockopt(s, SOL_SOCKET, SO_OOBINLINE,
> 			    (char *)&one, sizeof(one));
> 	if (status < 0)
> 	    err(EXIT_FAILURE, "SO_OOBINLINE");
>    }
> 
>    status = connect(s, (struct sockaddr *)&sin, sizeof(sin));
>    if (status < 0)
> 	err(EXIT_FAILURE, "connect");
> 
>    return s;
> }
> 
> 
> void
> oobtest(int s)
> {
>    char buf[128];
>    int status;
> 
>    printf("reading urgent data with %s\n",
> 	   oobinline ? "SO_OOBINLINE" : "MSG_OOB");
> 
>    for (;;) {
> 	int revents = pollsock(s);
> 	if (revents == 0)
> 	    continue;
> 
> 	if (revents & POLLNVAL) {
> 	    errno = EBADF;
> 	    err(EXIT_FAILURE, "poll");
> 	}
> 
> 	if (revents & POLLERR) {
> 	    int sockerr;
> 	    socklen_t optlen = (socklen_t)sizeof(sockerr);
> 
>            status = getsockopt(s, SOL_SOCKET, SO_ERROR,
> 				(char *)&sockerr, &optlen);
> 	    if (status < 0)
> 		err(EXIT_FAILURE, "SO_ERROR");
> 
> 	    errno = sockerr;
> 	    err(EXIT_FAILURE, NULL);
> 	}
> 
> 	int flags = 0;
> 	if (!oobinline && (revents & (POLLPRI | POLLRDBAND)))
> 	    flags = MSG_OOB;
> 
> 	ssize_t nread = recv(s, buf, sizeof(buf), flags);
> 	if (nread < 0)
> 	    err(EXIT_FAILURE, "recv%s",
> 		flags & MSG_OOB ? "(MSG_OOB)" : "");
> 
> 	printf("recv(%s) = %zd\n",
> 	       flags & MSG_OOB ? "MSG_OOB" : "",
> 	       nread);
> 
> 	if (nread == 0)
> 	    return;
> 
> 	fwrite(buf, 1, nread, stdout);
> 	printf("\n");
>    }
> }
> 
> 
> int
> pollsock(int s)
> {
>    struct pollfd fds[1];
> 
>    fds[0].fd = s;
>    fds[0].events = POLLIN | POLLPRI | POLLRDBAND;
>    fds[0].revents = 0;
> 
>    int nready = poll(fds, 1, -1);
>    if (nready < 0)
> 	err(EXIT_FAILURE, "poll");
> 
>    if (nready == 0)
> 	return 0;
> 
>    int revents = fds[0].revents;
> 
>    printf("poll: revents = 0x%x", revents);
>    if (fds[0].revents != 0) {
> 	printf(":");
> 	if (revents & POLLNVAL)   printf(" NVAL");
> 	if (revents & POLLERR) 	  printf(" ERR");
> 	if (revents & POLLHUP) 	  printf(" HUP");
> 	if (revents & POLLIN)  	  printf(" IN");
> 	if (revents & POLLPRI) 	  printf(" PRI");
> 	if (revents & POLLRDNORM) printf(" RDNORM");
> 	if (revents & POLLRDBAND) printf(" RDBAND");
>    }
>    printf("\n");
> 
>    return revents;
> }
> __EOF__
> 
> cc -o oobserv oobserv.c
> cc -o oobrecv oobrecv.c
> 
> ./oobserv &
> ./oobrecv oob
> 
> 
> The server sends "a", then send "b" as urgent/MSG_OOB, then "c".
> 
> The test invocation fails with:
> 
> reading urgent data with MSG_OOB
> poll: revents = 0x1: IN
> recv() = 1
> a
> poll: revents = 0x82: PRI RDBAND
> recv(MSG_OOB) = 1
> b
> poll: revents = 0x82: PRI RDBAND
> oobrecv: recv(MSG_OOB): Invalid argument
> 
> So after "b" was read (with MSG_OOB), the poll still reports POLLPRI.
> 
> This doesn't happen with SO_OOBINLINE:
> 
> $ ./oobrecv inline
> reading urgent data with SO_OOBINLINE
> poll: revents = 0x1: IN
> recv() = 1
> a
> poll: revents = 0x83: IN PRI RDBAND
> recv() = 1
> b
> poll: revents = 0x1: IN
> recv() = 1
> c
> poll: revents = 0x1: IN
> recv() = 0
> 
> here POLLPRI | POLLRDBAND are cleared after "b" was read.
> 
>> Fix: