kern/54435: reading TCP urgent data with MSG_OOB doesn't clear poll(2) status

[uwe%stderr.spb.ru@localhost]

>Number:         54435
>Category:       kern
>Synopsis:       reading TCP urgent data with MSG_OOB doesn't clear poll(2) status
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    kern-bug-people
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Sat Aug 03 14:50:00 +0000 2019
>Originator:     Valery Ushakov
>Release:        NetBSD-8
>Organization:
>Environment:
NetBSD pony 8.1_STABLE NetBSD 8.1_STABLE (GENERIC) #0: Sat Jun 15 07:30:17 MSK 2019  uwe@sampo:/home/uwe/work/netbsd/build8/obj/macppc/sys/arch/macppc/compile/GENERIC macppc

>Description:
It looks like POLLPRI | POLLRDBAND status in not cleared from the
socket when TCP urgent data is read with MSG_OOB

>How-To-Repeat:
cat >> oobserv.c <<'__EOF__'
#include <sys/types.h>
#include <sys/socket.h>

#include <netinet/in.h>
#include <arpa/inet.h>

#include <err.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>


int
main(void)
{
    int status;

    int server = socket(PF_INET, SOCK_STREAM, 0);
    if (server < 0)
	err(EXIT_FAILURE, "socket");

    struct sockaddr_in sin;
    memset(&sin, 0, sizeof(sin));
#if !defined(__linux__) && !defined(__sun__)
    sin.sin_len = sizeof(sin);
#endif
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
    sin.sin_port = htons(12345);

    status = bind(server, (struct sockaddr *)&sin, sizeof(sin));
    if (status < 0)
	err(EXIT_FAILURE, "bind");

    status = listen(server, 1);
    if (status < 0)
 	err(EXIT_FAILURE, "listen");

    for (;;) {
	int client = accept(server, NULL, 0);
	if (client < 0)
	    err(EXIT_FAILURE, "accept");

	send(client, "a", 1, 0);
	send(client, "b", 1, MSG_OOB);
	send(client, "c", 1, 0);

	close(client);
    }

    return 0;
}
__EOF__

cat >> oobrecv.c <<'__EOF__'
#include <sys/types.h>
#include <sys/socket.h>

#include <err.h>
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <netinet/in.h>
#include <arpa/inet.h>

int usage(void);
int getsocket(void);
void oobtest(int);
int pollsock(int);

bool oobinline = false;

int
usage(void)
{
    fprintf(stderr, "usage: oobrecv [oob | inline]\n");
    fprintf(stderr, "default is \"%s\"\n",
	    oobinline ? "inline" : "oob");

    return EXIT_FAILURE;
}


int
main(int argc, char **argv)
{
    if (argc > 1) {
	if (strcasecmp(argv[1], "oob") == 0)
	    oobinline = false;
	else if (strcasecmp(argv[1], "inline") == 0)
	    oobinline = true;
	else
	    return usage();
    }

    oobtest(getsocket());
    return 0;
}


int
getsocket(void)
{
    int s;
    int status;

    s = socket(PF_INET, SOCK_STREAM, 0);
    if (s < 0)
	err(EXIT_FAILURE, "socket");

    struct sockaddr_in sin;
    memset(&sin, 0, sizeof(sin));
#if !defined(__linux__) && !defined(__sun__)
    sin.sin_len = sizeof(sin);
#endif
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
    sin.sin_port = htons(12345);

    if (oobinline) {
	int one = 1;
	status = setsockopt(s, SOL_SOCKET, SO_OOBINLINE,
			    (char *)&one, sizeof(one));
	if (status < 0)
	    err(EXIT_FAILURE, "SO_OOBINLINE");
    }

    status = connect(s, (struct sockaddr *)&sin, sizeof(sin));
    if (status < 0)
	err(EXIT_FAILURE, "connect");

    return s;
}


void
oobtest(int s)
{
    char buf[128];
    int status;

    printf("reading urgent data with %s\n",
	   oobinline ? "SO_OOBINLINE" : "MSG_OOB");

    for (;;) {
	int revents = pollsock(s);
	if (revents == 0)
	    continue;

	if (revents & POLLNVAL) {
	    errno = EBADF;
	    err(EXIT_FAILURE, "poll");
	}

	if (revents & POLLERR) {
	    int sockerr;
	    socklen_t optlen = (socklen_t)sizeof(sockerr);

            status = getsockopt(s, SOL_SOCKET, SO_ERROR,
				(char *)&sockerr, &optlen);
	    if (status < 0)
		err(EXIT_FAILURE, "SO_ERROR");

	    errno = sockerr;
	    err(EXIT_FAILURE, NULL);
	}

	int flags = 0;
	if (!oobinline && (revents & (POLLPRI | POLLRDBAND)))
	    flags = MSG_OOB;

	ssize_t nread = recv(s, buf, sizeof(buf), flags);
	if (nread < 0)
	    err(EXIT_FAILURE, "recv%s",
		flags & MSG_OOB ? "(MSG_OOB)" : "");

	printf("recv(%s) = %zd\n",
	       flags & MSG_OOB ? "MSG_OOB" : "",
	       nread);

	if (nread == 0)
	    return;

	fwrite(buf, 1, nread, stdout);
	printf("\n");
    }
}


int
pollsock(int s)
{
    struct pollfd fds[1];

    fds[0].fd = s;
    fds[0].events = POLLIN | POLLPRI | POLLRDBAND;
    fds[0].revents = 0;

    int nready = poll(fds, 1, -1);
    if (nready < 0)
	err(EXIT_FAILURE, "poll");

    if (nready == 0)
	return 0;

    int revents = fds[0].revents;

    printf("poll: revents = 0x%x", revents);
    if (fds[0].revents != 0) {
	printf(":");
	if (revents & POLLNVAL)   printf(" NVAL");
	if (revents & POLLERR) 	  printf(" ERR");
	if (revents & POLLHUP) 	  printf(" HUP");
	if (revents & POLLIN)  	  printf(" IN");
	if (revents & POLLPRI) 	  printf(" PRI");
	if (revents & POLLRDNORM) printf(" RDNORM");
	if (revents & POLLRDBAND) printf(" RDBAND");
    }
    printf("\n");

    return revents;
}
__EOF__

cc -o oobserv oobserv.c
cc -o oobrecv oobrecv.c

./oobserv &
./oobrecv oob


The server sends "a", then send "b" as urgent/MSG_OOB, then "c".

The test invocation fails with:

reading urgent data with MSG_OOB
poll: revents = 0x1: IN
recv() = 1
a
poll: revents = 0x82: PRI RDBAND
recv(MSG_OOB) = 1
b
poll: revents = 0x82: PRI RDBAND
oobrecv: recv(MSG_OOB): Invalid argument

So after "b" was read (with MSG_OOB), the poll still reports POLLPRI.

This doesn't happen with SO_OOBINLINE:

$ ./oobrecv inline
reading urgent data with SO_OOBINLINE
poll: revents = 0x1: IN
recv() = 1
a
poll: revents = 0x83: IN PRI RDBAND
recv() = 1
b
poll: revents = 0x1: IN
recv() = 1
c
poll: revents = 0x1: IN
recv() = 0

here POLLPRI | POLLRDBAND are cleared after "b" was read.

>Fix: