kern/59129: futex(3): missing sign extension in FUTEX_WAIT_OP

[campbell+netbsd%mumble.net@localhost]

>Number:         59129
>Category:       kern
>Synopsis:       futex(3): missing sign extension in FUTEX_WAIT_OP
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    kern-bug-people
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Tue Mar 04 19:15:00 +0000 2025
>Originator:     Taylor R Campbell
>Release:        current, 10
>Organization:
The FutexBSigneD Extension
>Environment:
>Description:
According to the Linux man page at https://www.man7.org/linux/man-pages/man2/futex.2.html, Linux's FUTEX_WAKE_OP performs an operation of the form:

                  uint32_t oldval = *(uint32_t *) uaddr2;
                  *(uint32_t *) uaddr2 = oldval op oparg;
                  futex(uaddr, FUTEX_WAKE, val, 0, 0, 0);
                  if (oldval cmp cmparg)
                      futex(uaddr2, FUTEX_WAKE, val2, 0, 0, 0);

The operation is parametrized by a 32-bit word that contains four parts: cmp, cmparg, op, oparg.  The layout, again from the man page, is:

                  +---+---+-----------+-----------+
                  |op |cmp|   oparg   |  cmparg   |
                  +---+---+-----------+-----------+
                    4   4       12          12    <== # of bits

              Expressed in code, the encoding is:

                  #define FUTEX_OP(op, oparg, cmp, cmparg) \
                                  (((op & 0xf) << 28) | \
                                  ((cmp & 0xf) << 24) | \
                                  ((oparg & 0xfff) << 12) | \
                                  (cmparg & 0xfff))

Left unstated in this documentation -- and any other documentation I can find -- is that oparg and cmparg are interpreted by Linux as _signed_ 12-bit quantities.  But currently NetBSD's futex implementation interprets them as _unsigned_ 12-bit quantities.
>How-To-Repeat:
#include <sys/syscall.h>

#include <err.h>
#include <limits.h>
#include <stdio.h>
#include <stdint.h>
#include <unistd.h>

#if defined __NetBSD__
#include <sys/futex.h>
static inline int
futex(volatile int *uaddr, int op, int val, const struct timespec *timeout,
    volatile int *uaddr2, int val2, int val3)
{
	return syscall(SYS___futex, uaddr, op, val, timeout, uaddr2,
	    val2, val3);
}
#elif defined __linux__
#include <linux/futex.h>
#include <sys/time.h>
#include <assert.h>
static inline int
futex(volatile int *uaddr, int op, int val, const struct timespec *timeout,
    volatile int *uaddr2, int val2, int val3)
{

	switch (op & FUTEX_CMD_MASK) {
	case FUTEX_WAIT:
	case FUTEX_LOCK_PI:
	case FUTEX_WAIT_BITSET:
	case FUTEX_WAIT_REQUEUE_PI:
		break;
	case FUTEX_WAKE:
	case FUTEX_FD:
	case FUTEX_REQUEUE:
	case FUTEX_CMP_REQUEUE:
	case FUTEX_WAKE_OP:
	case FUTEX_UNLOCK_PI:
	case FUTEX_TRYLOCK_PI:
	case FUTEX_WAKE_BITSET:
	case FUTEX_CMP_REQUEUE_PI:
		assert(timeout == NULL);
		timeout = (void *)(intptr_t)val;
		break;
	}
	return syscall(SYS_futex, uaddr, op, val, timeout, uaddr2, val3);
}
#endif

static int
futex_cas11s(volatile int *word, int old, int new)
{
	volatile int word1;

	return futex(/*uaddr*/&word1, /*op*/FUTEX_WAKE_OP,
	    /*nr_wake=val*/INT_MAX, /*timeout*/NULL, /*uaddr2*/word,
	    /*nr_wake2=val2*/INT_MAX,
	    /*op=val3*/FUTEX_OP(FUTEX_OP_SET, new, FUTEX_OP_CMP_EQ, old));
}

int
main(void)
{
	volatile int word = 0;

	if (futex_cas11s(&word, 0, 4095) == -1)
		err(1, "futex(FUTEX_WAKE_OP)");
	printf("%d\n", word);
	fflush(stdout);
	return ferror(stdout);
}

Linux prints -1; NetBSD>=10 prints 4095.
>Fix:
1. Add some tests for this.
2. Sign-extend in futex_compute_cmp and futex_compute_op.