kern/60948: WSDISPLAYIO_GMODE returns wrong value once MODE_DUMBFB is set

["Izumi Tsutsui via gnats" <gnats-admin%NetBSD.org@localhost>]

>Number:         60948
>Category:       kern
>Synopsis:       WSDISPLAYIO_GMODE returns wrong value once MODE_DUMBFB is set
>Confidential:   no
>Severity:       non-critical
>Priority:       medium
>Responsible:    kern-bug-people
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Sat Oct 31 17:00:00 +0000 2026
>Originator:     Izumi Tsutsui
>Release:        NetBSD 11.0
>Organization:
>Environment:
System: NetBSD mirage 11.0 NetBSD 11.0 (GENERIC) #0:
Architecture: all
Machine: all (with wscons)
>Description:

wscons(4) WSDISPLAYIO_GMODE ioctl returns a incorrect value in
some case (see below).

>How-To-Repeat:

Save and build the following test code:

```c
#include <sys/types.h>
#include <sys/ioctl.h>

#include <dev/wscons/wsconsio.h>

#include <err.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

int
main(int argc, char **argv)
{
	const int modes[] = {
		WSDISPLAYIO_MODE_DUMBFB,
		WSDISPLAYIO_MODE_EMUL,
		WSDISPLAYIO_MODE_MAPPED
	};
	const char *names[] = { "DUMBFB", "EMUL", "MAPPED" };
	int fd, saved, mode, got[3], i, n = 0, status = 0;

	if (argc != 2)
		errx(EXIT_FAILURE, "usage: %s /dev/ttyE0", argv[0]);
	if ((fd = open(argv[1], O_RDWR)) == -1)
		err(EXIT_FAILURE, "%s", argv[1]);
	if (ioctl(fd, WSDISPLAYIO_GMODE, &saved) == -1)
		err(EXIT_FAILURE, "initial GMODE");

	/* SMODE (write) then GMODE (read back) */
	for (i = 0; i < 3; i++) {
		mode = modes[i];
		if (ioctl(fd, WSDISPLAYIO_SMODE, &mode) == -1) {
			warn("SMODE %s", names[i]);
			status = 1;
			break;
		}
		if (ioctl(fd, WSDISPLAYIO_GMODE, &got[i]) == -1) {
			warn("GMODE after %s", names[i]);
			status = 1;
			break;
		}
		n++;
		if (got[i] != modes[i])
			status = 1;
	}
	/* Restore SMODE */
	if (ioctl(fd, WSDISPLAYIO_SMODE, &saved) == -1) {
		warn("restore SMODE");
		status = 1;
	}
	close(fd);
	/* Print results */
	for (i = 0; i < n; i++)
		printf("%s: requested %d, GMODE returned %d (%s)\n",
		    names[i], modes[i], got[i],
		    got[i] == modes[i] ? "OK" : "FAIL");
	return status;
}
```

# cc -O -o wsdisplay-mode-test wsdisplay-mode-test.c
#  ./wsdisplay-mode-test
wsdisplay-mode-test: usage: ./wsdisplay-mode-test /dev/ttyE0
# ./wsdisplay-mode-test /dev/ttyE0
DUMBFB: requested 2, GMODE returned 2 (OK)
EMUL: requested 0, GMODE returned 0 (OK)
MAPPED: requested 1, GMODE returned 2 (FAIL)
#  uname -mrs
NetBSD 11.99.8 news68k
# 

>Fix:

Clear not only SCR_GRAPHICS but also SCR_DUMBFB in both
wsdisplayclose() and wsdisplay_internal_ioctl()?

https://nxr.netbsd.org/xref/src/sys/dev/wscons/wsdisplay.c?r=1.167#1111
```
   1072 wsdisplayclose(dev_t dev, int flag, int mode, struct lwp *l)

   1110 	if (WSSCREEN_HAS_EMULATOR(scr)) {
   1111 		scr->scr_flags &= ~SCR_GRAPHICS;
```

https://nxr.netbsd.org/xref/src/sys/dev/wscons/wsdisplay.c?r=1.167#1364
```
   1313 wsdisplay_internal_ioctl(struct wsdisplay_softc *sc, struct wsscreen *scr,
   1314 	u_long cmd, void *data, int flag, struct lwp *l)

   1363 	    if (WSSCREEN_HAS_EMULATOR(scr)) {
   1364 		    scr->scr_flags &= ~SCR_GRAPHICS;
   1365 		    if (d == WSDISPLAYIO_MODE_MAPPED ||
   1366 			d == WSDISPLAYIO_MODE_DUMBFB)
   1367 			    scr->scr_flags |= SCR_GRAPHICS |
   1368 				    ((d == WSDISPLAYIO_MODE_DUMBFB) ? SCR_DUMBFB : 0);
   1369 	    } else if (d == WSDISPLAYIO_MODE_EMUL)
   1370 		    return EINVAL;
```

---
Izumi Tsutsui