bin/56530: fstyp(8): Segmentation fault in the HAMMER2 filesystem check

[rvp%SDF.ORG@localhost]

>Number:         56530
>Category:       bin
>Synopsis:       fstyp(8): Segmentation fault in the HAMMER2 filesystem check
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    bin-bug-people
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Wed Dec 01 09:40:00 +0000 2021
>Originator:     RVP
>Release:        NetBSD/9.99.92
>Organization:
>Environment:
NetBSD x202e.localdomain 9.99.92 NetBSD 9.99.92 (MYKERNEL) #0: Wed Dec  1 00:05:59 UTC 2021  bld@x202e.localdomain:/usr/obj/usr/src/sys/arch/amd64/compile/MYKERNEL amd64
>Description:
fstyp(8) dumps core in the HAMMER2 filesystem check when read_voldata()
returns NULL--because ftell() returns 0 for /dev/dk* wedges and block
devices.

Before:

# dkctl wd0 listwedges | fgrep dk7
dk7: FreeBSD_swap, 8390656 blocks at 106973184, type: swap
# fstyp /dev/dk7
Segmentation fault (core dumped)
#

After:

# fstyp /dev/dk7
fstyp: /dev/dk7: filesystem not recognized: Device busy

Also affects NetBSD-9.2.
>How-To-Repeat:
See above.
>Fix:
diff -urN usr.sbin/fstyp.orig/hammer2.c usr.sbin/fstyp/hammer2.c
--- usr.sbin/fstyp.orig/hammer2.c	2021-01-10 13:44:57.000000000 +0000
+++ usr.sbin/fstyp/hammer2.c	2021-11-30 07:36:22.198347288 +0000
@@ -40,9 +40,40 @@
 #include "fstyp.h"
 #include "hammer2_disk.h"
 
+#ifdef __NetBSD__
+#include <sys/param.h>
+#include <sys/ioctl.h>
+#include <sys/disk.h>
+#include <sys/stat.h>
+
+static ssize_t
+get_nbsd_file_size(FILE* fp)
+{
+	struct dkwedge_info dkw;
+	ssize_t siz;
+	int fd;
+
+	if ((fd = fileno(fp)) == -1) {
+		warnx("hammer2: invalid file descriptor");
+		return -1;
+	}
+	if (ioctl(fd, DIOCGWEDGEINFO, &dkw) == 0) {
+		return (ssize_t)(dkw.dkw_size * DEV_BSIZE);
+	}
+	if ((siz = lseek(fd, 0, SEEK_END)) == -1) {
+		warnx("hammer2: lseek failed");
+		return -1;
+	}
+	return siz;
+}
+#endif
+
 static ssize_t
 get_file_size(FILE *fp)
 {
+#ifdef __NetBSD__
+	return get_nbsd_file_size(fp);
+#else
 	ssize_t siz;
 
 	if (fseek(fp, 0, SEEK_END) == -1) {
@@ -57,6 +88,7 @@
 	}
 
 	return (siz);
+#endif
 }
 
 static hammer2_volume_data_t *
@@ -386,6 +418,8 @@
 	hammer2_volume_data_t *voldata = read_voldata(fp, 0);
 	int error = 1;
 
+	if (voldata == NULL)
+		goto fail;
 	if (voldata->volu_id != HAMMER2_ROOT_VOLUME)
 		goto fail;
 	if (voldata->nvolumes != 0)