Hi,
some time ago I added zfs verification to cgdconfig which I'd like to
share in case others find it useful.
This lets you use zpools/vdevs on cgd encrypted disks without an
intermediate gpt label. The verification method is the same as used in
usr.sbin/fstyp.
The downside is that this modification requires linking cgdconfig
against libnvpair when compiling with MKZFS, which adds about 100kb.
I've attached 3 patches:
0001-cgdconfig-Add-zfs-verification-method.patch
Everything necessary to add ZFS verification for cgdconfig,
including changes to the man page, Makefile, and linking libnvpair
for rescue and cgdroot when building with MKZFS.
0002-Add-cgd-zfs-ramdisk.patch
Adds a cgdzfsroot ramdisk, a simple combination of the existing
cgdroot and zfsroot ramdisks which lets you boot a system from a
cgd-encrypted zfs root pool.
0003-fstyp-Don-t-link-libzfs.patch
Don't link usr.sbin/fstyp against libzfs. It really just needs
libnvpair.
You can also pull the changes from here:
https://github.com/mdehling/netbsd-src/tree/mdehling_cgd-zfs
Cheers,
--
Malte Dehling
From 3b9daeb69e8db94cbd5321e903d0a0cdce504b0f Mon Sep 17 00:00:00 2001
From: Malte Dehling <mdehling%gmail.com@localhost>
Date: Sun, 14 Apr 2024 10:04:27 -0700
Subject: [PATCH 1/3] cgdconfig: Add zfs verification method
This allows us to create a zpool directly on a cgd device without having
to place a gpt label in between.
---
.../amd64/ramdisks/ramdisk-cgdroot/Makefile | 3 ++
.../amd64/ramdisks/ramdisk-cgdroot/list.zfs | 3 ++
rescue/Makefile | 4 ++
rescue/list.zfs | 3 ++
sbin/cgdconfig/Makefile | 18 +++++++
sbin/cgdconfig/cgdconfig.8 | 2 +
sbin/cgdconfig/cgdconfig.c | 53 +++++++++++++++++++
sbin/cgdconfig/params.c | 9 ++++
sbin/cgdconfig/params.h | 1 +
9 files changed, 96 insertions(+)
create mode 100644 distrib/amd64/ramdisks/ramdisk-cgdroot/list.zfs
create mode 100644 rescue/list.zfs
diff --git a/distrib/amd64/ramdisks/ramdisk-cgdroot/Makefile b/distrib/amd64/ramdisks/ramdisk-cgdroot/Makefile
index 7be7a009f28..75b07c6ebf0 100644
--- a/distrib/amd64/ramdisks/ramdisk-cgdroot/Makefile
+++ b/distrib/amd64/ramdisks/ramdisk-cgdroot/Makefile
@@ -9,6 +9,9 @@ SMALLPROG_INET6=1
.include "${.CURDIR}/../common/Makefile.ramdisk"
LISTS+= ${DISTRIBDIR}/common/list.cgdroot
+.if ${MKZFS} != "no"
+LISTS+= ${.CURDIR}/list.zfs
+.endif
MTREECONF+= ${DISTRIBDIR}/common/mtree.cgdroot
.if ${USE_INET6} != "no"
diff --git a/distrib/amd64/ramdisks/ramdisk-cgdroot/list.zfs b/distrib/amd64/ramdisks/ramdisk-cgdroot/list.zfs
new file mode 100644
index 00000000000..8247065d020
--- /dev/null
+++ b/distrib/amd64/ramdisks/ramdisk-cgdroot/list.zfs
@@ -0,0 +1,3 @@
+# $NetBSD$
+
+LIBS -lnvpair
diff --git a/rescue/Makefile b/rescue/Makefile
index 459e8a7135a..6ed6c178f3f 100644
--- a/rescue/Makefile
+++ b/rescue/Makefile
@@ -42,6 +42,10 @@ LISTS+= ${.CURDIR}/list.${f}
LISTS+= ${.CURDIR}/list.inet6
.endif
+.if ${MKZFS} != "no"
+LISTS+= ${.CURDIR}/list.zfs
+.endif
+
LISTS+= ${.CURDIR}/list.crypto
CRUNCHENV+= MKKERBEROS=no # for ssh
diff --git a/rescue/list.zfs b/rescue/list.zfs
new file mode 100644
index 00000000000..02bf8bc362c
--- /dev/null
+++ b/rescue/list.zfs
@@ -0,0 +1,3 @@
+# $NetBSD$
+
+LIBS -lnvpair
diff --git a/sbin/cgdconfig/Makefile b/sbin/cgdconfig/Makefile
index fec75b4e375..bf1be8109e3 100644
--- a/sbin/cgdconfig/Makefile
+++ b/sbin/cgdconfig/Makefile
@@ -29,4 +29,22 @@ ARGON2_NO_THREADS=1
.include "${NETBSDSRCDIR}/external/apache2/argon2/lib/libargon2/Makefile.inc"
.endif
+.if ${MKZFS} != "no"
+DPADD+= ${LIBNVPAIR}
+LDADD+= -lnvpair
+CPPFLAGS+= -DHAVE_ZFS
+
+OSNET=${NETBSDSRCDIR}/external/cddl/osnet
+CPPFLAGS.cgdconfig.c+= -I${OSNET}/include
+CPPFLAGS.cgdconfig.c+= -I${OSNET}/sys
+CPPFLAGS.cgdconfig.c+= -I${OSNET}/dist/head
+CPPFLAGS.cgdconfig.c+= -I${OSNET}/dist/lib/libzpool/common
+CPPFLAGS.cgdconfig.c+= -I${OSNET}/dist/uts/common
+CPPFLAGS.cgdconfig.c+= -I${OSNET}/dist/uts/common/fs/zfs
+CPPFLAGS.cgdconfig.c+= -I${OSNET}/dist/lib/libnvpair
+
+COPTS.cgdconfig.c+= -Wno-unknown-pragmas
+COPTS.cgdconfig.c+= -Wno-strict-prototypes
+.endif
+
.include <bsd.prog.mk>
diff --git a/sbin/cgdconfig/cgdconfig.8 b/sbin/cgdconfig/cgdconfig.8
index 2b0da2cb732..e48ccde02c5 100644
--- a/sbin/cgdconfig/cgdconfig.8
+++ b/sbin/cgdconfig/cgdconfig.8
@@ -270,6 +270,8 @@ scan for a valid Master Boot Record.
scan for a valid GUID partition table.
.It ffs
scan for a valid FFS file system.
+.It zfs
+scan for a valid ZFS vdev label (if compiled with MKZFS).
.It re-enter
prompt for passphrase twice, and ensure entered passphrases are
identical.
diff --git a/sbin/cgdconfig/cgdconfig.c b/sbin/cgdconfig/cgdconfig.c
index 9a634ef37e2..4baccb1d68f 100644
--- a/sbin/cgdconfig/cgdconfig.c
+++ b/sbin/cgdconfig/cgdconfig.c
@@ -73,6 +73,11 @@ __RCSID("$NetBSD: cgdconfig.c,v 1.61 2022/11/17 06:40:38 chs Exp $");
#include <ufs/ffs/fs.h>
+#ifdef HAVE_ZFS
+#include <libnvpair.h>
+#include <sys/vdev_impl.h>
+#endif
+
#include "params.h"
#include "pkcs5_pbkdf2.h"
#include "utils.h"
@@ -170,6 +175,9 @@ static int verify_ffs(int);
static int verify_reenter(struct params *);
static int verify_mbr(int);
static int verify_gpt(int);
+#ifdef HAVE_ZFS
+static int verify_zfs(int);
+#endif
__dead static void usage(void);
@@ -1024,6 +1032,10 @@ verify(struct params *p, int fd)
return verify_mbr(fd);
case VERIFY_GPT:
return verify_gpt(fd);
+#ifdef HAVE_ZFS
+ case VERIFY_ZFS:
+ return verify_zfs(fd);
+#endif
default:
warnx("unimplemented verification method");
return -1;
@@ -1182,6 +1194,47 @@ verify_gpt(int fd)
return ret;
}
+#ifdef HAVE_ZFS
+static int
+verify_zfs(int fd)
+{
+ vdev_label_t *vdev_label;
+ vdev_phys_t *vdev_phys;
+ nvlist_t *config = NULL;
+ ssize_t ret;
+
+ /*
+ * Read the first ZFS vdev label located at offset 0.
+ */
+ vdev_label = emalloc(sizeof *vdev_label);
+ ret = prog_pread(fd, vdev_label, sizeof(*vdev_label), 0);
+ if (ret < 0) {
+ warn("pread");
+ ret = 1;
+ goto bail;
+ } else if ((size_t)ret < sizeof(*vdev_label)) {
+ warnx("pread: incomplete block");
+ ret = 1;
+ goto bail;
+ };
+
+ ret = 0;
+
+ vdev_phys = &(vdev_label->vl_vdev_phys);
+ if ((nvlist_unpack(vdev_phys->vp_nvlist,
+ sizeof(vdev_phys->vp_nvlist), &config, 0)) != 0 ||
+ !nvlist_exists(config, "name"))
+ {
+ ret = 1;
+ };
+
+ nvlist_free(config);
+ bail:
+ free(vdev_label);
+ return ret;
+}
+#endif
+
static off_t sblock_try[] = SBLOCKSEARCH;
static int
diff --git a/sbin/cgdconfig/params.c b/sbin/cgdconfig/params.c
index 91af02c21cc..5da25ec0595 100644
--- a/sbin/cgdconfig/params.c
+++ b/sbin/cgdconfig/params.c
@@ -287,6 +287,10 @@ params_verify_method(string_t *in)
p->verify_method = VERIFY_MBR;
if (!strcmp("gpt", vm))
p->verify_method = VERIFY_GPT;
+#ifdef HAVE_ZFS
+ if (!strcmp("zfs", vm))
+ p->verify_method = VERIFY_ZFS;
+#endif
string_free(in);
@@ -1065,6 +1069,11 @@ params_fput(struct params *p, FILE *f)
case VERIFY_GPT:
print_kvpair_cstr(f, ts, "verify_method", "gpt");
break;
+#ifdef HAVE_ZFS
+ case VERIFY_ZFS:
+ print_kvpair_cstr(f, ts, "verify_method", "zfs");
+ break;
+#endif
default:
warnx("unsupported verify_method (%d)", p->verify_method);
return -1;
diff --git a/sbin/cgdconfig/params.h b/sbin/cgdconfig/params.h
index fb79a0deb80..ef49dde6aba 100644
--- a/sbin/cgdconfig/params.h
+++ b/sbin/cgdconfig/params.h
@@ -81,6 +81,7 @@ struct params {
#define VERIFY_REENTER 0x4
#define VERIFY_MBR 0x5
#define VERIFY_GPT 0x6
+#define VERIFY_ZFS 0x7
/* shared key derivation methods */
--
2.44.0
From 31fdb4c03d3dd0ee8236b3e7a37653522a47c75d Mon Sep 17 00:00:00 2001
From: Malte Dehling <mdehling%gmail.com@localhost>
Date: Mon, 15 Apr 2024 11:34:30 -0700
Subject: [PATCH 2/3] Add cgd/zfs ramdisk
Allows booting into a zfs root pool on a cgd encrypted device.
---
distrib/amd64/ramdisks/Makefile | 7 +++-
.../ramdisks/ramdisk-cgdzfsroot/Makefile | 23 ++++++++++++
.../amd64/ramdisks/ramdisk-cgdzfsroot/list | 12 +++++++
distrib/common/cgdzfsroot.rc | 36 +++++++++++++++++++
distrib/common/list.cgdzfsroot | 35 ++++++++++++++++++
distrib/common/mtree.cgdzfsroot | 6 ++++
6 files changed, 118 insertions(+), 1 deletion(-)
create mode 100644 distrib/amd64/ramdisks/ramdisk-cgdzfsroot/Makefile
create mode 100644 distrib/amd64/ramdisks/ramdisk-cgdzfsroot/list
create mode 100644 distrib/common/cgdzfsroot.rc
create mode 100644 distrib/common/list.cgdzfsroot
create mode 100644 distrib/common/mtree.cgdzfsroot
diff --git a/distrib/amd64/ramdisks/Makefile b/distrib/amd64/ramdisks/Makefile
index d99db1158da..2addb5e9870 100644
--- a/distrib/amd64/ramdisks/Makefile
+++ b/distrib/amd64/ramdisks/Makefile
@@ -1,8 +1,13 @@
-# $NetBSD: Makefile,v 1.4.20.1 2024/02/23 18:07:24 martin Exp $
+# $NetBSD$
+
+.include <bsd.own.mk>
SUBDIR=
SUBDIR+= ramdisk
SUBDIR+= ramdisk-cgdroot
+.if ${MKZFS} != "no"
+SUBDIR+= ramdisk-cgdzfsroot
+.endif
SUBDIR+= ramdisk-zfsroot
TARGETS+= release
diff --git a/distrib/amd64/ramdisks/ramdisk-cgdzfsroot/Makefile b/distrib/amd64/ramdisks/ramdisk-cgdzfsroot/Makefile
new file mode 100644
index 00000000000..4983bc04790
--- /dev/null
+++ b/distrib/amd64/ramdisks/ramdisk-cgdzfsroot/Makefile
@@ -0,0 +1,23 @@
+# $NetBSD$
+
+IMAGE= ramdisk-cgdzfsroot.fs
+IMAGESIZE= 5000k
+IMAGEDEPENDS=
+MAKEDEVTARGETS= all
+CRUNCHENV= INIT_CHROOT=1
+SMALLPROG_INET6= 1
+
+# Build as re-entrant because zpool requires libhack built this way.
+CPPFLAGS+= -D_REENTRANT
+
+LISTS+= ${.CURDIR}/list
+LISTS+= ${DISTRIBDIR}/common/list.cgdzfsroot
+
+.include "${.CURDIR}/../common/Makefile.ramdisk"
+.include "${DISTRIBDIR}/common/Makefile.makedev"
+
+MTREECONF+= ${DISTRIBDIR}/common/mtree.cgdzfsroot
+
+.if ${USE_INET6} != "no"
+LISTS+= ${DISTRIBDIR}/common/list.inet6
+.endif
diff --git a/distrib/amd64/ramdisks/ramdisk-cgdzfsroot/list b/distrib/amd64/ramdisks/ramdisk-cgdzfsroot/list
new file mode 100644
index 00000000000..bbcaa577901
--- /dev/null
+++ b/distrib/amd64/ramdisks/ramdisk-cgdzfsroot/list
@@ -0,0 +1,12 @@
+# $NetBSD$
+
+PROG bin/sync
+
+PROG sbin/fdisk
+PROG sbin/gpt
+PROG sbin/mbrlabel
+PROG sbin/shutdown
+
+PROG usr/bin/less usr/bin/more
+
+PROG usr/sbin/installboot
diff --git a/distrib/common/cgdzfsroot.rc b/distrib/common/cgdzfsroot.rc
new file mode 100644
index 00000000000..cf08a123f5e
--- /dev/null
+++ b/distrib/common/cgdzfsroot.rc
@@ -0,0 +1,36 @@
+# $NetBSD$
+
+# Assumption - boot.cfg loads this ramdisk.
+# Assumption - The needed kernel modules (solaris, and zfs) are either on this
+# ramdisk or loaded by boot.cfg.
+# Assumption - The root pool has mountpoint=legacy set.
+
+#
+# XXX: Can these be set in boot.cfg? Or should we read them from /efi?
+#
+rpool="rpool"
+rdataset="ROOT/default"
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin
+export HOME=/
+export TERM=wsvt25
+export BLOCKSIZE=1k
+export EDITOR=ed
+
+umask 022
+
+echo "Mounting /boot read-only."
+mount -o ro NAME=boot /boot || exit 1
+
+echo "Running cgdconfig..."
+cgdconfig -C || exit 1
+umount /boot
+
+echo "Importing ${rpool}."
+zpool import -f -N "${rpool}" || exit 1
+
+echo "Mounting ${rpool}/${rdataset} to /altroot."
+mount_zfs "${rpool}/${rdataset}" /altroot || exit 1
+
+echo "Booting into /altroot..."
+sysctl -w init.root=/altroot
diff --git a/distrib/common/list.cgdzfsroot b/distrib/common/list.cgdzfsroot
new file mode 100644
index 00000000000..29db5b77b1f
--- /dev/null
+++ b/distrib/common/list.cgdzfsroot
@@ -0,0 +1,35 @@
+# $NetBSD$
+#
+# list file (c.f. parselist.awk) for root on CGD encrypted ZFS.
+#
+
+PROG sbin/cgdconfig
+
+SYMLINK /boot/cgd etc/cgd
+
+PROG sbin/modload
+PROG sbin/modunload
+PROG sbin/modstat
+
+SRCDIRS external/cddl/osnet/sbin
+PROG sbin/zpool
+PROG sbin/zfs sbin/mount_zfs
+
+PROG sbin/sysctl
+
+LIBS -lcrypto
+LIBS -lnvpair
+LIBS -luutil
+LIBS -lzfs
+LIBS -lavl
+LIBS -lm
+LIBS -lpthread
+LIBS -lumem
+#LIBS -lutil # replaced by libhack
+LIBS -lz
+LIBS -lzfs_core
+
+COPY ${NETBSDSRCDIR}/distrib/common/cgdzfsroot.rc etc/rc
+
+SYMLINK altroot/stand stand
+SYMLINK altroot/libdata libdata
diff --git a/distrib/common/mtree.cgdzfsroot b/distrib/common/mtree.cgdzfsroot
new file mode 100644
index 00000000000..882cda1fdf8
--- /dev/null
+++ b/distrib/common/mtree.cgdzfsroot
@@ -0,0 +1,6 @@
+# $NetBSD$
+
+.
+./altroot
+./boot
+./etc
--
2.44.0
From d50b05a300851be4e83702ac42658999592b4e1c Mon Sep 17 00:00:00 2001
From: Malte Dehling <mdehling%gmail.com@localhost>
Date: Tue, 16 Apr 2024 18:02:54 -0700
Subject: [PATCH 3/3] fstyp: Don't link libzfs
---
usr.sbin/fstyp/Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/usr.sbin/fstyp/Makefile b/usr.sbin/fstyp/Makefile
index 27482d2ee47..03d4ed9c199 100644
--- a/usr.sbin/fstyp/Makefile
+++ b/usr.sbin/fstyp/Makefile
@@ -17,7 +17,7 @@ WARNS?= 6
COPTS.zfs.c+= -Wno-unknown-pragmas
COPTS.zfs.c+= -Wno-sign-conversion
COPTS.zfs.c+= -Wno-strict-prototypes
-LDADD+= -lnvpair -lzfs
+LDADD+= -lnvpair
OSNET=${NETBSDSRCDIR}/external/cddl/osnet
CPPFLAGS+= -DHAVE_ZFS
--
2.44.0
Attachment:
signature.asc
Description: PGP signature