Re: zpool import skips wedges due to a race condition

[Taylor R Campbell <campbell+netbsd-current-users%mumble.net@localhost>]

> Date: Tue, 7 Sep 2021 22:30:45 +0100
> From: Alexander Nasonov <alnsn%yandex.ru@localhost>
> 
> When I run zfs import, it launches 32 threads and opens 32 disks in
> parallel, including cgd1 and dk24. But it can't open dk24 while
> cgd1 is still open (it fails with EBUSY).

I have the attached patch in my tree from a while ago to scan
non-wedge disks first, then wedges.  For some reason I didn't commit
it but I forget why.  (Well, it's a little kludgey to look at the
pathname to distinguish dkN devices, but...)

From dbee261ca7b5837946e93ec601acb04ae0b0a8b1 Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh%NetBSD.org@localhost>
Date: Sat, 29 Feb 2020 18:19:45 +0000
Subject: [PATCH] Scan primary disks first; then dkwedges.

---
 .../dist/lib/libzfs/common/libzfs_import.c    | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/external/cddl/osnet/dist/lib/libzfs/common/libzfs_import.c b/external/cddl/osnet/dist/lib/libzfs/common/libzfs_import.c
index b935e49076a5..e8d29b86f4b2 100644
--- a/external/cddl/osnet/dist/lib/libzfs/common/libzfs_import.c
+++ b/external/cddl/osnet/dist/lib/libzfs/common/libzfs_import.c
@@ -1175,6 +1175,9 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 	config_entry_t *ce, *cenext;
 	name_entry_t *ne, *nenext;
 	avl_tree_t slice_cache;
+#ifdef __NetBSD__
+	avl_tree_t dkwedge_cache;
+#endif
 	rdsk_node_t *slice;
 	void *cookie;
 
@@ -1266,6 +1269,8 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 		}
 #endif
 #ifdef __NetBSD__
+		avl_create(&dkwedge_cache, slice_cache_compare,
+		    sizeof (rdsk_node_t), offsetof(rdsk_node_t, rn_node));
 		if (strcmp(rdsk, "/dev/") == 0) {
 			static const char mib_name[] = "hw.disknames";
 			size_t len;
@@ -1291,7 +1296,10 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 				slice->rn_dfd = dfd;
 				slice->rn_hdl = hdl;
 				slice->rn_nozpool = B_FALSE;
-				avl_add(&slice_cache, slice);
+				if (strncmp(name, "dk", 2) == 0)
+					avl_add(&dkwedge_cache, slice);
+				else
+					avl_add(&slice_cache, slice);
 			}
 			free(disknames);
 
@@ -1333,6 +1341,16 @@ skipdir:
 		    AVL_AFTER)))
 			(void) tpool_dispatch(t, zpool_open_func, slice);
 		tpool_wait(t);
+#ifdef __NetBSD__
+		for (slice = avl_first(&dkwedge_cache); slice;
+		    (slice = avl_walk(&dkwedge_cache, slice,
+		    AVL_AFTER)))
+			(void) tpool_dispatch(t, zpool_open_func, slice);
+		tpool_wait(t);
+		while ((slice = avl_destroy_nodes(&dkwedge_cache,
+		    &cookie)) != NULL)
+			avl_add(&slice_cache, slice);
+#endif
 		tpool_destroy(t);
 
 		cookie = NULL;
@@ -1373,6 +1391,9 @@ skipdir:
 			free(slice->rn_name);
 			free(slice);
 		}
+#ifdef __NetBSD__
+		avl_destroy(&dkwedge_cache);
+#endif
 		avl_destroy(&slice_cache);
 
 		(void) closedir(dirp);