kern/60453: sparc: enable gem on sun4m and IOMMU mbuf DMA support

["erik%eriklabine.com@localhost via gnats" <gnats-admin%NetBSD.org@localhost>]

>Number:         60453
>Category:       kern
>Synopsis:       sparc: enable gem on sun4m and IOMMU mbuf DMA support
>Confidential:   no
>Severity:       non-critical
>Priority:       medium
>Responsible:    kern-bug-people
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Fri Jul 17 05:50:00 +0000 2026
>Originator:     Erik LaBine
>Release:        NetBSD 11.99.7 (NetBSD-current)
>Organization:
>Environment:
NetBSD 11.99.7 NetBSD 11.99.7 (GENERIC) #4: Wed Jul 15 01:44:58 EDT 2026
>Description:
I've been experimenting with attaching the gem driver for SBus gigabit ethernet devices on sun4m; surprisingly, despite a lack of official support from Sun on these machines, it appears to work fine.

The gem driver for SBus only required a minor change to config values for burst size. There was also the iommu_dmamap_load_mbuf() function, which sparc SBus didn't implement.

In my original testing earlier this year, my implementation of the IOMMU code was a stub just to get things running. I later learned from miod on the OpenBSD project that Mark Kettenis got these cards working on sparc in 2009 before OpenBSD dropped support for the architecture. Since his code is more well-proven, I ported it back to NetBSD's IOMMU handling for sparc.

The existing iommu_dmamap_load function was split to extract the functionality shared between that function and the _mbuf version. The mbuf code only seems to be required for IPv6 functionality in gem.

For the gem driver itself, GEM_SBUS_CFG_BSIZE64 was picked experimentally. I had difficulty tracking down the appropriate documentation on the appropriate size, but it tolerated this value fine in the subsequent testing.
>How-To-Repeat:
After building on an amd64 system, I booted a series of sparc systems to cover at least the MicroSPARC II, HyperSPARC, and TurboSPARC CPUs, and with a variety of memory configs. I also built the sparc64 kernel and tested it on an UltraSPARC II CPU to ensure there were no obvious regressions.

The gem interface was tested for both IPv4 and IPv6, and across both TCP traffic as well as UDP traffic at 300 kbit/s and 4 Mbit/s. I also tested TCP traffic for a 5 minute duration to make sure it remained stable over a longer period. These were synthetic loads produced by iperf3.

Testing seems to indicate that the interface is able to handle all tested traffic without instability. The network also seemed to remain stable during normal SSH/SFTP use.
>Fix:
Index: sys/arch/sparc/sparc/iommu.c
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc/sparc/iommu.c,v
retrieving revision 1.102
diff -u -r1.102 iommu.c
--- sys/arch/sparc/sparc/iommu.c	1 Dec 2023 05:22:01 -0000	1.102
+++ sys/arch/sparc/sparc/iommu.c	15 Jul 2026 05:45:04 -0000
@@ -48,6 +48,7 @@
 #include <sys/device.h>
 #include <sys/proc.h>
 #include <sys/vmem.h>
+#include <sys/mbuf.h>
 
 #include <uvm/uvm.h>
 
@@ -90,6 +91,9 @@
     iommu_match, iommu_attach, NULL, NULL);
 
 /* IOMMU DMA map functions */
+static int	iommu_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *,
+                                 bus_size_t, struct proc *, int);
+
 int	iommu_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
 			bus_size_t, int, bus_dmamap_t *);
 int	iommu_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *,
@@ -565,13 +569,9 @@
 	return (error);
 }
 
-/*
- * Prepare buffer for DMA transfer.
- */
-int
-iommu_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map,
-		  void *buf, bus_size_t buflen,
-		  struct proc *p, int flags)
+static int
+iommu_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
+    bus_size_t buflen, struct proc *p, int flags)
 {
 	struct iommu_softc *sc = t->_cookie;
 	bus_size_t sgsize;
@@ -581,10 +581,8 @@
 	pmap_t pmap;
 	int error;
 
-	/*
-	 * Make sure that on error condition we return "no valid mappings".
-	 */
-	map->dm_nsegs = 0;
+	if (map->dm_nsegs >= map->_dm_segcnt)
+		return SET_ERROR(EFBIG);
 
 	/* Allocate IOMMU resources */
 	if ((error = iommu_dvma_alloc(sc, map, va, buflen, flags,
@@ -598,11 +596,10 @@
 	/*
 	 * We always use just one segment.
 	 */
-	map->dm_mapsize = buflen;
-	map->dm_nsegs = 1;
-	map->dm_segs[0].ds_addr = dva + (va & (pagesz - 1));
-	map->dm_segs[0].ds_len = buflen;
-	map->dm_segs[0]._ds_sgsize = sgsize;
+	map->dm_segs[map->dm_nsegs].ds_addr = dva + (va & (pagesz - 1));
+	map->dm_segs[map->dm_nsegs].ds_len = buflen;
+	map->dm_segs[map->dm_nsegs]._ds_sgsize = sgsize;
+	map->dm_nsegs++;
 
 	if (p != NULL)
 		pmap = p->p_vmspace->vm_map.pmap;
@@ -614,10 +611,8 @@
 		/*
 		 * Get the physical address for this page.
 		 */
-		if (!pmap_extract(pmap, va, &pa)) {
-			iommu_dmamap_unload(t, map);
-			return (EFAULT);
-		}
+		if (!pmap_extract(pmap, va, &pa))
+			return SET_ERROR(EFAULT);
 
 		iommu_enter(sc, dva, pa);
 
@@ -630,14 +625,55 @@
 }
 
 /*
+ * Prepare buffer for DMA transfer.
+ */
+int
+iommu_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map,
+		  void *buf, bus_size_t buflen,
+		  struct proc *p, int flags)
+{
+	int error;
+
+	/*
+	 * Make sure that on error condition we return "no valid mappings".
+	 */
+	map->dm_mapsize = buflen;
+	map->dm_nsegs = 0;
+
+	error = iommu_dmamap_load_buffer(t, map, buf, buflen, p, flags);
+	if (error)
+		iommu_dmamap_unload(t, map);
+
+	return (error);
+}
+
+/*
  * Like _bus_dmamap_load(), but for mbufs.
  */
 int
 iommu_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map,
-		       struct mbuf *m, int flags)
+		       struct mbuf *m0, int flags)
 {
+	struct mbuf *m;
+	int error = 0;
 
-	panic("_bus_dmamap_load_mbuf: not implemented");
+	/*
+	 * Make sure that on error condition we return "no valid mappings".
+	 */
+	map->dm_mapsize = m0->m_pkthdr.len;
+	map->dm_nsegs = 0;
+
+	for (m = m0; m != NULL && error == 0; m = m->m_next) {
+		if (m->m_len == 0)
+			continue;
+		error = iommu_dmamap_load_buffer(t, map, m->m_data, m->m_len,
+		    NULL, flags);
+	}
+
+	if (error)
+		iommu_dmamap_unload(t, map);
+
+	return (error);
 }
 
 /*
Index: sys/arch/sparc/conf/GENERIC
===================================================================
RCS file: /cvsroot/src/sys/arch/sparc/conf/GENERIC,v
retrieving revision 1.276
diff -u -r1.276 GENERIC
--- sys/arch/sparc/conf/GENERIC	2 May 2026 21:45:34 -0000	1.276
+++ sys/arch/sparc/conf/GENERIC	15 Jul 2026 05:45:04 -0000
@@ -532,6 +532,9 @@
 ## Happy Meal Ethernet
 hme*	at sbus? slot ? offset ?
 
+## Gigabit Ethernet
+gem*	at sbus? slot ? offset ?
+
 # PCMCIA ethernet devices
 #ep*	at pcmcia?
 #mbe*	at pcmcia?
Index: sys/dev/sbus/if_gem_sbus.c
===================================================================
RCS file: /cvsroot/src/sys/dev/sbus/if_gem_sbus.c,v
retrieving revision 1.15
diff -u -r1.15 if_gem_sbus.c
--- sys/dev/sbus/if_gem_sbus.c	15 Oct 2025 01:36:25 -0000	1.15
+++ sys/dev/sbus/if_gem_sbus.c	15 Jul 2026 05:45:04 -0000
@@ -86,6 +86,7 @@
 	struct gem_sbus_softc *gsc = device_private(self);
 	struct gem_softc *sc = &gsc->gsc_gem;
 	uint8_t enaddr[ETHER_ADDR_LEN];
+	uint32_t sbus_cfg;
 
 	sc->sc_dev = self;
 
@@ -138,8 +139,14 @@
 	 */
 	(void) bus_space_read_4(sa->sa_bustag, sc->sc_h2, GEM_SBUS_RESET);
 	delay(100);
-	bus_space_write_4(sa->sa_bustag, sc->sc_h2, GEM_SBUS_CONFIG,
-	    GEM_SBUS_CFG_BSIZE128|GEM_SBUS_CFG_PARITY|GEM_SBUS_CFG_BMODE64);
+
+#if defined(__sparc64__)
+	sbus_cfg = GEM_SBUS_CFG_BSIZE128 | GEM_SBUS_CFG_PARITY |
+	    GEM_SBUS_CFG_BMODE64;
+#else
+	sbus_cfg = GEM_SBUS_CFG_BSIZE64;
+#endif
+	bus_space_write_4(sa->sa_bustag, sc->sc_h2, GEM_SBUS_CONFIG, sbus_cfg);
 	sc->sc_chiprev = bus_space_read_4(sa->sa_bustag, sc->sc_h2,
 	    GEM_SBUS_REVISION);