Subject: bce(4) and memory > 1GB problem
To: None <tech-kern@netbsd.org>
From: Yorick Hardy <yhardy@uj.ac.za>
List: tech-kern
Date: 01/08/2007 11:17:01
This is a multi-part message in MIME format.
--------------000704090809020700010607
Content-Type: text/plain; charset=ISO-8859-1; format=flowed
Content-Transfer-Encoding: 7bit

I encountered this problem with 2GB of RAM on an HP nx6310.

The attached patch (for 4 BETA) may not be the right way to solve
the problem, but it does get the network working for me.

FreeBSD solved the problem with bus_dma_tag_create:

http://www.freebsd.org/cgi/man.cgi?query=bus_dma_tag_create&apropos=0&sektion=0&manpath=FreeBSD+6.1-RELEASE&format=html

-- 
Kind regards,

Yorick Hardy


--------------000704090809020700010607
Content-Type: text/plain;
 name="bce.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="bce.patch"

--- sys/arch/x86/include/bus.h.orig	2007-01-07 14:35:24.000000000 +0200
+++ sys/arch/x86/include/bus.h	2007-01-07 18:44:12.000000000 +0200
@@ -1029,6 +1029,19 @@
 #define	BUS_DMA_WRITE		0x200	/* mapping is memory -> device only */
 #define	BUS_DMA_NOCACHE		0x400	/* hint: map non-cached memory */
 
+/*
+ * BUS_DMA_USEWIDTH indicates that the number of bits used
+ * for addressing is specified in the high 16 bits of flags.
+ * For example, for a device which can only address 24 bits
+ * (16MB) we use (BUS_DMA_USEWIDTH | BUS_DMA_WIDTH(24)).
+ * Only necessary if the device cannot use the full width of
+ * the host bus.
+ */
+
+#define BUS_DMA_USEWIDTH	BUS_DMA_BUS1
+#define BUS_DMA_WIDTH(x)	((x) << 16)
+#define BUS_DMA_HIGHADDR(x)	(1 << ((x) >> 16))
+
 /* Forwards needed by prototypes below. */
 struct mbuf;
 struct uio;
--- sys/arch/x86/pci/pci_machdep.c.orig	2007-01-06 13:42:22.000000000 +0200
+++ sys/arch/x86/pci/pci_machdep.c	2007-01-07 18:05:18.000000000 +0200
@@ -198,11 +198,7 @@
 	_bus_dmamap_load_uio,
 	_bus_dmamap_load_raw,
 	_bus_dmamap_unload,
-#if defined(_LP64) || defined(PAE)
 	_bus_dmamap_sync,
-#else
-	NULL,
-#endif
 	_bus_dmamem_alloc,
 	_bus_dmamem_free,
 	_bus_dmamem_map,
--- sys/arch/x86/x86/bus_dma.c.orig	2007-01-06 12:04:01.000000000 +0200
+++ sys/arch/x86/x86/bus_dma.c	2007-01-07 18:07:03.000000000 +0200
@@ -268,6 +268,18 @@
 			goto out;
 	}
 
+#ifdef BUS_DMA_USEWIDTH
+	/* address width limit was requested */
+	if(flags & BUS_DMA_USEWIDTH)
+	{
+		if(map->_dm_bounce_thresh == 0)
+			map->_dm_bounce_thresh = BUS_DMA_HIGHADDR(flags);
+		else
+			map->_dm_bounce_thresh = MIN(map->_dm_bounce_thresh,
+			                             BUS_DMA_HIGHADDR(flags));
+	}
+#endif
+
 	if (map->_dm_bounce_thresh != 0)
 		cookieflags |= X86_DMA_MIGHT_NEED_BOUNCE;
 
@@ -864,10 +876,20 @@
     bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
     int flags)
 {
-	bus_addr_t high;
+	bus_addr_t high = t->_bounce_alloc_hi;
+
+#ifdef BUS_DMA_USEWIDTH
+	if(flags & BUS_DMA_USEWIDTH)
+	{
+		if(high == 0)
+			high = BUS_DMA_HIGHADDR(flags);
+		else
+			high = MIN(high, BUS_DMA_HIGHADDR(flags));
+	}
+#endif
 
-	if (t->_bounce_alloc_hi != 0 && _BUS_AVAIL_END > t->_bounce_alloc_hi)
-		high = trunc_page(t->_bounce_alloc_hi);
+	if (high != 0 && _BUS_AVAIL_END > high)
+		high = trunc_page(high);
 	else
 		high = trunc_page(_BUS_AVAIL_END);
 
--- sys/dev/pci/if_bce.c.orig	2007-01-06 11:49:53.000000000 +0200
+++ sys/dev/pci/if_bce.c	2007-01-07 20:26:27.000000000 +0200
@@ -158,6 +158,13 @@
 	    BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);			\
 } while (/* CONSTCOND */ 0)
 
+#ifdef BUS_DMA_USEWIDTH
+/* BCM440x can only address up to 1GB, i.e. 30 bits */
+#define BCE_DMA	(BUS_DMA_USEWIDTH | BUS_DMA_WIDTH(30))
+#else 
+#define BCE_DMA	0
+#endif
+
 static	int	bce_probe(struct device *, struct cfdata *, void *);
 static	void	bce_attach(struct device *, struct device *, void *);
 static	int	bce_ioctl(struct ifnet *, u_long, caddr_t);
@@ -377,8 +384,8 @@
 	 * due to the limition above. ??
 	 */
 	if ((error = bus_dmamem_alloc(sc->bce_dmatag,
-	    2 * PAGE_SIZE, PAGE_SIZE, 2 * PAGE_SIZE,
-				      &seg, 1, &rseg, BUS_DMA_NOWAIT))) {
+	    2 * PAGE_SIZE, PAGE_SIZE, 2 * PAGE_SIZE, &seg, 1, &rseg,
+	    BUS_DMA_NOWAIT | BCE_DMA))) {
 		printf("%s: unable to alloc space for ring descriptors, "
 		       "error = %d\n", sc->bce_dev.dv_xname, error);
 		return;
@@ -393,7 +400,7 @@
 	}
 	/* create a dma map for the ring */
 	if ((error = bus_dmamap_create(sc->bce_dmatag,
-	    2 * PAGE_SIZE, 1, 2 * PAGE_SIZE, 0, BUS_DMA_NOWAIT,
+	    2 * PAGE_SIZE, 1, 2 * PAGE_SIZE, 0, BUS_DMA_NOWAIT | BCE_DMA,
 				       &sc->bce_ring_map))) {
 		printf("%s: unable to create ring DMA map, error = %d\n",
 		    sc->bce_dev.dv_xname, error);
@@ -403,7 +410,7 @@
 	}
 	/* connect the ring space to the dma map */
 	if (bus_dmamap_load(sc->bce_dmatag, sc->bce_ring_map, kva,
-	    2 * PAGE_SIZE, NULL, BUS_DMA_NOWAIT)) {
+	    2 * PAGE_SIZE, NULL, BUS_DMA_NOWAIT | BCE_DMA)) {
 		bus_dmamap_destroy(sc->bce_dmatag, sc->bce_ring_map);
 		bus_dmamem_unmap(sc->bce_dmatag, kva, 2 * PAGE_SIZE);
 		bus_dmamem_free(sc->bce_dmatag, &seg, rseg);
@@ -416,7 +423,7 @@
 	/* Create the transmit buffer DMA maps. */
 	for (i = 0; i < BCE_NTXDESC; i++) {
 		if ((error = bus_dmamap_create(sc->bce_dmatag, MCLBYTES,
-		    BCE_NTXFRAGS, MCLBYTES, 0, 0, &sc->bce_cdata.bce_tx_map[i])) != 0) {
+		    BCE_NTXFRAGS, MCLBYTES, 0, BCE_DMA, &sc->bce_cdata.bce_tx_map[i])) != 0) {
 			printf("%s: unable to create tx DMA map, error = %d\n",
 			    sc->bce_dev.dv_xname, error);
 		}
@@ -426,7 +433,7 @@
 	/* Create the receive buffer DMA maps. */
 	for (i = 0; i < BCE_NRXDESC; i++) {
 		if ((error = bus_dmamap_create(sc->bce_dmatag, MCLBYTES, 1,
-		    MCLBYTES, 0, 0, &sc->bce_cdata.bce_rx_map[i])) != 0) {
+		    MCLBYTES, 0, BCE_DMA, &sc->bce_cdata.bce_rx_map[i])) != 0) {
 			printf("%s: unable to create rx DMA map, error = %d\n",
 			    sc->bce_dev.dv_xname, error);
 		}
@@ -573,7 +580,7 @@
 		 * be tried again later.
 		 */
 		error = bus_dmamap_load_mbuf(sc->bce_dmatag, dmamap, m0,
-		    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
+		    BUS_DMA_WRITE | BUS_DMA_NOWAIT | BCE_DMA);
 		if (error == EFBIG) {
 			printf("%s: Tx packet consumes too many DMA segments, "
 			    "dropping...\n", sc->bce_dev.dv_xname);
@@ -1074,7 +1081,7 @@
 
 	error = bus_dmamap_load(sc->bce_dmatag, sc->bce_cdata.bce_rx_map[idx],
 	    m->m_ext.ext_buf, m->m_ext.ext_size, NULL,
-	    BUS_DMA_READ | BUS_DMA_NOWAIT);
+	    BUS_DMA_READ | BUS_DMA_NOWAIT | BCE_DMA);
 	if (error)
 		return (error);
 

--------------000704090809020700010607--