Subject: Re: bce(4) and memory > 1GB problem
To: Daniel Carosone <dan@geek.com.au>
From: Manuel Bouyer <bouyer@antioche.eu.org>
List: tech-kern
Date: 01/21/2007 23:17:27
--2oS5YaxWCcQjTEyO
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline

On Thu, Jan 18, 2007 at 07:59:01AM +1100, Daniel Carosone wrote:
> 
> I can happily test both native and Xen cases on my laptop.  At the
> moment, if I need to use bce, I use xen to limit the overall memory
> available to the system to 1g.  I was hoping that just restricting
> dom0 would be enough, but they get placed from the top down.

Hi,
can you please try the attached patch for Xen ? Obvisouly this will only
work for Xen-3, the Xen-2 hypervisor interface doesn't allow to express
a memory range contraint.

-- 
Manuel Bouyer <bouyer@antioche.eu.org>
     NetBSD: 26 ans d'experience feront toujours la difference
--

--2oS5YaxWCcQjTEyO
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename=diff

Index: xen_bus_dma.c
===================================================================
RCS file: /cvsroot/src/sys/arch/xen/x86/xen_bus_dma.c,v
retrieving revision 1.8
diff -u -r1.8 xen_bus_dma.c
--- xen_bus_dma.c	3 Sep 2006 19:04:20 -0000	1.8
+++ xen_bus_dma.c	21 Jan 2007 22:13:55 -0000
@@ -69,7 +69,7 @@
 
 static int
 _xen_alloc_contig(bus_size_t size, bus_size_t alignment, bus_size_t boundary,
-    struct pglist *mlistp, int flags)
+    struct pglist *mlistp, int flags, bus_addr_t low, bus_addr_t high)
 {
 	int order, i;
 	unsigned long npagesreq, npages, mfn;
@@ -138,8 +138,10 @@
 	res.extent_start = &mfn;
 	res.nr_extents = 1;
 	res.extent_order = order;
-	res.address_bits = 31;
+	res.address_bits = get_order(high) + PAGE_SHIFT;
 	res.domid = DOMID_SELF;
+	printf("_xen_alloc_contig low %x high %x address_bits %d\n",
+	    (u_int)low, (u_int)high, res.address_bits);
 	if (HYPERVISOR_memory_op(XENMEM_increase_reservation, &res) < 0) {
 #ifdef DEBUG
 		printf("xen_alloc_contig: XENMEM_increase_reservation "
@@ -205,7 +207,7 @@
 		res.extent_start = &mfn;
 		res.nr_extents = 1;
 		res.extent_order = 0;
-		res.address_bits = 31;
+		res.address_bits = 32;
 		res.domid = DOMID_SELF;
 		if (HYPERVISOR_memory_op(XENMEM_increase_reservation, &res)
 		    < 0) {
@@ -277,7 +279,9 @@
 	 */
 	m = mlist.tqh_first;
 	curseg = 0;
-	lastaddr = segs[curseg].ds_addr = _BUS_VM_PAGE_TO_BUS(m);
+	curaddr = lastaddr = segs[curseg].ds_addr = _BUS_VM_PAGE_TO_BUS(m);
+	if (curaddr < low || curaddr >= high)
+		goto badaddr;
 	segs[curseg].ds_len = PAGE_SIZE;
 	m = m->pageq.tqe_next;
 	if ((segs[curseg].ds_addr & (alignment - 1)) != 0)
@@ -285,49 +289,16 @@
 
 	for (; m != NULL; m = m->pageq.tqe_next) {
 		curaddr = _BUS_VM_PAGE_TO_BUS(m);
-		if ((lastaddr < low || lastaddr >= high) ||
-		    (curaddr < low || curaddr >= high)) {
-			/*
-			 * If machine addresses are outside the allowed
-			 * range we have to bail. Xen2 doesn't offer an
-			 * interface to get memory in a specific address
-			 * range.
-			 */
-			printf("_xen_bus_dmamem_alloc_range: no way to "
-			    "enforce address range\n");
-			uvm_pglistfree(&mlist);
-			return EINVAL;
-		}
+		if (curaddr < low || curaddr >= high)
+			goto badaddr;
 		if (curaddr == (lastaddr + PAGE_SIZE)) {
 			segs[curseg].ds_len += PAGE_SIZE;
-			if ((lastaddr & boundary) !=
-			    (curaddr & boundary))
+			if ((lastaddr & boundary) != (curaddr & boundary))
 				goto dorealloc;
 		} else {
 			curseg++;
-			if (curseg >= nsegs ||
-			    (curaddr & (alignment - 1)) != 0) {
-dorealloc:
-				if (doingrealloc == 1)
-					panic("_xen_bus_dmamem_alloc_range: "
-					   "xen_alloc_contig returned "
-					   "too much segments");
-				doingrealloc = 1;
-				/*
-				 * Too much segments. Free this memory and
-				 * get a contigous segment from the hypervisor.
-				 */
-				uvm_pglistfree(&mlist);
-				for (curseg = 0; curseg < nsegs; curseg++) {
-					segs[curseg].ds_addr = 0;
-					segs[curseg].ds_len = 0;
-				}
-				error = _xen_alloc_contig(size, alignment,
-				    boundary, &mlist, flags);
-				if (error)
-					return error;
-				goto again;
-			}
+			if (curseg >= nsegs || (curaddr & (alignment - 1)) != 0)
+				goto dorealloc;
 			segs[curseg].ds_addr = curaddr;
 			segs[curseg].ds_len = PAGE_SIZE;
 		}
@@ -335,6 +306,54 @@
 	}
 
 	*rsegs = curseg + 1;
-
 	return (0);
+
+badaddr:
+#ifdef XEN3
+	if (doingrealloc == 0)
+		goto dorealloc;
+	if (curaddr < low) {
+		/* no way to enforce this */
+		printf("_xen_bus_dmamem_alloc_range: no way to "
+		    "enforce address range\n");
+		uvm_pglistfree(&mlist);
+		return EINVAL;
+	}
+	printf("xen_bus_dmamem_alloc_range: "
+	    "curraddr=0x%lx > high=0x%lx\n",
+	    (u_long)curaddr, (u_long)high);
+	panic("xen_bus_dmamem_alloc_range 1");
+#else /* !XEN3 */
+	/*
+	 * If machine addresses are outside the allowed
+	 * range we have to bail. Xen2 doesn't offer an
+	 * interface to get memory in a specific address
+	 * range.
+	 */
+	printf("_xen_bus_dmamem_alloc_range: no way to "
+	    "enforce address range\n");
+	uvm_pglistfree(&mlist);
+	return EINVAL;
+#endif /* XEN3 */
+dorealloc:
+	if (doingrealloc == 1)
+		panic("_xen_bus_dmamem_alloc_range: "
+		   "xen_alloc_contig returned "
+		   "too much segments");
+	doingrealloc = 1;
+	/*
+	 * Too much segments, or memory doesn't fit
+	 * constraints. Free this memory and
+	 * get a contigous segment from the hypervisor.
+	 */
+	uvm_pglistfree(&mlist);
+	for (curseg = 0; curseg < nsegs; curseg++) {
+		segs[curseg].ds_addr = 0;
+		segs[curseg].ds_len = 0;
+	}
+	error = _xen_alloc_contig(size, alignment,
+	    boundary, &mlist, flags, low, high);
+	if (error)
+		return error;
+	goto again;
 }

--2oS5YaxWCcQjTEyO--