I've taken another whack at working around the O2 R10K instability issues, and
have come up with something that seems to keep things from crashing.

Mapping all DMA buffers to KSEG1, adding a few wbflush() calls to mace
bus_space operations, and disallowing buffers below the eight-meg line has
resulted in a stable machine.  By "stable" I mean "a machine that has survived
three parallel makes and a flood ping for the past three hours".  I still see
random crime cpu errors, but they appear to be harmless.

Patch appended.  Feedback appreciated.
-- 

-- Chris
	GPG key FEB9DE7F (91AF 4534 4529 4BCC 31A5  938E 023E EEFB FEB9 DE7F)

Index: bus.c
===================================================================
RCS file: /cvsroot/src/sys/arch/sgimips/sgimips/bus.c,v
retrieving revision 1.41
diff -u -r1.41 bus.c
--- bus.c	11 Dec 2005 12:18:58 -0000	1.41
+++ bus.c	13 Dec 2005 12:36:16 -0000
@@ -243,6 +244,7 @@
 		case SGIMIPS_BUS_SPACE_MACE:
 			s = splhigh();
 			delay(10);
+			wbflush();
 			reg = mips3_ld( (u_int64_t *)(bsh + o));
 			delay(10);
 			splx(s);
@@ -264,6 +266,7 @@
 			s = splhigh();
 			delay(10);
 			mips3_sd( (u_int64_t *)(bsh + o), v);
+			wbflush();
 			delay(10);
 			splx(s);
 			break;
@@ -934,6 +937,7 @@
 		  int nsegs, int *rsegs, int flags)
 {
 	extern paddr_t avail_start, avail_end;
+	paddr_t dma_start;
 	vaddr_t curaddr, lastaddr;
 	psize_t high;
 	struct vm_page *m;
@@ -945,10 +949,16 @@
 
 	high = avail_end - PAGE_SIZE;
 
+	/* Workaround for non-coherent R10K systems */
+	if (MIPS_PRID_IMPL(cpu_id) == MIPS_R10000)
+		dma_start = 0x800000;
+	else
+		dma_start = avail_start;
+
 	/*
 	 * Allocate pages from the VM system.
 	 */
-	error = uvm_pglistalloc(size, avail_start, high, alignment, boundary,
+	error = uvm_pglistalloc(size, dma_start, high, alignment, boundary,
 	    &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0);
 	if (error)
 		return error;
@@ -966,7 +976,7 @@
 	for (; m != NULL; m = m->pageq.tqe_next) {
 		curaddr = VM_PAGE_TO_PHYS(m);
 #ifdef DIAGNOSTIC
-		if (curaddr < avail_start || curaddr >= high) {
+		if (curaddr < dma_start || curaddr >= high) {
 			printf("uvm_pglistalloc returned non-sensical"
 			    " address 0x%lx\n", curaddr);
 			panic("_bus_dmamem_alloc");
@@ -1034,13 +1044,17 @@
 	 * TLB thrashing.
 	 */
 	if (nsegs == 1) {
-		if (flags & BUS_DMA_COHERENT)
+		if ( MIPS_PRID_IMPL(cpu_id) == MIPS_R10000 )
+			*kvap = (caddr_t)MIPS_PHYS_TO_KSEG1(segs[0].ds_addr);
+		else if (flags & BUS_DMA_COHERENT)
 			*kvap = (caddr_t)MIPS_PHYS_TO_KSEG1(segs[0].ds_addr);
 		else
 			*kvap = (caddr_t)MIPS_PHYS_TO_KSEG0(segs[0].ds_addr);
 		return 0;
 	}
 
+	/* The following does not appear to ever be used */
+
 	size = round_page(size);
 
 	va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);