--n8g4imXOkfNTN/H1
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline

On Tue, Jan 25, 2005 at 10:24:50PM -0500, Stephan Uphoff wrote:
> Just to make sure - did you also modify sys/arch/x86/include/intrdefs.h?
> -#define IDT_INTR_HIGH  0xef
> +#define IDT_INTR_HIGH  0xdf

Yes.

> 
> OK - I guess it is time for some experiments since I am out of ideas
> where to look in the source code.
> 
> Could you add ci_ipis to your CPU printout?

Done, see below

> Maybe there is no IPI pending for CPU 0 and we can stop looking at the
> interrupt delivery.
> 
> Replacing the panic in pmap_tlb_shootnow with some printf statements and
> a goto to the start of the function may show us if this is a deadlock or
> a race condition  (Maybe panic after 5 retries?)

Indeed, with the attached change, which makes pmap_tlb_shootnow() retry the
IPI, I got:
CPU 0 interrupt level 0x0d pending 0x400 depth 1 ci_ipis 16
but no panic. The dump completed. I ran another amanda dump, and then
got:
panic: fp_save ipi didn't
CPU 0 interrupt level 0x6 pending 0x10000000 depth 1 ci_ipis 8
CPU 1 interrupt level 0x0 pending 0x0 depth 0 ci_ipis 0

stack trace for CPU 1:
panic
npxsave_lwp
cpu_lwp_fork
newlwp
fork1
sys_vfork
stack trace for CPU 0:
Xdoreti
usb_all_tasks

(note that I have no USB devices connected, but the USB controllers share
their interrupts with the network adapter).

I'm going to add the same kind of patch to npxsave_lwp() as to
pmap_tlb_shootnow(), and retry some dumps.

> 
> There are also some nice Intel "Specification Updates" that warn of some
> order violations when mixing atomic and non-atomic operations.
> Just to make sure that this is not our problem can you replace the line
> 	self->ci_tlb_ipi_mask = cpumask;
> in pmap_tlb_shootnow with 
> 	x86_atomic_setbits(&self->ci_tlb_ipi_mask,cpumask);

I'll try this too.

-- 
Manuel Bouyer <bouyer@antioche.eu.org>
     NetBSD: 26 ans d'experience feront toujours la difference
--

--n8g4imXOkfNTN/H1
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename=diff

Index: sys/arch/i386/i386/db_interface.c
===================================================================
RCS file: /cvsroot/src/sys/arch/i386/i386/db_interface.c,v
retrieving revision 1.42
diff -u -r1.42 db_interface.c
--- sys/arch/i386/i386/db_interface.c	13 Feb 2004 11:36:13 -0000	1.42
+++ sys/arch/i386/i386/db_interface.c	27 Jan 2005 14:16:15 -0000
@@ -262,6 +262,19 @@
 void
 cpu_Debugger()
 {
+	volatile struct cpu_info *ci;
+	int i;
+	
+	for (i=0; i < X86_MAXPROCS; i++) {
+		ci = cpu_info[i];
+		if (ci == NULL)
+			continue;
+
+		printf("CPU %ld interrupt level 0x%x pending 0x%x depth %d "
+		   "ci_ipis %d\n",
+		   ci->ci_cpuid, ci->ci_ilevel, ci->ci_ipending, ci->ci_idepth,
+		   ci->ci_ipis);
+	}
 	breakpoint();
 }
 
@@ -365,6 +378,9 @@
 			db_printf("CPU %ld not paused\n", addr);
 			return;
 		}
+		printf("CPU %ld current interrupt level 0x%x pending 0x%x "
+		    "depth %d ci_ipis %d\n", ci->ci_cpuid, ci->ci_ilevel,
+		    ci->ci_ipending, ci->ci_idepth, ci->ci_ipis);
 	}
 	if (ci->ci_ddb_regs == 0) {
 		db_printf("CPU %ld has no saved regs\n", addr);
Index: sys/arch/i386/i386/machdep.c
===================================================================
RCS file: /cvsroot/src/sys/arch/i386/i386/machdep.c,v
retrieving revision 1.559
diff -u -r1.559 machdep.c
--- sys/arch/i386/i386/machdep.c	20 Oct 2004 04:20:05 -0000	1.559
+++ sys/arch/i386/i386/machdep.c	27 Jan 2005 14:16:15 -0000
@@ -1823,6 +1823,8 @@
 #endif
 	pmap_update(pmap_kernel());
 
+	memset(idt_allocmap, 0, sizeof(idt_allocmap));
+
 	tgdt = gdt;
 	gdt = (union descriptor *)
 		    ((char *)idt + NIDT * sizeof (struct gate_descriptor));
Index: sys/arch/i386/i386/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/arch/i386/i386/pmap.c,v
retrieving revision 1.180
diff -u -r1.180 pmap.c
--- sys/arch/i386/i386/pmap.c	1 Jan 2005 21:00:06 -0000	1.180
+++ sys/arch/i386/i386/pmap.c	27 Jan 2005 14:16:15 -0000
@@ -3659,6 +3659,7 @@
 	int s;
 #ifdef DIAGNOSTIC
 	int count = 0;
+	int ipi_retry = 0;
 #endif
 #endif
 
@@ -3679,6 +3680,7 @@
 	/*
 	 * Send the TLB IPI to other CPUs pending shootdowns.
 	 */
+ipi_again:
 	for (CPU_INFO_FOREACH(cii, ci)) {
 		if (ci == self)
 			continue;
@@ -3690,9 +3692,20 @@
 
 	while (self->ci_tlb_ipi_mask != 0) {
 #ifdef DIAGNOSTIC
-		if (count++ > 10000000)
+		if (count++ > 10000000) {
+			for (CPU_INFO_FOREACH(cii, ci)) {
+				if (ci == self)
+					continue;
+				printf("CPU %ld interrupt level 0x%x pending "
+				    "0x%x depth %d ci_ipis %d\n", ci->ci_cpuid,
+				    ci->ci_ilevel, ci->ci_ipending,
+				    ci->ci_idepth, ci->ci_ipis);
+			}
+			if (ipi_retry++ < 5)
+				goto ipi_again;
 			panic("TLB IPI rendezvous failed (mask %x)",
 			    self->ci_tlb_ipi_mask);
+		}
 #endif
 		x86_pause();
 	}

--n8g4imXOkfNTN/H1--