Re: PCU panic on V240

[Takeshi Nakayama <tn%catvmics.ne.jp@localhost>]

>>> "J. Hannken-Illjes" <hannken%eis.cs.tu-bs.de@localhost> wrote

> Running -current on a Sparc V240 I get a diagnostic assertion running
> the script attached:
> 
> panic: kernel diagnostic assertion "!cpu_intr_p()" failed: file 
> "/work/build/src/sys/kern/subr_xcall.c", line 351 
:
:
> This is the high priority xcall from subr_pcu.c and
> curcpu()->ci_idepth is zero.
> 
> What is going wrong here?

There is some trick to handle ci_idepth in the case of softint.

In locore.s::softint_fastintr, decrease ci_idepth before calling
softint handlers then increase ci_idepth.  But it seems that there is
a chance to interrupt SOFTINT_SERIAL while handling other SOFTINT_*,
and ci_idepth is increased in the meantime.


locore.s:
-> 3456         wrpr    %g0, PSTATE_INTR, %pstate       ! Reenable interrupts
   3457         jmpl    %o4, %o7                ! handled = (*ih->ih_fun)(...)
   3458          movrz  %o0, %o2, %o0           ! arg = (arg == 0) ? arg : tf
   3459         wrpr    %g0, PSTATE_KERN, %pstate       ! Disable interrupts
    :
   5236 ENTRY(softint_fastintr)
   5237         save    %sp, -CC64FSZ, %sp
   5238         set     CPUINFO_VA, %l0                 ! l0 = curcpu()
   5239         rdpr    %pil, %l7                       ! l7 = splhigh()
-> 5240         wrpr    %g0, PIL_HIGH, %pil
   5241         ld      [%l0 + CI_IDEPTH], %l1
   5242         LDPTR   [%l0 + CI_EINTSTACK], %l6       ! l6 = ci_eintstack
   5243         dec     %l1
   5244         add     %sp, -CC64FSZ, %l2              ! ci_eintstack = sp - 
CC64FSZ
   5245         st      %l1, [%l0 + CI_IDEPTH]          ! adjust ci_idepth
   5246         STPTR   %l2, [%l0 + CI_EINTSTACK]       ! save intstack for 
nexted intr


So, does the attached patch fix the panic?

-- Takeshi Nakayama


--- sparc64/locore.s.orig       2013-05-25 08:14:47.000000000 +0900
+++ sparc64/locore.s    2013-12-06 13:16:42.000000000 +0900
@@ -3288,12 +3288,6 @@
         LDPTR  [%g3 + %lo(CPUINFO_VA+CI_TICK_IH)], %g5
 0:
 
-       ! Increment the per-cpu interrupt level
-       sethi   %hi(CPUINFO_VA+CI_IDEPTH), %g1
-       ld      [%g1 + %lo(CPUINFO_VA+CI_IDEPTH)], %g2
-       inc     %g2
-       st      %g2, [%g1 + %lo(CPUINFO_VA+CI_IDEPTH)]
-
 #ifdef TRAPSTATS
        sethi   %hi(_C_LABEL(kintrcnt)), %g1
        sethi   %hi(_C_LABEL(uintrcnt)), %g2
@@ -3382,6 +3376,17 @@
        
        wrpr    %l6, %pil
 
+#define SOFTINT_INT \
+       (1<<IPL_SOFTCLOCK|1<<IPL_SOFTBIO|1<<IPL_SOFTNET|1<<IPL_SOFTSERIAL)
+
+       ! Increment the per-cpu interrupt depth in case of hardintrs
+       btst    SOFTINT_INT, %l3
+       bnz,pn  %icc, sparc_intr_retry
+        sethi  %hi(CPUINFO_VA+CI_IDEPTH), %l1
+       ld      [%l1 + %lo(CPUINFO_VA+CI_IDEPTH)], %l2
+       inc     %l2
+       st      %l2, [%l1 + %lo(CPUINFO_VA+CI_IDEPTH)]
+
 sparc_intr_retry:
        wr      %l3, 0, CLEAR_SOFTINT   ! (don't clear possible %tick IRQ)
        sethi   %hi(CPUINFO_VA+CI_INTRPENDING), %l4
@@ -3481,11 +3486,14 @@
        bnz,pn  %icc, sparc_intr_retry
         mov    1, %l5                  ! initialize intr count for next run
 
-       ! Decrement this cpu's interrupt depth
-       sethi   %hi(CPUINFO_VA+CI_IDEPTH), %l4
+       ! Decrement this cpu's interrupt depth in case of hardintrs
+       btst    SOFTINT_INT, %l3
+       bnz,pn  %icc, 1f
+        sethi  %hi(CPUINFO_VA+CI_IDEPTH), %l4
        ld      [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)], %l5
        dec     %l5
        st      %l5, [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)]
+1:
 
 #ifdef NOT_DEBUG
        set     _C_LABEL(intrdebug), %o2
@@ -5238,11 +5246,8 @@
        set     CPUINFO_VA, %l0                 ! l0 = curcpu()
        rdpr    %pil, %l7                       ! l7 = splhigh()
        wrpr    %g0, PIL_HIGH, %pil
-       ld      [%l0 + CI_IDEPTH], %l1
        LDPTR   [%l0 + CI_EINTSTACK], %l6       ! l6 = ci_eintstack
-       dec     %l1
        add     %sp, -CC64FSZ, %l2              ! ci_eintstack = sp - CC64FSZ
-       st      %l1, [%l0 + CI_IDEPTH]          ! adjust ci_idepth
        STPTR   %l2, [%l0 + CI_EINTSTACK]       ! save intstack for nexted intr
 
        mov     %i0, %o0                        ! o0/i0 = softint lwp
@@ -5287,10 +5292,7 @@
 
        restore                                 ! rewind register window
 
-       ld      [%l0 + CI_IDEPTH], %l1
        STPTR   %l6, [%l0 + CI_EINTSTACK]       ! restore ci_eintstack
-       inc     %l1
-       st      %l1, [%l0 + CI_IDEPTH]          ! re-adjust ci_idepth
        wrpr    %g0, %l7, %pil                  ! restore ipl
        ret
         restore        %g0, 1, %o0
@@ -5314,10 +5316,7 @@
        st      %o1, [%l0 + CI_MTX_COUNT]
        st      %g0, [%o0 + L_CTXSWTCH]         ! prev->l_ctxswtch = 0
 
-       ld      [%l0 + CI_IDEPTH], %l1
        STPTR   %l6, [%l0 + CI_EINTSTACK]       ! restore ci_eintstack
-       inc     %l1
-       st      %l1, [%l0 + CI_IDEPTH]          ! re-adjust ci_idepth
        wrpr    %g0, %l7, %pil                  ! restore ipl
        ret
         restore        %g0, 1, %o0