[src/trunk]: src/sys/dev/pci/ixgbe Don't free and reallocate bus_dmamem when ...

[msaitoh <msaitoh%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/ad6109c000a3
branches:  trunk
changeset: 831997:ad6109c000a3
user:      msaitoh <msaitoh%NetBSD.org@localhost>
date:      Wed Apr 25 08:46:19 2018 +0000

description:
Don't free and reallocate bus_dmamem when it's not required. Currently,
the watchdog timer is completely broken and never fire (it's from FreeBSD
(pre iflib)). If the problem is fixed and watchdog fired, ixgbe_init() always
calls ixgbe_jcl_reinit() and it causes panic. The reason is that
ixgbe_local_timer1(it includes watchdog function) is softint and
xgbe_jcl_reinit() calls bus_dmamem*() functions. bus_dmamem*() can't be called
from interrupt context.

 One of the way to prevent panic is use worqueue for the timer, but it's
not a small change. (I'll do it in future).

 Another way is not reallocate dmamem if it's not required. If both the MTU
(rx_mbuf_sz in reality) and the number of RX descriptors are not changed, it's
not required to call bus_dmamem_{unmap,free}(). Even if we use workque, this
change save time of ixgbe_init().

 I have a code to fix broken watchdog timer but it sometime causes watchdog
timeout, so I don't commit it yet.

diffstat:

 sys/dev/pci/ixgbe/ix_txrx.c      |   4 ++--
 sys/dev/pci/ixgbe/ixgbe.h        |   5 ++++-
 sys/dev/pci/ixgbe/ixgbe_netbsd.c |  25 +++++++++++++++++++++----
 sys/dev/pci/ixgbe/ixgbe_netbsd.h |   3 +--
 sys/dev/pci/ixgbe/ixgbe_osdep.h  |   4 +++-
 5 files changed, 31 insertions(+), 10 deletions(-)

diffs (135 lines):

diff -r 6c6ec204fa09 -r ad6109c000a3 sys/dev/pci/ixgbe/ix_txrx.c
--- a/sys/dev/pci/ixgbe/ix_txrx.c       Wed Apr 25 08:29:45 2018 +0000
+++ b/sys/dev/pci/ixgbe/ix_txrx.c       Wed Apr 25 08:46:19 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ix_txrx.c,v 1.40 2018/04/17 08:38:05 msaitoh Exp $ */
+/* $NetBSD: ix_txrx.c,v 1.41 2018/04/25 08:46:19 msaitoh Exp $ */
 
 /******************************************************************************
 
@@ -1607,7 +1607,7 @@
         * or size of jumbo mbufs may have changed.
         * Assume all of rxr->ptag are the same.
         */
-       ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat,
+       ixgbe_jcl_reinit(adapter, rxr->ptag->dt_dmat,
            (2 * adapter->num_rx_desc) * adapter->num_queues,
            adapter->rx_mbuf_sz);
 
diff -r 6c6ec204fa09 -r ad6109c000a3 sys/dev/pci/ixgbe/ixgbe.h
--- a/sys/dev/pci/ixgbe/ixgbe.h Wed Apr 25 08:29:45 2018 +0000
+++ b/sys/dev/pci/ixgbe/ixgbe.h Wed Apr 25 08:46:19 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ixgbe.h,v 1.45 2018/04/19 21:50:09 christos Exp $ */
+/* $NetBSD: ixgbe.h,v 1.46 2018/04/25 08:46:19 msaitoh Exp $ */
 
 /******************************************************************************
   SPDX-License-Identifier: BSD-3-Clause
@@ -751,6 +751,9 @@
 
 const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *);
 
+/* For NetBSD */
+void ixgbe_jcl_reinit(struct adapter *, bus_dma_tag_t, int, size_t);
+
 #include "ixgbe_bypass.h"
 #include "ixgbe_fdir.h"
 #include "ixgbe_rss.h"
diff -r 6c6ec204fa09 -r ad6109c000a3 sys/dev/pci/ixgbe/ixgbe_netbsd.c
--- a/sys/dev/pci/ixgbe/ixgbe_netbsd.c  Wed Apr 25 08:29:45 2018 +0000
+++ b/sys/dev/pci/ixgbe/ixgbe_netbsd.c  Wed Apr 25 08:46:19 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ixgbe_netbsd.c,v 1.6 2017/06/01 02:45:11 chs Exp $ */
+/* $NetBSD: ixgbe_netbsd.c,v 1.7 2018/04/25 08:46:19 msaitoh Exp $ */
 /*
  * Copyright (c) 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -40,7 +40,7 @@
 #include <sys/workqueue.h>
 #include <dev/pci/pcivar.h>
 
-#include "ixgbe_netbsd.h"
+#include "ixgbe.h"
 
 void
 ixgbe_dma_tag_destroy(ixgbe_dma_tag_t *dt)
@@ -162,11 +162,12 @@
 }
 
 void
-ixgbe_jcl_reinit(ixgbe_extmem_head_t *eh, bus_dma_tag_t dmat, int nbuf,
+ixgbe_jcl_reinit(struct adapter *adapter, bus_dma_tag_t dmat, int nbuf,
     size_t size)
 {
+       ixgbe_extmem_head_t *eh = &adapter->jcl_head;
+       ixgbe_extmem_t *em;
        int i;
-       ixgbe_extmem_t *em;
 
        if (!eh->eh_initialized) {
                TAILQ_INIT(&eh->eh_freelist);
@@ -174,6 +175,18 @@
                eh->eh_initialized = true;
        }
 
+       /*
+        *  Check previous parameters. If it's not required to reinit, just
+        * return.
+        *
+        *  Note that the num_rx_desc is currently fixed value. It's never
+        * changed after device is attached.
+        */
+       if ((adapter->osdep.last_rx_mbuf_sz == adapter->rx_mbuf_sz)
+           && (adapter->osdep.last_num_rx_desc == adapter->num_rx_desc))
+               return;
+
+       /* Free all dmamem */
        while ((em = ixgbe_getext(eh, 0)) != NULL) {
                KASSERT(em->em_vaddr != NULL);
                bus_dmamem_unmap(dmat, em->em_vaddr, em->em_size);
@@ -190,6 +203,10 @@
                }
                ixgbe_putext(em);
        }
+
+       /* Keep current parameters */
+       adapter->osdep.last_rx_mbuf_sz = adapter->rx_mbuf_sz;
+       adapter->osdep.last_num_rx_desc = adapter->num_rx_desc;
 }
 
 static void
diff -r 6c6ec204fa09 -r ad6109c000a3 sys/dev/pci/ixgbe/ixgbe_netbsd.h
--- a/sys/dev/pci/ixgbe/ixgbe_netbsd.h  Wed Apr 25 08:29:45 2018 +0000
+++ b/sys/dev/pci/ixgbe/ixgbe_netbsd.h  Wed Apr 25 08:46:19 2018 +0000
@@ -1,4 +1,4 @@
-/*$NetBSD: ixgbe_netbsd.h,v 1.7 2017/02/08 04:32:43 msaitoh Exp $*/
+/*$NetBSD: ixgbe_netbsd.h,v 1.8 2018/04/25 08:46:19 msaitoh Exp $*/
 /*
  * Copyright (c) 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -93,7 +93,6 @@
 void ixgbe_dmamap_sync(ixgbe_dma_tag_t *, bus_dmamap_t, int);
 void ixgbe_dmamap_unload(ixgbe_dma_tag_t *, bus_dmamap_t);
 
-void ixgbe_jcl_reinit(ixgbe_extmem_head_t *, bus_dma_tag_t, int, size_t);
 struct mbuf *ixgbe_getjcl(ixgbe_extmem_head_t *, int, int, int, size_t);
 void ixgbe_pci_enable_busmaster(pci_chipset_tag_t, pcitag_t);
 
diff -r 6c6ec204fa09 -r ad6109c000a3 sys/dev/pci/ixgbe/ixgbe_osdep.h
--- a/sys/dev/pci/ixgbe/ixgbe_osdep.h   Wed Apr 25 08:29:45 2018 +0000
+++ b/sys/dev/pci/ixgbe/ixgbe_osdep.h   Wed Apr 25 08:46:19 2018 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: ixgbe_osdep.h,v 1.21 2018/04/04 08:13:07 msaitoh Exp $ */
+/* $NetBSD: ixgbe_osdep.h,v 1.22 2018/04/25 08:46:19 msaitoh Exp $ */
 
 /******************************************************************************
   SPDX-License-Identifier: BSD-3-Clause
@@ -207,6 +207,8 @@
        bus_space_handle_t mem_bus_space_handle;
        bus_size_t         mem_size;
        bus_dma_tag_t      dmat;
+       u16                last_rx_mbuf_sz;
+       u32                last_num_rx_desc;
        pci_intr_handle_t  *intrs;
        int                nintrs;
        void               *ihs[IXG_MAX_NINTR];