Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/arch add support for using MSI for XenPV Dom0



details:   https://anonhg.NetBSD.org/src/rev/4ff850e9474e
branches:  trunk
changeset: 1009818:4ff850e9474e
user:      jdolecek <jdolecek%NetBSD.org@localhost>
date:      Mon May 04 15:55:56 2020 +0000

description:
add support for using MSI for XenPV Dom0

use PHYSDEVOP_map_pirq to get the pirq/gsi for MSI/MSI-X, switch also INTx
to use it instead of PHYSDEVOP_alloc_irq_vector

MSI confirmed working with single-vector MSI for wm(4), ahcisata(4), bge(4)

XXX added some provision for MSI-X, but it doesn't actually work (no interrupts
delivered), needs some further investigation; disable MSI-X for XENPV
via flag in x86/pci/pci_machdep.c

diffstat:

 sys/arch/amd64/include/types.h |   8 ++--
 sys/arch/i386/include/types.h  |  12 ++----
 sys/arch/x86/pci/msipic.c      |  45 ++++++++++++++++------
 sys/arch/x86/pci/msipic.h      |  10 ++++-
 sys/arch/x86/pci/pci_machdep.c |  13 +++++-
 sys/arch/xen/include/evtchn.h  |   4 +-
 sys/arch/xen/x86/pintr.c       |  82 ++++++++++++++++++++++++++---------------
 sys/arch/xen/x86/xen_intr.c    |  12 +++---
 sys/arch/xen/xen/evtchn.c      |   8 ++-
 9 files changed, 124 insertions(+), 70 deletions(-)

diffs (truncated from 505 to 300 lines):

diff -r c8874d10131d -r 4ff850e9474e sys/arch/amd64/include/types.h
--- a/sys/arch/amd64/include/types.h    Mon May 04 15:13:45 2020 +0000
+++ b/sys/arch/amd64/include/types.h    Mon May 04 15:55:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: types.h,v 1.67 2020/04/25 15:26:16 bouyer Exp $        */
+/*     $NetBSD: types.h,v 1.68 2020/05/04 15:55:56 jdolecek Exp $      */
 
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
@@ -100,6 +100,9 @@
 #define        __HAVE_MM_MD_DIRECT_MAPPED_IO
 #define        __HAVE_MM_MD_DIRECT_MAPPED_PHYS
 #define        __HAVE_UCAS_FULL
+#if !defined(NO_PCI_MSI_MSIX)
+#define        __HAVE_PCI_MSI_MSIX
+#endif
 
 #ifdef _KERNEL_OPT
 #define        __HAVE_RAS
@@ -116,9 +119,6 @@
 #define        __HAVE_DIRECT_MAP 1
 #endif
 #define        __HAVE_CPU_UAREA_ROUTINES 1
-#if !defined(NO_PCI_MSI_MSIX)
-#define        __HAVE_PCI_MSI_MSIX
-#endif
 #endif
 #endif
 
diff -r c8874d10131d -r 4ff850e9474e sys/arch/i386/include/types.h
--- a/sys/arch/i386/include/types.h     Mon May 04 15:13:45 2020 +0000
+++ b/sys/arch/i386/include/types.h     Mon May 04 15:55:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: types.h,v 1.90 2019/09/23 23:06:26 kamil Exp $ */
+/*     $NetBSD: types.h,v 1.91 2020/05/04 15:55:56 jdolecek Exp $      */
 
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
@@ -133,13 +133,9 @@
 #define        __HAVE_TLS_VARIANT_II
 #define        __HAVE_COMMON___TLS_GET_ADDR
 #define        __HAVE_UCAS_FULL
-
-#if defined(_KERNEL)
+#if !defined(NO_PCI_MSI_MSIX)
+#define __HAVE_PCI_MSI_MSIX
+#endif
 #define        __HAVE_RAS
 
-#if !defined(XEN) && !defined(NO_PCI_MSI_MSIX)
-#define __HAVE_PCI_MSI_MSIX
-#endif
-#endif
-
 #endif /* _I386_MACHTYPES_H_ */
diff -r c8874d10131d -r 4ff850e9474e sys/arch/x86/pci/msipic.c
--- a/sys/arch/x86/pci/msipic.c Mon May 04 15:13:45 2020 +0000
+++ b/sys/arch/x86/pci/msipic.c Mon May 04 15:55:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: msipic.c,v 1.22 2020/05/04 09:34:37 jdolecek Exp $     */
+/*     $NetBSD: msipic.c,v 1.23 2020/05/04 15:55:56 jdolecek Exp $     */
 
 /*
  * Copyright (c) 2015 Internet Initiative Japan Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: msipic.c,v 1.22 2020/05/04 09:34:37 jdolecek Exp $");
+__KERNEL_RCSID(0, "$NetBSD: msipic.c,v 1.23 2020/05/04 15:55:56 jdolecek Exp $");
 
 #include "opt_intrdebug.h"
 
@@ -70,12 +70,9 @@
  * is managed by below "dev_seqs".
  */
 struct msipic {
-       int mp_bus;
-       int mp_dev;
-       int mp_fun;
+       struct msipic_pci_info mp_i;
 
        int mp_devid; /* The device id for the MSI/MSI-X device. */
-       int mp_veccnt; /* The number of MSI/MSI-X vectors. */
 
        char mp_pic_name[MSIPICNAMEBUF]; /* The MSI/MSI-X device's name. */
 
@@ -256,7 +253,7 @@
        pic->pic_msipic = msipic;
        msipic->mp_pic = pic;
        pci_decompose_tag(pa->pa_pc, pa->pa_tag,
-           &msipic->mp_bus, &msipic->mp_dev, &msipic->mp_fun);
+           &msipic->mp_i.mp_bus, &msipic->mp_i.mp_dev, &msipic->mp_i.mp_fun);
        memcpy(&msipic->mp_pa, pa, sizeof(msipic->mp_pa));
        msipic->mp_devid = devid;
        /*
@@ -311,6 +308,17 @@
        return pic->pic_msipic->mp_devid;
 }
 
+/*
+ * Return the PCI bus/dev/func info for the device.
+ */
+const struct msipic_pci_info *
+msipic_get_pci_info(struct pic *pic)
+{
+       KASSERT(msipic_is_msi_pic(pic));
+
+       return &pic->pic_msipic->mp_i;
+}
+
 #define MSI_MSICTL_ENABLE 1
 #define MSI_MSICTL_DISABLE 0
 static void
@@ -366,7 +374,10 @@
        pci_chipset_tag_t pc;
        struct pci_attach_args *pa;
        pcitag_t tag;
-       pcireg_t addr, data, ctl;
+#ifndef XENPV
+       pcireg_t addr, data;
+#endif
+       pcireg_t ctl;
        int off, err __diagused;
 
        pc = NULL;
@@ -375,6 +386,8 @@
        err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
        KASSERT(err != 0);
 
+       ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
+#ifndef XENPV
        /*
         * See Intel 64 and IA-32 Architectures Software Developer's Manual
         * Volume 3 10.11 Message Signalled Interrupts.
@@ -398,7 +411,6 @@
         * spec, so it's OK just to write it regardless of the value of the
         * upper 16bit.
         */
-       ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
        if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
                pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, addr);
                pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 0);
@@ -407,6 +419,7 @@
                pci_conf_write(pc, tag, off + PCI_MSI_MADDR, addr);
                pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data);
        }
+#endif /* !XENPV */
        ctl |= PCI_MSI_CTL_MSI_ENABLE;
        pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
 }
@@ -536,7 +549,10 @@
        bus_space_tag_t bstag;
        bus_space_handle_t bshandle;
        uint64_t entry_base;
-       pcireg_t addr, data, ctl;
+#ifndef XENPV
+       pcireg_t addr, data;
+#endif
+       pcireg_t ctl;
        int off, err __diagused;
 
        if (msix_vec < 0) {
@@ -556,8 +572,11 @@
        ctl &= ~PCI_MSIX_CTL_ENABLE;
        pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
 
+       bstag = pic->pic_msipic->mp_bstag;
+       bshandle = pic->pic_msipic->mp_bshandle;
        entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
 
+#ifndef XENPV
        /*
         * See Intel 64 and IA-32 Architectures Software Developer's Manual
         * Volume 3 10.11 Message Signalled Interrupts.
@@ -572,14 +591,13 @@
        data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
                | LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;
 
-       bstag = pic->pic_msipic->mp_bstag;
-       bshandle = pic->pic_msipic->mp_bshandle;
        bus_space_write_4(bstag, bshandle,
            entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, addr);
        bus_space_write_4(bstag, bshandle,
            entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 0);
        bus_space_write_4(bstag, bshandle,
            entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data);
+#endif /* !XENPV */
        bus_space_write_4(bstag, bshandle,
            entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0);
        BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
@@ -734,6 +752,7 @@
        msix_pic->pic_msipic->mp_bstag = bstag;
        msix_pic->pic_msipic->mp_bshandle = bshandle;
        msix_pic->pic_msipic->mp_bssize = bssize;
+       msix_pic->pic_msipic->mp_i.mp_table_base = memaddr + table_offset;
 
        return msix_pic;
 }
@@ -785,7 +804,7 @@
                pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
        }
 
-       msi_pic->pic_msipic->mp_veccnt = count;
+       msi_pic->pic_msipic->mp_i.mp_veccnt = count;
        return 0;
 }
 
diff -r c8874d10131d -r 4ff850e9474e sys/arch/x86/pci/msipic.h
--- a/sys/arch/x86/pci/msipic.h Mon May 04 15:13:45 2020 +0000
+++ b/sys/arch/x86/pci/msipic.h Mon May 04 15:55:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: msipic.h,v 1.2 2015/05/08 04:27:48 knakahara Exp $     */
+/*     $NetBSD: msipic.h,v 1.3 2020/05/04 15:55:56 jdolecek Exp $      */
 
 /*
  * Copyright (c) 2015 Internet Initiative Japan Inc.
@@ -41,6 +41,14 @@
 bool           msipic_is_msi_pic(struct pic *);
 int            msipic_get_devid(struct pic *);
 
+struct msipic_pci_info {
+       int mp_bus, mp_dev, mp_fun;
+       int mp_veccnt; /* The number of MSI/MSI-X vectors. */
+       uint32_t mp_table_base; /* MSI-X table location in memory space */
+};
+
+const struct msipic_pci_info *msipic_get_pci_info(struct pic *);
+
 void           msipic_init(void);
 
 #endif /* _X86_PCI_MSIPIC_H_ */
diff -r c8874d10131d -r 4ff850e9474e sys/arch/x86/pci/pci_machdep.c
--- a/sys/arch/x86/pci/pci_machdep.c    Mon May 04 15:13:45 2020 +0000
+++ b/sys/arch/x86/pci/pci_machdep.c    Mon May 04 15:55:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: pci_machdep.c,v 1.86 2019/05/24 14:28:48 nonaka Exp $  */
+/*     $NetBSD: pci_machdep.c,v 1.87 2020/05/04 15:55:56 jdolecek Exp $        */
 
 /*-
  * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc.
@@ -73,7 +73,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pci_machdep.c,v 1.86 2019/05/24 14:28:48 nonaka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pci_machdep.c,v 1.87 2020/05/04 15:55:56 jdolecek Exp $");
 
 #include <sys/types.h>
 #include <sys/param.h>
@@ -554,6 +554,15 @@
                        pba->pba_flags &= ~PCI_FLAGS_MSIX_OKAY;
                }
        }
+
+#ifdef XENPV
+       /*
+        * XXX MSI-X doesn't work for XenPV yet - setup seems to be correct,
+        * XXX but no interrupts are actually delivered.
+        */
+       pba->pba_flags &= ~PCI_FLAGS_MSIX_OKAY;
+#endif
+
 #endif /* __HAVE_PCI_MSI_MSIX */
 }
 
diff -r c8874d10131d -r 4ff850e9474e sys/arch/xen/include/evtchn.h
--- a/sys/arch/xen/include/evtchn.h     Mon May 04 15:13:45 2020 +0000
+++ b/sys/arch/xen/include/evtchn.h     Mon May 04 15:55:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: evtchn.h,v 1.30 2020/04/25 15:26:17 bouyer Exp $       */
+/*     $NetBSD: evtchn.h,v 1.31 2020/05/04 15:55:56 jdolecek Exp $     */
 
 /*
  *
@@ -29,8 +29,6 @@
 #ifndef _XEN_EVENTS_H_
 #define _XEN_EVENTS_H_
 
-#define NR_PIRQS       256
-
 extern struct evtsource *evtsource[];
 
 void events_default_setup(void);
diff -r c8874d10131d -r 4ff850e9474e sys/arch/xen/x86/pintr.c
--- a/sys/arch/xen/x86/pintr.c  Mon May 04 15:13:45 2020 +0000
+++ b/sys/arch/xen/x86/pintr.c  Mon May 04 15:55:56 2020 +0000
@@ -103,7 +103,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pintr.c,v 1.13 2020/04/25 15:26:17 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pintr.c,v 1.14 2020/05/04 15:55:56 jdolecek Exp $");
 
 #include "opt_multiprocessor.h"
 #include "opt_xen.h"
@@ -126,6 +126,10 @@
 #include <xen/evtchn.h>
 #include <xen/intr.h>
 



Home | Main Index | Thread Index | Old Index