Re: VisionFive 2 hangs at boot with NVMe

[berte <behzaterte%pm.me@localhost>]

Hi folks,

diff -u a/sys/dev/ic/nvme.c b/sys/dev/ic/nvme.c
--- a/sys/dev/ic/nvme.c
+++ b/sys/dev/ic/nvme.c
@@ -2025,6 +2025,29 @@
 	kmem_free(q, sizeof(*q));
 }
 
+static bool
+nvme_q_pending(struct nvme_softc *sc, struct nvme_queue *q)
+{
+	struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem);
+	uint16_t flags;
+
+	mutex_enter(&q->q_cq_mtx);
+	nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
+	flags = lemtoh16(&ring[q->q_cq_head].flags);
+	nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
+	mutex_exit(&q->q_cq_mtx);
+
+	return (flags & NVME_CQE_PHASE) == q->q_cq_phase;
+}
+
+bool
+nvme_intx_pending(struct nvme_softc *sc)
+{
+
+	return nvme_q_pending(sc, sc->sc_admin_q) ||
+	    (sc->sc_q != NULL && nvme_q_pending(sc, sc->sc_q[0]));
+}
+
 int
 nvme_intr(void *xsc)
 {
@@ -2058,13 +2081,12 @@
 	if (sc->sc_q != NULL)
 	        nvme_q_complete(sc, sc->sc_q[0]);
 
-	/*
-	 * Processing done, tell controller to issue interrupts again. There
-	 * is no race, as NVMe spec requires the controller to maintain state,
-	 * and assert the interrupt whenever there are unacknowledged
-	 * completion queue entries.
-	 */
+	/* Processing done, tell controller to issue interrupts again. */
 	nvme_write4(sc, NVME_INTMC, 1);
+
+	/* Some controllers do not reassert for entries posted while masked. */
+	if (nvme_intx_pending(sc))
+		softint_schedule(sc->sc_softih[0]);
 }
 
 int
diff -u a/sys/dev/ic/nvmevar.h b/sys/dev/ic/nvmevar.h
--- a/sys/dev/ic/nvmevar.h
+++ b/sys/dev/ic/nvmevar.h
@@ -193,6 +193,7 @@
 int	nvme_resume(struct nvme_softc *);
 int	nvme_intr(void *);
 void	nvme_softintr_intx(void *);
+bool	nvme_intx_pending(struct nvme_softc *);
 int	nvme_intr_msi(void *);
 void	nvme_softintr_msi(void *);
 
diff -u a/sys/dev/pci/nvme_pci.c b/sys/dev/pci/nvme_pci.c
--- a/sys/dev/pci/nvme_pci.c
+++ b/sys/dev/pci/nvme_pci.c
@@ -74,6 +74,7 @@
 	struct nvme_softc	psc_nvme;
 
 	pci_chipset_tag_t	psc_pc;
+	pcitag_t		psc_tag;
 	pci_intr_handle_t	*psc_intrs;
 	int			psc_nintrs;
 };
@@ -157,6 +158,7 @@
 
 	sc->sc_dev = self;
 	psc->psc_pc = pa->pa_pc;
+	psc->psc_tag = pa->pa_tag;
 	if (pci_dma64_available(pa))
 		sc->sc_dmat = pa->pa_dmat64;
 	else
@@ -309,6 +311,46 @@
 	return 0;
 }
 
+static void
+nvme_pci_intx_disable(struct nvme_pci_softc *psc, bool disable)
+{
+	pcireg_t command;
+
+	/* Write only the Command half to keep W1C Status bits intact. */
+	command = pci_conf_read(psc->psc_pc, psc->psc_tag,
+	    PCI_COMMAND_STATUS_REG) & PCI_COMMAND_MASK;
+	if (disable)
+		command |= PCI_COMMAND_INTERRUPT_DISABLE;
+	else
+		command &= ~PCI_COMMAND_INTERRUPT_DISABLE;
+	pci_conf_write(psc->psc_pc, psc->psc_tag, PCI_COMMAND_STATUS_REG,
+	    command);
+	(void)pci_conf_read(psc->psc_pc, psc->psc_tag, PCI_COMMAND_STATUS_REG);
+}
+
+static int
+nvme_pci_intr_intx(void *arg)
+{
+	struct nvme_softc *sc = arg;
+
+	/* Some controllers keep INTx asserted after INTMS is set. */
+	nvme_pci_intx_disable((struct nvme_pci_softc *)sc, true);
+	return nvme_intr(sc);
+}
+
+static void
+nvme_pci_softintr_intx(void *arg)
+{
+	struct nvme_queue *q = arg;
+	struct nvme_softc *sc = q->q_sc;
+
+	nvme_softintr_intx(q);
+	nvme_pci_intx_disable((struct nvme_pci_softc *)sc, false);
+
+	if (nvme_intx_pending(sc))
+		softint_schedule(sc->sc_softih[0]);
+}
+
 static int
 nvme_pci_intr_establish(struct nvme_softc *sc, uint16_t qid,
     struct nvme_queue *q)
@@ -336,6 +378,11 @@
 		ih_arg = sc;
 		ih_func = nvme_intr;
 		ih_func_soft = nvme_softintr_intx;
+		if (pci_intr_type(psc->psc_pc, psc->psc_intrs[0]) ==
+		    PCI_INTR_TYPE_INTX) {
+			ih_func = nvme_pci_intr_intx;
+			ih_func_soft = nvme_pci_softintr_intx;
+		}
 	} else {
 		if (qid == NVME_ADMIN_Q) {
 			snprintf(intr_xname, sizeof(intr_xname), "%s adminq",


Could you please try this and ping me back ?

Cheers/Esenlikler,
berte
>En Taro Adun<

On Saturday, October 17th, 2026 at 9:32 PM, Robert Swindells <rjs%fdy2.co.uk@localhost> wrote:

> 
> Benny Siegert <bsiegert%netbsd.org@localhost> wrote:
> > On Sat, 17 Oct 2026, Martin Husemann wrote:
> >
> >> On Sat, Oct 17, 2026 at 04:53:38PM +0000, Benny Siegert wrote:
> >>> The one that does not work is an NVMe 1.4.
> >>
> >> Same here, I tried two different ones but both are 1.4.
> >
> > Oddly enough, it looks like it's not specific to RISC-V!
> 
> Have you tried the 1.4 one with x86_64.
> 
> > When I put the NVMe 1.4 SSD into an Orange Pi 5 (so aarch64), I can also
> > reproduce the crash -- when initializing the SSD, nothing happens for 16
> > seconds, then a heartbeat panic. Unfortunately, the keyboard does not work
> > in ddb.
> >
> > This same machine is otherwise running in production with the NVMe 1.1
> > SSD.
> 
> The RK3588 has a fairly complicated setup process to use MSI interrupts,
> you need to go back to set some device registers after you know which
> interrupt number has been chosen.
> 
> Someone could check what UEFI is doing on an RK3588 but it is a
> different software stack below NetBSD to that on the VisionFive 2.
> 
> I am running my native RK3588 pcie driver with MSI&MSI-X disabled for
> now, it does work with a 1.4 ssd though.
>