Raspberry Pi 3B: /dev/mem mmap GPIO works on earmv7hf but not on aarch64

[Izumi Tsutsui <tsutsui%ceres.dti.ne.jp@localhost>]

Hi,

I'm trying to control Raspberry Pi GPIO via /dev/mem + mmap(2)
from userland of NetBSD 11.0_RC1, to control YM2149F PSG chip:
 https://x.com/tsutsuii/status/2022913343913390424

On Raspberry Pi 3 Model B, the same user program behaves
correctly on an earmv7hf kernel, but behaves incorrectly
on an aarch64 kernel.

Hardware
--------

- Raspberry Pi 3 Model B (can boot both 32-bit and 64-bit NetBSD)

Kernels
-------

- NetBSD 11.0_RC1 tagged source tree

- evbarm 32-bit (earmv7hf):
   sys/arch/evbarm/conf/GENERIC
    + options INSECURE (force kern.securelevel=-1 for mmap /dev/mem)
    + options HZ=1000  (to get 1ms tick to access YM2149F in each ~2ms)

- evbarm 64-bit (aarch64):
   sys/arch/evbarm/conf/GENERIC64
    + options INSECURE
    + options HZ=1000

Reproduction
------------

- Boot the earmv7hf kernel and run the attached gpio_mmap_test.c:
 -> GPIO20 LED turns ON then OFF as expected.

- Boot the aarch64 kernel and run the same test program
 -> GPIO20 LED does not turn on.
    Sometimes an unrelated GPIO (e.g. GPIO22) turns on.
    In some cases the kernel SDHC driver starts reporting CRC errors
    repeatedly and stalls.
``` (typed from video screen)
sdhc0: cmd crc error
sdhc0: cmd timeout error
sdmmc1: extended I/O error 60, r=40992 p=0xffffc000b1680dac I=4 read
```
 - The same earmv7hf binary (that works on the earmv7hf kernel) on aarch64
   (via COMPAT_NETBSD32) also gets the same failures.

Test program
------------

This program maps the GPIO registers and toggles GPIO20 via GPSET0/GPCLR0.
(Uses PERI_BASE = 0x3F000000 for Pi 2/3.)

https://gist.github.com/tsutsui/0c0d6c1d7f4e2d5dbaaf210789c51ad7

```c
/*
 * gpio_mmap_test.c
 *  Minimal Raspberry Pi GPIO test using GPIO access via /dev/mem mmap(2)
 *
 * Wiring:
 *   GPIO20     -> LED (1: on, 0: off)
 *
 * Build:
 *   cc -O2 -Wall -Wextra -o gpio_mmap_test gpio_mmap_test.c
 */

#include <sys/mman.h>

#include <fcntl.h>
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

// #define PERI_BASE   0x20000000u /* BCM2835 (Pi 1/Zero) */
#define PERI_BASE   0x3F000000u /* BCM2836/2837 (Pi 2/3) */
// #define PERI_BASE   0xFE000000u /* BCM2711 (Pi 4) */

#define GPIO_BASE   (PERI_BASE + 0x200000u)
#define GPIO_SIZE   0x1000u

/* GPIO registers */
#define GPFSEL0     0x00
#define GPFSEL1     0x04
#define GPFSEL2     0x08
#define GPSET0      0x1c
#define GPCLR0      0x28
#define GPLEV0      0x34

enum {
    PIN_D0   = 20, /* DA0 */
};

#define MASK_D0 (1u << PIN_D0)

static volatile uint32_t *gpio;

static void
die(const char *msg)
{
    perror(msg);
    exit(EXIT_FAILURE);
}

/* Minimal memory barrier (ordering for MMIO) */
static inline void
mmio_barrier(void)
{
#if (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__)
    __asm__ volatile("dmb ish" ::: "memory");
#else
    __sync_synchronize();
#endif
}

/* Set GPIO function to output: fsel=001 */
static void
gpio_config_output(int pin)
{
    uint32_t reg = pin / 10;
    uint32_t shift = (pin % 10) * 3;
    volatile uint32_t *fsel = &gpio[GPFSEL0 / 4 + reg];

    uint32_t v = *fsel;
    v &= ~(7u << shift);
    v |=  (1u << shift);
    *fsel = v;
    mmio_barrier();
}

static void
gpio_config(void)
{
    gpio_config_output(PIN_D0);
}

static inline void
gpio_write_masks(uint32_t set_mask, uint32_t clr_mask)
{
    if (clr_mask)
        gpio[GPCLR0 / 4] = clr_mask;
    if (set_mask)
        gpio[GPSET0 / 4] = set_mask;
    mmio_barrier();
}

static inline void
gpio_d0_on()
{
    gpio_write_masks(MASK_D0, 0);
}

static inline void
gpio_d0_off()
{
    gpio_write_masks(0, MASK_D0);
}

int
main(int argc, char **argv)
{

    int fd = open("/dev/mem", O_RDWR | O_SYNC);
    if (fd == -1)
        die("open(/dev/mem)");

    void *p = mmap(NULL, GPIO_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED,
      fd, GPIO_BASE);
    if (p == MAP_FAILED)
        die("mmap(GPIO)");

    gpio = (volatile uint32_t *)p;

    gpio_config();

    printf("D0 (GPIO%d): ON\n", PIN_D0);
    gpio_d0_on();

    sleep(2);

    printf("D0 (GPIO%d): OFF\n", PIN_D0);
    gpio_d0_off();

    munmap((void*)gpio, GPIO_SIZE);
    close(fd);
    exit(EXIT_SUCCESS);
}
```

Questions
---------

- Is /dev/mem + mmap expected to work for BCM283x/2711 peripheral MMIO
  on evbarm/aarch64?
- If yes, where should I look for the correct mapping attributes
  (device vs normal cacheable) for /dev/mem mappings on aarch64?
- Does the described behavior sound like a cache attribute / pmap issue,
  or could it be an address translation issue?

Any comments would be appreciated. If you need additional information
(dmesg etc.) I can provide it.

Thanks,
---
Izumi Tsutsui