toolchain/59994: gcc/m68k generates bad code for stack protector

["mlelstv%netbsd.org@localhost via gnats" <gnats-admin%NetBSD.org@localhost>]

>Number:         59994
>Category:       toolchain
>Synopsis:       gcc/m68k generates bad code for stack protector
>Confidential:   no
>Severity:       serious
>Priority:       high
>Responsible:    toolchain-manager
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Wed Feb 11 07:00:00 +0000 2026
>Originator:     Michael van Elst
>Release:        NetBSD 11.0_RC1
>Organization:
-- 
                                Michael van Elst
Internet: mlelstv%serpens.de@localhost
                                "A potential Snark may lurk in every tree."
>Environment:
	
	
System: NetBSD serpens.de 11.0_RC1 NetBSD 11.0_RC1 (SERPENS) #0: Sat Feb 7 12:30:40 UTC 2026 root%aries.NetBSD.de@localhost:/home/netbsd/11/arch/amiga/obj/sys/arch/amiga/compile/SERPENS amiga
Architecture: m68k
Machine: amiga
>Description:

mutt from pkgsrc aborts when trying to forward a mail on a NetBSD/m68k-11.0RC1
system:

> #0  0x0ec23d8c in _lwp_kill () from /usr/lib/libc.so.12
> #1  0x0ec23702 in raise () from /usr/lib/libc.so.12
> #2  0x0ec0c274 in ?? () from /usr/lib/libc.so.12
> #3  0x0ec0c296 in __stack_chk_fail () from /usr/lib/libc.so.12

This is caused by gcc generating bad code for the stack protector.
A simple reproducer is:

#include <stdio.h>

extern void bar(const char *);

void foo(unsigned x)
{
        char buf[1024];

        snprintf(buf, sizeof(buf), "%u\n", x);
        bar(buf);
}

Compiling with:

cc -fstack-protector -O -S c.c

generates the following code:

#NO_APP
	.file	"c.c"
	.text
	.section	.rodata.str1.1,"aMS",@progbits,1
.LC0:
	.string	"%u\n"
	.text
	.align	2
	.globl	foo
	.type	foo, @function
foo:
.LFB3:
	.cfi_startproc
	lea (-1028,%sp),%sp
	.cfi_def_cfa_offset 1032
	move.l %d2,-(%sp)
	.cfi_def_cfa_offset 1036
	.cfi_offset 2, -1036
	move.w 1028(%sp),%d0
	move.w 1030(%sp),%d0
	move.l __stack_chk_guard,%d0
	move.l 1036(%sp),-(%sp)
	.cfi_def_cfa_offset 1040
	pea .LC0
	.cfi_def_cfa_offset 1044
	pea 1024.w
	.cfi_def_cfa_offset 1048
	moveq #16,%d2
	add.l %sp,%d2
	move.l %d2,-(%sp)
	.cfi_def_cfa_offset 1052
	jsr snprintf
	move.l %d2,-(%sp)
	.cfi_def_cfa_offset 1056
	jsr bar
	lea (20,%sp),%sp
	.cfi_def_cfa_offset 1036
	move.w 1028(%sp),%d0
	swap %d0
	clr.w %d0
	move.w 1030(%sp),%d1
	or.w %d1,%d0
	move.l __stack_chk_guard,%d1
	cmp.l %d1,%d0
	jne .L3
	move.l (%sp)+,%d2
	lea (1028,%sp),%sp
	rts
.L3:
	jsr __stack_chk_fail
	.cfi_endproc
.LFE3:
	.size	foo, .-foo
	.ident	"GCC: (nb1 20260119) 12.5.0"

The stack check code at the end of the function is bad but correct,
maybe a consequence of trying to be coldfire compatible.

- assemble 32bit value from stack
        move.w 1028(%sp),%d0
        swap %d0
        clr.w %d0
        move.w 1030(%sp),%d1
        or.w %d1,%d0

- fetch 32bit guard value
        move.l __stack_chk_guard,%d1

- compare both
        cmp.l %d1,%d0


The setup code is nonsense:

- read 32bit value from stack, but throw it away
        move.w 1028(%sp),%d0
        move.w 1030(%sp),%d0

- read 32bit guard value (and don't use it either).
        move.l __stack_chk_guard,%d0

Instead of seeding the longword on the stack with the random
guard value, it just _reads_ the stack and the guard value.


>How-To-Repeat:

Compile reproducer.

Adding a simple main program:

#include <stdio.h>

extern void foo(unsigned);

void bar(const char *p)
{
        puts(p);
}

int main()
{
        foo(42);
        return 0;
}

results in an abort when compiled with stack protector:

% ./a.out
42

Abort


>Fix:

>Unformatted: