Here are updated diffs that:

(a) use sigreturn() to enter VM86 mode [no more need for the new gate
    and new sysarch() function]
(b) define new macros USERMODE and KERNELMODE, which look at PSL_VM if
    usermode isn't set.
(c) unifies the trapframe and intrframe structures to use same
    substructure (may be needed if other places need to examine
    USERMODE/KERNELMODE on an interrupt frame rather than a trap frame)
(d) squeezes in fs/gs segment selectors into struct sigcontext.  [We can
    do this becuase the selectors are 16 bits--we don't need 32 bits for
    %es and %ds].  This is kind of gross, but it maintains binary
    compatibility with NetBSD 1.0 signal handling code/programs.
(e) If running on a kernel with VM86 mode enabled, all processes have an
    extra 4 longwords at the top of kstack for use when returning to
    VM86 mode.

All the kernel stuff is conditional on 'options VM86'.

Again, a sample (stupid) program is at the end.

===================================================================
RCS file: include/RCS/cpu.h,v
retrieving revision 1.1
diff -ubw -r1.1 include/cpu.h
--- 1.1	1995/02/25 20:21:33
+++ include/cpu.h	1995/03/06 02:40:25
@@ -68,7 +68,12 @@
  */
 #define clockframe intrframe
 
+#ifdef VM86
+#define	CLKF_USERMODE(frame)	(ISPL((frame)->if_cs) == SEL_UPL || ((frame)->if_eflags & PSL_VM))
+#else
 #define	CLKF_USERMODE(frame)	(ISPL((frame)->if_cs) == SEL_UPL)
+#endif
+
 #define	CLKF_BASEPRI(frame)	((frame)->if_ppl == 0)
 #define	CLKF_PC(frame)		((frame)->if_eip)
 #define	CLKF_INTR(frame)	(0)	/* XXX should have an interrupt stack */
===================================================================
RCS file: include/RCS/frame.h,v
retrieving revision 1.1
diff -ubw -r1.1 include/frame.h
--- 1.1	1995/02/25 22:27:19
+++ include/frame.h	1995/03/06 02:52:22
@@ -38,6 +38,7 @@
  *	@(#)frame.h	5.2 (Berkeley) 1/18/91
  */
 
+#ifndef LOCORE
 #include <sys/signal.h>
 
 /*
@@ -71,24 +72,29 @@
 /* Interrupt stack frame */
 struct intrframe {
 	int	if_ppl;
-	int	if_es;
-	int	if_ds;
-	int	if_edi;
-	int	if_esi;
-	int	if_ebp;
-	int	if_ebx;
-	int	if_edx;
-	int	if_ecx;
-	int	if_eax;
-	int	:32;		/* for compat with trap frame - trapno */
-	int	:32;		/* for compat with trap frame - err */
-	/* below portion defined in 386 hardware */
-	int	if_eip;
-	int	if_cs;
-	int	if_eflags;
-	/* below only when transitting rings (e.g. user to kernel) */
-	int	if_esp;
-	int	if_ss;
+	struct trapframe if_tf;
+#define if_es if_tf.tf_es
+#define if_ds if_tf.tf_ds
+#define if_edi if_tf.tf_edi
+#define if_esi if_tf.tf_esi
+#define if_ebp if_tf.tf_ebp
+#define if_ebx if_tf.tf_ebx
+#define if_edx if_tf.tf_edx
+#define if_ecx if_tf.tf_ecx
+#define if_eax if_tf.tf_eax
+#define if_eip if_tf.tf_eip
+#define if_cs if_tf.tf_cs
+#define if_eflags if_tf.tf_eflags
+#define if_esp if_tf.tf_esp
+#define if_ss if_tf.tf_ss
+};
+
+struct vm86_frame {
+    struct trapframe tframe;
+    int vm_es;
+    int vm_ds;
+    int vm_fs;
+    int vm_gs;
 };
 
 /*
@@ -101,3 +107,6 @@
 	sig_t	sf_handler;
 	struct	sigcontext sf_sc;
 };
+#endif /* LOCORE */
+
+#define	VM86_EXTRA_FRAMESIZE	(4*sizeof(long)) /* extra size for VM86 */
===================================================================
RCS file: include/RCS/signal.h,v
retrieving revision 1.1
diff -ubw -r1.1 include/signal.h
--- 1.1	1995/03/07 03:43:30
+++ include/signal.h	1995/03/07 06:25:14
@@ -57,8 +57,10 @@
 	int	sc_onstack;		/* sigstack state to restore */
 	int	sc_mask;		/* signal mask to restore */
 
-	int	sc_es;
-	int	sc_ds;
+	short	sc_es;
+	short	sc_fs;			/* MBZ if not VM86 */
+	short	sc_ds;
+	short	sc_gs;			/* MBZ if not VM86 */
 	int	sc_edi;
 	int	sc_esi;
 	int	sc_ebp;
===================================================================
RCS file: isa/RCS/icu.s,v
retrieving revision 1.1
diff -ubw -r1.1 isa/icu.s
--- 1.1	1995/02/26 04:15:16
+++ isa/icu.s	1995/03/06 02:44:28
@@ -132,8 +132,12 @@
 	cmpb	$0,_astpending
 	je	3f
 	testb   $SEL_RPL_MASK,TF_CS(%esp)
+#ifdef VM86
+	jnz	4f
+	testl	$PSL_VM,TF_EFLAGS(%esp)
+#endif
 	jz	3f
-	movb	$0,_astpending
+4:	movb	$0,_astpending
 	sti
 	/* Pushed T_ASTFLT into tf_trapno on entry. */
 	call	_trap
===================================================================
RCS file: isa/RCS/npx.c,v
retrieving revision 1.1
diff -ubw -r1.1 isa/npx.c
--- 1.1	1995/02/25 20:21:56
+++ isa/npx.c	1995/03/06 03:37:59
@@ -419,7 +419,7 @@
 	 * Pass exception to process.  If it's the current process, try to do
 	 * it immediately.
 	 */
-	if (p == curproc && ISPL(frame->if_cs) == SEL_UPL) {
+	if (p == curproc && USERMODE(&frame->if_tf)) {
 		/*
 		 * Interrupt is essentially a trap, so we can afford to call
 		 * the SIGFPE handler (if any) as soon as the interrupt
===================================================================
RCS file: i386/RCS/db_interface.c,v
retrieving revision 1.2
diff -ubw -r1.2 i386/db_interface.c
--- 1.2	1995/01/30 02:58:58
+++ i386/db_interface.c	1995/03/08 03:33:46
@@ -79,7 +79,7 @@
 	/* XXX Should switch to kdb`s own stack here. */
 
 	ddb_regs = *regs;
-	if (ISPL(regs->tf_cs) == SEL_KPL) {
+	if (KERNELMODE(regs)) {
 		/*
 		 * Kernel mode - esp and ss not saved
 		 */
@@ -107,7 +107,7 @@
 	regs->tf_eip    = ddb_regs.tf_eip;
 	regs->tf_cs     = ddb_regs.tf_cs;
 	regs->tf_eflags = ddb_regs.tf_eflags;
-	if (ISPL(regs->tf_cs) != SEL_KPL) {
+	if (!KERNELMODE(regs)) {
 		/* ring transit - saved esp and ss valid */
 		regs->tf_esp    = ddb_regs.tf_esp;
 		regs->tf_ss     = ddb_regs.tf_ss;
===================================================================
RCS file: i386/RCS/locore.s,v
retrieving revision 1.1
diff -ubw -r1.1 i386/locore.s
--- 1.1	1995/02/26 04:18:17
+++ i386/locore.s	1995/03/08 03:34:36
@@ -1973,8 +1973,12 @@
 	cmpb	$0,_astpending
 	je	1f
 	testb	$SEL_RPL_MASK,TF_CS(%esp)
+#ifdef VM86
+	jnz	5f
+	testl	$PSL_VM,TF_EFLAGS(%esp)
+#endif
 	jz	1f
-	movb	$0,_astpending
+5:	movb	$0,_astpending
 	sti
 	movl	$T_ASTFLT,TF_TRAPNO(%esp)
 	call	_trap
===================================================================
RCS file: i386/RCS/machdep.c,v
retrieving revision 1.1.1.3
diff -ubw -r1.1.1.3 i386/machdep.c
--- 1.1.1.3	1995/03/07 02:40:12
+++ i386/machdep.c	1995/03/08 03:25:58
@@ -468,6 +468,9 @@
 {
 	register struct proc *p = curproc;
 	register struct trapframe *tf;
+#ifdef VM86
+	register struct vm86_frame *vf;
+#endif
 	struct sigframe *fp, frame;
 	struct sigacts *psp = p->p_sigacts;
 	int oonstack;
@@ -521,8 +524,21 @@
 	 */
 	frame.sf_sc.sc_onstack = oonstack;
 	frame.sf_sc.sc_mask = mask;
+#ifdef VM86
+	if (tf->tf_eflags & PSL_VM) {
+	    vf = (struct vm86_frame *)tf;
+	    frame.sf_sc.sc_es     = vf->vm_es;
+	    frame.sf_sc.sc_fs     = vf->vm_fs;
+	    frame.sf_sc.sc_ds     = vf->vm_ds;
+	    frame.sf_sc.sc_gs     = vf->vm_gs;
+	} else
+#endif
+	{
 	frame.sf_sc.sc_es     = tf->tf_es;
+	    frame.sf_sc.sc_fs     = 0;
 	frame.sf_sc.sc_ds     = tf->tf_ds;
+	    frame.sf_sc.sc_gs     = 0;
+	}
 	frame.sf_sc.sc_edi    = tf->tf_edi;
 	frame.sf_sc.sc_esi    = tf->tf_esi;
 	frame.sf_sc.sc_ebp    = tf->tf_ebp;
@@ -555,6 +571,11 @@
 	tf->tf_ds = _udatasel;
 	tf->tf_es = _udatasel;
 	tf->tf_ss = _udatasel;
+	/* when we got out of VM86 mode, the fs & gs selectors were set to
+	   zero by the processor (or so says the i386 book I have!).
+	   see also the locore switching routines--%fs and %gs are handled
+	   there--though for trapping from vm86 mode,
+	   they should always be zero, */
 }
 
 /*
@@ -576,6 +597,10 @@
 {
 	struct sigcontext *scp, context;
 	register struct trapframe *tf;
+#ifdef VM86
+	register struct vm86_frame *vf;
+	vf = (struct vm86_frame *)p->p_md.md_regs;
+#endif
 
 	tf = (struct trapframe *)p->p_md.md_regs;
 
@@ -591,9 +616,22 @@
 	/*
 	 * Check for security violations.
 	 */
+#ifdef VM86
+	/* if turning on PSL_VM (vm86 mode), allow code selector
+	   to be whatever. */
+	if (((context.sc_eflags ^ tf->tf_eflags) & PSL_USERSTATIC) != 0 ||
+	    (ISPL(context.sc_cs) != SEL_UPL && !(context.sc_eflags & PSL_VM)))
+		return (EINVAL);
+
+	/* If _not_ returning to PSL_VM, non-code selectors are
+	   validated by processor before loading the associated
+	   descriptor, and a fault occurs if a security violation is
+	   attempted. */
+#else
 	if (((context.sc_eflags ^ tf->tf_eflags) & PSL_USERSTATIC) != 0 ||
 	    ISPL(context.sc_cs) != SEL_UPL)
 		return (EINVAL);
+#endif
 
 	if (context.sc_onstack & 01)
 		p->p_sigacts->ps_sigstk.ss_flags |= SA_ONSTACK;
@@ -604,8 +642,20 @@
 	/*
 	 * Restore signal context.
 	 */
+#ifdef VM86
+	if (context.sc_eflags & PSL_VM) {
+	    vf->vm_es = context.sc_es;
+	    vf->vm_fs = context.sc_fs;
+	    vf->vm_ds = context.sc_ds;
+	    vf->vm_gs = context.sc_gs;
+	    tf->tf_es     = 0;
+	    tf->tf_ds     = 0;		/* set by VM86 part of frame */
+	} else
+#endif
+        {
 	tf->tf_es     = context.sc_es;
 	tf->tf_ds     = context.sc_ds;
+	}
 	tf->tf_edi    = context.sc_edi;
 	tf->tf_esi    = context.sc_esi;
 	tf->tf_ebp    = context.sc_ebp;
@@ -1168,7 +1223,12 @@
 	/* now running on new page tables, configured,and u/iom is accessible */
 
 	/* make a initial tss so microp can get interrupt stack on syscall! */
+#ifdef VM86
+	proc0.p_addr->u_pcb.pcb_tss.tss_esp0 = (int) USRSTACK + UPAGES*NBPG
+	    - VM86_EXTRA_FRAMESIZE;
+#else
 	proc0.p_addr->u_pcb.pcb_tss.tss_esp0 = (int) USRSTACK + UPAGES*NBPG;
+#endif
 	proc0.p_addr->u_pcb.pcb_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
 	_gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
 
===================================================================
RCS file: i386/RCS/trap.c,v
retrieving revision 1.1.1.1
diff -ubw -r1.1.1.1 i386/trap.c
--- 1.1.1.1	1995/03/07 02:38:46
+++ i386/trap.c	1995/03/08 03:46:52
@@ -183,7 +183,7 @@
 	}
 #endif
 
-	if (ISPL(frame.tf_cs) != SEL_KPL) {
+	if (!KERNELMODE(&frame)) {
 		type |= T_USER;
 		sticks = p->p_sticks;
 		p->p_md.md_regs = (int *)&frame;
@@ -511,8 +511,9 @@
 	int fromtramp;
 #endif
 
+
 	cnt.v_syscall++;
-	if (ISPL(frame.tf_cs) != SEL_UPL)
+	if (!USERMODE(&frame))
 		panic("syscall");
 	p = curproc;
 	sticks = p->p_sticks;
@@ -567,6 +568,16 @@
 #endif
 	}
 
+#ifdef VM86
+	/*
+	 * VM86 mode application found our syscall trap gate by accident...
+	 * let it get a sigsys() and have the VM86 handler in the process
+	 * take care of it.
+	 */
+	if (frame.tf_eflags & PSL_VM)
+	    code = -1;
+	else
+#endif
 	switch (code) {
 	case SYS_syscall:
 #ifdef COMPAT_LINUX
===================================================================
RCS file: include/RCS/psl.h,v
retrieving revision 1.1
diff -ubw -r1.1 include/psl.h
--- 1.1	1995/03/07 03:44:19
+++ include/psl.h	1995/03/07 06:26:18
@@ -66,7 +66,11 @@
 #define	PSL_MBZ		0xffc08028	/* must be zero bits */
 
 #define	PSL_USERSET	(PSL_MBO | PSL_I)
+#ifdef VM86
+#define	PSL_USERSTATIC	(PSL_MBO | PSL_MBZ | PSL_I | PSL_IOPL | PSL_VIF | PSL_VIP)
+#else
 #define	PSL_USERSTATIC	(PSL_MBO | PSL_MBZ | PSL_I | PSL_IOPL | PSL_VM | PSL_VIF | PSL_VIP)
+#endif
 
 #ifdef KERNEL
 
===================================================================
RCS file: include/RCS/segments.h,v
retrieving revision 1.1
diff -ubw -r1.1 include/segments.h
--- 1.1	1995/03/06 02:45:24
+++ include/segments.h	1995/03/07 06:19:11
@@ -58,6 +58,13 @@
 #define	ISPL(s)	((s)&3)		/* what is the priority level of a selector */
 #define	SEL_KPL	0		/* kernel priority level */	
 #define	SEL_UPL	3		/* user priority level */	
+#ifdef VM86
+#define	USERMODE(trapframe) (ISPL((trapframe)->tf_cs) == SEL_UPL || ((trapframe)->tf_eflags & PSL_VM))
+#define	KERNELMODE(trapframe) (ISPL((trapframe)->tf_cs) == SEL_KPL && !((trapframe)->tf_eflags & PSL_VM))
+#else
+#define	USERMODE(trapframe) (ISPL((trapframe)->tf_cs) == SEL_UPL)
+#define	KERNELMODE(trapframe) (ISPL((trapframe)->tf_cs) == SEL_KPL)
+#endif
 #define	ISLDT(s)	((s)&SEL_LDT)	/* is it local or global */
 #define	SEL_LDT	4		/* local descriptor table */	
 #define	IDXSEL(s)	(((s)>>3) & 0x1fff)		/* index of selector */
--- /dev/null	Tue Mar  7 22:13:05 1995
+++ /var/tmp/Makefile	Tue Mar  7 01:33:36 1995
@@ -0,0 +1,3 @@
+sigill: sigill.c
+#	$(CC) -static -I/usr/include.current -g -o sigill sigill.c -li386
+	$(CC) -static -I/usr/include.current -g -o sigill sigill.c
--- /dev/null	Tue Mar  7 22:13:05 1995
+++ /var/tmp/sigill.c	Tue Mar  7 22:27:20 1995
@@ -0,0 +1,61 @@
+#include <unistd.h>
+#include <signal.h>
+#include <string.h>
+#include <setjmp.h>
+#include <machine/segments.h>
+#include <machine/sysarch.h>
+#include <machine/psl.h>
+
+jmp_buf jbuf;
+
+void
+sigill(int sig, int code, struct sigcontext *scp)
+{
+    printf("sig %d, code %d, eip %#08lx, eflags=%#08lx\n",
+	   sig, code, scp->sc_eip, scp->sc_eflags);
+    longjmp(jbuf, 1);
+    _exit(2);
+}
+
+unsigned long vmstack[256];
+
+unsigned char ill[] = { 144, 144, 144, /* 0xfb, */ 0xeb, 0xfb /* nop, nop, nop, sti, loop */ };
+
+foo()
+{
+    asm("1: nop; nop; nop");
+    asm("jmp 1b");
+}
+
+main()
+{
+    struct sigcontext vmctx;
+    int rval;
+    int psl;
+
+    signal(SIGILL, sigill);
+    signal(SIGBUS, sigill);
+    signal(SIGQUIT, sigill);
+    memset(&vmctx, 0, sizeof(vmctx));
+
+
+    vmctx.sc_esp = (int) &vmstack[sizeof(vmstack)/4-1];
+    vmctx.sc_eip = (int) &ill[0];
+    vmctx.sc_cs = 0;
+    vmctx.sc_ds = 0;
+    vmctx.sc_es = 0;
+    vmctx.sc_fs = 0;
+    vmctx.sc_gs = 0;
+    __asm("pushfl");
+    __asm("popl %0" : "=r" (psl));
+    vmctx.sc_eflags = psl | PSL_VM;
+
+    if (setjmp(jbuf)) {
+	printf("jumped out of sighandler, exiting\n");
+	exit(2);
+    }
+    rval = sigreturn(&vmctx);
+    printf("returning from vm86 with rval=%d?\n", rval);
+    exit(1);
+}
+