Re: New line discipline flag to prevent userland open/close of the device

[Martin Husemann <martin%duskware.de@localhost>]

To give a better idea of what changes: here is a tested patch that solves
the original problem for me.

It is kinda ugly, but I don't see an easier solution.

Anyone?

Martin

Index: dev/ic/com.c
===================================================================
RCS file: /cvsroot/src/sys/dev/ic/com.c,v
retrieving revision 1.343
diff -u -p -r1.343 com.c
--- dev/ic/com.c	28 Oct 2017 04:53:55 -0000	1.343
+++ dev/ic/com.c	29 Oct 2017 13:43:11 -0000
@@ -879,6 +879,13 @@ comopen(dev_t dev, int flag, int mode, s
 
 	tp = sc->sc_tty;
 
+	/*
+	 * If the device is exclusively for kernel use, deny userland
+	 * open.
+	 */
+	if (ISSET(tp->t_state, TS_NO_USER_OPEN))
+		return (EBUSY);
+
 	if (kauth_authorize_device_tty(l->l_cred, KAUTH_DEVICE_TTY_OPEN, tp))
 		return (EBUSY);
 
@@ -1017,6 +1024,12 @@ comclose(dev_t dev, int flag, int mode, 
 	/* XXX This is for cons.c. */
 	if (!ISSET(tp->t_state, TS_ISOPEN))
 		return (0);
+	/*
+	 * If the device is exclusively for kernel use, deny userland
+	 * close.
+	 */
+	if (ISSET(tp->t_state, TS_NO_USER_OPEN))
+		return (0);
 
 	(*tp->t_linesw->l_close)(tp, flag);
 	ttyclose(tp);
Index: dev/sun/sunkbd.c
===================================================================
RCS file: /cvsroot/src/sys/dev/sun/sunkbd.c,v
retrieving revision 1.29
diff -u -p -r1.29 sunkbd.c
--- dev/sun/sunkbd.c	15 Sep 2013 14:10:04 -0000	1.29
+++ dev/sun/sunkbd.c	29 Oct 2017 13:43:10 -0000
@@ -145,6 +145,7 @@ sunkbd_attach(device_t parent, device_t 
 	KASSERT(tp->t_linesw == &sunkbd_disc);
 	tp->t_oflag &= ~OPOST;
 	tp->t_dev = args->kmta_dev;
+	SET(tp->t_state, TS_NO_USER_OPEN);
 
 	/* link the structures together. */
 	k->k_priv = tp;
@@ -218,6 +219,7 @@ sunkbdiopen(device_t dev, int flags)
 	int error;
 
 	/* Open the lower device */
+	CLR(tp->t_state, TS_NO_USER_OPEN);
 	if ((error = cdev_open(tp->t_dev, O_NONBLOCK|flags,
 				     0/* ignored? */, l)) != 0)
 		return (error);
@@ -228,6 +230,7 @@ sunkbdiopen(device_t dev, int flags)
 	t.c_ospeed = sunkbd_bps;
 	t.c_cflag =  CLOCAL|CS8;
 	(*tp->t_param)(tp, &t);
+	SET(tp->t_state, TS_NO_USER_OPEN);
 
 	return (0);
 }
Index: dev/sun/sunms.c
===================================================================
RCS file: /cvsroot/src/sys/dev/sun/sunms.c,v
retrieving revision 1.32
diff -u -p -r1.32 sunms.c
--- dev/sun/sunms.c	15 Sep 2013 14:13:19 -0000	1.32
+++ dev/sun/sunms.c	29 Oct 2017 13:43:10 -0000
@@ -162,6 +162,7 @@ sunms_attach(device_t parent, device_t s
 	tp->t_linesw = ttyldisc_lookup(sunms_disc.l_name);
 	KASSERT(tp->t_linesw == &sunms_disc);
 	tp->t_oflag &= ~OPOST;
+	SET(tp->t_state, TS_NO_USER_OPEN);
 
 	/* Initialize translator. */
 	ms->ms_byteno = -1;
@@ -192,6 +193,7 @@ sunmsiopen(device_t dev, int flags)
 	int error;
 
 	/* Open the lower device */
+	CLR(tp->t_state, TS_NO_USER_OPEN);
 	if ((error = cdev_open(tp->t_dev, O_NONBLOCK|flags,
 				     0/* ignored? */, l)) != 0)
 		return (error);
@@ -202,6 +204,7 @@ sunmsiopen(device_t dev, int flags)
 	t.c_ospeed = sunms_bps;
 	t.c_cflag =  CLOCAL|CS8;
 	(*tp->t_param)(tp, &t);
+	SET(tp->t_state, TS_NO_USER_OPEN);
 
 	return (0);
 }
Index: sys/tty.h
===================================================================
RCS file: /cvsroot/src/sys/sys/tty.h,v
retrieving revision 1.93
diff -u -p -r1.93 tty.h
--- sys/tty.h	15 Nov 2014 19:17:05 -0000	1.93
+++ sys/tty.h	29 Oct 2017 13:43:11 -0000
@@ -203,6 +203,9 @@ struct tty {
 #define	TS_TYPEN	0x08000		/* Retyping suspended input (PENDIN). */
 #define	TS_LOCAL	(TS_BKSL | TS_CNTTB | TS_ERASE | TS_LNCH | TS_TYPEN)
 
+/* for special line disciplines, like dev/sun/sunkbd.c */
+#define	TS_NO_USER_OPEN	0x10000		/* Device is kernel only */
+
 /* Character type information. */
 #define	ORDINARY	0
 #define	CONTROL		1