Just for the sake of completeness, here's a complete set of diffs to
put dialout handling in tty.c, and modify the com driver to use it.

This has not been extensively tested.


Index: dev/ic/com.c
===================================================================
RCS file: /cvsroot/src/sys/dev/ic/com.c,v
retrieving revision 1.142
diff -c -2 -r1.142 com.c
*** com.c	1998/03/21 04:27:58	1.142
--- com.c	1998/03/21 14:12:33
***************
*** 199,203 ****
  #endif /* KGDB */
  
! #define	COMUNIT(x)	(minor(x))
  
  int
--- 199,204 ----
  #endif /* KGDB */
  
! #define	COMUNIT(x)	(minor(x) & 0x7ffff)
! #define	COMFLAG(x)	(minor(x) >> 19)
  
  int
***************
*** 743,764 ****
  		splx(s2);
  	}
! 
! 	/* If we're doing a blocking open... */
! 	if (!ISSET(flag, O_NONBLOCK))
! 		/* ...then wait for carrier. */
! 		while (!ISSET(tp->t_state, TS_CARR_ON) &&
! 		    !ISSET(tp->t_cflag, CLOCAL | MDMBUF)) {
! 			tp->t_wopen++;
! 			error = ttysleep(tp, &tp->t_rawq, TTIPRI | PCATCH,
! 			    ttopen, 0);
! 			tp->t_wopen--;
! 			if (error) {
! 				splx(s);
! 				goto bad;
! 			}
! 		}
! 
  	splx(s);
  
  	error = (*linesw[tp->t_line].l_open)(dev, tp);
  	if (error)
--- 744,754 ----
  		splx(s2);
  	}
! 	
  	splx(s);
  
+ 	error = ttyopen(tp, COMFLAG(dev), ISSET(flag, O_NONBLOCK));
+ 	if (error)
+ 		goto bad;
+ 
  	error = (*linesw[tp->t_line].l_open)(dev, tp);
  	if (error)
***************
*** 795,799 ****
  	ttyclose(tp);
  
! 	com_shutdown(sc);
  
  	return (0);
--- 785,796 ----
  	ttyclose(tp);
  
! 	if (!ISSET(tp->t_state, TS_ISOPEN) && tp->t_wopen == 0) {
! 		/*
! 		 * Although we got a last close, the device may still be in
! 		 * use; e.g. if this was the dialout node, and there are still
! 		 * processes waiting for carrier on the non-dialout node.
! 		 */
! 		com_shutdown(sc);
! 	}
  
  	return (0);
Index: kern/tty.c
===================================================================
RCS file: /cvsroot/src/sys/kern/tty.c,v
retrieving revision 1.106
diff -c -2 -r1.106 tty.c
*** tty.c	1998/03/21 04:02:47	1.106
--- tty.c	1998/03/21 14:12:33
***************
*** 171,179 ****
  int tty_count;
  
  /*
   * Initial open of tty, or (re)entry to standard tty line discipline.
   */
  int
! ttyopen(device, tp)
  	dev_t device;
  	register struct tty *tp;
--- 171,242 ----
  int tty_count;
  
+ int
+ ttyopen(tp, dialout, nonblock)
+ 	struct tty *tp;
+ 	int dialout, nonblock;
+ {
+ 	int s;
+ 	int error;
+ 
+ 	s = spltty();
+ 
+ 	if (dialout) {
+ 		/*
+ 		 * If the device is already open for non-dialout, fail.
+ 		 * Otherwise, set TS_DIALOUT here so that a dialout open always
+ 		 * has precedence, even if we have to wait for carrier.
+ 		 */
+ 		if (ISSET(tp->t_state, TS_ISOPEN)) {
+ 			if (!ISSET(tp->t_state, TS_DIALOUT)) {
+ 				splx(s);
+ 				return (EBUSY);
+ 			}
+ 		} else
+ 			SET(tp->t_state, TS_DIALOUT);
+ 	}
+ 
+ 	 if (!nonblock) {
+ 		/*
+ 		 * Wait for carrier.  If this is a non-dialout open, also wait
+ 		 * for any dialout processes to close the tty first.
+ 		 */
+ 		while ((!dialout && ISSET(tp->t_state, TS_DIALOUT)) ||
+ 		       (!ISSET(tp->t_state, TS_CARR_ON) && 
+ 			!ISSET(tp->t_cflag, CLOCAL | MDMBUF))) {
+ 			tp->t_wopen++;
+ 			error = ttysleep(tp, &tp->t_rawq, TTIPRI | PCATCH,
+ 			    ttopen, 0);
+ 			tp->t_wopen--;
+ 			if (error) {
+ 				/*
+ 				 * If there are no other dialout users, clean
+ 				 * up.
+ 				 */
+ 				if (dialout && !ISSET(tp->t_state, TS_ISOPEN))
+ 					CLR(tp->t_state, TS_DIALOUT);
+ 				splx(s);
+ 				return (error);
+ 			}
+ 		}
+ 	} else {
+ 		/*
+ 		 * Don't allow a non-blocking non-dialout open if the device is
+ 		 * already open for dialout.
+ 		 */
+ 	        if (!dialout && ISSET(tp->t_state, TS_DIALOUT)) {
+ 			splx(s);
+ 			return (EBUSY);
+ 		}
+ 	}
+ 
+ 	splx(s);
+ 	return (0);
+ }
+ 
  /*
   * Initial open of tty, or (re)entry to standard tty line discipline.
   */
  int
! ttylopen(device, tp)
  	dev_t device;
  	register struct tty *tp;
Index: kern/tty_conf.c
===================================================================
RCS file: /cvsroot/src/sys/kern/tty_conf.c,v
retrieving revision 1.20
diff -c -2 -r1.20 tty_conf.c
*** tty_conf.c	1998/03/01 02:22:32	1.20
--- tty_conf.c	1998/03/21 14:12:33
***************
*** 101,105 ****
  struct	linesw linesw[] =
  {
! 	{ ttyopen, ttylclose, ttread, ttwrite, nullioctl,
  	  ttyinput, ttstart, ttymodem },		/* 0- termios */
  
--- 101,105 ----
  struct	linesw linesw[] =
  {
! 	{ ttylopen, ttylclose, ttread, ttwrite, nullioctl,
  	  ttyinput, ttstart, ttymodem },		/* 0- termios */
  
***************
*** 108,112 ****
  
  #if defined(COMPAT_43) || defined(COMPAT_FREEBSD)
! 	{ ttyopen, ttylclose, ttread, ttwrite, nullioctl,
  	  ttyinput, ttstart, ttymodem },		/* 2- old NTTYDISC */
  #else
--- 108,112 ----
  
  #if defined(COMPAT_43) || defined(COMPAT_FREEBSD)
! 	{ ttylopen, ttylclose, ttread, ttwrite, nullioctl,
  	  ttyinput, ttstart, ttymodem },		/* 2- old NTTYDISC */
  #else
Index: sys/tty.h
===================================================================
RCS file: /cvsroot/src/sys/sys/tty.h,v
retrieving revision 1.40
diff -c -2 -r1.40 tty.h
*** tty.h	1998/03/21 04:02:47	1.40
--- tty.h	1998/03/21 14:12:33
***************
*** 145,153 ****
  #define	TS_BUSY		0x00004		/* Draining output. */
  #define	TS_CARR_ON	0x00008		/* Carrier is present. */
! #define	TS_FLUSH	0x00010		/* Outq has been flushed during DMA. */
! #define	TS_ISOPEN	0x00020		/* Open has completed. */
! #define	TS_TBLOCK	0x00040		/* Further input blocked. */
! #define	TS_TIMEOUT	0x00080		/* Wait for output char processing. */
! #define	TS_TTSTOP	0x00100		/* Output paused. */
  #define	TS_XCLUDE	0x00400		/* Tty requires exclusivity. */
  
--- 145,154 ----
  #define	TS_BUSY		0x00004		/* Draining output. */
  #define	TS_CARR_ON	0x00008		/* Carrier is present. */
! #define	TS_DIALOUT	0x00010		/* Tty used for dialout. */
! #define	TS_FLUSH	0x00020		/* Outq has been flushed during DMA. */
! #define	TS_ISOPEN	0x00040		/* Open has completed. */
! #define	TS_TBLOCK	0x00080		/* Further input blocked. */
! #define	TS_TIMEOUT	0x00100		/* Wait for output char processing. */
! #define	TS_TTSTOP	0x00200		/* Output paused. */
  #define	TS_XCLUDE	0x00400		/* Tty requires exclusivity. */
  
***************
*** 238,243 ****
  int	 ttyinput __P((int c, struct tty *tp));
  int	 ttylclose __P((struct tty *tp, int flag));
  int	 ttymodem __P((struct tty *tp, int flag));
! int	 ttyopen __P((dev_t device, struct tty *tp));
  int	 ttyoutput __P((int c, struct tty *tp));
  void	 ttypend __P((struct tty *tp));
--- 239,245 ----
  int	 ttyinput __P((int c, struct tty *tp));
  int	 ttylclose __P((struct tty *tp, int flag));
+ int	 ttylopen __P((dev_t device, struct tty *tp));
  int	 ttymodem __P((struct tty *tp, int flag));
! int	 ttyopen __P((struct tty *tp, int dialout, int nonblock));
  int	 ttyoutput __P((int c, struct tty *tp));
  void	 ttypend __P((struct tty *tp));