Proposal: Remove filemon(4); switch make meta to ktrace

[Taylor R Campbell <riastradh%NetBSD.org@localhost>]

[followups on tech-kern]

I propose to remove the filemon(4) device.

- Why?  filemon(4), which writes records of file-related system calls
  to a file, is redundant with ktrace, except potentially for
  performance concerns (but I don't know whether anyone has measured).
  Our implementation raises some security and correctness concerns by
  munging the syscall table, and doesn't compose well.

  We could devote some engineering effort to fixing those issues, but
  as far as I know, filemon(4) is not often exercised in NetBSD so it
  doesn't seem that fixing filemon(4) is worth the effort.

- What instead?  The attached patch (patch set make-meta-v2.patch;
  combined diff make-meta-v2.diff) replaces make's use of /dev/filemon
  by ktrace, in meta mode.  Other applications could use the same
  logic verbatim; if it turns out that there are other applications we
  could migrate the logic from usr.bin/make/filemon/filemon_*.c to a
  library.

  At sjg's request, there's a compile-time option -- setting the make
  variable USE_FILEMON=ktrace vs USE_FILEMON=dev -- to switch between
  filemon back ends, since Juniper still uses /dev/filemon on FreeBSD
  which has seen more maintenance, security, and performance updates.
  (This might also enable future strace, dtrace, &c., back ends.)

  So this patch would still leave a small amount of unexercised code
  in our tree, but it's much smaller than before and only in userland,
  not in kernel.

Do you use filemon(4)?  Objections?  Thoughts?

>From 656117e6a394087bbf134a8a1c09b0ff9248f4cc Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh%NetBSD.org@localhost>
Date: Mon, 23 Dec 2019 05:41:16 +0000
Subject: [PATCH 1/2] Reimplement make(1) meta mode without filemon(4).

---
 usr.bin/make/Makefile  |   7 +-
 usr.bin/make/compat.c  |   2 +-
 usr.bin/make/filemon.c | 870 +++++++++++++++++++++++++++++++++++++++++
 usr.bin/make/filemon.h |  50 +++
 usr.bin/make/job.c     |  66 +++-
 usr.bin/make/meta.c    | 168 ++++++--
 usr.bin/make/meta.h    |   7 +-
 7 files changed, 1120 insertions(+), 50 deletions(-)
 create mode 100644 usr.bin/make/filemon.c
 create mode 100644 usr.bin/make/filemon.h

diff --git a/usr.bin/make/Makefile b/usr.bin/make/Makefile
index bb0ce15f967b..98c4a13d797d 100644
--- a/usr.bin/make/Makefile
+++ b/usr.bin/make/Makefile
@@ -17,9 +17,10 @@ USE_META ?= yes
 .if ${USE_META:tl} != "no"
 SRCS+=	meta.c
 CPPFLAGS+= -DUSE_META
-FILEMON_H ?= ${.CURDIR:H:H}/sys/dev/filemon/filemon.h
-.if exists(${FILEMON_H}) && ${FILEMON_H:T} == "filemon.h"
-COPTS.meta.c += -DHAVE_FILEMON_H -I${FILEMON_H:H}
+USE_FILEMON ?= yes
+.if ${USE_FILEMON:tl} != "no"
+SRCS+=	filemon.c
+CPPFLAGS+= -DUSE_FILEMON
 .endif
 .endif
 
diff --git a/usr.bin/make/compat.c b/usr.bin/make/compat.c
index 9e34e6e351e8..14affa0b552e 100644
--- a/usr.bin/make/compat.c
+++ b/usr.bin/make/compat.c
@@ -404,7 +404,7 @@ again:
 
 #ifdef USE_META
     if (useMeta) {
-	meta_compat_parent();
+	meta_compat_parent(cpid);
     }
 #endif
 
diff --git a/usr.bin/make/filemon.c b/usr.bin/make/filemon.c
new file mode 100644
index 000000000000..5a30afa4980e
--- /dev/null
+++ b/usr.bin/make/filemon.c
@@ -0,0 +1,870 @@
+/*	$NetBSD$	*/
+
+/*-
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef USE_FILEMON
+
+#include "filemon.h"
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/rbtree.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/uio.h>
+#include <sys/wait.h>
+
+#include <sys/ktrace.h>
+
+#include <assert.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "make.h"
+
+#ifndef AT_CWD
+#define AT_CWD -1
+#endif
+
+struct filemon;
+struct filemon_key;
+struct filemon_state;
+
+typedef struct filemon_state *filemon_syscall_t(struct filemon *,
+    const struct filemon_key *, const struct ktr_syscall *);
+
+static filemon_syscall_t filemon_sys_chdir;
+static filemon_syscall_t filemon_sys_execve;
+static filemon_syscall_t filemon_sys_exit;
+static filemon_syscall_t filemon_sys_fork;
+static filemon_syscall_t filemon_sys_link;
+static filemon_syscall_t filemon_sys_open;
+static filemon_syscall_t filemon_sys_openat;
+static filemon_syscall_t filemon_sys_symlink;
+static filemon_syscall_t filemon_sys_unlink;
+static filemon_syscall_t filemon_sys_rename;
+
+static filemon_syscall_t *const filemon_syscalls[] = {
+	[SYS_chdir] = &filemon_sys_chdir,
+	[SYS_execve] = &filemon_sys_execve,
+	[SYS_exit] = &filemon_sys_exit,
+	[SYS_fork] = &filemon_sys_fork,
+	[SYS_link] = &filemon_sys_link,
+	[SYS_open] = &filemon_sys_open,
+	[SYS_openat] = &filemon_sys_openat,
+	[SYS_symlink] = &filemon_sys_symlink,
+	[SYS_unlink] = &filemon_sys_unlink,
+	[SYS_rename] = &filemon_sys_rename,
+};
+
+struct filemon {
+	int			ktrfd; /* kernel writes ktrace events here */
+	FILE			*in;   /* we read ktrace events from here */
+	FILE			*out;  /* we write filemon events to here */
+	rb_tree_t		active;
+	pid_t			child;
+
+	/* I/O state machine.  */
+	enum {
+		FILEMON_START = 0,
+		FILEMON_HEADER,
+		FILEMON_PAYLOAD,
+		FILEMON_ERROR,
+	}			state;
+	unsigned char		*p;
+	size_t			resid;
+
+	/* I/O buffer.  */
+	struct ktr_header	hdr;
+	union {
+		struct ktr_syscall	syscall;
+		struct ktr_sysret	sysret;
+		char			namei[PATH_MAX];
+		unsigned char		buf[4096];
+	}			payload;
+};
+
+struct filemon_state {
+	struct filemon_key {
+		pid_t		pid;
+		lwpid_t		lid;
+	}		key;
+	struct rb_node	node;
+	int		syscode;
+	void		(*show)(struct filemon *, const struct filemon_state *,
+			    const struct ktr_sysret *);
+	unsigned	i;
+	unsigned	npath;
+	char		*path[/*npath*/];
+};
+
+static int
+compare_filemon_states(void *cookie MAKE_ATTR_UNUSED, const void *na,
+    const void *nb)
+{
+	const struct filemon_state *Sa = na;
+	const struct filemon_state *Sb = nb;
+
+	if (Sa->key.pid < Sb->key.pid)
+		return -1;
+	if (Sa->key.pid > Sb->key.pid)
+		return +1;
+	if (Sa->key.lid < Sb->key.lid)
+		return -1;
+	if (Sa->key.lid > Sb->key.lid)
+		return +1;
+	return 0;
+}
+
+static int
+compare_filemon_key(void *cookie MAKE_ATTR_UNUSED, const void *n,
+    const void *k)
+{
+	const struct filemon_state *S = n;
+	const struct filemon_key *key = k;
+
+	if (S->key.pid < key->pid)
+		return -1;
+	if (S->key.pid > key->pid)
+		return +1;
+	if (S->key.lid < key->lid)
+		return -1;
+	if (S->key.lid > key->lid)
+		return +1;
+	return 0;
+}
+
+static const rb_tree_ops_t filemon_rb_ops = {
+	.rbto_compare_nodes = &compare_filemon_states,
+	.rbto_compare_key = &compare_filemon_key,
+	.rbto_node_offset = offsetof(struct filemon_state, node),
+	.rbto_context = NULL,
+};
+
+/*
+ * filemon_open()
+ *
+ *	Allocate a filemon descriptor.  Returns NULL and sets errno on
+ *	failure.
+ */
+struct filemon *
+filemon_open(void)
+{
+	struct filemon *F;
+	int ktrpipe[2];
+	int error;
+
+	/* Allocate and zero a struct filemon object.  */
+	F = calloc(1, sizeof(*F));
+	if (F == NULL)
+		return NULL;
+
+	/* Create a pipe for ktrace events.  */
+	if (pipe2(ktrpipe, O_CLOEXEC|O_NONBLOCK) == -1) {
+		error = errno;
+		goto fail0;
+	}
+
+	/* Create a file stream for reading the ktrace events.  */
+	if ((F->in = fdopen(ktrpipe[0], "r")) == NULL) {
+		error = errno;
+		goto fail1;
+	}
+	ktrpipe[0] = -1;	/* claimed by fdopen */
+
+	/*
+	 * Set the fd for writing ktrace events and initialize the
+	 * rbtree.  The rest can be safely initialized to zero.
+	 */
+	F->ktrfd = ktrpipe[1];
+	rb_tree_init(&F->active, &filemon_rb_ops);
+
+	/* Success!  */
+	return F;
+
+fail2: __unused
+	(void)fclose(F->in);
+fail1:	(void)close(ktrpipe[0]);
+	(void)close(ktrpipe[1]);
+fail0:	free(F);
+	errno = error;
+	return NULL;
+}
+
+/*
+ * filemon_closefd(F)
+ *
+ *	Internal subroutine to try to flush and close the output file.
+ *	If F is not open for output, do nothing.  Never leaves F open
+ *	for output even on failure.  Returns 0 on success; sets errno
+ *	and return -1 on failure.
+ */
+static int
+filemon_closefd(struct filemon *F)
+{
+	int error = 0;
+
+	/* If we're not open, nothing to do.  */
+	if (F->out == NULL)
+		return 0;
+
+	/*
+	 * Flush it, close it, and null it unconditionally, but be
+	 * careful to return the earliest error in errno.
+	 */
+	if (fflush(F->out) == EOF && error == 0)
+		error = errno;
+	if (fclose(F->out) == EOF && error == 0)
+		error = errno;
+	F->out = NULL;
+
+	/* Set errno and return -1 if anything went wrong.  */
+	if (error) {
+		errno = error;
+		return -1;
+	}
+
+	/* Success!  */
+	return 0;
+}
+
+/*
+ * filemon_setfd(F, fd)
+ *
+ *	Cause filemon activity on F to be sent to fd.  Claims ownership
+ *	of fd; caller should not use fd afterward, and any duplicates
+ *	of fd may see their file positions changed.
+ */
+int
+filemon_setfd(struct filemon *F, int fd)
+{
+
+	/*
+	 * Close an existing output file if done.  Fail now if there's
+	 * an error closing.
+	 */
+	if ((filemon_closefd(F)) == -1)
+		return -1;
+	assert(F->out == NULL);
+
+	/* Open a file stream and claim ownership of the fd.  */
+	if ((F->out = fdopen(fd, "a")) == NULL)
+		return -1;
+
+	/*
+	 * Print the opening output.  Any failure will be deferred
+	 * until closing.  For hysterical raisins, we show the parent
+	 * pid, not the child pid.
+	 */
+	fprintf(F->out, "# filemon version 4\n");
+	fprintf(F->out, "# Target pid %jd\n", (intmax_t)getpid());
+	fprintf(F->out, "V 4\n");
+
+	/* Success!  */
+	return 0;
+}
+
+/*
+ * filemon_setpid_parent(F, pid)
+ *
+ *	Set the traced pid, from the parent.  Never fails.
+ */
+void
+filemon_setpid_parent(struct filemon *F, pid_t pid)
+{
+
+	F->child = pid;
+}
+
+/*
+ * filemon_setpid_child(F, pid)
+ *
+ *	Set the traced pid, from the child.  Returns 0 on success; sets
+ *	errno and returns -1 on failure.
+ */
+int
+filemon_setpid_child(const struct filemon *F, pid_t pid)
+{
+	int ops, trpoints;
+
+	ops = KTROP_SET|KTRFLAG_DESCEND;
+	trpoints = KTRFACv2;
+	trpoints |= KTRFAC_SYSCALL|KTRFAC_NAMEI|KTRFAC_SYSRET;
+	trpoints |= KTRFAC_INHERIT;
+	if (fktrace(F->ktrfd, ops, trpoints, pid) == -1)
+		return -1;
+
+	return 0;
+}
+
+/*
+ * filemon_close(F)
+ *
+ *	Close F for output if necessary, and free a filemon descriptor.
+ *	Returns 0 on success; sets errno and returns -1 on failure, but
+ *	frees the filemon descriptor either way;
+ */
+int
+filemon_close(struct filemon *F)
+{
+	struct filemon_state *S;
+	int error = 0;
+
+	/* Close for output.  */
+	if (filemon_closefd(F) == -1 && error == 0)
+		error = errno;
+
+	/* Close the ktrace pipe.  */
+	if (fclose(F->in) == EOF && error == 0)
+		error = errno;
+	if (close(F->ktrfd) == -1 && error == 0)
+		error = errno;
+
+	/* Free any active records.  */
+	while ((S = RB_TREE_MIN(&F->active)) != NULL) {
+		rb_tree_remove_node(&F->active, S);
+		free(S);
+	}
+
+	/* Free the filemon descriptor.  */
+	free(F);
+
+	/* Set errno and return -1 if anything went wrong.  */
+	if (error) {
+		errno = error;
+		return -1;
+	}
+
+	/* Success!  */
+	return 0;
+}
+
+/*
+ * filemon_readfd(F)
+ *
+ *	Returns a file descriptor which will select/poll ready for read
+ *	when there are filemon events to be processed by
+ *	filemon_process, or -1 if anything has gone wrong.
+ */
+int
+filemon_readfd(const struct filemon *F)
+{
+
+	if (F->state == FILEMON_ERROR)
+		return -1;
+	return fileno(F->in);
+}
+
+/*
+ * filemon_dispatch(F)
+ *
+ *	Internal subroutine to dispatch a filemon ktrace event.
+ *	Silently ignore events that we don't recognize.
+ */
+static void
+filemon_dispatch(struct filemon *F)
+{
+	const struct filemon_key key = {
+		.pid = F->hdr.ktr_pid,
+		.lid = F->hdr.ktr_lid,
+	};
+	struct filemon_state *S;
+
+	switch (F->hdr.ktr_type) {
+	case KTR_SYSCALL: {
+		struct ktr_syscall *call = &F->payload.syscall;
+		struct filemon_state *S1;
+
+		/* Validate the syscall code.  */
+		if (call->ktr_code < 0 ||
+		    (size_t)call->ktr_code >= __arraycount(filemon_syscalls) ||
+		    filemon_syscalls[call->ktr_code] == NULL)
+			break;
+
+		/*
+		 * Invoke the syscall-specific logic to create a new
+		 * active state.
+		 */
+		S = (*filemon_syscalls[call->ktr_code])(F, &key, call);
+		if (S == NULL)
+			break;
+
+		/*
+		 * Insert the active state, or ignore it if there
+		 * already is one.
+		 *
+		 * Collisions shouldn't happen because the states are
+		 * keyed by <pid,lid>, in which syscalls should happen
+		 * sequentially in CALL/RET pairs, but let's be
+		 * defensive.
+		 */
+		S1 = rb_tree_insert_node(&F->active, S);
+		if (S1 != S) {
+			/* XXX Which one to drop?  */
+			free(S);
+			break;
+		}
+		break;
+	}
+	case KTR_NAMEI:
+		/* Find an active syscall state, or drop it.  */
+		S = rb_tree_find_node(&F->active, &key);
+		if (S == NULL)
+			break;
+		/* Find the position of the next path, or drop it.  */
+		if (S->i >= S->npath)
+			break;
+		/* Record the path.  */
+		S->path[S->i++] = strndup(F->payload.namei,
+		    sizeof F->payload.namei);
+		break;
+	case KTR_SYSRET: {
+		struct ktr_sysret *ret = &F->payload.sysret;
+		unsigned i;
+
+		/* Find and remove an active syscall state, or drop it.  */
+		S = rb_tree_find_node(&F->active, &key);
+		if (S == NULL)
+			break;
+		rb_tree_remove_node(&F->active, S);
+
+		/*
+		 * If the active syscall state matches this return,
+		 * invoke the syscall-specific logic to show a filemon
+		 * event.
+		 */
+		/* XXX What to do if syscall code doesn't match?  */
+		if (S->i == S->npath && S->syscode == ret->ktr_code)
+			(*S->show)(F, S, ret);
+
+		/* Free the state now that it is no longer active.  */
+		for (i = 0; i < S->i; i++)
+			free(S->path[i]);
+		free(S);
+		break;
+	}
+	default:
+		/* Ignore all other ktrace events.  */
+		break;
+	}
+}
+
+/*
+ * filemon_process(F)
+ *
+ *	Process all pending events after filemon_readfd(F) has
+ *	selected/polled ready for read.
+ *
+ *	Returns -1 on failure, 0 on end of events, and anything else if
+ *	there may be more events.
+ *
+ *	XXX What about fairness to other activities in the event loop?
+ *	If we stop while there's events buffered in F->in, then select
+ *	or poll may not return ready even though there's work queued up
+ *	in the buffer of F->in, but if we don't stop then ktrace events
+ *	may overwhelm all other activity in the event loop.
+ */
+int
+filemon_process(struct filemon *F)
+{
+	size_t nread;
+
+top:	/* If the child has exited, nothing to do.  */
+	/* XXX What if one thread calls exit while another is running?  */
+	if (F->child == 0)
+		return 0;
+
+	/* If we're waiting for input, read some.  */
+	if (F->resid) {
+		nread = fread(F->p, 1, F->resid, F->in);
+		if (nread == 0) {
+			if (feof(F->in))
+				return 0;
+			assert(ferror(F->in));
+			/*
+			 * If interrupted or would block, there may be
+			 * more events.  Otherwise fail.
+			 */
+			if (errno == EAGAIN || errno == EINTR)
+				return 1;
+			F->state = FILEMON_ERROR;
+			F->p = NULL;
+			F->resid = 0;
+			return -1;
+		}
+		assert(nread <= F->resid);
+		F->p += nread;
+		F->resid -= nread;
+		if (F->resid)	/* may be more events */
+			return 1;
+	}
+
+	/* Process a state transition now that we've read a buffer.  */
+	switch (F->state) {
+	case FILEMON_START:	/* just started filemon; read header next */
+		F->state = FILEMON_HEADER;
+		F->p = (void *)&F->hdr;
+		F->resid = sizeof F->hdr;
+		goto top;
+	case FILEMON_HEADER:	/* read header */
+		/* Sanity-check ktrace header; then read payload.  */
+		if (F->hdr.ktr_len < 0 ||
+		    (size_t)F->hdr.ktr_len > sizeof F->payload) {
+			F->state = FILEMON_ERROR;
+			F->p = NULL;
+			F->resid = 0;
+			errno = EIO;
+			return -1;
+		}
+		F->state = FILEMON_PAYLOAD;
+		F->p = (void *)&F->payload;
+		F->resid = (size_t)F->hdr.ktr_len;
+		goto top;
+	case FILEMON_PAYLOAD:	/* read header and payload */
+		/* Dispatch ktrace event; then read next header.  */
+		filemon_dispatch(F);
+		F->state = FILEMON_HEADER;
+		F->p = (void *)&F->hdr;
+		F->resid = sizeof F->hdr;
+		goto top;
+	default:		/* paranoia */
+		F->state = FILEMON_ERROR;
+		/*FALLTHROUGH*/
+	case FILEMON_ERROR:	/* persistent error indicator */
+		F->p = NULL;
+		F->resid = 0;
+		errno = EIO;
+		return -1;
+	}
+}
+
+static struct filemon_state *
+syscall_enter(struct filemon *F MAKE_ATTR_UNUSED,
+    const struct filemon_key *key, const struct ktr_syscall *call,
+    unsigned npath,
+    void (*show)(struct filemon *, const struct filemon_state *,
+	const struct ktr_sysret *))
+{
+	struct filemon_state *S;
+	unsigned i;
+
+	S = calloc(1, offsetof(struct filemon_state, path[npath]));
+	if (S == NULL)
+		return NULL;
+	S->key = *key;
+	S->show = show;
+	S->syscode = call->ktr_code;
+	S->i = 0;
+	S->npath = npath;
+	for (i = 0; i < npath; i++)
+		 S->path[i] = NULL; /* paranoia */
+
+	return S;
+}
+
+static void
+show_paths(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret, const char *prefix)
+{
+	unsigned i;
+
+	/* Caller must ensure all paths have been specified.  */
+	assert(S->i == S->npath);
+
+	/*
+	 * Ignore it if it failed or yielded EJUSTRETURN (-2), or if
+	 * we're not producing output.
+	 */
+	if (ret->ktr_error && ret->ktr_error != -2)
+		return;
+	if (F->out == NULL)
+		return;
+
+	/*
+	 * Print the prefix, pid, and paths -- with the paths quoted if
+	 * there's more than one.
+	 */
+	fprintf(F->out, "%s %jd", prefix, (intmax_t)S->key.pid);
+	for (i = 0; i < S->npath; i++) {
+		const char *q = S->npath > 1 ? "'" : "";
+		fprintf(F->out, " %s%s%s", q, S->path[i], q);
+	}
+	fprintf(F->out, "\n");
+}
+
+static void
+show_retval(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret, const char *prefix)
+{
+
+	/*
+	 * Ignore it if it failed or yielded EJUSTRETURN (-2), or if
+	 * we're not producing output.
+	 */
+	if (ret->ktr_error && ret->ktr_error != -2)
+		return;
+	if (F->out == NULL)
+		return;
+
+	fprintf(F->out, "%s %jd %jd\n", prefix, (intmax_t)S->key.pid,
+	    (intmax_t)ret->ktr_retval);
+}
+
+static void
+show_chdir(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "C");
+}
+
+static void
+show_execve(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	return show_paths(F, S, ret, "E");
+}
+
+static void
+show_fork(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_retval(F, S, ret, "F");
+}
+
+static void
+show_link(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "L"); /* XXX same as symlink */
+}
+
+static void
+show_open_read(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "R");
+}
+
+static void
+show_open_write(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "W");
+}
+
+static void
+show_open_readwrite(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "R");
+	show_paths(F, S, ret, "W");
+}
+
+static void
+show_openat_read(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	if (S->path[0][0] != '/')
+		show_paths(F, S, ret, "A");
+	show_paths(F, S, ret, "R");
+}
+
+static void
+show_openat_write(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	if (S->path[0][0] != '/')
+		show_paths(F, S, ret, "A");
+	show_paths(F, S, ret, "W");
+}
+
+static void
+show_openat_readwrite(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	if (S->path[0][0] != '/')
+		show_paths(F, S, ret, "A");
+	show_paths(F, S, ret, "R");
+	show_paths(F, S, ret, "W");
+}
+
+static void
+show_symlink(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "L"); /* XXX same as link */
+}
+
+static void
+show_unlink(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "D");
+}
+
+static void
+show_rename(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "M");
+}
+
+static struct filemon_state *
+filemon_sys_chdir(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 1, &show_chdir);
+}
+
+static struct filemon_state *
+filemon_sys_execve(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 1, &show_execve);
+}
+
+static struct filemon_state *
+filemon_sys_exit(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	const register_t *args = (const void *)&call[1];
+	int status = args[0];
+
+	if (F->out) {
+		fprintf(F->out, "X %jd %d\n", (intmax_t)key->pid, status);
+		if (key->pid == F->child) {
+			fprintf(F->out, "# Bye bye\n");
+			F->child = 0;
+		}
+	}
+	return NULL;
+}
+
+static struct filemon_state *
+filemon_sys_fork(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 0, &show_fork);
+}
+
+static struct filemon_state *
+filemon_sys_link(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 2, &show_link);
+}
+
+static struct filemon_state *
+filemon_sys_open(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	const register_t *args = (const void *)&call[1];
+	int flags;
+
+	if (call->ktr_argsize < 2)
+		return NULL;
+	flags = args[1];
+
+	if ((flags & O_RDWR) == O_RDWR)
+		return syscall_enter(F, key, call, 1, &show_open_readwrite);
+	else if ((flags & O_WRONLY) == O_WRONLY)
+		return syscall_enter(F, key, call, 1, &show_open_write);
+	else if ((flags & O_RDONLY) == O_RDONLY)
+		return syscall_enter(F, key, call, 1, &show_open_read);
+	else
+		return NULL;	/* XXX Do we care if no read or write?  */
+}
+
+static struct filemon_state *
+filemon_sys_openat(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	const register_t *args = (const void *)&call[1];
+	int flags, fd;
+
+	if (call->ktr_argsize < 3)
+		return NULL;
+	fd = args[0];
+	flags = args[2];
+
+	if (fd == AT_CWD) {
+		if ((flags & O_RDWR) == O_RDWR)
+			return syscall_enter(F, key, call, 1,
+			    &show_open_readwrite);
+		else if ((flags & O_WRONLY) == O_WRONLY)
+			return syscall_enter(F, key, call, 1,
+			    &show_open_write);
+		else if ((flags & O_RDONLY) == O_RDONLY)
+			return syscall_enter(F, key, call, 1, &show_open_read);
+		else
+			return NULL;
+	} else {
+		if ((flags & O_RDWR) == O_RDWR)
+			return syscall_enter(F, key, call, 1,
+			    &show_openat_readwrite);
+		else if ((flags & O_WRONLY) == O_WRONLY)
+			return syscall_enter(F, key, call, 1,
+			    &show_openat_write);
+		else if ((flags & O_RDONLY) == O_RDONLY)
+			return syscall_enter(F, key, call, 1,
+			    &show_openat_read);
+		else
+			return NULL;
+	}
+}
+
+static struct filemon_state *
+filemon_sys_symlink(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 2, &show_symlink);
+}
+
+static struct filemon_state *
+filemon_sys_unlink(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 1, &show_unlink);
+}
+
+static struct filemon_state *
+filemon_sys_rename(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 2, &show_rename);
+}
+
+#endif	/* USE_META */
diff --git a/usr.bin/make/filemon.h b/usr.bin/make/filemon.h
new file mode 100644
index 000000000000..82b7ca73c72f
--- /dev/null
+++ b/usr.bin/make/filemon.h
@@ -0,0 +1,50 @@
+/*	$NetBSD$	*/
+
+/*-
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef FILEMON_H
+#define	FILEMON_H
+
+#include <sys/types.h>
+
+struct filemon;
+
+struct filemon *
+	filemon_open(void);
+int	filemon_close(struct filemon *);
+
+int	filemon_setfd(struct filemon *, int);
+void	filemon_setpid_parent(struct filemon *, pid_t);
+int	filemon_setpid_child(const struct filemon *, pid_t);
+
+int	filemon_readfd(const struct filemon *);
+int	filemon_process(struct filemon *);
+
+#endif	/* FILEMON_H */
diff --git a/usr.bin/make/job.c b/usr.bin/make/job.c
index 48ec2840e8bf..dac04206aab4 100644
--- a/usr.bin/make/job.c
+++ b/usr.bin/make/job.c
@@ -329,6 +329,8 @@ static Job childExitJob;	/* child exit pseudo-job */
 #define	CHILD_EXIT	"."
 #define	DO_JOB_RESUME	"R"
 
+static const int npseudojobs = 2; /* number of pseudo-jobs */
+
 #define TARG_FMT  "%s %s ---\n" /* Default format */
 #define MESSAGE(fp, gn) \
 	if (maxJobs != 1 && targPrefix && *targPrefix) \
@@ -357,6 +359,16 @@ static void JobSigReset(void);
 
 const char *malloc_options="A";
 
+static unsigned
+nfds_per_job(void)
+{
+#ifdef USE_META
+    if (useMeta)
+	return 2;
+#endif
+    return 1;
+}
+
 static void
 job_table_dump(const char *where)
 {
@@ -1439,6 +1451,12 @@ JobExec(Job *job, char **argv)
 
     Trace_Log(JOBSTART, job);
 
+#ifdef USE_META
+    if (useMeta) {
+	meta_job_parent(job, cpid);
+    }
+#endif
+
     /*
      * Set the current position in the buffer to the beginning
      * and mark another stream to watch in the outputs mask
@@ -2121,12 +2139,24 @@ Job_CatchOutput(void)
     if (nready == 0)
 	    return;
 
-    for (i = 2; i < nfds; i++) {
+    for (i = npseudojobs*nfds_per_job(); i < nfds; i++) {
 	if (!fds[i].revents)
 	    continue;
 	job = jobfds[i];
 	if (job->job_state == JOB_ST_RUNNING)
 	    JobDoOutput(job, FALSE);
+#ifdef USE_META
+	/*
+	 * With meta mode, we may have activity on the job's filemon
+	 * descriptor too, which at the moment is any pollfd other than
+	 * job->inPollfd.
+	 */
+	if (useMeta && job->inPollfd != &fds[i]) {
+	    if (meta_job_event(job) <= 0) {
+		fds[i].events = 0; /* never mind */
+	    }
+	}
+#endif
 	if (--nready == 0)
 		return;
     }
@@ -2271,9 +2301,11 @@ Job_Init(void)
 
     JobCreatePipe(&childExitJob, 3);
 
-    /* We can only need to wait for tokens, children and output from each job */
-    fds = bmake_malloc(sizeof (*fds) * (2 + maxJobs));
-    jobfds = bmake_malloc(sizeof (*jobfds) * (2 + maxJobs));
+    /* Preallocate enough for the maximum number of jobs.  */
+    fds = bmake_malloc(sizeof(*fds) *
+	(npseudojobs + maxJobs) * nfds_per_job());
+    jobfds = bmake_malloc(sizeof(*jobfds) *
+	(npseudojobs + maxJobs) * nfds_per_job());
 
     /* These are permanent entries and take slots 0 and 1 */
     watchfd(&tokenWaitJob);
@@ -2792,6 +2824,14 @@ watchfd(Job *job)
     jobfds[nfds] = job;
     job->inPollfd = &fds[nfds];
     nfds++;
+#ifdef USE_META
+    if (useMeta) {
+	fds[nfds].fd = meta_job_fd(job);
+	fds[nfds].events = fds[nfds].fd == -1 ? 0 : POLLIN;
+	jobfds[nfds] = job;
+	nfds++;
+    }
+#endif
 }
 
 static void
@@ -2802,6 +2842,18 @@ clearfd(Job *job)
 	Punt("Unwatching unwatched job");
     i = job->inPollfd - fds;
     nfds--;
+#ifdef USE_META
+    if (useMeta) {
+	/*
+	 * Sanity check: there should be two fds per job, so the job's
+	 * pollfd number should be even.
+	 */
+	assert(nfds_per_job() == 2);
+	if (i % 2)
+	    Punt("odd-numbered fd with meta");
+	nfds--;
+    }
+#endif
     /*
      * Move last job in table into hole made by dead job.
      */
@@ -2809,6 +2861,12 @@ clearfd(Job *job)
 	fds[i] = fds[nfds];
 	jobfds[i] = jobfds[nfds];
 	jobfds[i]->inPollfd = &fds[i];
+#ifdef USE_META
+	if (useMeta) {
+	    fds[i + 1] = fds[nfds + 1];
+	    jobfds[i + 1] = jobfds[nfds + 1];
+	}
+#endif
     }
     job->inPollfd = NULL;
 }
diff --git a/usr.bin/make/meta.c b/usr.bin/make/meta.c
index c350a655653a..4d146a38ac03 100644
--- a/usr.bin/make/meta.c
+++ b/usr.bin/make/meta.c
@@ -36,7 +36,6 @@
 # include "config.h"
 #endif
 #include <sys/stat.h>
-#include <sys/ioctl.h>
 #include <libgen.h>
 #include <errno.h>
 #if !defined(HAVE_CONFIG_H) || defined(HAVE_ERR_H)
@@ -46,11 +45,8 @@
 #include "make.h"
 #include "job.h"
 
-#ifdef HAVE_FILEMON_H
-# include <filemon.h>
-#endif
-#if !defined(USE_FILEMON) && defined(FILEMON_SET_FD)
-# define USE_FILEMON
+#ifdef USE_FILEMON
+#include "filemon.h"
 #endif
 
 static BuildMon Mybm;			/* for compat */
@@ -117,30 +113,24 @@ extern char    **environ;
  * the benefits are more limited.
  */
 #ifdef USE_FILEMON
-# ifndef _PATH_FILEMON
-#   define _PATH_FILEMON "/dev/filemon"
-# endif
 
 /*
  * Open the filemon device.
  */
 static void
-filemon_open(BuildMon *pbm)
+meta_open_filemon(BuildMon *pbm)
 {
-    int retry;
-    
-    pbm->mon_fd = pbm->filemon_fd = -1;
+    int dupfd;
+
+    pbm->mon_fd = -1;
+    pbm->filemon = NULL;
     if (!useFilemon)
 	return;
 
-    for (retry = 5; retry >= 0; retry--) {
-	if ((pbm->filemon_fd = open(_PATH_FILEMON, O_RDWR)) >= 0)
-	    break;
-    }
-
-    if (pbm->filemon_fd < 0) {
+    pbm->filemon = filemon_open();
+    if (pbm->filemon == NULL) {
 	useFilemon = FALSE;
-	warn("Could not open %s", _PATH_FILEMON);
+	warn("Could not open filemon");
 	return;
     }
 
@@ -151,12 +141,15 @@ filemon_open(BuildMon *pbm)
      * We only care about the descriptor.
      */
     pbm->mon_fd = mkTempFile("filemon.XXXXXX", NULL);
-    if (ioctl(pbm->filemon_fd, FILEMON_SET_FD, &pbm->mon_fd) < 0) {
+    if ((dupfd = dup(pbm->mon_fd)) == -1) {
+	err(1, "Could not dup filemon output!");
+    }
+    (void)fcntl(dupfd, F_SETFD, FD_CLOEXEC);
+    if (filemon_setfd(pbm->filemon, dupfd) == -1) {
 	err(1, "Could not set filemon file descriptor!");
     }
     /* we don't need these once we exec */
     (void)fcntl(pbm->mon_fd, F_SETFD, FD_CLOEXEC);
-    (void)fcntl(pbm->filemon_fd, F_SETFD, FD_CLOEXEC);
 }
 
 /*
@@ -570,7 +563,7 @@ meta_init(void)
 {
 #ifdef USE_FILEMON
 	/* this allows makefiles to test if we have filemon support */
-	Var_Set(".MAKE.PATH_FILEMON", _PATH_FILEMON, VAR_GLOBAL, 0);
+	Var_Set(".MAKE.PATH_FILEMON", "ktrace", VAR_GLOBAL, 0); /* XXX */
 #endif
 }
 
@@ -680,9 +673,10 @@ meta_job_start(Job *job, GNode *gn)
 #endif
 #ifdef USE_FILEMON
     if (pbm->mfp != NULL && useFilemon) {
-	filemon_open(pbm);
+	meta_open_filemon(pbm);
     } else {
-	pbm->mon_fd = pbm->filemon_fd = -1;
+	pbm->mon_fd = -1;
+	pbm->filemon = NULL;
     }
 #endif
 }
@@ -708,7 +702,7 @@ meta_job_child(Job *job)
 	    pid_t pid;
 
 	    pid = getpid();
-	    if (ioctl(pbm->filemon_fd, FILEMON_SET_PID, &pid) < 0) {
+	    if (filemon_setpid_child(pbm->filemon, pid) == -1) {
 		err(1, "Could not set filemon pid!");
 	    }
 	}
@@ -716,6 +710,59 @@ meta_job_child(Job *job)
 #endif
 }
 
+void
+meta_job_parent(Job *job, pid_t pid)
+{
+#ifdef USE_FILEMON
+    BuildMon *pbm;
+
+    if (job != NULL) {
+	pbm = &job->bm;
+    } else {
+	pbm = &Mybm;
+    }
+    if (useFilemon) {
+	filemon_setpid_parent(pbm->filemon, pid);
+    }
+#endif
+}
+
+int
+meta_job_fd(Job *job)
+{
+#ifdef USE_FILEMON
+    BuildMon *pbm;
+
+    if (job != NULL) {
+	pbm = &job->bm;
+    } else {
+	pbm = &Mybm;
+    }
+    if (useFilemon && pbm->filemon) {
+	return filemon_readfd(pbm->filemon);
+    }
+#endif
+    return -1;
+}
+
+int
+meta_job_event(Job *job)
+{
+#ifdef USE_FILEMON
+    BuildMon *pbm;
+
+    if (job != NULL) {
+	pbm = &job->bm;
+    } else {
+	pbm = &Mybm;
+    }
+    if (useFilemon && pbm->filemon) {
+	return filemon_process(pbm->filemon);
+    }
+#endif
+    return 0;
+}
+
 void
 meta_job_error(Job *job, GNode *gn, int flags, int status)
 {
@@ -794,13 +841,16 @@ meta_cmd_finish(void *pbmp)
 	pbm = &Mybm;
 
 #ifdef USE_FILEMON
-    if (pbm->filemon_fd >= 0) {
-	if (close(pbm->filemon_fd) < 0)
+    if (pbm->filemon) {
+	while (filemon_process(pbm->filemon) > 0)
+	    continue;
+	if (filemon_close(pbm->filemon) == -1)
 	    error = errno;
 	x = filemon_read(pbm->mfp, pbm->mon_fd);
 	if (error == 0 && x != 0)
 	    error = x;
-	pbm->filemon_fd = pbm->mon_fd = -1;
+	pbm->mon_fd = -1;
+	pbm->filemon = NULL;
     } else
 #endif
 	fprintf(pbm->mfp, "\n");	/* ensure end with newline */
@@ -1599,9 +1649,10 @@ meta_compat_start(void)
     BuildMon *pbm = &Mybm;
     
     if (pbm->mfp != NULL && useFilemon) {
-	filemon_open(pbm);
+	meta_open_filemon(pbm);
     } else {
-	pbm->mon_fd = pbm->filemon_fd = -1;
+	pbm->mon_fd = -1;
+	pbm->filemon = NULL;
     }
 #endif
     if (pipe(childPipe) < 0)
@@ -1623,19 +1674,56 @@ meta_compat_child(void)
 }
 
 void
-meta_compat_parent(void)
+meta_compat_parent(pid_t child)
 {
-    FILE *fp;
+    int outfd, metafd, maxfd, nfds;
     char buf[BUFSIZ];
-    
+    fd_set readfds;
+
+    meta_job_parent(NULL, child);
     close(childPipe[1]);			/* child side */
-    fp = fdopen(childPipe[0], "r");
-    while (fgets(buf, sizeof(buf), fp)) {
-	meta_job_output(NULL, buf, "");
-	printf("%s", buf);
-	fflush(stdout);
+    outfd = childPipe[0];
+    metafd = filemon_readfd(Mybm.filemon);
+
+    maxfd = -1;
+    if (outfd > maxfd)
+	    maxfd = outfd;
+    if (metafd > maxfd)
+	    maxfd = metafd;
+
+    while (outfd != -1 || metafd != -1) {
+	FD_ZERO(&readfds);
+	if (outfd != -1) {
+	    FD_SET(outfd, &readfds);
+	}
+	if (metafd != -1) {
+	    FD_SET(metafd, &readfds);
+	}
+	nfds = select(maxfd + 1, &readfds, NULL, NULL, NULL);
+	if (nfds == -1) {
+	    if (errno == EINTR)
+		continue;
+	    err(1, "select");
+	}
+
+	if (outfd != -1 && FD_ISSET(outfd, &readfds)) do {
+	    /* XXX this is not line-buffered */
+	    ssize_t nread = read(outfd, buf, sizeof buf);
+	    if (nread == -1)
+		err(1, "read");
+	    if (nread == 0) {
+		close(outfd);
+		outfd = -1;
+		break;
+	    }
+	    fwrite(buf, 1, (size_t)nread, stdout);
+	    fflush(stdout);
+	} while (0);
+	if (metafd != -1 && FD_ISSET(metafd, &readfds)) {
+	    if (meta_job_event(NULL) <= 0)
+		metafd = -1;
+	}
     }
-    fclose(fp);
 }
 
 #endif	/* USE_META */
diff --git a/usr.bin/make/meta.h b/usr.bin/make/meta.h
index 8f1018e89a7d..fc9c9fd07276 100644
--- a/usr.bin/make/meta.h
+++ b/usr.bin/make/meta.h
@@ -33,7 +33,7 @@
 
 typedef struct BuildMon {
     char	meta_fname[MAXPATHLEN];
-    int		filemon_fd;
+    struct filemon *filemon;
     int		mon_fd;
     FILE	*mfp;
 } BuildMon;
@@ -46,6 +46,9 @@ void meta_finish(void);
 void meta_mode_init(const char *);
 void meta_job_start(struct Job *, GNode *);
 void meta_job_child(struct Job *);
+void meta_job_parent(struct Job *, pid_t);
+int  meta_job_fd(struct Job *);
+int  meta_job_event(struct Job *);
 void meta_job_error(struct Job *, GNode *, int, int);
 void meta_job_output(struct Job *, char *, const char *);
 int  meta_cmd_finish(void *);
@@ -53,4 +56,4 @@ int  meta_job_finish(struct Job *);
 Boolean meta_oodate(GNode *, Boolean);
 void meta_compat_start(void);
 void meta_compat_child(void);
-void meta_compat_parent(void);
+void meta_compat_parent(pid_t);

>From f7257a99a0cc4dc7b07e08687cb33497a118adb3 Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastradh%NetBSD.org@localhost>
Date: Sun, 12 Jan 2020 20:30:44 +0000
Subject: [PATCH 2/2] Per sjg's suggestion, split filemon API into separate
 back ends.

By default we use the ktrace back end, but the /dev/filemon back end
is available as a compile-time option, by setting USE_FILEMON=dev in
make.  sjg raised concerns about ktrace performance and would like to
continue using /dev/filemon on FreeBSD (which has seen more
maintenance kernel-side) without forking make.
---
 usr.bin/make/Makefile                         |   5 +-
 usr.bin/make/{ => filemon}/filemon.h          |   3 +
 usr.bin/make/filemon/filemon_dev.c            | 151 ++++++++++++++++++
 .../{filemon.c => filemon/filemon_ktrace.c}   |  30 ++--
 usr.bin/make/meta.c                           |   6 +-
 5 files changed, 178 insertions(+), 17 deletions(-)
 rename usr.bin/make/{ => filemon}/filemon.h (98%)
 create mode 100644 usr.bin/make/filemon/filemon_dev.c
 rename usr.bin/make/{filemon.c => filemon/filemon_ktrace.c} (98%)

diff --git a/usr.bin/make/Makefile b/usr.bin/make/Makefile
index 98c4a13d797d..c715419124f4 100644
--- a/usr.bin/make/Makefile
+++ b/usr.bin/make/Makefile
@@ -17,9 +17,10 @@ USE_META ?= yes
 .if ${USE_META:tl} != "no"
 SRCS+=	meta.c
 CPPFLAGS+= -DUSE_META
-USE_FILEMON ?= yes
+USE_FILEMON ?= ktrace
 .if ${USE_FILEMON:tl} != "no"
-SRCS+=	filemon.c
+.PATH:	${.CURDIR}/filemon
+SRCS+=	filemon_${USE_FILEMON}.c
 CPPFLAGS+= -DUSE_FILEMON
 .endif
 .endif
diff --git a/usr.bin/make/filemon.h b/usr.bin/make/filemon/filemon.h
similarity index 98%
rename from usr.bin/make/filemon.h
rename to usr.bin/make/filemon/filemon.h
index 82b7ca73c72f..bd2d56f804f5 100644
--- a/usr.bin/make/filemon.h
+++ b/usr.bin/make/filemon/filemon.h
@@ -36,6 +36,9 @@
 
 struct filemon;
 
+const char *
+	filemon_path(void);
+
 struct filemon *
 	filemon_open(void);
 int	filemon_close(struct filemon *);
diff --git a/usr.bin/make/filemon/filemon_dev.c b/usr.bin/make/filemon/filemon_dev.c
new file mode 100644
index 000000000000..12c50379f698
--- /dev/null
+++ b/usr.bin/make/filemon/filemon_dev.c
@@ -0,0 +1,151 @@
+/*	$NetBSD$	*/
+
+/*-
+ * Copyright (c) 2020 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "filemon.h"
+
+#include <sys/ioctl.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#ifdef HAVE_FILEMON_H
+#  include <filemon.h>
+#endif
+
+#ifndef _PATH_FILEMON
+#define	_PATH_FILEMON	"/dev/filemon"
+#endif
+
+struct filemon {
+	int	fd;
+};
+
+const char *
+filemon_path(void)
+{
+
+	return _PATH_FILEMON;
+}
+
+struct filemon *
+filemon_open(void)
+{
+	struct filemon *F;
+	unsigned i;
+	int error;
+
+	/* Allocate and zero a struct filemon object.  */
+	F = calloc(1, sizeof(*F));
+	if (F == NULL)
+		return NULL;
+
+	/* Try opening /dev/filemon, up to six times (cargo cult!).  */
+	for (i = 0; (F->fd = open(_PATH_FILEMON, O_RDWR)) == -1; i++) {
+		if (i == 5) {
+			error = errno;
+			goto fail0;
+		}
+	}
+
+	/* Success!  */
+	return F;
+
+fail0:	free(F);
+	errno = error;
+	return NULL;
+}
+
+int
+filemon_setfd(struct filemon *F, int fd)
+{
+
+	/* Point the kernel at this file descriptor.  */
+	if (ioctl(F->fd, FILEMON_SET_FD, &fd) == -1)
+		return -1;
+
+	/* No need for it in userland any more; close it.  */
+	(void)close(fd);
+
+	/* Success!  */
+	return 0;
+}
+
+void
+filemon_setpid_parent(struct filemon *F, pid_t pid)
+{
+	/* Nothing to do!  */
+}
+
+int
+filemon_setpid_child(const struct filemon *F, pid_t pid)
+{
+
+	/* Just pass it on to the kernel.  */
+	return ioctl(F->fd, FILEMON_SET_PID, &pid);
+}
+
+int
+filemon_close(struct filemon *F)
+{
+	int error = 0;
+
+	/* Close the filemon device fd.  */
+	if (close(F->fd) == -1 && error == 0)
+		error = errno;
+
+	/* Free the filemon descriptor.  */
+	free(F);
+
+	/* Set errno and return -1 if anything went wrong.  */
+	if (error) {
+		errno = error;
+		return -1;
+	}
+
+	/* Success!  */
+	return 0;
+}
+
+int
+filemon_readfd(const struct filemon *F)
+{
+
+	return -1;
+}
+
+int
+filemon_process(struct filemon *F)
+{
+
+	return 0;
+}
diff --git a/usr.bin/make/filemon.c b/usr.bin/make/filemon/filemon_ktrace.c
similarity index 98%
rename from usr.bin/make/filemon.c
rename to usr.bin/make/filemon/filemon_ktrace.c
index 5a30afa4980e..aa4a8332f3b5 100644
--- a/usr.bin/make/filemon.c
+++ b/usr.bin/make/filemon/filemon_ktrace.c
@@ -1,7 +1,7 @@
 /*	$NetBSD$	*/
 
 /*-
- * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * Copyright (c) 2020 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
@@ -29,8 +29,6 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifdef USE_FILEMON
-
 #include "filemon.h"
 
 #include <sys/param.h>
@@ -52,8 +50,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-
-#include "make.h"
+#include <unistd.h>
 
 #ifndef AT_CWD
 #define AT_CWD -1
@@ -132,8 +129,7 @@ struct filemon_state {
 };
 
 static int
-compare_filemon_states(void *cookie MAKE_ATTR_UNUSED, const void *na,
-    const void *nb)
+compare_filemon_states(void *cookie, const void *na, const void *nb)
 {
 	const struct filemon_state *Sa = na;
 	const struct filemon_state *Sb = nb;
@@ -150,8 +146,7 @@ compare_filemon_states(void *cookie MAKE_ATTR_UNUSED, const void *na,
 }
 
 static int
-compare_filemon_key(void *cookie MAKE_ATTR_UNUSED, const void *n,
-    const void *k)
+compare_filemon_key(void *cookie, const void *n, const void *k)
 {
 	const struct filemon_state *S = n;
 	const struct filemon_key *key = k;
@@ -174,6 +169,19 @@ static const rb_tree_ops_t filemon_rb_ops = {
 	.rbto_context = NULL,
 };
 
+/*
+ * filemon_path()
+ *
+ *	Return a pointer to a constant string denoting the `path' of
+ *	the filemon.
+ */
+const char *
+filemon_path(void)
+{
+
+	return "ktrace";
+}
+
 /*
  * filemon_open()
  *
@@ -572,7 +580,7 @@ top:	/* If the child has exited, nothing to do.  */
 }
 
 static struct filemon_state *
-syscall_enter(struct filemon *F MAKE_ATTR_UNUSED,
+syscall_enter(struct filemon *F,
     const struct filemon_key *key, const struct ktr_syscall *call,
     unsigned npath,
     void (*show)(struct filemon *, const struct filemon_state *,
@@ -866,5 +874,3 @@ filemon_sys_rename(struct filemon *F, const struct filemon_key *key,
 {
 	return syscall_enter(F, key, call, 2, &show_rename);
 }
-
-#endif	/* USE_META */
diff --git a/usr.bin/make/meta.c b/usr.bin/make/meta.c
index 4d146a38ac03..c0cda600642d 100644
--- a/usr.bin/make/meta.c
+++ b/usr.bin/make/meta.c
@@ -46,7 +46,7 @@
 #include "job.h"
 
 #ifdef USE_FILEMON
-#include "filemon.h"
+#include "filemon/filemon.h"
 #endif
 
 static BuildMon Mybm;			/* for compat */
@@ -130,7 +130,7 @@ meta_open_filemon(BuildMon *pbm)
     pbm->filemon = filemon_open();
     if (pbm->filemon == NULL) {
 	useFilemon = FALSE;
-	warn("Could not open filemon");
+	warn("Could not open filemon %s", filemon_path());
 	return;
     }
 
@@ -563,7 +563,7 @@ meta_init(void)
 {
 #ifdef USE_FILEMON
 	/* this allows makefiles to test if we have filemon support */
-	Var_Set(".MAKE.PATH_FILEMON", "ktrace", VAR_GLOBAL, 0); /* XXX */
+	Var_Set(".MAKE.PATH_FILEMON", filemon_path(), VAR_GLOBAL, 0);
 #endif
 }
 

diff --git a/usr.bin/make/Makefile b/usr.bin/make/Makefile
index bb0ce15f967b..c715419124f4 100644
--- a/usr.bin/make/Makefile
+++ b/usr.bin/make/Makefile
@@ -17,9 +17,11 @@ USE_META ?= yes
 .if ${USE_META:tl} != "no"
 SRCS+=	meta.c
 CPPFLAGS+= -DUSE_META
-FILEMON_H ?= ${.CURDIR:H:H}/sys/dev/filemon/filemon.h
-.if exists(${FILEMON_H}) && ${FILEMON_H:T} == "filemon.h"
-COPTS.meta.c += -DHAVE_FILEMON_H -I${FILEMON_H:H}
+USE_FILEMON ?= ktrace
+.if ${USE_FILEMON:tl} != "no"
+.PATH:	${.CURDIR}/filemon
+SRCS+=	filemon_${USE_FILEMON}.c
+CPPFLAGS+= -DUSE_FILEMON
 .endif
 .endif
 
diff --git a/usr.bin/make/compat.c b/usr.bin/make/compat.c
index 9e34e6e351e8..14affa0b552e 100644
--- a/usr.bin/make/compat.c
+++ b/usr.bin/make/compat.c
@@ -404,7 +404,7 @@ again:
 
 #ifdef USE_META
     if (useMeta) {
-	meta_compat_parent();
+	meta_compat_parent(cpid);
     }
 #endif
 
diff --git a/usr.bin/make/filemon/filemon.h b/usr.bin/make/filemon/filemon.h
new file mode 100644
index 000000000000..bd2d56f804f5
--- /dev/null
+++ b/usr.bin/make/filemon/filemon.h
@@ -0,0 +1,53 @@
+/*	$NetBSD$	*/
+
+/*-
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef FILEMON_H
+#define	FILEMON_H
+
+#include <sys/types.h>
+
+struct filemon;
+
+const char *
+	filemon_path(void);
+
+struct filemon *
+	filemon_open(void);
+int	filemon_close(struct filemon *);
+
+int	filemon_setfd(struct filemon *, int);
+void	filemon_setpid_parent(struct filemon *, pid_t);
+int	filemon_setpid_child(const struct filemon *, pid_t);
+
+int	filemon_readfd(const struct filemon *);
+int	filemon_process(struct filemon *);
+
+#endif	/* FILEMON_H */
diff --git a/usr.bin/make/filemon/filemon_dev.c b/usr.bin/make/filemon/filemon_dev.c
new file mode 100644
index 000000000000..12c50379f698
--- /dev/null
+++ b/usr.bin/make/filemon/filemon_dev.c
@@ -0,0 +1,151 @@
+/*	$NetBSD$	*/
+
+/*-
+ * Copyright (c) 2020 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "filemon.h"
+
+#include <sys/ioctl.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#ifdef HAVE_FILEMON_H
+#  include <filemon.h>
+#endif
+
+#ifndef _PATH_FILEMON
+#define	_PATH_FILEMON	"/dev/filemon"
+#endif
+
+struct filemon {
+	int	fd;
+};
+
+const char *
+filemon_path(void)
+{
+
+	return _PATH_FILEMON;
+}
+
+struct filemon *
+filemon_open(void)
+{
+	struct filemon *F;
+	unsigned i;
+	int error;
+
+	/* Allocate and zero a struct filemon object.  */
+	F = calloc(1, sizeof(*F));
+	if (F == NULL)
+		return NULL;
+
+	/* Try opening /dev/filemon, up to six times (cargo cult!).  */
+	for (i = 0; (F->fd = open(_PATH_FILEMON, O_RDWR)) == -1; i++) {
+		if (i == 5) {
+			error = errno;
+			goto fail0;
+		}
+	}
+
+	/* Success!  */
+	return F;
+
+fail0:	free(F);
+	errno = error;
+	return NULL;
+}
+
+int
+filemon_setfd(struct filemon *F, int fd)
+{
+
+	/* Point the kernel at this file descriptor.  */
+	if (ioctl(F->fd, FILEMON_SET_FD, &fd) == -1)
+		return -1;
+
+	/* No need for it in userland any more; close it.  */
+	(void)close(fd);
+
+	/* Success!  */
+	return 0;
+}
+
+void
+filemon_setpid_parent(struct filemon *F, pid_t pid)
+{
+	/* Nothing to do!  */
+}
+
+int
+filemon_setpid_child(const struct filemon *F, pid_t pid)
+{
+
+	/* Just pass it on to the kernel.  */
+	return ioctl(F->fd, FILEMON_SET_PID, &pid);
+}
+
+int
+filemon_close(struct filemon *F)
+{
+	int error = 0;
+
+	/* Close the filemon device fd.  */
+	if (close(F->fd) == -1 && error == 0)
+		error = errno;
+
+	/* Free the filemon descriptor.  */
+	free(F);
+
+	/* Set errno and return -1 if anything went wrong.  */
+	if (error) {
+		errno = error;
+		return -1;
+	}
+
+	/* Success!  */
+	return 0;
+}
+
+int
+filemon_readfd(const struct filemon *F)
+{
+
+	return -1;
+}
+
+int
+filemon_process(struct filemon *F)
+{
+
+	return 0;
+}
diff --git a/usr.bin/make/filemon/filemon_ktrace.c b/usr.bin/make/filemon/filemon_ktrace.c
new file mode 100644
index 000000000000..aa4a8332f3b5
--- /dev/null
+++ b/usr.bin/make/filemon/filemon_ktrace.c
@@ -0,0 +1,876 @@
+/*	$NetBSD$	*/
+
+/*-
+ * Copyright (c) 2020 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "filemon.h"
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/rbtree.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/uio.h>
+#include <sys/wait.h>
+
+#include <sys/ktrace.h>
+
+#include <assert.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#ifndef AT_CWD
+#define AT_CWD -1
+#endif
+
+struct filemon;
+struct filemon_key;
+struct filemon_state;
+
+typedef struct filemon_state *filemon_syscall_t(struct filemon *,
+    const struct filemon_key *, const struct ktr_syscall *);
+
+static filemon_syscall_t filemon_sys_chdir;
+static filemon_syscall_t filemon_sys_execve;
+static filemon_syscall_t filemon_sys_exit;
+static filemon_syscall_t filemon_sys_fork;
+static filemon_syscall_t filemon_sys_link;
+static filemon_syscall_t filemon_sys_open;
+static filemon_syscall_t filemon_sys_openat;
+static filemon_syscall_t filemon_sys_symlink;
+static filemon_syscall_t filemon_sys_unlink;
+static filemon_syscall_t filemon_sys_rename;
+
+static filemon_syscall_t *const filemon_syscalls[] = {
+	[SYS_chdir] = &filemon_sys_chdir,
+	[SYS_execve] = &filemon_sys_execve,
+	[SYS_exit] = &filemon_sys_exit,
+	[SYS_fork] = &filemon_sys_fork,
+	[SYS_link] = &filemon_sys_link,
+	[SYS_open] = &filemon_sys_open,
+	[SYS_openat] = &filemon_sys_openat,
+	[SYS_symlink] = &filemon_sys_symlink,
+	[SYS_unlink] = &filemon_sys_unlink,
+	[SYS_rename] = &filemon_sys_rename,
+};
+
+struct filemon {
+	int			ktrfd; /* kernel writes ktrace events here */
+	FILE			*in;   /* we read ktrace events from here */
+	FILE			*out;  /* we write filemon events to here */
+	rb_tree_t		active;
+	pid_t			child;
+
+	/* I/O state machine.  */
+	enum {
+		FILEMON_START = 0,
+		FILEMON_HEADER,
+		FILEMON_PAYLOAD,
+		FILEMON_ERROR,
+	}			state;
+	unsigned char		*p;
+	size_t			resid;
+
+	/* I/O buffer.  */
+	struct ktr_header	hdr;
+	union {
+		struct ktr_syscall	syscall;
+		struct ktr_sysret	sysret;
+		char			namei[PATH_MAX];
+		unsigned char		buf[4096];
+	}			payload;
+};
+
+struct filemon_state {
+	struct filemon_key {
+		pid_t		pid;
+		lwpid_t		lid;
+	}		key;
+	struct rb_node	node;
+	int		syscode;
+	void		(*show)(struct filemon *, const struct filemon_state *,
+			    const struct ktr_sysret *);
+	unsigned	i;
+	unsigned	npath;
+	char		*path[/*npath*/];
+};
+
+static int
+compare_filemon_states(void *cookie, const void *na, const void *nb)
+{
+	const struct filemon_state *Sa = na;
+	const struct filemon_state *Sb = nb;
+
+	if (Sa->key.pid < Sb->key.pid)
+		return -1;
+	if (Sa->key.pid > Sb->key.pid)
+		return +1;
+	if (Sa->key.lid < Sb->key.lid)
+		return -1;
+	if (Sa->key.lid > Sb->key.lid)
+		return +1;
+	return 0;
+}
+
+static int
+compare_filemon_key(void *cookie, const void *n, const void *k)
+{
+	const struct filemon_state *S = n;
+	const struct filemon_key *key = k;
+
+	if (S->key.pid < key->pid)
+		return -1;
+	if (S->key.pid > key->pid)
+		return +1;
+	if (S->key.lid < key->lid)
+		return -1;
+	if (S->key.lid > key->lid)
+		return +1;
+	return 0;
+}
+
+static const rb_tree_ops_t filemon_rb_ops = {
+	.rbto_compare_nodes = &compare_filemon_states,
+	.rbto_compare_key = &compare_filemon_key,
+	.rbto_node_offset = offsetof(struct filemon_state, node),
+	.rbto_context = NULL,
+};
+
+/*
+ * filemon_path()
+ *
+ *	Return a pointer to a constant string denoting the `path' of
+ *	the filemon.
+ */
+const char *
+filemon_path(void)
+{
+
+	return "ktrace";
+}
+
+/*
+ * filemon_open()
+ *
+ *	Allocate a filemon descriptor.  Returns NULL and sets errno on
+ *	failure.
+ */
+struct filemon *
+filemon_open(void)
+{
+	struct filemon *F;
+	int ktrpipe[2];
+	int error;
+
+	/* Allocate and zero a struct filemon object.  */
+	F = calloc(1, sizeof(*F));
+	if (F == NULL)
+		return NULL;
+
+	/* Create a pipe for ktrace events.  */
+	if (pipe2(ktrpipe, O_CLOEXEC|O_NONBLOCK) == -1) {
+		error = errno;
+		goto fail0;
+	}
+
+	/* Create a file stream for reading the ktrace events.  */
+	if ((F->in = fdopen(ktrpipe[0], "r")) == NULL) {
+		error = errno;
+		goto fail1;
+	}
+	ktrpipe[0] = -1;	/* claimed by fdopen */
+
+	/*
+	 * Set the fd for writing ktrace events and initialize the
+	 * rbtree.  The rest can be safely initialized to zero.
+	 */
+	F->ktrfd = ktrpipe[1];
+	rb_tree_init(&F->active, &filemon_rb_ops);
+
+	/* Success!  */
+	return F;
+
+fail2: __unused
+	(void)fclose(F->in);
+fail1:	(void)close(ktrpipe[0]);
+	(void)close(ktrpipe[1]);
+fail0:	free(F);
+	errno = error;
+	return NULL;
+}
+
+/*
+ * filemon_closefd(F)
+ *
+ *	Internal subroutine to try to flush and close the output file.
+ *	If F is not open for output, do nothing.  Never leaves F open
+ *	for output even on failure.  Returns 0 on success; sets errno
+ *	and return -1 on failure.
+ */
+static int
+filemon_closefd(struct filemon *F)
+{
+	int error = 0;
+
+	/* If we're not open, nothing to do.  */
+	if (F->out == NULL)
+		return 0;
+
+	/*
+	 * Flush it, close it, and null it unconditionally, but be
+	 * careful to return the earliest error in errno.
+	 */
+	if (fflush(F->out) == EOF && error == 0)
+		error = errno;
+	if (fclose(F->out) == EOF && error == 0)
+		error = errno;
+	F->out = NULL;
+
+	/* Set errno and return -1 if anything went wrong.  */
+	if (error) {
+		errno = error;
+		return -1;
+	}
+
+	/* Success!  */
+	return 0;
+}
+
+/*
+ * filemon_setfd(F, fd)
+ *
+ *	Cause filemon activity on F to be sent to fd.  Claims ownership
+ *	of fd; caller should not use fd afterward, and any duplicates
+ *	of fd may see their file positions changed.
+ */
+int
+filemon_setfd(struct filemon *F, int fd)
+{
+
+	/*
+	 * Close an existing output file if done.  Fail now if there's
+	 * an error closing.
+	 */
+	if ((filemon_closefd(F)) == -1)
+		return -1;
+	assert(F->out == NULL);
+
+	/* Open a file stream and claim ownership of the fd.  */
+	if ((F->out = fdopen(fd, "a")) == NULL)
+		return -1;
+
+	/*
+	 * Print the opening output.  Any failure will be deferred
+	 * until closing.  For hysterical raisins, we show the parent
+	 * pid, not the child pid.
+	 */
+	fprintf(F->out, "# filemon version 4\n");
+	fprintf(F->out, "# Target pid %jd\n", (intmax_t)getpid());
+	fprintf(F->out, "V 4\n");
+
+	/* Success!  */
+	return 0;
+}
+
+/*
+ * filemon_setpid_parent(F, pid)
+ *
+ *	Set the traced pid, from the parent.  Never fails.
+ */
+void
+filemon_setpid_parent(struct filemon *F, pid_t pid)
+{
+
+	F->child = pid;
+}
+
+/*
+ * filemon_setpid_child(F, pid)
+ *
+ *	Set the traced pid, from the child.  Returns 0 on success; sets
+ *	errno and returns -1 on failure.
+ */
+int
+filemon_setpid_child(const struct filemon *F, pid_t pid)
+{
+	int ops, trpoints;
+
+	ops = KTROP_SET|KTRFLAG_DESCEND;
+	trpoints = KTRFACv2;
+	trpoints |= KTRFAC_SYSCALL|KTRFAC_NAMEI|KTRFAC_SYSRET;
+	trpoints |= KTRFAC_INHERIT;
+	if (fktrace(F->ktrfd, ops, trpoints, pid) == -1)
+		return -1;
+
+	return 0;
+}
+
+/*
+ * filemon_close(F)
+ *
+ *	Close F for output if necessary, and free a filemon descriptor.
+ *	Returns 0 on success; sets errno and returns -1 on failure, but
+ *	frees the filemon descriptor either way;
+ */
+int
+filemon_close(struct filemon *F)
+{
+	struct filemon_state *S;
+	int error = 0;
+
+	/* Close for output.  */
+	if (filemon_closefd(F) == -1 && error == 0)
+		error = errno;
+
+	/* Close the ktrace pipe.  */
+	if (fclose(F->in) == EOF && error == 0)
+		error = errno;
+	if (close(F->ktrfd) == -1 && error == 0)
+		error = errno;
+
+	/* Free any active records.  */
+	while ((S = RB_TREE_MIN(&F->active)) != NULL) {
+		rb_tree_remove_node(&F->active, S);
+		free(S);
+	}
+
+	/* Free the filemon descriptor.  */
+	free(F);
+
+	/* Set errno and return -1 if anything went wrong.  */
+	if (error) {
+		errno = error;
+		return -1;
+	}
+
+	/* Success!  */
+	return 0;
+}
+
+/*
+ * filemon_readfd(F)
+ *
+ *	Returns a file descriptor which will select/poll ready for read
+ *	when there are filemon events to be processed by
+ *	filemon_process, or -1 if anything has gone wrong.
+ */
+int
+filemon_readfd(const struct filemon *F)
+{
+
+	if (F->state == FILEMON_ERROR)
+		return -1;
+	return fileno(F->in);
+}
+
+/*
+ * filemon_dispatch(F)
+ *
+ *	Internal subroutine to dispatch a filemon ktrace event.
+ *	Silently ignore events that we don't recognize.
+ */
+static void
+filemon_dispatch(struct filemon *F)
+{
+	const struct filemon_key key = {
+		.pid = F->hdr.ktr_pid,
+		.lid = F->hdr.ktr_lid,
+	};
+	struct filemon_state *S;
+
+	switch (F->hdr.ktr_type) {
+	case KTR_SYSCALL: {
+		struct ktr_syscall *call = &F->payload.syscall;
+		struct filemon_state *S1;
+
+		/* Validate the syscall code.  */
+		if (call->ktr_code < 0 ||
+		    (size_t)call->ktr_code >= __arraycount(filemon_syscalls) ||
+		    filemon_syscalls[call->ktr_code] == NULL)
+			break;
+
+		/*
+		 * Invoke the syscall-specific logic to create a new
+		 * active state.
+		 */
+		S = (*filemon_syscalls[call->ktr_code])(F, &key, call);
+		if (S == NULL)
+			break;
+
+		/*
+		 * Insert the active state, or ignore it if there
+		 * already is one.
+		 *
+		 * Collisions shouldn't happen because the states are
+		 * keyed by <pid,lid>, in which syscalls should happen
+		 * sequentially in CALL/RET pairs, but let's be
+		 * defensive.
+		 */
+		S1 = rb_tree_insert_node(&F->active, S);
+		if (S1 != S) {
+			/* XXX Which one to drop?  */
+			free(S);
+			break;
+		}
+		break;
+	}
+	case KTR_NAMEI:
+		/* Find an active syscall state, or drop it.  */
+		S = rb_tree_find_node(&F->active, &key);
+		if (S == NULL)
+			break;
+		/* Find the position of the next path, or drop it.  */
+		if (S->i >= S->npath)
+			break;
+		/* Record the path.  */
+		S->path[S->i++] = strndup(F->payload.namei,
+		    sizeof F->payload.namei);
+		break;
+	case KTR_SYSRET: {
+		struct ktr_sysret *ret = &F->payload.sysret;
+		unsigned i;
+
+		/* Find and remove an active syscall state, or drop it.  */
+		S = rb_tree_find_node(&F->active, &key);
+		if (S == NULL)
+			break;
+		rb_tree_remove_node(&F->active, S);
+
+		/*
+		 * If the active syscall state matches this return,
+		 * invoke the syscall-specific logic to show a filemon
+		 * event.
+		 */
+		/* XXX What to do if syscall code doesn't match?  */
+		if (S->i == S->npath && S->syscode == ret->ktr_code)
+			(*S->show)(F, S, ret);
+
+		/* Free the state now that it is no longer active.  */
+		for (i = 0; i < S->i; i++)
+			free(S->path[i]);
+		free(S);
+		break;
+	}
+	default:
+		/* Ignore all other ktrace events.  */
+		break;
+	}
+}
+
+/*
+ * filemon_process(F)
+ *
+ *	Process all pending events after filemon_readfd(F) has
+ *	selected/polled ready for read.
+ *
+ *	Returns -1 on failure, 0 on end of events, and anything else if
+ *	there may be more events.
+ *
+ *	XXX What about fairness to other activities in the event loop?
+ *	If we stop while there's events buffered in F->in, then select
+ *	or poll may not return ready even though there's work queued up
+ *	in the buffer of F->in, but if we don't stop then ktrace events
+ *	may overwhelm all other activity in the event loop.
+ */
+int
+filemon_process(struct filemon *F)
+{
+	size_t nread;
+
+top:	/* If the child has exited, nothing to do.  */
+	/* XXX What if one thread calls exit while another is running?  */
+	if (F->child == 0)
+		return 0;
+
+	/* If we're waiting for input, read some.  */
+	if (F->resid) {
+		nread = fread(F->p, 1, F->resid, F->in);
+		if (nread == 0) {
+			if (feof(F->in))
+				return 0;
+			assert(ferror(F->in));
+			/*
+			 * If interrupted or would block, there may be
+			 * more events.  Otherwise fail.
+			 */
+			if (errno == EAGAIN || errno == EINTR)
+				return 1;
+			F->state = FILEMON_ERROR;
+			F->p = NULL;
+			F->resid = 0;
+			return -1;
+		}
+		assert(nread <= F->resid);
+		F->p += nread;
+		F->resid -= nread;
+		if (F->resid)	/* may be more events */
+			return 1;
+	}
+
+	/* Process a state transition now that we've read a buffer.  */
+	switch (F->state) {
+	case FILEMON_START:	/* just started filemon; read header next */
+		F->state = FILEMON_HEADER;
+		F->p = (void *)&F->hdr;
+		F->resid = sizeof F->hdr;
+		goto top;
+	case FILEMON_HEADER:	/* read header */
+		/* Sanity-check ktrace header; then read payload.  */
+		if (F->hdr.ktr_len < 0 ||
+		    (size_t)F->hdr.ktr_len > sizeof F->payload) {
+			F->state = FILEMON_ERROR;
+			F->p = NULL;
+			F->resid = 0;
+			errno = EIO;
+			return -1;
+		}
+		F->state = FILEMON_PAYLOAD;
+		F->p = (void *)&F->payload;
+		F->resid = (size_t)F->hdr.ktr_len;
+		goto top;
+	case FILEMON_PAYLOAD:	/* read header and payload */
+		/* Dispatch ktrace event; then read next header.  */
+		filemon_dispatch(F);
+		F->state = FILEMON_HEADER;
+		F->p = (void *)&F->hdr;
+		F->resid = sizeof F->hdr;
+		goto top;
+	default:		/* paranoia */
+		F->state = FILEMON_ERROR;
+		/*FALLTHROUGH*/
+	case FILEMON_ERROR:	/* persistent error indicator */
+		F->p = NULL;
+		F->resid = 0;
+		errno = EIO;
+		return -1;
+	}
+}
+
+static struct filemon_state *
+syscall_enter(struct filemon *F,
+    const struct filemon_key *key, const struct ktr_syscall *call,
+    unsigned npath,
+    void (*show)(struct filemon *, const struct filemon_state *,
+	const struct ktr_sysret *))
+{
+	struct filemon_state *S;
+	unsigned i;
+
+	S = calloc(1, offsetof(struct filemon_state, path[npath]));
+	if (S == NULL)
+		return NULL;
+	S->key = *key;
+	S->show = show;
+	S->syscode = call->ktr_code;
+	S->i = 0;
+	S->npath = npath;
+	for (i = 0; i < npath; i++)
+		 S->path[i] = NULL; /* paranoia */
+
+	return S;
+}
+
+static void
+show_paths(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret, const char *prefix)
+{
+	unsigned i;
+
+	/* Caller must ensure all paths have been specified.  */
+	assert(S->i == S->npath);
+
+	/*
+	 * Ignore it if it failed or yielded EJUSTRETURN (-2), or if
+	 * we're not producing output.
+	 */
+	if (ret->ktr_error && ret->ktr_error != -2)
+		return;
+	if (F->out == NULL)
+		return;
+
+	/*
+	 * Print the prefix, pid, and paths -- with the paths quoted if
+	 * there's more than one.
+	 */
+	fprintf(F->out, "%s %jd", prefix, (intmax_t)S->key.pid);
+	for (i = 0; i < S->npath; i++) {
+		const char *q = S->npath > 1 ? "'" : "";
+		fprintf(F->out, " %s%s%s", q, S->path[i], q);
+	}
+	fprintf(F->out, "\n");
+}
+
+static void
+show_retval(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret, const char *prefix)
+{
+
+	/*
+	 * Ignore it if it failed or yielded EJUSTRETURN (-2), or if
+	 * we're not producing output.
+	 */
+	if (ret->ktr_error && ret->ktr_error != -2)
+		return;
+	if (F->out == NULL)
+		return;
+
+	fprintf(F->out, "%s %jd %jd\n", prefix, (intmax_t)S->key.pid,
+	    (intmax_t)ret->ktr_retval);
+}
+
+static void
+show_chdir(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "C");
+}
+
+static void
+show_execve(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	return show_paths(F, S, ret, "E");
+}
+
+static void
+show_fork(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_retval(F, S, ret, "F");
+}
+
+static void
+show_link(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "L"); /* XXX same as symlink */
+}
+
+static void
+show_open_read(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "R");
+}
+
+static void
+show_open_write(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "W");
+}
+
+static void
+show_open_readwrite(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "R");
+	show_paths(F, S, ret, "W");
+}
+
+static void
+show_openat_read(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	if (S->path[0][0] != '/')
+		show_paths(F, S, ret, "A");
+	show_paths(F, S, ret, "R");
+}
+
+static void
+show_openat_write(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	if (S->path[0][0] != '/')
+		show_paths(F, S, ret, "A");
+	show_paths(F, S, ret, "W");
+}
+
+static void
+show_openat_readwrite(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	if (S->path[0][0] != '/')
+		show_paths(F, S, ret, "A");
+	show_paths(F, S, ret, "R");
+	show_paths(F, S, ret, "W");
+}
+
+static void
+show_symlink(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "L"); /* XXX same as link */
+}
+
+static void
+show_unlink(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "D");
+}
+
+static void
+show_rename(struct filemon *F, const struct filemon_state *S,
+    const struct ktr_sysret *ret)
+{
+	show_paths(F, S, ret, "M");
+}
+
+static struct filemon_state *
+filemon_sys_chdir(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 1, &show_chdir);
+}
+
+static struct filemon_state *
+filemon_sys_execve(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 1, &show_execve);
+}
+
+static struct filemon_state *
+filemon_sys_exit(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	const register_t *args = (const void *)&call[1];
+	int status = args[0];
+
+	if (F->out) {
+		fprintf(F->out, "X %jd %d\n", (intmax_t)key->pid, status);
+		if (key->pid == F->child) {
+			fprintf(F->out, "# Bye bye\n");
+			F->child = 0;
+		}
+	}
+	return NULL;
+}
+
+static struct filemon_state *
+filemon_sys_fork(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 0, &show_fork);
+}
+
+static struct filemon_state *
+filemon_sys_link(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 2, &show_link);
+}
+
+static struct filemon_state *
+filemon_sys_open(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	const register_t *args = (const void *)&call[1];
+	int flags;
+
+	if (call->ktr_argsize < 2)
+		return NULL;
+	flags = args[1];
+
+	if ((flags & O_RDWR) == O_RDWR)
+		return syscall_enter(F, key, call, 1, &show_open_readwrite);
+	else if ((flags & O_WRONLY) == O_WRONLY)
+		return syscall_enter(F, key, call, 1, &show_open_write);
+	else if ((flags & O_RDONLY) == O_RDONLY)
+		return syscall_enter(F, key, call, 1, &show_open_read);
+	else
+		return NULL;	/* XXX Do we care if no read or write?  */
+}
+
+static struct filemon_state *
+filemon_sys_openat(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	const register_t *args = (const void *)&call[1];
+	int flags, fd;
+
+	if (call->ktr_argsize < 3)
+		return NULL;
+	fd = args[0];
+	flags = args[2];
+
+	if (fd == AT_CWD) {
+		if ((flags & O_RDWR) == O_RDWR)
+			return syscall_enter(F, key, call, 1,
+			    &show_open_readwrite);
+		else if ((flags & O_WRONLY) == O_WRONLY)
+			return syscall_enter(F, key, call, 1,
+			    &show_open_write);
+		else if ((flags & O_RDONLY) == O_RDONLY)
+			return syscall_enter(F, key, call, 1, &show_open_read);
+		else
+			return NULL;
+	} else {
+		if ((flags & O_RDWR) == O_RDWR)
+			return syscall_enter(F, key, call, 1,
+			    &show_openat_readwrite);
+		else if ((flags & O_WRONLY) == O_WRONLY)
+			return syscall_enter(F, key, call, 1,
+			    &show_openat_write);
+		else if ((flags & O_RDONLY) == O_RDONLY)
+			return syscall_enter(F, key, call, 1,
+			    &show_openat_read);
+		else
+			return NULL;
+	}
+}
+
+static struct filemon_state *
+filemon_sys_symlink(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 2, &show_symlink);
+}
+
+static struct filemon_state *
+filemon_sys_unlink(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 1, &show_unlink);
+}
+
+static struct filemon_state *
+filemon_sys_rename(struct filemon *F, const struct filemon_key *key,
+    const struct ktr_syscall *call)
+{
+	return syscall_enter(F, key, call, 2, &show_rename);
+}
diff --git a/usr.bin/make/job.c b/usr.bin/make/job.c
index 48ec2840e8bf..dac04206aab4 100644
--- a/usr.bin/make/job.c
+++ b/usr.bin/make/job.c
@@ -329,6 +329,8 @@ static Job childExitJob;	/* child exit pseudo-job */
 #define	CHILD_EXIT	"."
 #define	DO_JOB_RESUME	"R"
 
+static const int npseudojobs = 2; /* number of pseudo-jobs */
+
 #define TARG_FMT  "%s %s ---\n" /* Default format */
 #define MESSAGE(fp, gn) \
 	if (maxJobs != 1 && targPrefix && *targPrefix) \
@@ -357,6 +359,16 @@ static void JobSigReset(void);
 
 const char *malloc_options="A";
 
+static unsigned
+nfds_per_job(void)
+{
+#ifdef USE_META
+    if (useMeta)
+	return 2;
+#endif
+    return 1;
+}
+
 static void
 job_table_dump(const char *where)
 {
@@ -1439,6 +1451,12 @@ JobExec(Job *job, char **argv)
 
     Trace_Log(JOBSTART, job);
 
+#ifdef USE_META
+    if (useMeta) {
+	meta_job_parent(job, cpid);
+    }
+#endif
+
     /*
      * Set the current position in the buffer to the beginning
      * and mark another stream to watch in the outputs mask
@@ -2121,12 +2139,24 @@ Job_CatchOutput(void)
     if (nready == 0)
 	    return;
 
-    for (i = 2; i < nfds; i++) {
+    for (i = npseudojobs*nfds_per_job(); i < nfds; i++) {
 	if (!fds[i].revents)
 	    continue;
 	job = jobfds[i];
 	if (job->job_state == JOB_ST_RUNNING)
 	    JobDoOutput(job, FALSE);
+#ifdef USE_META
+	/*
+	 * With meta mode, we may have activity on the job's filemon
+	 * descriptor too, which at the moment is any pollfd other than
+	 * job->inPollfd.
+	 */
+	if (useMeta && job->inPollfd != &fds[i]) {
+	    if (meta_job_event(job) <= 0) {
+		fds[i].events = 0; /* never mind */
+	    }
+	}
+#endif
 	if (--nready == 0)
 		return;
     }
@@ -2271,9 +2301,11 @@ Job_Init(void)
 
     JobCreatePipe(&childExitJob, 3);
 
-    /* We can only need to wait for tokens, children and output from each job */
-    fds = bmake_malloc(sizeof (*fds) * (2 + maxJobs));
-    jobfds = bmake_malloc(sizeof (*jobfds) * (2 + maxJobs));
+    /* Preallocate enough for the maximum number of jobs.  */
+    fds = bmake_malloc(sizeof(*fds) *
+	(npseudojobs + maxJobs) * nfds_per_job());
+    jobfds = bmake_malloc(sizeof(*jobfds) *
+	(npseudojobs + maxJobs) * nfds_per_job());
 
     /* These are permanent entries and take slots 0 and 1 */
     watchfd(&tokenWaitJob);
@@ -2792,6 +2824,14 @@ watchfd(Job *job)
     jobfds[nfds] = job;
     job->inPollfd = &fds[nfds];
     nfds++;
+#ifdef USE_META
+    if (useMeta) {
+	fds[nfds].fd = meta_job_fd(job);
+	fds[nfds].events = fds[nfds].fd == -1 ? 0 : POLLIN;
+	jobfds[nfds] = job;
+	nfds++;
+    }
+#endif
 }
 
 static void
@@ -2802,6 +2842,18 @@ clearfd(Job *job)
 	Punt("Unwatching unwatched job");
     i = job->inPollfd - fds;
     nfds--;
+#ifdef USE_META
+    if (useMeta) {
+	/*
+	 * Sanity check: there should be two fds per job, so the job's
+	 * pollfd number should be even.
+	 */
+	assert(nfds_per_job() == 2);
+	if (i % 2)
+	    Punt("odd-numbered fd with meta");
+	nfds--;
+    }
+#endif
     /*
      * Move last job in table into hole made by dead job.
      */
@@ -2809,6 +2861,12 @@ clearfd(Job *job)
 	fds[i] = fds[nfds];
 	jobfds[i] = jobfds[nfds];
 	jobfds[i]->inPollfd = &fds[i];
+#ifdef USE_META
+	if (useMeta) {
+	    fds[i + 1] = fds[nfds + 1];
+	    jobfds[i + 1] = jobfds[nfds + 1];
+	}
+#endif
     }
     job->inPollfd = NULL;
 }
diff --git a/usr.bin/make/meta.c b/usr.bin/make/meta.c
index c350a655653a..c0cda600642d 100644
--- a/usr.bin/make/meta.c
+++ b/usr.bin/make/meta.c
@@ -36,7 +36,6 @@
 # include "config.h"
 #endif
 #include <sys/stat.h>
-#include <sys/ioctl.h>
 #include <libgen.h>
 #include <errno.h>
 #if !defined(HAVE_CONFIG_H) || defined(HAVE_ERR_H)
@@ -46,11 +45,8 @@
 #include "make.h"
 #include "job.h"
 
-#ifdef HAVE_FILEMON_H
-# include <filemon.h>
-#endif
-#if !defined(USE_FILEMON) && defined(FILEMON_SET_FD)
-# define USE_FILEMON
+#ifdef USE_FILEMON
+#include "filemon/filemon.h"
 #endif
 
 static BuildMon Mybm;			/* for compat */
@@ -117,30 +113,24 @@ extern char    **environ;
  * the benefits are more limited.
  */
 #ifdef USE_FILEMON
-# ifndef _PATH_FILEMON
-#   define _PATH_FILEMON "/dev/filemon"
-# endif
 
 /*
  * Open the filemon device.
  */
 static void
-filemon_open(BuildMon *pbm)
+meta_open_filemon(BuildMon *pbm)
 {
-    int retry;
-    
-    pbm->mon_fd = pbm->filemon_fd = -1;
+    int dupfd;
+
+    pbm->mon_fd = -1;
+    pbm->filemon = NULL;
     if (!useFilemon)
 	return;
 
-    for (retry = 5; retry >= 0; retry--) {
-	if ((pbm->filemon_fd = open(_PATH_FILEMON, O_RDWR)) >= 0)
-	    break;
-    }
-
-    if (pbm->filemon_fd < 0) {
+    pbm->filemon = filemon_open();
+    if (pbm->filemon == NULL) {
 	useFilemon = FALSE;
-	warn("Could not open %s", _PATH_FILEMON);
+	warn("Could not open filemon %s", filemon_path());
 	return;
     }
 
@@ -151,12 +141,15 @@ filemon_open(BuildMon *pbm)
      * We only care about the descriptor.
      */
     pbm->mon_fd = mkTempFile("filemon.XXXXXX", NULL);
-    if (ioctl(pbm->filemon_fd, FILEMON_SET_FD, &pbm->mon_fd) < 0) {
+    if ((dupfd = dup(pbm->mon_fd)) == -1) {
+	err(1, "Could not dup filemon output!");
+    }
+    (void)fcntl(dupfd, F_SETFD, FD_CLOEXEC);
+    if (filemon_setfd(pbm->filemon, dupfd) == -1) {
 	err(1, "Could not set filemon file descriptor!");
     }
     /* we don't need these once we exec */
     (void)fcntl(pbm->mon_fd, F_SETFD, FD_CLOEXEC);
-    (void)fcntl(pbm->filemon_fd, F_SETFD, FD_CLOEXEC);
 }
 
 /*
@@ -570,7 +563,7 @@ meta_init(void)
 {
 #ifdef USE_FILEMON
 	/* this allows makefiles to test if we have filemon support */
-	Var_Set(".MAKE.PATH_FILEMON", _PATH_FILEMON, VAR_GLOBAL, 0);
+	Var_Set(".MAKE.PATH_FILEMON", filemon_path(), VAR_GLOBAL, 0);
 #endif
 }
 
@@ -680,9 +673,10 @@ meta_job_start(Job *job, GNode *gn)
 #endif
 #ifdef USE_FILEMON
     if (pbm->mfp != NULL && useFilemon) {
-	filemon_open(pbm);
+	meta_open_filemon(pbm);
     } else {
-	pbm->mon_fd = pbm->filemon_fd = -1;
+	pbm->mon_fd = -1;
+	pbm->filemon = NULL;
     }
 #endif
 }
@@ -708,7 +702,7 @@ meta_job_child(Job *job)
 	    pid_t pid;
 
 	    pid = getpid();
-	    if (ioctl(pbm->filemon_fd, FILEMON_SET_PID, &pid) < 0) {
+	    if (filemon_setpid_child(pbm->filemon, pid) == -1) {
 		err(1, "Could not set filemon pid!");
 	    }
 	}
@@ -716,6 +710,59 @@ meta_job_child(Job *job)
 #endif
 }
 
+void
+meta_job_parent(Job *job, pid_t pid)
+{
+#ifdef USE_FILEMON
+    BuildMon *pbm;
+
+    if (job != NULL) {
+	pbm = &job->bm;
+    } else {
+	pbm = &Mybm;
+    }
+    if (useFilemon) {
+	filemon_setpid_parent(pbm->filemon, pid);
+    }
+#endif
+}
+
+int
+meta_job_fd(Job *job)
+{
+#ifdef USE_FILEMON
+    BuildMon *pbm;
+
+    if (job != NULL) {
+	pbm = &job->bm;
+    } else {
+	pbm = &Mybm;
+    }
+    if (useFilemon && pbm->filemon) {
+	return filemon_readfd(pbm->filemon);
+    }
+#endif
+    return -1;
+}
+
+int
+meta_job_event(Job *job)
+{
+#ifdef USE_FILEMON
+    BuildMon *pbm;
+
+    if (job != NULL) {
+	pbm = &job->bm;
+    } else {
+	pbm = &Mybm;
+    }
+    if (useFilemon && pbm->filemon) {
+	return filemon_process(pbm->filemon);
+    }
+#endif
+    return 0;
+}
+
 void
 meta_job_error(Job *job, GNode *gn, int flags, int status)
 {
@@ -794,13 +841,16 @@ meta_cmd_finish(void *pbmp)
 	pbm = &Mybm;
 
 #ifdef USE_FILEMON
-    if (pbm->filemon_fd >= 0) {
-	if (close(pbm->filemon_fd) < 0)
+    if (pbm->filemon) {
+	while (filemon_process(pbm->filemon) > 0)
+	    continue;
+	if (filemon_close(pbm->filemon) == -1)
 	    error = errno;
 	x = filemon_read(pbm->mfp, pbm->mon_fd);
 	if (error == 0 && x != 0)
 	    error = x;
-	pbm->filemon_fd = pbm->mon_fd = -1;
+	pbm->mon_fd = -1;
+	pbm->filemon = NULL;
     } else
 #endif
 	fprintf(pbm->mfp, "\n");	/* ensure end with newline */
@@ -1599,9 +1649,10 @@ meta_compat_start(void)
     BuildMon *pbm = &Mybm;
     
     if (pbm->mfp != NULL && useFilemon) {
-	filemon_open(pbm);
+	meta_open_filemon(pbm);
     } else {
-	pbm->mon_fd = pbm->filemon_fd = -1;
+	pbm->mon_fd = -1;
+	pbm->filemon = NULL;
     }
 #endif
     if (pipe(childPipe) < 0)
@@ -1623,19 +1674,56 @@ meta_compat_child(void)
 }
 
 void
-meta_compat_parent(void)
+meta_compat_parent(pid_t child)
 {
-    FILE *fp;
+    int outfd, metafd, maxfd, nfds;
     char buf[BUFSIZ];
-    
+    fd_set readfds;
+
+    meta_job_parent(NULL, child);
     close(childPipe[1]);			/* child side */
-    fp = fdopen(childPipe[0], "r");
-    while (fgets(buf, sizeof(buf), fp)) {
-	meta_job_output(NULL, buf, "");
-	printf("%s", buf);
-	fflush(stdout);
+    outfd = childPipe[0];
+    metafd = filemon_readfd(Mybm.filemon);
+
+    maxfd = -1;
+    if (outfd > maxfd)
+	    maxfd = outfd;
+    if (metafd > maxfd)
+	    maxfd = metafd;
+
+    while (outfd != -1 || metafd != -1) {
+	FD_ZERO(&readfds);
+	if (outfd != -1) {
+	    FD_SET(outfd, &readfds);
+	}
+	if (metafd != -1) {
+	    FD_SET(metafd, &readfds);
+	}
+	nfds = select(maxfd + 1, &readfds, NULL, NULL, NULL);
+	if (nfds == -1) {
+	    if (errno == EINTR)
+		continue;
+	    err(1, "select");
+	}
+
+	if (outfd != -1 && FD_ISSET(outfd, &readfds)) do {
+	    /* XXX this is not line-buffered */
+	    ssize_t nread = read(outfd, buf, sizeof buf);
+	    if (nread == -1)
+		err(1, "read");
+	    if (nread == 0) {
+		close(outfd);
+		outfd = -1;
+		break;
+	    }
+	    fwrite(buf, 1, (size_t)nread, stdout);
+	    fflush(stdout);
+	} while (0);
+	if (metafd != -1 && FD_ISSET(metafd, &readfds)) {
+	    if (meta_job_event(NULL) <= 0)
+		metafd = -1;
+	}
     }
-    fclose(fp);
 }
 
 #endif	/* USE_META */
diff --git a/usr.bin/make/meta.h b/usr.bin/make/meta.h
index 8f1018e89a7d..fc9c9fd07276 100644
--- a/usr.bin/make/meta.h
+++ b/usr.bin/make/meta.h
@@ -33,7 +33,7 @@
 
 typedef struct BuildMon {
     char	meta_fname[MAXPATHLEN];
-    int		filemon_fd;
+    struct filemon *filemon;
     int		mon_fd;
     FILE	*mfp;
 } BuildMon;
@@ -46,6 +46,9 @@ void meta_finish(void);
 void meta_mode_init(const char *);
 void meta_job_start(struct Job *, GNode *);
 void meta_job_child(struct Job *);
+void meta_job_parent(struct Job *, pid_t);
+int  meta_job_fd(struct Job *);
+int  meta_job_event(struct Job *);
 void meta_job_error(struct Job *, GNode *, int, int);
 void meta_job_output(struct Job *, char *, const char *);
 int  meta_cmd_finish(void *);
@@ -53,4 +56,4 @@ int  meta_job_finish(struct Job *);
 Boolean meta_oodate(GNode *, Boolean);
 void meta_compat_start(void);
 void meta_compat_child(void);
-void meta_compat_parent(void);
+void meta_compat_parent(pid_t);