tech-toolchain archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
Re: make: dynamic scaling of maxJobs ?
On Fri, 16 Aug 2013 16:00:25 -0700, "Simon J. Gerraty" writes:
>Actually, by simply making a call to Job_maxTokens() from
>Job_CatchOutput() you ensure that any style of build gets regular
>opportunities to adjust the token pool.
Removing the ability to set via -j arg, simplifies the diff quite a bit.
Index: job.c
===================================================================
RCS file: /cvsroot/src/usr.bin/make/job.c,v
retrieving revision 1.176
diff -u -p -r1.176 job.c
--- job.c 4 Aug 2013 16:48:15 -0000 1.176
+++ job.c 17 Aug 2013 00:14:55 -0000
@@ -175,6 +175,15 @@ int jobTokensRunning = 0;
int not_parallel = 0; /* set if .NOT_PARALLEL */
/*
+ * If set, we run it occasionally to adjust maxJobTokens
+ */
+static char *jobs_cmd = NULL;
+static int jobTokensAdjust = 0; /* only relevant if jobs_cmd set */
+#ifndef DEFAULT_MAKE_JOBS_CMD_INTERVAL
+# define DEFAULT_MAKE_JOBS_CMD_INTERVAL 300
+#endif
+
+/*
* XXX: Avoid SunOS bug... FILENO() is fp->_file, and file
* is a char! So when we go above 127 we turn negative!
*/
@@ -363,6 +372,122 @@ static void JobSigReset(void);
const char *malloc_options="A";
+/*
+ * Job_maxTokens - dynamically adjust maxJobTokens
+ *
+ * This is only used by the initial instance of make which is the
+ * master of the token pool.
+ * We run an external tool which reports the maxJobTokens
+ * value we should use.
+ * We get called from JobFinish() as well as Job_CatchOutput()
+ * and if it has been more than ${MAKE_JOBS_CMD_INTERVAL} seconds we
+ * re-run ${MAKE_JOBS_CMD}.
+ * If the result is higher than before (limited by maxJobs) we add
+ * tokens to the queue, and if less avoid re-adding until the
+ * adjustment is completed.
+ */
+static void
+Job_maxTokens(void)
+{
+ static time_t interval, last = 0, utc;
+ char *cp = NULL;
+ const char *ep;
+ int max_tokens = maxJobTokens;
+
+ if (!last) {
+ /* First time. */
+ interval = 0; /* not initialized */
+ jobs_cmd = Var_Subst(NULL, "${MAKE_JOBS_CMD}", VAR_GLOBAL, 0);
+ if (jobs_cmd) {
+ if (!jobs_cmd[0]) {
+ free(jobs_cmd);
+ jobs_cmd = NULL;
+ }
+ }
+ }
+ if (!jobs_cmd) {
+ last = 1; /* avoid wasting time above */
+ return;
+ }
+ time(&utc);
+ /* makefiles have been read by now */
+ if (!interval) {
+ interval = getInt("MAKE_JOBS_CMD_INTERVAL",
+ DEFAULT_MAKE_JOBS_CMD_INTERVAL);
+ }
+ if (utc - last < interval)
+ return;
+
+ cp = Cmd_Exec(jobs_cmd, &ep);
+ if (cp) {
+ if (*cp) {
+ max_tokens = atoi(cp);
+ if (max_tokens < 1 && !last) {
+ if (DEBUG(JOB)) {
+ (void)fprintf(debug_file,
+ "Job_maxTokens: %s failed: '%s'\n",
+ jobs_cmd, cp);
+ }
+ last = 1; /* don't come back */
+ free(cp);
+ free(jobs_cmd);
+ jobs_cmd = NULL;
+ return;
+ }
+ }
+ free(cp);
+ }
+ last = utc;
+ if (DEBUG(JOB)) {
+ (void)fprintf(debug_file,
+ "Job_maxTokens: cmd=%s interval=%u old=%d new=%d\n",
+ jobs_cmd, (unsigned int)interval,
+ maxJobTokens, max_tokens);
+ }
+ /* sanity checks */
+ if (max_tokens > maxJobs)
+ max_tokens = maxJobs;
+ else if (max_tokens < 1)
+ max_tokens = maxJobTokens;
+ if (max_tokens == maxJobTokens) {
+ jobTokensAdjust = 0;
+ } else {
+ jobTokensAdjust = max_tokens - maxJobTokens;
+ if (DEBUG(JOB)) {
+ (void)fprintf(debug_file,
+ "Job_maxTokens: adjust %+d tokens\n",
+ jobTokensAdjust);
+ }
+ }
+ /*
+ * Token addition is done here.
+ */
+ if (jobTokensAdjust > 0) {
+ while (jobTokensAdjust-- > 0) {
+ JobTokenAdd();
+ maxJobTokens++;
+ }
+ }
+ /*
+ * Token reduction mainly happens via Job_TokenReturn
+ * but try to take some here anyway - until we block.
+ */
+ if (jobTokensAdjust < 0) {
+ /*
+ * Job_TokenWithdraw() won't really pull a token if
+ * jobTokensRunning is 0
+ */
+ int saved = jobTokensRunning;
+
+ jobTokensRunning = maxJobs + jobTokensAdjust;
+ while (jobTokensAdjust < 0 && Job_TokenWithdraw()) {
+ jobTokensAdjust++;
+ maxJobTokens--;
+ }
+ jobTokensRunning = saved;
+ }
+}
+
static void
job_table_dump(const char *where)
{
@@ -1102,6 +1227,8 @@ JobFinish(Job *job, int status)
*/
Finish(errors);
}
+ if (jobs_cmd)
+ Job_maxTokens();
}
/*-
@@ -2050,6 +2177,9 @@ Job_CatchOutput(void)
(void)fflush(stdout);
+ if (jobs_cmd)
+ Job_maxTokens();
+
/* The first fd in the list is the job token pipe */
do {
nready = poll(fds + 1 - wantToken, nfds - 1 + wantToken, POLL_MSEC);
@@ -2199,6 +2329,12 @@ void
Job_Init(void)
{
Job_SetPrefix();
+
+ if (makelevel == 0) {
+ /* See if makefiles or environment set MAKE_JOBS_CMD */
+ Job_maxTokens();
+ }
+
/* Allocate space for all the job info */
job_table = bmake_malloc(maxJobs * sizeof *job_table);
memset(job_table, 0, maxJobs * sizeof *job_table);
@@ -2873,8 +3009,14 @@ Job_TokenReturn(void)
jobTokensRunning--;
if (jobTokensRunning < 0)
Punt("token botch");
- if (jobTokensRunning || JOB_TOKENS[aborting] != '+')
- JobTokenAdd();
+ if (jobTokensRunning || JOB_TOKENS[aborting] != '+') {
+ if (jobTokensAdjust < 0 && maxJobTokens > 1) {
+ jobTokensAdjust++;
+ maxJobTokens--;
+ } else {
+ JobTokenAdd();
+ }
+ }
}
/*-
Index: job.h
===================================================================
RCS file: /cvsroot/src/usr.bin/make/job.h,v
retrieving revision 1.42
diff -u -p -r1.42 job.h
--- job.h 5 Jul 2013 22:14:56 -0000 1.42
+++ job.h 17 Aug 2013 00:14:55 -0000
@@ -247,6 +247,7 @@ extern char *shellErrFlag;
extern int jobTokensRunning; /* tokens currently "out" */
extern int maxJobs; /* Max jobs we can run */
+extern int maxJobTokens; /* almost the same thing */
void Shell_Init(void);
const char *Shell_GetNewline(void);
Index: main.c
===================================================================
RCS file: /cvsroot/src/usr.bin/make/main.c,v
retrieving revision 1.223
diff -u -p -r1.223 main.c
--- main.c 4 Aug 2013 16:48:15 -0000 1.223
+++ main.c 17 Aug 2013 00:14:55 -0000
@@ -155,7 +155,7 @@ static Lst makefiles; /* ordered list o
static Boolean printVars; /* print value of one or more vars */
static Lst variables; /* list of variables to print */
int maxJobs; /* -j argument */
-static int maxJobTokens; /* -j argument */
+int maxJobTokens; /* -j argument */
Boolean compatMake; /* -B argument */
int debug; /* -d argument */
Boolean debugVflag; /* -dV */
@@ -1861,6 +1861,8 @@ Main_ExportMAKEFLAGS(Boolean first)
setenv("MAKE", s, 1);
#endif
}
+ if (s)
+ free(s);
}
char *
@@ -1960,3 +1962,21 @@ getBoolean(const char *name, Boolean bf)
}
return (bf);
}
+
+int
+getInt(const char *name, int i)
+{
+ char tmp[64];
+ char *cp;
+
+ if (snprintf(tmp, sizeof(tmp), "${%s}", name) < (int)(sizeof(tmp))) {
+ cp = Var_Subst(NULL, tmp, VAR_GLOBAL, 0);
+
+ if (cp) {
+ if (*cp)
+ i = atoi(cp);
+ free(cp);
+ }
+ }
+ return (i);
+}
Index: make.1
===================================================================
RCS file: /cvsroot/src/usr.bin/make/make.1,v
retrieving revision 1.220
diff -u -p -r1.220 make.1
--- make.1 30 Jul 2013 19:09:57 -0000 1.220
+++ make.1 17 Aug 2013 00:14:55 -0000
@@ -29,7 +29,7 @@
.\"
.\" from: @(#)make.1 8.4 (Berkeley) 3/19/94
.\"
-.Dd July 30, 2013
+.Dd August 10, 2013
.Dt MAKE 1
.Os
.Sh NAME
@@ -736,6 +736,13 @@ The list of variables exported by
The argument to the
.Fl j
option.
+.It Va MAKE_JOBS_CMD
+names a command to be run occasionally by the initial instance (level 0) of
+.Nm
+to adjust the number of job tokens available.
+.It Va MAKE_JOBS_CMD_INTERVAL
+Sets the minimum number of seconds between runs of
+.Va MAKE_JOBS_CMD .
.It Va .MAKE.JOB.PREFIX
If
.Nm
Index: make.h
===================================================================
RCS file: /cvsroot/src/usr.bin/make/make.h,v
retrieving revision 1.91
diff -u -p -r1.91 make.h
--- make.h 18 Jun 2013 20:06:09 -0000 1.91
+++ make.h 17 Aug 2013 00:14:55 -0000
@@ -410,6 +410,8 @@ extern char *progname; /* The program na
extern char *makeDependfile; /* .depend */
extern char **savedEnv; /* if we replaced environ this will be
non-NULL */
+extern int makelevel;
+
/*
* We cannot vfork() in a child of vfork().
* Most systems do not enforce this but some do.
@@ -479,6 +481,8 @@ void Main_ExportMAKEFLAGS(Boolean);
Boolean Main_SetObjdir(const char *);
int mkTempFile(const char *, char **);
int str2Lst_Append(Lst, char *, const char *);
+int getInt(const char *, int);
+
#ifdef __GNUC__
#define UNCONST(ptr) ({ \
Index: nonints.h
===================================================================
RCS file: /cvsroot/src/usr.bin/make/nonints.h,v
retrieving revision 1.65
diff -u -p -r1.65 nonints.h
--- nonints.h 30 Aug 2012 21:17:05 -0000 1.65
+++ nonints.h 17 Aug 2013 00:14:55 -0000
@@ -179,6 +179,7 @@ void Targ_Propagate_Wait(void);
/* var.c */
void Var_Delete(const char *, GNode *);
void Var_Set(const char *, const char *, GNode *, int);
+void Var_SetInt(const char *, int, GNode *, int);
void Var_Append(const char *, const char *, GNode *);
Boolean Var_Exists(const char *, GNode *);
char *Var_Value(const char *, GNode *, char **);
Index: var.c
===================================================================
RCS file: /cvsroot/src/usr.bin/make/var.c,v
retrieving revision 1.183
diff -u -p -r1.183 var.c
--- var.c 16 Jul 2013 20:00:56 -0000 1.183
+++ var.c 17 Aug 2013 00:14:55 -0000
@@ -139,7 +139,6 @@ __RCSID("$NetBSD: var.c,v 1.183 2013/07/
#include "dir.h"
#include "job.h"
-extern int makelevel;
/*
* This lets us tell if we have replaced the original environ
* (which we cannot free).
@@ -979,6 +978,16 @@ Var_Set(const char *name, const char *va
VarFreeEnv(v, TRUE);
}
+
+void
+Var_SetInt(const char *name, int val, GNode *ctxt, int flags)
+{
+ char tmp[64];
+
+ snprintf(tmp, sizeof(tmp), "%d", val);
+ Var_Set(name, tmp, ctxt, flags);
+}
+
/*-
*-----------------------------------------------------------------------
* Var_Append --
Home |
Main Index |
Thread Index |
Old Index