tech-toolchain archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
Re: make: dynamic scaling of maxJobs ?
Below is the patch I would like to commit.
I renamed the operative variables so that they can be set in the
environment.
This makes it easy to impose this strategy on old branches that know
nothing of it, this is likely the most useful approach.
Only level 0 make will pay attention - since it would be a bad idea for
more than one instance of make to attempt to adjust the token pool.
Also, since the -j value passed to submakes has no effect other than as
an upper bound on their token consumption, it is fine to leave it alone,
just make sure it is equal to maxJobs.
As the man page says, it is handy if the maxJobs tool can be asked to
report the max value so one can:
.if ${.MAKE.LEVEL} == 0 && !empty(MAKE_JOBS_CMD)
MAKE_JOBS_CMD_MAX!= ${MAKE_JOBS_CMD} --max
.endif
or in the environment
export MAKE_JOBS_CMD_MAX=`maxJobs --max`
I have't provided maxJobs.sh but could - the one I have should work
with any BSD or Linux.
For our build at $work we see at least 10% improvement for concurrent
builds.
--sjg
Index: job.c
===================================================================
RCS file: /cvsroot/src/usr.bin/make/job.c,v
retrieving revision 1.176
diff -u -p -r1.176 job.c
--- job.c 4 Aug 2013 16:48:15 -0000 1.176
+++ job.c 11 Aug 2013 00:15:55 -0000
@@ -175,6 +175,18 @@ int jobTokensRunning = 0;
int not_parallel = 0; /* set if .NOT_PARALLEL */
/*
+ * If set, we run it occasionally to adjust maxJobTokens
+ */
+static char *jobs_cmd = NULL;
+static int jobTokensAdjust = 0; /* only relevant if jobs_cmd set */
+#ifndef DEFAULT_MAKE_JOBS_CMD_INTERVAL
+# define DEFAULT_MAKE_JOBS_CMD_INTERVAL 300
+#endif
+#ifndef DEFAULT_MAKE_JOBS_CMD_MAX
+# define DEFAULT_MAKE_JOBS_CMD_MAX 64
+#endif
+
+/*
* XXX: Avoid SunOS bug... FILENO() is fp->_file, and file
* is a char! So when we go above 127 we turn negative!
*/
@@ -363,6 +375,115 @@ static void JobSigReset(void);
const char *malloc_options="A";
+/*
+ * Job_maxTokens - dynamically determine .MAKE.JOBS
+ *
+ * This is only used by the initial instance of make
+ * which usually the one that runs the longest.
+ * We run an external tool which reports the maxJobTokens
+ * value we should use.
+ * We get called from JobFinish() and if it has been more than
+ * ${MAKE_JOBS_CMD_INTERVAL} seconds we re-run ${MAKE_JOBS_CMD}.
+ * If the result is higher than before (limited by maxJobs) we add
+ * tokens to the queue, and if less avoid re-adding until the
+ * adjustment is completed.
+ */
+int
+Job_maxTokens(int old)
+{
+ static time_t interval, last = 0, utc;
+ char *cp = NULL;
+ const char *ep;
+ int max_tokens = old;
+
+ if (!last) {
+ /*
+ * First time.
+ * We split initializtion since we can be called before
+ * and after makefiles have been read.
+ */
+ interval = 0; /* not initialized */
+ jobs_cmd = Var_Subst(NULL, "${MAKE_JOBS_CMD}", VAR_GLOBAL, 0);
+ if (jobs_cmd) {
+ if (!jobs_cmd[0]) {
+ free(jobs_cmd);
+ jobs_cmd = NULL;
+ }
+ }
+ }
+ if (!jobs_cmd) {
+ last = 1; /* avoid wasting time above */
+ return old;
+ }
+ time(&utc);
+ if (old) {
+ /* makefiles have been read by now */
+ if (!interval) {
+ interval = getInt("MAKE_JOBS_CMD_INTERVAL",
+ DEFAULT_MAKE_JOBS_CMD_INTERVAL);
+ }
+ if (utc - last < interval)
+ return old;
+ }
+ cp = Cmd_Exec(jobs_cmd, &ep);
+ if (cp && *cp) {
+ max_tokens = atoi(cp);
+ if (max_tokens < 1 && !last) {
+ if (DEBUG(JOB)) {
+ (void)fprintf(debug_file,
+ "Job_maxTokens: %s failed: '%s'\n",
+ jobs_cmd, cp);
+ }
+ last = 1; /* don't come back */
+ free(cp);
+ free(jobs_cmd);
+ jobs_cmd = NULL;
+ return old;
+ }
+ }
+ if (DEBUG(JOB)) {
+ (void)fprintf(debug_file,
+ "Job_maxTokens: cmd=%s interval=%u old=%d new=%d\n",
+ jobs_cmd, (unsigned int)interval, old, max_tokens);
+ }
+ if (old) {
+ /* sanity checks */
+ if (max_tokens > maxJobs)
+ max_tokens = maxJobs;
+ else if (max_tokens < 1)
+ max_tokens = old;
+ if (max_tokens == old) {
+ jobTokensAdjust = 0;
+ } else {
+ jobTokensAdjust = max_tokens - old;
+ if (DEBUG(JOB)) {
+ (void)fprintf(debug_file,
+ "Job_maxTokens: adjust %+d tokens\n",
+ jobTokensAdjust);
+ }
+ }
+ /*
+ * Token reduction happens via Job_TokenReturn
+ * addition is done here.
+ */
+ if (jobTokensAdjust > 0) {
+ while (jobTokensAdjust-- > 0) {
+ JobTokenAdd();
+ maxJobTokens++;
+ }
+ }
+ }
+ if (cp) {
+ if (!old) {
+ /* ensure this has a value, we may adjust later */
+ Var_Set(".MAKE.JOBS", cp, VAR_GLOBAL, 0);
+ }
+ free(cp);
+ }
+ last = utc;
+ return max_tokens;
+}
+
static void
job_table_dump(const char *where)
{
@@ -1102,6 +1223,8 @@ JobFinish(Job *job, int status)
*/
Finish(errors);
}
+ if (jobs_cmd)
+ Job_maxTokens(maxJobTokens);
}
/*-
@@ -2199,6 +2322,28 @@ void
Job_Init(void)
{
Job_SetPrefix();
+
+ if (jobs_cmd) {
+ /*
+ * We may not yet have a decent maxJobs
+ * which we need to pass to sub-makes.
+ */
+ maxJobs = getInt("MAKE_JOBS_CMD_MAX",
+ MAX(maxJobTokens, DEFAULT_MAKE_JOBS_CMD_MAX));
+ if (DEBUG(JOB)) {
+ (void)fprintf(debug_file, "Job_Init: tokens=%d max=%d\n",
+ maxJobTokens, maxJobs);
+ }
+ if (maxJobs > maxJobTokens) {
+ Var_SetInt(".MAKE.JOBS", maxJobs, VAR_CMD, 0);
+ /* we need to re-export MAKEFLAGS */
+ Main_ExportMAKEFLAGS(FALSE);
+ }
+ } else if (makelevel == 0) {
+ /* in case makefiles or environment set MAKE_JOBS_CMD */
+ maxJobTokens = Job_maxTokens(maxJobTokens);
+ }
+
/* Allocate space for all the job info */
job_table = bmake_malloc(maxJobs * sizeof *job_table);
memset(job_table, 0, maxJobs * sizeof *job_table);
@@ -2841,10 +2986,9 @@ Job_ServerStart(int max_tokens, int jp_0
JobCreatePipe(&tokenWaitJob, 15);
- snprintf(jobarg, sizeof(jobarg), "%d,%d",
+ snprintf(jobarg, sizeof(jobarg), "-J%d,%d",
tokenWaitJob.inPipe, tokenWaitJob.outPipe);
- Var_Append(MAKEFLAGS, "-J", VAR_GLOBAL);
Var_Append(MAKEFLAGS, jobarg, VAR_GLOBAL);
/*
@@ -2873,8 +3017,14 @@ Job_TokenReturn(void)
jobTokensRunning--;
if (jobTokensRunning < 0)
Punt("token botch");
- if (jobTokensRunning || JOB_TOKENS[aborting] != '+')
- JobTokenAdd();
+ if (jobTokensRunning || JOB_TOKENS[aborting] != '+') {
+ if (jobTokensAdjust < 0 && maxJobTokens > 1) {
+ jobTokensAdjust++;
+ maxJobTokens--;
+ } else {
+ JobTokenAdd();
+ }
+ }
}
/*-
Index: job.h
===================================================================
RCS file: /cvsroot/src/usr.bin/make/job.h,v
retrieving revision 1.42
diff -u -p -r1.42 job.h
--- job.h 5 Jul 2013 22:14:56 -0000 1.42
+++ job.h 11 Aug 2013 00:15:55 -0000
@@ -247,6 +247,7 @@ extern char *shellErrFlag;
extern int jobTokensRunning; /* tokens currently "out" */
extern int maxJobs; /* Max jobs we can run */
+extern int maxJobTokens; /* almost the same thing */
void Shell_Init(void);
const char *Shell_GetNewline(void);
Index: main.c
===================================================================
RCS file: /cvsroot/src/usr.bin/make/main.c,v
retrieving revision 1.223
diff -u -p -r1.223 main.c
--- main.c 4 Aug 2013 16:48:15 -0000 1.223
+++ main.c 11 Aug 2013 00:15:55 -0000
@@ -155,7 +155,7 @@ static Lst makefiles; /* ordered list o
static Boolean printVars; /* print value of one or more vars */
static Lst variables; /* list of variables to print */
int maxJobs; /* -j argument */
-static int maxJobTokens; /* -j argument */
+int maxJobTokens; /* -j argument */
Boolean compatMake; /* -B argument */
int debug; /* -d argument */
Boolean debugVflag; /* -dV */
@@ -534,15 +534,20 @@ rearg:
case 'j':
if (argvalue == NULL) goto noarg;
forceJobs = TRUE;
- maxJobs = strtol(argvalue, &p, 0);
- if (*p != '\0' || maxJobs < 1) {
+ if (argvalue[0] == '/') {
+ /* we are going to do dynamic scaling */
+ Var_Set("MAKE_JOBS_CMD", argvalue, VAR_GLOBAL, 0);
+ maxJobs = Job_maxTokens(0);
+ } else {
+ maxJobs = strtol(argvalue, &p, 0);
+ if (*p != '\0' || maxJobs < 1) {
(void)fprintf(stderr, "%s: illegal argument to
-j -- must be positive integer!\n",
progname);
exit(1);
+ }
+ Var_Set(".MAKE.JOBS", argvalue, VAR_CMD, 0);
}
- Var_Append(MAKEFLAGS, "-j", VAR_GLOBAL);
- Var_Append(MAKEFLAGS, argvalue, VAR_GLOBAL);
- Var_Set(".MAKE.JOBS", argvalue, VAR_GLOBAL, 0);
+ Var_Append(MAKEFLAGS, "-j${.MAKE.JOBS}", VAR_GLOBAL);
maxJobTokens = maxJobs;
break;
case 'k':
@@ -1861,6 +1866,8 @@ Main_ExportMAKEFLAGS(Boolean first)
setenv("MAKE", s, 1);
#endif
}
+ if (s)
+ free(s);
}
char *
@@ -1960,3 +1967,21 @@ getBoolean(const char *name, Boolean bf)
}
return (bf);
}
+
+int
+getInt(const char *name, int i)
+{
+ char tmp[64];
+ char *cp;
+
+ if (snprintf(tmp, sizeof(tmp), "${%s}", name) < (int)(sizeof(tmp))) {
+ cp = Var_Subst(NULL, tmp, VAR_GLOBAL, 0);
+
+ if (cp) {
+ if (*cp)
+ i = atoi(cp);
+ free(cp);
+ }
+ }
+ return (i);
+}
Index: make.1
===================================================================
RCS file: /cvsroot/src/usr.bin/make/make.1,v
retrieving revision 1.220
diff -u -p -r1.220 make.1
--- make.1 30 Jul 2013 19:09:57 -0000 1.220
+++ make.1 11 Aug 2013 00:15:55 -0000
@@ -29,7 +29,7 @@
.\"
.\" from: @(#)make.1 8.4 (Berkeley) 3/19/94
.\"
-.Dd July 30, 2013
+.Dd August 10, 2013
.Dt MAKE 1
.Os
.Sh NAME
@@ -257,6 +257,11 @@ The value is saved in
Turns compatibility mode off, unless the
.Ar B
flag is also specified.
+If the argument is the absolute path to a program it will be
+assigned to
+.Va MAKE_JOBS_CMD
+and used to obtain the value for
+.Va .MAKE.JOBS .
When compatibility mode is off, all commands associated with a
target are executed in a single shell invocation as opposed to the
traditional one shell invocation per line.
@@ -736,6 +741,40 @@ The list of variables exported by
The argument to the
.Fl j
option.
+.It Va MAKE_JOBS_CMD
+names a command to run to get the value of
+.Va .MAKE.JOBS .
+It will be re-run occasionally by the initial instance (level 0) of
+.Nm
+to adjust the number of job tokens available.
+The value of
+.Va .MAKE.JOBS
+passed to sub-makes is not adjusted, it serves only as an upper bound.
+The number of job tokens controlled by the initial instance of
+.Nm
+is all that matters.
+.It Va MAKE_JOBS_CMD_INTERVAL
+Sets the minimum number of seconds between runs of
+.Va MAKE_JOBS_CMD .
+.It Va MAKE_JOBS_CMD_MAX
+Sets an upper bounds on the value accepted from
+.Va MAKE_JOBS_CMD .
+If
+.Va MAKE_JOBS_CMD
+was set via the
+.Ar j
+option, then it is actually the value of
+.Va MAKE_JOBS_CMD_MAX
+that is passed to sub-makes.
+It is useful if the command indicated by
+.Va MAKE_JOBS_CMD
+can be leveraged to set this.
+For example:
+.Bd -literal -offset indent
+.Dv .if ${.MAKE.LEVEL} == 0 && !empty(MAKE_JOBS_CMD)
+MAKE_JOBS_CMD_MAX!= ${MAKE_JOBS_CMD} --max
+.Dv .endif
+.Ed
.It Va .MAKE.JOB.PREFIX
If
.Nm
Index: make.h
===================================================================
RCS file: /cvsroot/src/usr.bin/make/make.h,v
retrieving revision 1.91
diff -u -p -r1.91 make.h
--- make.h 18 Jun 2013 20:06:09 -0000 1.91
+++ make.h 11 Aug 2013 00:15:55 -0000
@@ -410,6 +410,8 @@ extern char *progname; /* The program na
extern char *makeDependfile; /* .depend */
extern char **savedEnv; /* if we replaced environ this will be
non-NULL */
+extern int makelevel;
+
/*
* We cannot vfork() in a child of vfork().
* Most systems do not enforce this but some do.
@@ -479,6 +481,9 @@ void Main_ExportMAKEFLAGS(Boolean);
Boolean Main_SetObjdir(const char *);
int mkTempFile(const char *, char **);
int str2Lst_Append(Lst, char *, const char *);
+int getInt(const char *, int);
+int Job_maxTokens(int);
+
#ifdef __GNUC__
#define UNCONST(ptr) ({ \
Index: nonints.h
===================================================================
RCS file: /cvsroot/src/usr.bin/make/nonints.h,v
retrieving revision 1.65
diff -u -p -r1.65 nonints.h
--- nonints.h 30 Aug 2012 21:17:05 -0000 1.65
+++ nonints.h 11 Aug 2013 00:15:55 -0000
@@ -179,6 +179,7 @@ void Targ_Propagate_Wait(void);
/* var.c */
void Var_Delete(const char *, GNode *);
void Var_Set(const char *, const char *, GNode *, int);
+void Var_SetInt(const char *, int, GNode *, int);
void Var_Append(const char *, const char *, GNode *);
Boolean Var_Exists(const char *, GNode *);
char *Var_Value(const char *, GNode *, char **);
Index: var.c
===================================================================
RCS file: /cvsroot/src/usr.bin/make/var.c,v
retrieving revision 1.183
diff -u -p -r1.183 var.c
--- var.c 16 Jul 2013 20:00:56 -0000 1.183
+++ var.c 11 Aug 2013 00:15:55 -0000
@@ -139,7 +139,6 @@ __RCSID("$NetBSD: var.c,v 1.183 2013/07/
#include "dir.h"
#include "job.h"
-extern int makelevel;
/*
* This lets us tell if we have replaced the original environ
* (which we cannot free).
@@ -979,6 +978,16 @@ Var_Set(const char *name, const char *va
VarFreeEnv(v, TRUE);
}
+
+void
+Var_SetInt(const char *name, int val, GNode *ctxt, int flags)
+{
+ char tmp[64];
+
+ snprintf(tmp, sizeof(tmp), "%d", val);
+ Var_Set(name, tmp, ctxt, flags);
+}
+
/*-
*-----------------------------------------------------------------------
* Var_Append --
Home |
Main Index |
Thread Index |
Old Index