CVS commit: pkgsrc/pkgtools

["Joerg Sonnenberger" <joerg%netbsd.org@localhost>]

Module Name:    pkgsrc
Committed By:   joerg
Date:           Sun Feb 12 04:12:54 UTC 2023

Modified Files:
        pkgsrc/pkgtools/pbulk-base: Makefile
        pkgsrc/pkgtools/pbulk/files/pbulk/pbuild: jobs.c pbuild.h

Log Message:
pbulk-base-0.56: Support for adjusting scheduling

Switch to a weighted scheduling algorithm. Before, build order was based
on number of reachable nodes in the dependency graph. This assumes that
heavy packages are required by other packages. Some big packages
nowadays violate that assumption and can result in long periods at the
end of a build where only a few builders are active. Annotating those
packages with PBULK_WEIGHT in the pbulk-index output can boost their
priority to let them be built earlier. The default weight is 100.

Note: the pbulk-build report has grown an extra field per line with the
computed effective weight of each package. This file is normally used
only internally.


To generate a diff of this commit:
cvs rdiff -u -r1.26 -r1.27 pkgsrc/pkgtools/pbulk-base/Makefile
cvs rdiff -u -r1.18 -r1.19 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c
cvs rdiff -u -r1.8 -r1.9 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: pkgsrc/pkgtools/pbulk-base/Makefile
diff -u pkgsrc/pkgtools/pbulk-base/Makefile:1.26 pkgsrc/pkgtools/pbulk-base/Makefile:1.27
--- pkgsrc/pkgtools/pbulk-base/Makefile:1.26    Fri Feb 10 23:14:32 2023
+++ pkgsrc/pkgtools/pbulk-base/Makefile Sun Feb 12 04:12:54 2023
@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.26 2023/02/10 23:14:32 joerg Exp $
+# $NetBSD: Makefile,v 1.27 2023/02/12 04:12:54 joerg Exp $
 
-DISTNAME=      pbulk-base-0.55
+DISTNAME=      pbulk-base-0.56
 COMMENT=       Core components of the modular bulk build framework
 
 .include "../../pkgtools/pbulk/Makefile.common"

Index: pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c
diff -u pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c:1.18 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c:1.19
--- pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c:1.18        Fri Feb 10 23:14:32 2023
+++ pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/jobs.c     Sun Feb 12 04:12:54 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: jobs.c,v 1.18 2023/02/10 23:14:32 joerg Exp $ */
+/* $NetBSD: jobs.c,v 1.19 2023/02/12 04:12:54 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007, 2009, 2011 Joerg Sonnenberger <joerg%NetBSD.org@localhost>.
@@ -164,6 +164,7 @@ compute_tree_depth_rec(struct build_job 
                return 0;
        seen[job - jobs] = 1;
        ++root->pkg_depth;
+       root->pkg_weighted_depth += job->pkg_weight;
        SLIST_FOREACH(dep_iter, &job->depending_pkgs, depends_link) {
                if (compute_tree_depth_rec(dep_iter->dependency, root, seen)) {
                        fprintf(stderr, "%s\n", job->pkgname);
@@ -179,6 +180,7 @@ compute_tree_depth(struct build_job *job
        memset(seen, 0, len_jobs);
 
        job->pkg_depth = 0;
+       job->pkg_weighted_depth = 0;
        if (compute_tree_depth_rec(job, job, seen))
                exit(1);
 }
@@ -281,6 +283,29 @@ mark_initial_state(int fd, enum job_stat
 }
 
 static void
+parse_weight(struct build_job *job)
+{
+       const char *line;
+       char *eos;
+       long long value;
+
+       line = find_content(job, "PBULK_WEIGHT=");
+       if (line == NULL) {
+               job->pkg_weight = 100;
+               return;
+       }
+       errno = 0;
+       value = strtoll(line, &eos, 10);
+       if (errno || line == eos || *eos != '\n')
+               errx(1, "Invalid PBULK_WEIGHT for package %s", job->pkgname);
+       if (value < 0 && value <= LLONG_MIN / len_jobs)
+               errx(1, "PBULK_WEIGHT too small for package %s", job->pkgname);
+       if (value > 0 && value >= LLONG_MAX / len_jobs)
+               errx(1, "PBULK_WEIGHT too large for package %s", job->pkgname);
+       job->pkg_weight = value;
+}
+
+static void
 mark_initial(void)
 {
        const char *line;
@@ -299,7 +324,7 @@ mark_initial(void)
                        process_job(&jobs[i], JOB_PREFAILED, 0);
                        continue;
                }
-       }               
+       }
 
        mark_initial_state(log_success, JOB_DONE, "successful");
        mark_initial_state(log_failed, JOB_FAILED, "failing");
@@ -315,7 +340,7 @@ add_to_build_list(struct build_job *job)
        struct build_job *iter;
 
        TAILQ_FOREACH(iter, &buildable_jobs, build_link) {
-               if (iter->pkg_depth < job->pkg_depth) {
+               if (iter->pkg_weighted_depth < job->pkg_weighted_depth) {
                        TAILQ_INSERT_BEFORE(iter, job, build_link);
                        return;
                }
@@ -334,6 +359,8 @@ build_tree(void)
        for (i = 0; i < len_jobs; ++i) {
                job = &jobs[i];
 
+               parse_weight(job);
+
                if ((depends = find_content(job, "DEPENDS=")) == NULL)
                        continue;
 
@@ -530,8 +557,8 @@ finish_build(const char *report_file)
                default:
                        errx(1, "internal error");
                }
-               fprintf(report, "%s|%s||%d\n", jobs[i].pkgname, status,
-                   jobs[i].pkg_depth);
+               fprintf(report, "%s|%s||%d|%lld\n", jobs[i].pkgname, status,
+                   jobs[i].pkg_depth, jobs[i].pkg_weighted_depth);
        }
 }
 

Index: pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h
diff -u pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h:1.8 pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h:1.9
--- pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h:1.8       Fri Feb 10 23:14:32 2023
+++ pkgsrc/pkgtools/pbulk/files/pbulk/pbuild/pbuild.h   Sun Feb 12 04:12:54 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: pbuild.h,v 1.8 2023/02/10 23:14:32 joerg Exp $ */
+/* $NetBSD: pbuild.h,v 1.9 2023/02/12 04:12:54 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg%NetBSD.org@localhost>.
@@ -96,6 +96,8 @@ struct build_job {
 
        enum job_state state;
        int pkg_depth;
+       long long pkg_weighted_depth;
+       long long pkg_weight;
 
        /**
         * The number of direct dependencies that must be built before