[pkgsrc/trunk]: pkgsrc/mk/bulk - Added a new top level file that saves the RE...

To: pkgsrc-changes-hg%NetBSD.org@localhost
Subject: [pkgsrc/trunk]: pkgsrc/mk/bulk - Added a new top level file that saves the RE...
From: rillig <rillig%pkgsrc.org@localhost>
Date: Thu, 14 Oct 2021 02:56:19 +0000
details:   https://anonhg.NetBSD.org/pkgsrc/rev/5c031dcd65cd
branches:  trunk
changeset: 506738:5c031dcd65cd
user:      rillig <rillig%pkgsrc.org@localhost>
date:      Sat Jan 21 11:29:31 2006 +0000

description:
- Added a new top level file that saves the REPORT_BASEDIR from the first
  call to the mk/bulk/build script. This is to allow restarting the bulk
  build without getting confused by the various timestamps that occur
  during a bulk build. It also fixes some problems where the text/plain
  report had ended up in a directory different from the other files.

- Changed the default filename for the text/plain report from
  "pkgsrc-results-${BUILDDATE}.txt" to simply "report.txt", as the
  ${BUILDDATE} variable didn't give a hint to when the bulk build had been
  started, which is often more important than when the bulk build stopped.

- Changed the way to specify the directory and the URL where the bulk build
  reports are made available. See

      http://mail-index.netbsd.org/tech-pkg/2005/12/05/0002.html

  for details. Specifying the directories using the FTPHOST and FTPURL
  variables will be possible for some months, after which it support for
  it will be dropped.

diffstat:

 mk/bulk/bsd.bulk-pkg.mk    |   6 ++-
 mk/bulk/build              |  27 +++++++++++----
 mk/bulk/build.conf-example |  30 ++++++++++-------
 mk/bulk/post-build         |  78 +++++++++++++++++++++++++++++++++------------
 mk/bulk/post-build-conf    |  30 ++++++++++++++--
 5 files changed, 124 insertions(+), 47 deletions(-)

diffs (truncated from 375 to 300 lines):

diff -r 5cb608e8c4a6 -r 5c031dcd65cd mk/bulk/bsd.bulk-pkg.mk
--- a/mk/bulk/bsd.bulk-pkg.mk   Sat Jan 21 10:42:21 2006 +0000
+++ b/mk/bulk/bsd.bulk-pkg.mk   Sat Jan 21 11:29:31 2006 +0000
@@ -1,4 +1,4 @@
-#      $NetBSD: bsd.bulk-pkg.mk,v 1.111 2006/01/17 22:35:13 jdc Exp $
+#      $NetBSD: bsd.bulk-pkg.mk,v 1.112 2006/01/21 11:29:31 rillig Exp $
 
 #
 # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf%NetBSD.org@localhost>
@@ -145,6 +145,10 @@
 # eventually for looking for leftover files (files not properly deinstalled)
 STARTFILE?=    ${BULKFILESDIR}/.start${BULK_ID}
 
+# This top level file saves the bulk build ID. As it most often has a time
+# stamp, it is only generated once and later retrieved from this file.
+BULK_BUILD_ID_FILE?=   ${BULKFILESDIR}/.bulk_build_id
+
 # File created and used by lintpkgsrc(8) to cache package metadata for
 # pruning and bulk-upload exclusions.
 LINTPKGSRC_DB?=        ${BULKFILESDIR}/.lintpkgsrc.db${BULK_ID}
diff -r 5cb608e8c4a6 -r 5c031dcd65cd mk/bulk/build
--- a/mk/bulk/build     Sat Jan 21 10:42:21 2006 +0000
+++ b/mk/bulk/build     Sat Jan 21 11:29:31 2006 +0000
@@ -1,5 +1,5 @@
 #!/bin/sh
-# $NetBSD: build,v 1.91 2005/12/05 10:19:14 rillig Exp $
+# $NetBSD: build,v 1.92 2006/01/21 11:29:32 rillig Exp $
 
 #
 # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf%NetBSD.org@localhost>
@@ -48,8 +48,7 @@
        one.
 
    -e | --no-email
-       Don't send email when the bulk build is finished, it will put
-       the results into a file (FTP/pkgsrc-results.txt).
+       Don't send email when the bulk build is finished.
 
    -h | --help
        Displays this message.
@@ -279,6 +278,7 @@
 load_pkgsrc_var ORDERFILE
 load_pkgsrc_var STARTFILE
 load_pkgsrc_var SUPPORTSFILE
+load_pkgsrc_var BULK_BUILD_ID_FILE
 
 load_pkgsrc_var BUILDLOG
 load_pkgsrc_var BROKENFILE
@@ -307,6 +307,15 @@
 fi
 
 #
+# Save the bulk build ID in a file, as it most often contains a time
+# stamp.
+#
+case $restart in
+no)    echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \
+       || die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";;
+esac
+
+#
 # Install prerequisite packages.
 #
 # Note: we do this _before_ the depends tree because some packages like
@@ -417,18 +426,22 @@
 # Generate the post-build report.
 #
 echo "build> Generating the bulk build report..."
-BUILDDATE=`date +%Y-%m-%d`
-mkdir -p "${FTP}"
+
+bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \
+|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}."
+
+report_dir="${REPORTS_DIR}/${bulk_build_id}"
+${MKDIR} "${report_dir}"
 ( cd "${pkgsrc_dir}" \
   && ${PERL5} mk/bulk/post-build \
-     > ${FTP}/pkgsrc-results-${BUILDDATE}.txt
+     > "${report_dir}/${REPORT_TXT_FILE}"
 ) || die "Could not write the results file."
 
 #
 # Notify the ADMIN of the finished build.
 #
 case $noemail in
-no)    cat "${FTP}/pkgsrc-results-${BUILDDATE}.txt" \
+no)    cat "${report_dir}/${REPORT_TXT_FILE}" \
        | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results $BUILDDATE" "$ADMIN"
 esac
 
diff -r 5cb608e8c4a6 -r 5c031dcd65cd mk/bulk/build.conf-example
--- a/mk/bulk/build.conf-example        Sat Jan 21 10:42:21 2006 +0000
+++ b/mk/bulk/build.conf-example        Sat Jan 21 11:29:31 2006 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: build.conf-example,v 1.32 2005/12/03 01:00:37 rillig Exp $
+# $NetBSD: build.conf-example,v 1.33 2006/01/21 11:29:32 rillig Exp $
 #
 # This is an example configuration file for pkgsrc bulk builds.
 # Actually it's a shell script that is sourced in by the pre-build,
@@ -61,11 +61,22 @@
 # Who the report is signed by
 ADMINSIG="-Your Name"
 
-# Some paths for output files and paths to log files
-FTPx=`date +%Y%m%d.%H%M`
-FTPURL="pub/NetBSD/pkgstat/${FTPx}"            # relative to ~ftp !
-FTP="/home/ftp/${FTPURL}"                      # absolute base path
-FTPHOST="ftp://ftp.machi.ne";                   # host for broken.html
+# The directory where the final reports are collected.
+#REPORTS_DIR="$HOME/bulk-logs"
+REPORTS_DIR="/home/ftp/pub/NetBSD/pkgstat"
+
+# The URL where the final reports will be available.
+REPORTS_URL="ftp://localhost/pub/NetBSD/pkgstat";
+
+# The basename of the directory of the current bulk build. This
+# directory will be created under ${REPORTS_DIR}. After uploading the
+# report, it will also be available under ${REPORTS_URL}.
+REPORT_BASEDIR=`date +%Y%m%d.%H%M`
+
+# The basenames of the the two report files, which will both be created
+# in ${REPORTS_DIR}/${REPORT_BASEDIR}.
+REPORT_HTML_FILE="report.html"
+REPORT_TXT_FILE="report.txt"
 
 #
 # Uploading binary packages
@@ -88,10 +99,3 @@
 # Destination for packages and rsync options 
 RSYNC_DST=ftp.NetBSD.org:/pub/NetBSD/packages/pkgsrc-200xQy/NetBSD-a.b.c/i386
 RSYNC_OPTS='-e ssh'
-
-###########################################################################
-### No changes should be needed below this line !!!
-###########################################################################
-
-# Name of the (generated) HTML file which lists all broken pkgs
-REPORT="$FTP/broken.html"
diff -r 5cb608e8c4a6 -r 5c031dcd65cd mk/bulk/post-build
--- a/mk/bulk/post-build        Sat Jan 21 10:42:21 2006 +0000
+++ b/mk/bulk/post-build        Sat Jan 21 11:29:31 2006 +0000
@@ -1,5 +1,5 @@
 #!/usr/pkg/bin/perl
-# $NetBSD: post-build,v 1.62 2005/12/03 13:39:04 rillig Exp $
+# $NetBSD: post-build,v 1.63 2006/01/21 11:29:32 rillig Exp $
 #
 # Collect stuff after a pkg bulk build
 #
@@ -41,6 +41,23 @@
        return system(@_);
 }
 
+sub readfirstline($) {
+       my ($fname) = @_;
+       my ($contents);
+
+       open(F, "<", $fname)
+       or pb_die($fname, "Cannot be read: $!");
+
+       defined($contents = <F>)
+       or pb_die($fname, "Must not be empty.");
+       chomp($contents);
+
+       close(F)
+       or pb_die($fname, "Cannot be closed: $!");
+
+       return $contents;
+}
+
 #
 # Load configuration variables from the bulk.conf file, which is a shell
 # script.
@@ -59,7 +76,16 @@
        my ($is_set, $value);
 
        foreach my $varname (@varnames) {
-               open(CMD, ". '$BULK_BUILD_CONF'; echo \"\${${varname}+set}\"; echo \"\${${varname}-}\" |")
+               my $cmd = join("\n", (
+                       #"set -eu",             # TODO: Should be enabled soon
+                       ". '${BULK_BUILD_CONF}'",
+                       ". mk/bulk/post-build-conf",
+                       "check_config_vars",
+                       "echo \"\${${varname}+set}\"",
+                       "echo \"\${${varname}-}\""
+               ));
+               
+               open(CMD, "${cmd} |")
                        or pb_die($BULK_BUILD_CONF, "Could not evaluate configuration file.");
 
                chomp($is_set = <CMD>);
@@ -91,15 +117,17 @@
 
 get_build_conf_vars(
        'ADMINSIG',             # "-Your Name"
-       'FTPURL',               # "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
-       'FTP',                  # "/disk1/ftp/${FTPURL}"
-       'FTPHOST',              # ftp://ftp.machi.ne/
-       'REPORT',               # "broken.html"
+       'REPORTS_URL',          # "ftp://ftp.example.com/pub/pkgsrc/misc/pkgstat";
+       'REPORTS_DIR',          # "$HOME/bulk-logs"
+# REPORT_BASEDIR often depends on a timestamp, which has been saved at
+# the beginning of the bulk build in the BULK_BUILD_ID_FILE. It will be
+# retrieved later.
+       'REPORT_HTML_FILE',     # "report.html"
+       'REPORT_TXT_FILE',      # "report.txt"
        'USR_PKGSRC',           # "/usr/pkgsrc"
        'osrev',                # `uname -r`
 );
 
-my $reportf = basename($vars{REPORT});
 
 my $os = `uname -s`;
 chomp $os;
@@ -151,6 +179,7 @@
        SUPPORTSFILE
        X11BASE
        PKG_TOOLS_BIN
+       BULK_BUILD_ID_FILE
 ));
 
 my $bulk_dbfile_base = basename($vars{BULK_DBFILE});
@@ -160,6 +189,17 @@
 my $indexfile_base = basename($vars{INDEXFILE});
 my $orderfile_base = basename($vars{ORDERFILE});
 
+my $reports_url = $vars{"REPORTS_URL"};
+my $reports_dir = $vars{"REPORTS_DIR"};
+my $report_basedir = readfirstline($vars{"BULK_BUILD_ID_FILE"});
+my $report_html_file = $vars{"REPORT_HTML_FILE"};
+my $report_txt_file = $vars{"REPORT_TXT_FILE"};
+
+my $report_url = "${reports_url}/${report_basedir}";
+my $report_dir = "${reports_dir}/${report_basedir}";
+my $report_html_path = "${report_dir}/${report_html_file}";
+my $report_txt_path = "${report_dir}/${report_txt_file}";
+
 my $startdate = (stat($vars{STARTFILE}))[9];
 my $enddate = '';
 if (!defined($startdate) || $startdate == 0) {
@@ -187,31 +227,27 @@
        print("--------------------------------------------------------------\n");
 }
 
-my_system("mkdir", "-p", "--", $vars{FTP});
+my_system("mkdir", "-p", "--", $report_dir);
 
 # Copy over the output from the build process
 chdir($vars{"BULKFILESDIR"}) or pb_die($vars{"BULKFILESDIR"}, "Cannot change directory.");
-my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X $vars{FTP}");
+my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X ${report_dir}");
 
 # Copy over the cache files used during the build
 foreach my $f qw(BULK_DBFILE DEPENDSTREEFILE DEPENDSFILE SUPPORTSFILE INDEXFILE ORDERFILE) {
        if (-f $vars{$f}) {
-               my_system("cp", "--", $vars{$f}, $vars{FTP});
+               my_system("cp", "--", $vars{$f}, $report_dir);
        }
 }
 
-chdir($vars{FTP}) or pb_die($vars{"FTP"}, "Cannot change directory.");
+chdir($report_dir) or pb_die($report_dir, "Cannot change directory.");
 writeReport();
 
 #
 # Adjust "last" symlink
 #
-{
-       my ($base, $dir) = ($vars{FTP} =~ m|^(.*)/([^/]*)$|);
-
-       unlink("$base/last");
-       symlink($dir, "$base/last");
-}
+unlink("${reports_dir}/last");
+symlink($report_basedir, "${reports_dir}/last");
 
 #
 # Generate leftovers-$vars{MACHINE_ARCH}.html: files not deleted
@@ -219,7 +255,7 @@
 # and linked from leftovers-$vars{MACHINE_ARCH}.html
 #
 {
-       chdir($vars{FTP});
+       chdir($report_dir) or pb_die($report_dir, "Cannot change directory.");
        my_system("mkdir", "-p", "leftovers-$vars{MACHINE_ARCH}");
 
        # Find files since last build:
@@ -270,7 +306,7 @@
 EOOUT
        foreach (@leftovers) {
                chomp;
-               print OUT "<a href=\"$vars{FTPHOST}/$vars{FTPURL}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n";
+               print OUT "<a href=\"${report_url}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n";
        }
        print OUT <<EOOUT2;
 </pre>
@@ -369,11 +405,11 @@
 Packages not listed here resulted in a binary package. The build
 report, including logs of failed/not-packaged is available from:
Prev by Date: [pkgsrc/trunk]: pkgsrc/x11/xorg-server Add missing entries to PLIST. Bump PK...
Next by Date: [pkgsrc/trunk]: pkgsrc/mk Added http://downloads.sourceforge.jp/ in MASTER_SI...
Previous by Thread: [pkgsrc/trunk]: pkgsrc/x11/xorg-server Add missing entries to PLIST. Bump PK...
Next by Thread: [pkgsrc/trunk]: pkgsrc/mk Added http://downloads.sourceforge.jp/ in MASTER_SI...
Indexes:
Home | Main Index | Thread Index | Old Index