pkgsrc-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[pkgsrc/trunk]: pkgsrc/mk/bulk Use Perl-based tflat, which is up to 90% faste...
details: https://anonhg.NetBSD.org/pkgsrc/rev/b3f6e6fecd3d
branches: trunk
changeset: 501713:b3f6e6fecd3d
user: tv <tv%pkgsrc.org@localhost>
date: Wed Oct 26 16:54:04 2005 +0000
description:
Use Perl-based tflat, which is up to 90% faster than the awk-based one
(necause it was forking off "sort" subprocesses on every package).
diffstat:
mk/bulk/bsd.bulk-pkg.mk | 17 +--
mk/bulk/tflat | 217 ++++++++++++++---------------------------------
2 files changed, 73 insertions(+), 161 deletions(-)
diffs (296 lines):
diff -r 620c001849e1 -r b3f6e6fecd3d mk/bulk/bsd.bulk-pkg.mk
--- a/mk/bulk/bsd.bulk-pkg.mk Wed Oct 26 16:44:24 2005 +0000
+++ b/mk/bulk/bsd.bulk-pkg.mk Wed Oct 26 16:54:04 2005 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.bulk-pkg.mk,v 1.86 2005/10/16 17:35:25 tv Exp $
+# $NetBSD: bsd.bulk-pkg.mk,v 1.87 2005/10/26 16:54:04 tv Exp $
#
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf%NetBSD.org@localhost>
@@ -47,13 +47,6 @@
WC?= wc
TO_HTML?= ${SED} -e 's,&,\&,g' -e 's,<,\<,g' -e 's,>,\>,g'
-# A sort(1) capable of very long lines is needed for full builds in "tflat".
-# Some platforms (namely, Interix) may not provide one, so override here.
-.if ${OPSYS} == "Interix"
-_SORT= ${LOCALBASE}/bin/${GNU_PROGRAM_PREFIX}sort
-.endif
-_SORT?= ${SORT}
-
# This variable is set to 'yes' by the pkgsrc/mk/bulk/build script. It enables
# the use of several cache files (DEPENDSTREEFILE, DEPENDSFILE, SUPPORTSFILE,
# and INDEXFILE) for speeding up the processing of dependencies. If we're not
@@ -199,15 +192,17 @@
}} END{ \
for(pkg in pkgs) {if( pkg in listed ) {} else{ print pkg " " pkg;}} \
}' \
- ${BULK_DBFILE} | ${_SORT} -u > ${DEPENDSTREEFILE}
+ ${BULK_DBFILE} | ${SORT} -u > ${DEPENDSTREEFILE}
@${ECHO_MSG} "BULK> Extracting package name <=> package directory cross reference file"
${AWK} '/^index/ {print $$2 " " $$3 " "}' ${BULK_DBFILE} > ${INDEXFILE}
.endif
@${ECHO_MSG} "BULK> Sorting build order."
${TSORT} ${DEPENDSTREEFILE} > ${ORDERFILE}
@${ECHO_MSG} "BULK> Generating up and down dependency files."
- ${SETENV} SORT=${_SORT:Q} ${AWK} -f ${PKGSRCDIR}/mk/bulk/tflat up ${DEPENDSTREEFILE} > ${SUPPORTSFILE}
- ${SETENV} SORT=${_SORT:Q} ${AWK} -f ${PKGSRCDIR}/mk/bulk/tflat down ${DEPENDSTREEFILE} > ${DEPENDSFILE}
+ ${LOCALBASE}/bin/perl ${PKGSRCDIR}/mk/bulk/tflat ${SUPPORTSFILE} ${DEPENDSFILE} < ${DEPENDSTREEFILE}
+
+# note explicit pathname on "perl" above, so that we do NOT auto-set
+# USE_TOOLS=perl (which may be undesirable for package building)
# remove the bulk cache files
clean-bulk-cache:
diff -r 620c001849e1 -r b3f6e6fecd3d mk/bulk/tflat
--- a/mk/bulk/tflat Wed Oct 26 16:44:24 2005 +0000
+++ b/mk/bulk/tflat Wed Oct 26 16:54:04 2005 +0000
@@ -1,10 +1,10 @@
-# $NetBSD: tflat,v 1.14 2005/03/24 16:47:35 tv Exp $
+# $NetBSD: tflat,v 1.15 2005/10/26 16:54:04 tv Exp $
#
-# Copyright (c) 2001 The NetBSD Foundation, Inc.
+# Copyright (c) 2001, 2005 The NetBSD Foundation, Inc.
# All rights reserved.
#
# This code is derived from software contributed to The NetBSD Foundation
-# by Dan McMahill.
+# by Dan McMahill and Todd Vierling.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -35,170 +35,87 @@
# POSSIBILITY OF SUCH DAMAGE.
#
-BEGIN {
- if (ARGC != 3){
- printf("tflat: wrong number of arguments\n");
- usage();
- exit(1);
- }
+#
+# This perl version is very fast compared to other alternatives, which would
+# likely use things like forking "sort" subprocesses.
+#
+use strict;
- if ( ARGV[1] == "up" ) {
- up=1;
- }
- else{
- if ( ARGV[1] == "down" ) { up=0; }
- else{
- printf("tflat: unknown option \"%s\"\n",ARGV[1]);
- usage();
- exit(1);
- }
- }
-
- InFile = ARGV[2];
+sub usage () {
+ print STDERR <<EOF ;
+tflat -- flatten a depends tree. tflat is used to show all
+ packages which depend upon a given package, and
+ all packages which are depend upon by a given package.
- #
- # Read in the entire depends tree
- #
- if (up){
- while(getline < InFile > 0){
- if ($1 in topdepends)
- topdepends[$1] = topdepends[$1] " " $2 " " ;
- else{
- topdepends[$1] = " " $2 " ";
- }
+usage: $0 upfile downfile < depfile
- # Note that it is possible for a package "foo/bar" to
- # never appear in $1. In fact if foo/bar is not
- # depended upon by anything and it has depends then it
- # will not show up in $1 however, we need to make sure
- # we get a topdepends[foo/bar] entry so add it here if
- # its not already there.
- if (!($2 in topdepends))
- topdepends[$2] = "";
- }
- depstr = " is depended on by: ";
- }
- else{
- while(getline < InFile > 0){
- if ($2 in topdepends)
- topdepends[$2] = topdepends[$2] " " $1 " " ;
- else
- topdepends[$2] = " " $1 " " ;
- }
- depstr = " depends on: ";
- }
+Input file format is in the form
+foo bar
+foo baz
+libbar bar
- close(InFile);
- #
- # Now recurse the tree to give a flattened depends list for each pkg
- #
-
+meaning "foo is depended upon by bar,
+ foo is depended upon by baz,
+ libbar is depended upon by bar"
- for (toppkg in topdepends){
- find_all_depends(toppkg);
- }
+The typical use is:
+cd /usr/pkgsrc
+./mk/bulk/printdepends > .depends
+perl ./mk/bulk/tflat .supports .requires < .depends
- for (x in alldepends){
- print x depstr alldepends[x] | "${SORT-sort}";
- }
-
- printf("\n");
- exit 0
+EOF
+ exit 1;
}
-function find_all_depends(pkg,pkgreg,i,deps){
- # If we find the package already has been fully depended
- # then return the depends list.
- if (pkg in alldepends){
- return(alldepends[pkg]);
- }
+my $upfile = shift(@ARGV) || usage();
+my $downfile = shift(@ARGV) || usage();
+scalar(@ARGV) && usage();
- # If we find the package listed in its own depends list, then
- # return an empty list if we're going down the depends tree.
- # When a package lists itself in the depends tree file, it simply
- # is a place holder and means the package has no depends. However
- # other packages may still depend upon it, so we need to keep looking.
- if ( (!up) && (topdepends[pkg]~reg2str(pkg)) ){
- alldepends[pkg] = " ";
- return(alldepends[pkg]);
- }
+open(UPF, ">$upfile") || die $!;
+open(DOWNF, ">$downfile") || die $!;
+
+# read in dependstree file
- # Otherwise recursively gather depends that each of the depends
- # has.
- pkgreg=reg2str(pkg);
- split(topdepends[pkg],deps);
- i=1;
- alldepends[pkg] = " ";
- while ( i in deps ){
- # Don't add ourselves to the list (a possibility when going up
- # the tree).
- if (" "deps[i]" "!~pkgreg){
- alldepends[pkg] = alldepends[pkg] " " deps[i] " " find_all_depends(deps[i]);
- }
- i=i+1;
- }
- alldepends[pkg] = uniq(alldepends[pkg]);
- return(alldepends[pkg]);
-}
+my %depended;
+my %depends;
+while (<>) {
+ chomp;
+ my ($dep, $pkg) = split;
-#
-# Take a string which has special characters like '+' in it and
-# escape them. Also put a space before and after since that's how
-# we'll distinguish things like gnome from gnome-libs
-#
-function reg2str(reg){
- gsub(/\+/,"\\+",reg);
- reg = " "reg" ";
- return(reg);
+ push(@{$depended{$dep}}, $pkg);
+ push(@{$depends{$pkg}}, $dep);
}
-#
-# Take the depends lists and uniq them.
-#
-function uniq(list,deps,i,ulist){
-
- # split out the depends
- split(list,deps);
+# print out upfile and downfile
+
+sub getdeps ($$$@) {
+ my $curhash = shift;
+ my $alldeps = shift;
+ my $what = shift;
- i=1;
- ulist = " ";
- while (i in deps){
- if (ulist !~reg2str(deps[i])){
- ulist = ulist deps[i]" ";
+ foreach my $dep (@{$curhash->{$what}}) {
+ if ($what eq $dep) {
+ next;
+ } elsif (grep { $_ eq $dep } @_) {
+ print STDERR "circular dependency in $dep\n";
+ } elsif (!grep { $_ eq $dep } @$alldeps) {
+ push(@$alldeps, $dep);
+ getdeps($curhash, $alldeps, $dep, $what, @_);
}
- i++;
}
- return(ulist);
}
-#
-# show usage
-#
-function usage(){
- printf("tflat -- flatten a depends tree. tflat is used to show all\n");
- printf(" packages which depend upon a given package or alternatively\n");
- printf(" all packages which are depend upon by a given package.\n");
- printf("\n");
- printf("Usage:\ttflat up|down depfile\n");
- printf("\n");
- printf("Options:\tdown\tgo down the depends tree (ie \"foo depends on:\")\n");
- printf(" \tup\tgo up the depends tree (ie \"foo is depended on by:\")\n");
- printf("\n");
- printf("Input file format is in the form\n");
- printf("foo bar\n");
- printf("foo baz\n");
- printf("libbar bar\n");
- printf("\n");
- printf("meaning \"foo is depended upon by bar,\n");
- printf(" foo is depended upon by baz,\n");
- printf(" libbar is depended upon by bar\"\n");
- printf("\n");
- printf("The typical use is:\n");
- printf("cd /usr/pkgsrc\n");
- printf("./mk/bulk/printdepends > .depends\n");
- printf("./mk/bulk/tflat up .depends > .supports\n");
- printf("./mk/bulk/tflat down .depends > .requires\n");
- printf("\n");
-
+foreach my $pkg (sort(keys(%depended))) {
+ my @alldeps;
+ getdeps(\%depended, \@alldeps, $pkg);
+ print UPF "$pkg is depended on by: ".join(' ', @alldeps)."\n";
}
+foreach my $pkg (sort(keys(%depends))) {
+ my @alldeps;
+ getdeps(\%depends, \@alldeps, $pkg);
+ print DOWNF "$pkg depends on: ".join(' ', @alldeps)."\n";
+}
+
+close(UPF);
+close(DOWNF);
Home |
Main Index |
Thread Index |
Old Index