Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[pkgsrc/trunk]: pkgsrc/pkgtools/pkg_install/files pkg_install: Fix and speed ...
details: https://anonhg.NetBSD.org/pkgsrc/rev/cdfe1edeced8
branches: trunk
changeset: 435145:cdfe1edeced8
user: jperkin <jperkin%pkgsrc.org@localhost>
date: Wed Jul 01 10:03:19 2020 +0000
description:
pkg_install: Fix and speed up "pkg_admin rebuild-tree".
In the pkg_admin front end, instead of adding +REQUIRED_BY entries as they
are found, which previously led to duplicate entries, cache the results and
write out the files at the end.
Underneath, add a caching version of iterate_pkg_db() that avoids the same
pkgdb directory lookup for every installed package, but is only suitable for
reads. Also add a cache for best_match lookups to avoid expensive matches
each time.
For all caches, use a simple hashing function to improve lookup performance.
In summary, as well as fixing +REQUIRED_BY files, these patches reduce the
wall/user/system time of "pkg_admin rebuild-tree" on a system with 12,762
packages installed down from 13m52s/11m20s/2m32s to just 1m4s/1m3s/0m1s.
diffstat:
pkgtools/pkg_install/files/add/perform.c | 12 +-
pkgtools/pkg_install/files/admin/main.c | 150 +++++++++++++++++++++++----
pkgtools/pkg_install/files/create/perform.c | 6 +-
pkgtools/pkg_install/files/info/perform.c | 8 +-
pkgtools/pkg_install/files/lib/iterate.c | 143 +++++++++++++++++++++++++-
pkgtools/pkg_install/files/lib/lib.h | 23 +++-
6 files changed, 298 insertions(+), 44 deletions(-)
diffs (truncated from 570 to 300 lines):
diff -r 93db25015226 -r cdfe1edeced8 pkgtools/pkg_install/files/add/perform.c
--- a/pkgtools/pkg_install/files/add/perform.c Wed Jul 01 09:50:16 2020 +0000
+++ b/pkgtools/pkg_install/files/add/perform.c Wed Jul 01 10:03:19 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: perform.c,v 1.111 2020/07/01 09:46:04 jperkin Exp $ */
+/* $NetBSD: perform.c,v 1.112 2020/07/01 10:03:19 jperkin Exp $ */
#if HAVE_CONFIG_H
#include "config.h"
#endif
@@ -6,7 +6,7 @@
#if HAVE_SYS_CDEFS_H
#include <sys/cdefs.h>
#endif
-__RCSID("$NetBSD: perform.c,v 1.111 2020/07/01 09:46:04 jperkin Exp $");
+__RCSID("$NetBSD: perform.c,v 1.112 2020/07/01 10:03:19 jperkin Exp $");
/*-
* Copyright (c) 2003 Grant Beattie <grant%NetBSD.org@localhost>
@@ -450,7 +450,7 @@
return -1;
}
*iter = '\0';
- pkg->other_version = find_best_matching_installed_pkg(pkgbase);
+ pkg->other_version = find_best_matching_installed_pkg(pkgbase, 0);
free(pkgbase);
if (pkg->other_version == NULL)
return 0;
@@ -1127,7 +1127,7 @@
warnx("Can't install dependency %s, continuing", dep);
}
- if (find_best_matching_installed_pkg(dep) == NULL) {
+ if (find_best_matching_installed_pkg(dep, 0) == NULL) {
if (!ForceDepends) {
warnx("Just installed dependency %s disappeared", dep);
return 1;
@@ -1158,7 +1158,7 @@
} else if (p->type != PLIST_PKGDEP)
continue;
- if (find_best_matching_installed_pkg(p->name) == NULL) {
+ if (find_best_matching_installed_pkg(p->name, 0) == NULL) {
if (install_depend_pkg(p->name) != 0) {
status = -1;
break;
@@ -1177,7 +1177,7 @@
} else if (p->type != PLIST_PKGDEP)
continue;
- best_installed = find_best_matching_installed_pkg(p->name);
+ best_installed = find_best_matching_installed_pkg(p->name, 0);
for (i = 0; i < pkg->dep_length; ++i) {
if (strcmp(best_installed, pkg->dependencies[i]) == 0)
diff -r 93db25015226 -r cdfe1edeced8 pkgtools/pkg_install/files/admin/main.c
--- a/pkgtools/pkg_install/files/admin/main.c Wed Jul 01 09:50:16 2020 +0000
+++ b/pkgtools/pkg_install/files/admin/main.c Wed Jul 01 10:03:19 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: main.c,v 1.67 2019/10/11 11:57:41 joerg Exp $ */
+/* $NetBSD: main.c,v 1.68 2020/07/01 10:03:20 jperkin Exp $ */
#if HAVE_CONFIG_H
#include "config.h"
@@ -7,7 +7,7 @@
#if HAVE_SYS_CDEFS_H
#include <sys/cdefs.h>
#endif
-__RCSID("$NetBSD: main.c,v 1.67 2019/10/11 11:57:41 joerg Exp $");
+__RCSID("$NetBSD: main.c,v 1.68 2020/07/01 10:03:20 jperkin Exp $");
/*-
* Copyright (c) 1999-2019 The NetBSD Foundation, Inc.
@@ -90,6 +90,25 @@
size_t packages;
};
+/*
+ * A hashed list of +REQUIRED_BY entries.
+ */
+struct reqd_by_entry {
+ char *pkgname;
+ SLIST_ENTRY(reqd_by_entry) entries;
+};
+SLIST_HEAD(reqd_by_entry_head, reqd_by_entry);
+
+/*
+ * A hashed list of packages that contain +REQUIRED_BY entries.
+ */
+struct pkg_reqd_by {
+ char *pkgname;
+ struct reqd_by_entry_head required_by[PKG_HASH_SIZE];
+ SLIST_ENTRY(pkg_reqd_by) entries;
+};
+SLIST_HEAD(pkg_reqd_by_head, pkg_reqd_by);
+
static const char Options[] = "C:K:SVbd:qs:v";
int quiet, verbose;
@@ -280,37 +299,79 @@
}
static void
-add_required_by(const char *pattern, const char *required_by)
+add_required_by(const char *pattern, const char *pkgname, struct pkg_reqd_by_head *hash)
{
- char *best_installed, *path;
- int fd;
- size_t len;
+ struct pkg_reqd_by_head *phead;
+ struct pkg_reqd_by *pkg;
+ struct reqd_by_entry_head *ehead;
+ struct reqd_by_entry *entry;
+ char *best_installed;
+ int i;
- best_installed = find_best_matching_installed_pkg(pattern);
+ best_installed = find_best_matching_installed_pkg(pattern, 1);
if (best_installed == NULL) {
- warnx("Dependency %s of %s unresolved", pattern, required_by);
+ warnx("Dependency %s of %s unresolved", pattern, pkgname);
return;
}
- path = pkgdb_pkg_file(best_installed, REQUIRED_BY_FNAME);
- free(best_installed);
+ /*
+ * Find correct reqd_by head based on hash of best_installed, which is
+ * the package in question that we are adding +REQUIRED_BY entries for.
+ */
+ phead = &hash[PKG_HASH_ENTRY(best_installed)];
+
+ /*
+ * Look for an existing entry in this hash list.
+ */
+ SLIST_FOREACH(pkg, phead, entries) {
+ if (strcmp(pkg->pkgname, best_installed) == 0) {
+
+ /*
+ * Found an entry, now see if it already has a
+ * +REQUIRED_BY entry recorded for this pkgname,
+ * and if not then add it.
+ */
+ ehead = &pkg->required_by[PKG_HASH_ENTRY(pkgname)];
+ SLIST_FOREACH(entry, ehead, entries) {
+ if (strcmp(entry->pkgname, pkgname) == 0)
+ break;
+ }
- if ((fd = open(path, O_WRONLY | O_APPEND | O_CREAT, 0644)) == -1)
- errx(EXIT_FAILURE, "Cannot write to %s", path);
- free(path);
-
- len = strlen(required_by);
- if (write(fd, required_by, len) != (ssize_t)len ||
- write(fd, "\n", 1) != 1 ||
- close(fd) == -1)
- errx(EXIT_FAILURE, "Cannot write to %s", path);
+ if (entry == NULL) {
+ entry = xmalloc(sizeof(*entry));
+ entry->pkgname = xstrdup(pkgname);
+ SLIST_INSERT_HEAD(ehead, entry, entries);
+ }
+
+ break;
+ }
+ }
+
+ /*
+ * Create new package containing its first +REQUIRED_BY entry.
+ */
+ if (pkg == NULL) {
+ pkg = xmalloc(sizeof(*pkg));
+ pkg->pkgname = xstrdup(best_installed);
+ for (i = 0; i < PKG_HASH_SIZE; i++)
+ SLIST_INIT(&pkg->required_by[i]);
+
+ ehead = &pkg->required_by[PKG_HASH_ENTRY(pkgname)];
+ entry = xmalloc(sizeof(*entry));
+ entry->pkgname = xstrdup(pkgname);
+ SLIST_INSERT_HEAD(ehead, entry, entries);
+
+ SLIST_INSERT_HEAD(phead, pkg, entries);
+ }
+
+ free(best_installed);
}
-
static int
add_depends_of(const char *pkgname, void *cookie)
{
FILE *fp;
+ struct pkg_reqd_by_head *h = cookie;
plist_t *p;
package_t plist;
char *path;
@@ -325,7 +386,7 @@
for (p = plist.head; p; p = p->next) {
if (p->type == PLIST_PKGDEP)
- add_required_by(p->name, pkgname);
+ add_required_by(p->name, pkgname, h);
}
free_plist(&plist);
@@ -336,10 +397,53 @@
static void
rebuild_tree(void)
{
+ FILE *fp;
+ struct pkg_reqd_by_head pkgs[PKG_HASH_SIZE];
+ struct pkg_reqd_by *p;
+ struct reqd_by_entry *e;
+ int fd, i, j;
+ char *path;
+
+ for (i = 0; i < PKG_HASH_SIZE; i++)
+ SLIST_INIT(&pkgs[i]);
+
+ /*
+ * First, calculate all of the +REQUIRED_BY entries and store in our
+ * pkgs hashed list.
+ */
+ if (iterate_pkg_db(add_depends_of, &pkgs) == -1)
+ errx(EXIT_FAILURE, "cannot iterate pkgdb");
+
+ /*
+ * Now we can remove all existing +REQUIRED_BY files.
+ */
if (iterate_pkg_db(remove_required_by, NULL) == -1)
errx(EXIT_FAILURE, "cannot iterate pkgdb");
- if (iterate_pkg_db(add_depends_of, NULL) == -1)
- errx(EXIT_FAILURE, "cannot iterate pkgdb");
+
+ /*
+ * Finally, write out all the new +REQUIRED_BY files.
+ */
+ for (i = 0; i < PKG_HASH_SIZE; i++) {
+ SLIST_FOREACH(p, &pkgs[i], entries) {
+ path = pkgdb_pkg_file(p->pkgname, REQUIRED_BY_FNAME);
+
+ if ((fd = open(path, O_WRONLY | O_APPEND | O_CREAT,
+ 0644)) == -1)
+ errx(EXIT_FAILURE, "cannot write to %s", path);
+
+ if ((fp = fdopen(fd, "a")) == NULL)
+ errx(EXIT_FAILURE, "cannot open %s", path);
+
+ for (j = 0; j < PKG_HASH_SIZE; j++) {
+ SLIST_FOREACH(e, &p->required_by[j], entries)
+ fprintf(fp, "%s\n", e->pkgname);
+ }
+ if (fclose(fp) == EOF) {
+ remove(path);
+ errx(EXIT_FAILURE, "cannot close %s", path);
+ }
+ }
+ }
}
int
diff -r 93db25015226 -r cdfe1edeced8 pkgtools/pkg_install/files/create/perform.c
--- a/pkgtools/pkg_install/files/create/perform.c Wed Jul 01 09:50:16 2020 +0000
+++ b/pkgtools/pkg_install/files/create/perform.c Wed Jul 01 10:03:19 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: perform.c,v 1.27 2015/12/27 12:36:42 joerg Exp $ */
+/* $NetBSD: perform.c,v 1.28 2020/07/01 10:03:20 jperkin Exp $ */
#if HAVE_CONFIG_H
#include "config.h"
@@ -7,7 +7,7 @@
#if HAVE_SYS_CDEFS_H
#include <sys/cdefs.h>
#endif
-__RCSID("$NetBSD: perform.c,v 1.27 2015/12/27 12:36:42 joerg Exp $");
+__RCSID("$NetBSD: perform.c,v 1.28 2020/07/01 10:03:20 jperkin Exp $");
/*
* FreeBSD install - a package for the installation and maintainance
@@ -68,7 +68,7 @@
cp = strsep(&deps, " \t\n");
if (*cp) {
char *best_installed;
- best_installed = find_best_matching_installed_pkg(cp);
+ best_installed = find_best_matching_installed_pkg(cp, 1);
if (best_installed != NULL) {
add_plist(plist, PLIST_BLDDEP, best_installed);
if (Verbose && !PlistOnly && build_only)
diff -r 93db25015226 -r cdfe1edeced8 pkgtools/pkg_install/files/info/perform.c
--- a/pkgtools/pkg_install/files/info/perform.c Wed Jul 01 09:50:16 2020 +0000
+++ b/pkgtools/pkg_install/files/info/perform.c Wed Jul 01 10:03:19 2020 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: perform.c,v 1.63 2017/04/19 21:42:50 joerg Exp $ */
+/* $NetBSD: perform.c,v 1.64 2020/07/01 10:03:20 jperkin Exp $ */
#if HAVE_CONFIG_H
#include "config.h"
@@ -7,7 +7,7 @@
#if HAVE_SYS_CDEFS_H
#include <sys/cdefs.h>
Home |
Main Index |
Thread Index |
Old Index