Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/usr.bin/vndcompress Implement machinery for fixed-size windo...
details: https://anonhg.NetBSD.org/src/rev/234c8c1726b4
branches: trunk
changeset: 792937:234c8c1726b4
user: riastradh <riastradh%NetBSD.org@localhost>
date: Wed Jan 22 06:15:22 2014 +0000
description:
Implement machinery for fixed-size windows into the offset table.
diffstat:
usr.bin/vndcompress/common.h | 3 +-
usr.bin/vndcompress/offtab.c | 312 ++++++++++++++++++++++++++++++------
usr.bin/vndcompress/offtab.h | 10 +-
usr.bin/vndcompress/utils.c | 41 ++++-
usr.bin/vndcompress/utils.h | 3 +-
usr.bin/vndcompress/vndcompress.c | 10 +-
usr.bin/vndcompress/vnduncompress.c | 7 +-
7 files changed, 319 insertions(+), 67 deletions(-)
diffs (truncated from 639 to 300 lines):
diff -r 4ce1bc997615 -r 234c8c1726b4 usr.bin/vndcompress/common.h
--- a/usr.bin/vndcompress/common.h Wed Jan 22 06:15:12 2014 +0000
+++ b/usr.bin/vndcompress/common.h Wed Jan 22 06:15:22 2014 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: common.h,v 1.1 2013/05/03 23:28:15 riastradh Exp $ */
+/* $NetBSD: common.h,v 1.2 2014/01/22 06:15:22 riastradh Exp $ */
/*-
* Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -109,6 +109,7 @@
#define MAX_N_BLOCKS \
(MIN(UINT32_MAX, (SIZE_MAX / sizeof(uint64_t))) - 1)
#define MAX_N_OFFSETS (MAX_N_BLOCKS + 1)
+#define MAX_WINDOW_SIZE MAX_N_OFFSETS
struct cloop2_header {
char cl2h_magic[128];
diff -r 4ce1bc997615 -r 234c8c1726b4 usr.bin/vndcompress/offtab.c
--- a/usr.bin/vndcompress/offtab.c Wed Jan 22 06:15:12 2014 +0000
+++ b/usr.bin/vndcompress/offtab.c Wed Jan 22 06:15:22 2014 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: offtab.c,v 1.1 2014/01/22 06:14:46 riastradh Exp $ */
+/* $NetBSD: offtab.c,v 1.2 2014/01/22 06:15:22 riastradh Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -62,13 +62,137 @@
errx(1, "bug in offtab, please report");
}
+
+static uint32_t
+offtab_compute_window_size(struct offtab *offtab, uint32_t start,
+ uint32_t end)
+{
+
+ if (end == 0)
+ end = offtab->ot_n_offsets;
+
+ assert(end <= offtab->ot_n_offsets);
+ assert(start < end);
+ return MIN(offtab->ot_window_size, (end - start));
+}
+
+static uint32_t
+offtab_current_window_size(struct offtab *offtab)
+{
+
+ return offtab_compute_window_size(offtab, offtab->ot_window_start, 0);
+}
+
+static uint32_t
+offtab_current_window_end(struct offtab *offtab)
+{
+
+ assert(offtab->ot_window_start < offtab->ot_n_offsets);
+ assert(offtab_current_window_size(offtab) <=
+ (offtab->ot_n_offsets - offtab->ot_window_start));
+ return (offtab->ot_window_start + offtab_current_window_size(offtab));
+}
+
+#define OFFTAB_READ_SEEK 0x01
+#define OFFTAB_READ_NOSEEK 0x00
+
+static bool
+offtab_read_window(struct offtab *offtab, uint32_t blkno, int read_flags)
+{
+
+ assert(offtab->ot_mode == OFFTAB_MODE_READ);
+
+ const uint32_t window_start = rounddown(blkno, offtab->ot_window_size);
+ const uint32_t window_size = offtab_compute_window_size(offtab,
+ window_start, 0);
+
+ __CTASSERT(MAX_WINDOW_SIZE <= (SIZE_MAX / sizeof(uint64_t)));
+ __CTASSERT(MAX_N_OFFSETS <= (OFF_MAX / sizeof(uint64_t)));
+ assert(window_start < offtab->ot_n_offsets);
+ assert(offtab->ot_fdpos <=
+ (OFF_MAX - (window_start * sizeof(uint64_t))));
+ assert(ISSET(read_flags, OFFTAB_READ_SEEK) ||
+ (lseek(offtab->ot_fd, 0, SEEK_CUR) == offtab->ot_fdpos) ||
+ ((lseek(offtab->ot_fd, 0, SEEK_CUR) == -1) && (errno == ESPIPE)));
+ const size_t n_req = (window_size * sizeof(uint64_t));
+ const ssize_t n_read = (ISSET(read_flags, OFFTAB_READ_SEEK)
+ ? pread_block(offtab->ot_fd, offtab->ot_window, n_req,
+ (offtab->ot_fdpos + (window_start * sizeof(uint64_t))))
+ : read_block(offtab->ot_fd, offtab->ot_window, n_req));
+ if (n_read == -1) {
+ (*offtab->ot_report)("read offset table at %"PRIuMAX,
+ (uintmax_t)(offtab->ot_fdpos +
+ (window_start * sizeof(uint64_t))));
+ return false;
+ }
+ assert(n_read >= 0);
+ if ((size_t)n_read != (window_size * sizeof(uint64_t))) {
+ (*offtab->ot_reportx)("partial read of offset table"
+ " at %"PRIuMAX": %zu != %zu",
+ (uintmax_t)(offtab->ot_fdpos +
+ (window_start * sizeof(uint64_t))),
+ (size_t)n_read,
+ (size_t)(window_size * sizeof(uint64_t)));
+ return false;
+ }
+ offtab->ot_window_start = window_start;
+
+ return true;
+}
+
+static bool
+offtab_maybe_read_window(struct offtab *offtab, uint32_t blkno, int read_flags)
+{
+
+ /* Don't bother if blkno is already in the window. */
+ if ((offtab->ot_window_start <= blkno) &&
+ (blkno < offtab_current_window_end(offtab)))
+ return true;
+
+ if (!offtab_read_window(offtab, blkno, read_flags))
+ return false;
+
+ return true;
+}
+
+static void
+offtab_write_window(struct offtab *offtab, uint32_t start, uint32_t end)
+{
+
+ assert(offtab->ot_mode == OFFTAB_MODE_WRITE);
+
+ /* Don't bother if [start, end) does not cover our window. */
+ if (end <= offtab->ot_window_start)
+ return;
+ if (offtab_current_window_end(offtab) < start)
+ return;
+
+ const uint32_t window_size = offtab_current_window_size(offtab);
+ __CTASSERT(MAX_WINDOW_SIZE <= (SIZE_MAX / sizeof(uint64_t)));
+ __CTASSERT(MAX_N_OFFSETS <= (OFF_MAX / sizeof(uint64_t)));
+ assert(offtab->ot_window_start < offtab->ot_n_offsets);
+ assert(offtab->ot_fdpos <=
+ (OFF_MAX - (offtab->ot_window_start * sizeof(uint64_t))));
+ const ssize_t n_written = pwrite(offtab->ot_fd, offtab->ot_window,
+ (window_size * sizeof(uint64_t)),
+ (offtab->ot_fdpos +
+ (offtab->ot_window_start * sizeof(uint64_t))));
+ if (n_written == -1)
+ err_ss(1, "write initial offset table");
+ assert(n_written >= 0);
+ if ((size_t)n_written != (window_size * sizeof(uint64_t)))
+ errx_ss(1, "partial write of initial offset bytes: %zu <= %zu",
+ (size_t)n_written,
+ (size_t)(window_size * sizeof(uint64_t)));
+}
/*
* Initialize an offtab to support the specified number of offsets read
* to or written from fd at byte position fdpos.
*/
void
-offtab_init(struct offtab *offtab, uint32_t n_offsets, int fd, off_t fdpos)
+offtab_init(struct offtab *offtab, uint32_t n_offsets, uint32_t window_size,
+ int fd, off_t fdpos)
{
assert(offtab != NULL);
@@ -77,9 +201,15 @@
assert(0 <= fdpos);
offtab->ot_n_offsets = n_offsets;
- __CTASSERT(MAX_N_OFFSETS <= (SIZE_MAX / sizeof(uint64_t)));
- offtab->ot_offsets = malloc(n_offsets * sizeof(uint64_t));
- if (offtab->ot_offsets == NULL)
+ if ((window_size == 0) || (n_offsets < window_size))
+ offtab->ot_window_size = n_offsets;
+ else
+ offtab->ot_window_size = window_size;
+ assert(offtab->ot_window_size <= offtab->ot_n_offsets);
+ offtab->ot_window_start = (uint32_t)-1;
+ __CTASSERT(MAX_WINDOW_SIZE <= (SIZE_MAX / sizeof(uint64_t)));
+ offtab->ot_window = malloc(offtab->ot_window_size * sizeof(uint64_t));
+ if (offtab->ot_window == NULL)
err(1, "malloc offset table");
offtab->ot_blkno = (uint32_t)-1;
offtab->ot_fd = fd;
@@ -96,21 +226,28 @@
offtab_destroy(struct offtab *offtab)
{
- free(offtab->ot_offsets);
+ free(offtab->ot_window);
}
/*
* For an offtab that has been used to read data from disk, convert it
* to an offtab that can be used to write subsequent data to disk.
+ * blkno is the last valid blkno read from disk.
*/
-void
-offtab_transmogrify_read_to_write(struct offtab *offtab)
+bool
+offtab_transmogrify_read_to_write(struct offtab *offtab, uint32_t blkno)
{
assert(offtab->ot_mode == OFFTAB_MODE_READ);
- assert(offtab->ot_offsets[0] == htobe64(offtab->ot_fdpos +
- (offtab->ot_n_offsets * sizeof(uint64_t))));
+ assert(0 < blkno);
+
+ if (!offtab_maybe_read_window(offtab, blkno, OFFTAB_READ_SEEK))
+ return false;
+
offtab->ot_mode = OFFTAB_MODE_WRITE;
+ offtab->ot_blkno = blkno;
+
+ return true;
}
/*
@@ -120,6 +257,10 @@
* will be reported by the report/reportx routines, which are called
* like warn/warnx. May fail; returns true on success, false on
* failure.
+ *
+ * This almost has copypasta of offtab_prepare_get, but this uses read,
+ * rather than pread, so that it will work on nonseekable input if the
+ * window is the whole offset table.
*/
bool
offtab_reset_read(struct offtab *offtab,
@@ -133,21 +274,10 @@
offtab->ot_report = report;
offtab->ot_reportx = reportx;
offtab->ot_mode = OFFTAB_MODE_READ;
+ offtab->ot_blkno = (uint32_t)-1;
- const ssize_t n_read = read_block(offtab->ot_fd, offtab->ot_offsets,
- (offtab->ot_n_offsets * sizeof(uint64_t)));
- if (n_read == -1) {
- (*offtab->ot_report)("read offset table");
+ if (!offtab_read_window(offtab, 0, OFFTAB_READ_NOSEEK))
return false;
- }
- assert(n_read >= 0);
- if ((size_t)n_read != (offtab->ot_n_offsets * sizeof(uint64_t))) {
- (*offtab->ot_reportx)("partial read of offset table"
- ": %zu != %zu",
- (size_t)n_read,
- (size_t)(offtab->ot_n_offsets * sizeof(uint64_t)));
- return false;
- }
return true;
}
@@ -163,6 +293,13 @@
assert(offtab->ot_mode == OFFTAB_MODE_READ);
assert(blkno < offtab->ot_n_offsets);
+
+ if (!offtab_maybe_read_window(offtab, blkno, OFFTAB_READ_SEEK))
+ return false;
+
+ assert(offtab->ot_window_start <= blkno);
+ assert(blkno < offtab_current_window_end(offtab));
+
offtab->ot_blkno = blkno;
return true;
}
@@ -176,9 +313,11 @@
{
assert(offtab->ot_mode == OFFTAB_MODE_READ);
- assert(blkno < offtab->ot_n_offsets);
assert(blkno == offtab->ot_blkno);
- return be64toh(offtab->ot_offsets[blkno]);
+ assert(offtab->ot_window_start <= blkno);
+ assert(blkno < offtab_current_window_end(offtab));
+
+ return be64toh(offtab->ot_window[blkno - offtab->ot_window_start]);
}
/*
@@ -195,6 +334,7 @@
assert(lseek(offtab->ot_fd, 0, SEEK_CUR) == offtab->ot_fdpos);
offtab->ot_mode = OFFTAB_MODE_WRITE;
+ offtab->ot_blkno = (uint32_t)-1;
/*
* Initialize the offset table to all ones (except for the
@@ -203,21 +343,36 @@
*/
__CTASSERT(MAX_N_OFFSETS <= UINT32_MAX);
assert(offtab->ot_n_offsets > 0);
- offtab->ot_offsets[0] = htobe64(offtab->ot_fdpos +
- (offtab->ot_n_offsets * sizeof(uint64_t)));
- for (i = 1; i < offtab->ot_n_offsets; i++)
- offtab->ot_offsets[i] = ~(uint64_t)0;
+
+ for (i = 0; i < offtab->ot_window_size; i++)
+ offtab->ot_window[i] = ~(uint64_t)0;
Home |
Main Index |
Thread Index |
Old Index