Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/usr.bin/vndcompress Implement machinery for fixed-size windo...



details:   https://anonhg.NetBSD.org/src/rev/7dded648894c
branches:  trunk
changeset: 326197:7dded648894c
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Wed Jan 22 06:15:22 2014 +0000

description:
Implement machinery for fixed-size windows into the offset table.

diffstat:

 usr.bin/vndcompress/common.h        |    3 +-
 usr.bin/vndcompress/offtab.c        |  312 ++++++++++++++++++++++++++++++------
 usr.bin/vndcompress/offtab.h        |   10 +-
 usr.bin/vndcompress/utils.c         |   41 ++++-
 usr.bin/vndcompress/utils.h         |    3 +-
 usr.bin/vndcompress/vndcompress.c   |   10 +-
 usr.bin/vndcompress/vnduncompress.c |    7 +-
 7 files changed, 319 insertions(+), 67 deletions(-)

diffs (truncated from 639 to 300 lines):

diff -r cf4d613dc906 -r 7dded648894c usr.bin/vndcompress/common.h
--- a/usr.bin/vndcompress/common.h      Wed Jan 22 06:15:12 2014 +0000
+++ b/usr.bin/vndcompress/common.h      Wed Jan 22 06:15:22 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: common.h,v 1.1 2013/05/03 23:28:15 riastradh Exp $     */
+/*     $NetBSD: common.h,v 1.2 2014/01/22 06:15:22 riastradh Exp $     */
 
 /*-
  * Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -109,6 +109,7 @@
 #define        MAX_N_BLOCKS                                                    \
        (MIN(UINT32_MAX, (SIZE_MAX / sizeof(uint64_t))) - 1)
 #define        MAX_N_OFFSETS           (MAX_N_BLOCKS + 1)
+#define        MAX_WINDOW_SIZE         MAX_N_OFFSETS
 
 struct cloop2_header {
        char            cl2h_magic[128];
diff -r cf4d613dc906 -r 7dded648894c usr.bin/vndcompress/offtab.c
--- a/usr.bin/vndcompress/offtab.c      Wed Jan 22 06:15:12 2014 +0000
+++ b/usr.bin/vndcompress/offtab.c      Wed Jan 22 06:15:22 2014 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: offtab.c,v 1.1 2014/01/22 06:14:46 riastradh Exp $     */
+/*     $NetBSD: offtab.c,v 1.2 2014/01/22 06:15:22 riastradh Exp $     */
 
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -62,13 +62,137 @@
 
        errx(1, "bug in offtab, please report");
 }
+
+static uint32_t
+offtab_compute_window_size(struct offtab *offtab, uint32_t start,
+    uint32_t end)
+{
+
+       if (end == 0)
+               end = offtab->ot_n_offsets;
+
+       assert(end <= offtab->ot_n_offsets);
+       assert(start < end);
+       return MIN(offtab->ot_window_size, (end - start));
+}
+
+static uint32_t
+offtab_current_window_size(struct offtab *offtab)
+{
+
+       return offtab_compute_window_size(offtab, offtab->ot_window_start, 0);
+}
+
+static uint32_t
+offtab_current_window_end(struct offtab *offtab)
+{
+
+       assert(offtab->ot_window_start < offtab->ot_n_offsets);
+       assert(offtab_current_window_size(offtab) <=
+           (offtab->ot_n_offsets - offtab->ot_window_start));
+       return (offtab->ot_window_start + offtab_current_window_size(offtab));
+}
+
+#define        OFFTAB_READ_SEEK        0x01
+#define        OFFTAB_READ_NOSEEK      0x00
+
+static bool
+offtab_read_window(struct offtab *offtab, uint32_t blkno, int read_flags)
+{
+
+       assert(offtab->ot_mode == OFFTAB_MODE_READ);
+
+       const uint32_t window_start = rounddown(blkno, offtab->ot_window_size);
+       const uint32_t window_size = offtab_compute_window_size(offtab,
+           window_start, 0);
+
+       __CTASSERT(MAX_WINDOW_SIZE <= (SIZE_MAX / sizeof(uint64_t)));
+       __CTASSERT(MAX_N_OFFSETS <= (OFF_MAX / sizeof(uint64_t)));
+       assert(window_start < offtab->ot_n_offsets);
+       assert(offtab->ot_fdpos <=
+           (OFF_MAX - (window_start * sizeof(uint64_t))));
+       assert(ISSET(read_flags, OFFTAB_READ_SEEK) ||
+           (lseek(offtab->ot_fd, 0, SEEK_CUR) == offtab->ot_fdpos) ||
+           ((lseek(offtab->ot_fd, 0, SEEK_CUR) == -1) && (errno == ESPIPE)));
+       const size_t n_req = (window_size * sizeof(uint64_t));
+       const ssize_t n_read = (ISSET(read_flags, OFFTAB_READ_SEEK)
+           ? pread_block(offtab->ot_fd, offtab->ot_window, n_req,
+               (offtab->ot_fdpos + (window_start * sizeof(uint64_t))))
+           : read_block(offtab->ot_fd, offtab->ot_window, n_req));
+       if (n_read == -1) {
+               (*offtab->ot_report)("read offset table at %"PRIuMAX,
+                   (uintmax_t)(offtab->ot_fdpos +
+                       (window_start * sizeof(uint64_t))));
+               return false;
+       }
+       assert(n_read >= 0);
+       if ((size_t)n_read != (window_size * sizeof(uint64_t))) {
+               (*offtab->ot_reportx)("partial read of offset table"
+                   " at %"PRIuMAX": %zu != %zu",
+                   (uintmax_t)(offtab->ot_fdpos +
+                       (window_start * sizeof(uint64_t))),
+                   (size_t)n_read,
+                   (size_t)(window_size * sizeof(uint64_t)));
+               return false;
+       }
+       offtab->ot_window_start = window_start;
+
+       return true;
+}
+
+static bool
+offtab_maybe_read_window(struct offtab *offtab, uint32_t blkno, int read_flags)
+{
+
+       /* Don't bother if blkno is already in the window.  */
+       if ((offtab->ot_window_start <= blkno) &&
+           (blkno < offtab_current_window_end(offtab)))
+               return true;
+
+       if (!offtab_read_window(offtab, blkno, read_flags))
+               return false;
+
+       return true;
+}
+
+static void
+offtab_write_window(struct offtab *offtab, uint32_t start, uint32_t end)
+{
+
+       assert(offtab->ot_mode == OFFTAB_MODE_WRITE);
+
+       /* Don't bother if [start, end) does not cover our window.  */
+       if (end <= offtab->ot_window_start)
+               return;
+       if (offtab_current_window_end(offtab) < start)
+               return;
+
+       const uint32_t window_size = offtab_current_window_size(offtab);
+       __CTASSERT(MAX_WINDOW_SIZE <= (SIZE_MAX / sizeof(uint64_t)));
+       __CTASSERT(MAX_N_OFFSETS <= (OFF_MAX / sizeof(uint64_t)));
+       assert(offtab->ot_window_start < offtab->ot_n_offsets);
+       assert(offtab->ot_fdpos <=
+           (OFF_MAX - (offtab->ot_window_start * sizeof(uint64_t))));
+       const ssize_t n_written = pwrite(offtab->ot_fd, offtab->ot_window,
+           (window_size * sizeof(uint64_t)),
+           (offtab->ot_fdpos +
+               (offtab->ot_window_start * sizeof(uint64_t))));
+       if (n_written == -1)
+               err_ss(1, "write initial offset table");
+       assert(n_written >= 0);
+       if ((size_t)n_written != (window_size * sizeof(uint64_t)))
+               errx_ss(1, "partial write of initial offset bytes: %zu <= %zu",
+                   (size_t)n_written,
+                   (size_t)(window_size * sizeof(uint64_t)));
+}
 
 /*
  * Initialize an offtab to support the specified number of offsets read
  * to or written from fd at byte position fdpos.
  */
 void
-offtab_init(struct offtab *offtab, uint32_t n_offsets, int fd, off_t fdpos)
+offtab_init(struct offtab *offtab, uint32_t n_offsets, uint32_t window_size,
+    int fd, off_t fdpos)
 {
 
        assert(offtab != NULL);
@@ -77,9 +201,15 @@
        assert(0 <= fdpos);
 
        offtab->ot_n_offsets = n_offsets;
-       __CTASSERT(MAX_N_OFFSETS <= (SIZE_MAX / sizeof(uint64_t)));
-       offtab->ot_offsets = malloc(n_offsets * sizeof(uint64_t));
-       if (offtab->ot_offsets == NULL)
+       if ((window_size == 0) || (n_offsets < window_size))
+               offtab->ot_window_size = n_offsets;
+       else
+               offtab->ot_window_size = window_size;
+       assert(offtab->ot_window_size <= offtab->ot_n_offsets);
+       offtab->ot_window_start = (uint32_t)-1;
+       __CTASSERT(MAX_WINDOW_SIZE <= (SIZE_MAX / sizeof(uint64_t)));
+       offtab->ot_window = malloc(offtab->ot_window_size * sizeof(uint64_t));
+       if (offtab->ot_window == NULL)
                err(1, "malloc offset table");
        offtab->ot_blkno = (uint32_t)-1;
        offtab->ot_fd = fd;
@@ -96,21 +226,28 @@
 offtab_destroy(struct offtab *offtab)
 {
 
-       free(offtab->ot_offsets);
+       free(offtab->ot_window);
 }
 
 /*
  * For an offtab that has been used to read data from disk, convert it
  * to an offtab that can be used to write subsequent data to disk.
+ * blkno is the last valid blkno read from disk.
  */
-void
-offtab_transmogrify_read_to_write(struct offtab *offtab)
+bool
+offtab_transmogrify_read_to_write(struct offtab *offtab, uint32_t blkno)
 {
 
        assert(offtab->ot_mode == OFFTAB_MODE_READ);
-       assert(offtab->ot_offsets[0] == htobe64(offtab->ot_fdpos +
-           (offtab->ot_n_offsets * sizeof(uint64_t))));
+       assert(0 < blkno);
+
+       if (!offtab_maybe_read_window(offtab, blkno, OFFTAB_READ_SEEK))
+               return false;
+
        offtab->ot_mode = OFFTAB_MODE_WRITE;
+       offtab->ot_blkno = blkno;
+
+       return true;
 }
 
 /*
@@ -120,6 +257,10 @@
  * will be reported by the report/reportx routines, which are called
  * like warn/warnx.  May fail; returns true on success, false on
  * failure.
+ *
+ * This almost has copypasta of offtab_prepare_get, but this uses read,
+ * rather than pread, so that it will work on nonseekable input if the
+ * window is the whole offset table.
  */
 bool
 offtab_reset_read(struct offtab *offtab,
@@ -133,21 +274,10 @@
        offtab->ot_report = report;
        offtab->ot_reportx = reportx;
        offtab->ot_mode = OFFTAB_MODE_READ;
+       offtab->ot_blkno = (uint32_t)-1;
 
-       const ssize_t n_read = read_block(offtab->ot_fd, offtab->ot_offsets,
-           (offtab->ot_n_offsets * sizeof(uint64_t)));
-       if (n_read == -1) {
-               (*offtab->ot_report)("read offset table");
+       if (!offtab_read_window(offtab, 0, OFFTAB_READ_NOSEEK))
                return false;
-       }
-       assert(n_read >= 0);
-       if ((size_t)n_read != (offtab->ot_n_offsets * sizeof(uint64_t))) {
-               (*offtab->ot_reportx)("partial read of offset table"
-                   ": %zu != %zu",
-                   (size_t)n_read,
-                   (size_t)(offtab->ot_n_offsets * sizeof(uint64_t)));
-               return false;
-       }
 
        return true;
 }
@@ -163,6 +293,13 @@
 
        assert(offtab->ot_mode == OFFTAB_MODE_READ);
        assert(blkno < offtab->ot_n_offsets);
+
+       if (!offtab_maybe_read_window(offtab, blkno, OFFTAB_READ_SEEK))
+               return false;
+
+       assert(offtab->ot_window_start <= blkno);
+       assert(blkno < offtab_current_window_end(offtab));
+
        offtab->ot_blkno = blkno;
        return true;
 }
@@ -176,9 +313,11 @@
 {
 
        assert(offtab->ot_mode == OFFTAB_MODE_READ);
-       assert(blkno < offtab->ot_n_offsets);
        assert(blkno == offtab->ot_blkno);
-       return be64toh(offtab->ot_offsets[blkno]);
+       assert(offtab->ot_window_start <= blkno);
+       assert(blkno < offtab_current_window_end(offtab));
+
+       return be64toh(offtab->ot_window[blkno - offtab->ot_window_start]);
 }
 
 /*
@@ -195,6 +334,7 @@
        assert(lseek(offtab->ot_fd, 0, SEEK_CUR) == offtab->ot_fdpos);
 
        offtab->ot_mode = OFFTAB_MODE_WRITE;
+       offtab->ot_blkno = (uint32_t)-1;
 
        /*
         * Initialize the offset table to all ones (except for the
@@ -203,21 +343,36 @@
         */
        __CTASSERT(MAX_N_OFFSETS <= UINT32_MAX);
        assert(offtab->ot_n_offsets > 0);
-       offtab->ot_offsets[0] = htobe64(offtab->ot_fdpos +
-           (offtab->ot_n_offsets * sizeof(uint64_t)));
-       for (i = 1; i < offtab->ot_n_offsets; i++)
-               offtab->ot_offsets[i] = ~(uint64_t)0;
+
+       for (i = 0; i < offtab->ot_window_size; i++)
+               offtab->ot_window[i] = ~(uint64_t)0;



Home | Main Index | Thread Index | Old Index