[src/trunk]: src/distrib/utils/embedded Make autosizing really work:

[christos <christos%NetBSD.org@localhost>]

details:   https://anonhg.NetBSD.org/src/rev/5429bc1d21e1
branches:  trunk
changeset: 784040:5429bc1d21e1
user:      christos <christos%NetBSD.org@localhost>
date:      Tue Jan 15 21:04:41 2013 +0000

description:
Make autosizing really work:
    - Use newfs to compute the actual filesystem required size.
    - Fix the computation of set sizes to account for blocks and fragments.
This results into a 95% full filesystem for my test (Total 566MB, 24MB free)
from which 8MB is my requested overhead. Not perfect, but good enough.

diffstat:

 distrib/utils/embedded/mkimage |  112 ++++++++++++++++++++++++++++++++++++----
 1 files changed, 101 insertions(+), 11 deletions(-)

diffs (168 lines):

diff -r 5af1651a94ac -r 5429bc1d21e1 distrib/utils/embedded/mkimage
--- a/distrib/utils/embedded/mkimage    Tue Jan 15 20:55:00 2013 +0000
+++ b/distrib/utils/embedded/mkimage    Tue Jan 15 21:04:41 2013 +0000
@@ -1,6 +1,6 @@
 #! /bin/sh
 
-# $NetBSD: mkimage,v 1.10 2013/01/15 03:26:27 christos Exp $
+# $NetBSD: mkimage,v 1.11 2013/01/15 21:04:41 christos Exp $
 
 # Copyright (c) 2012 Alistair Crooks <agc%NetBSD.org@localhost>
 # All rights reserved.
@@ -69,6 +69,8 @@
 }
 
 # find the size of the gzipped files in a .tgz archive
+# Directories appear as 0, so count them as one block
+# and round up files to a fragment.
 sizeone() {
        if [ ! -f "$1" ]
        then
@@ -77,11 +79,18 @@
                return;
        fi
         case "$1" in 
-        *.tgz|*.tar.gz)
-                tar tvzf "$1" | awk '{ tot += $5 } END { print tot }'
-                ;;
-        *.tbz|*.tar.bz2)
-                tar tvjf "$1" | awk '{ tot += $5 } END { print tot }' 
+        *.tgz|*.tar.gz|*.tbz|*.tar.bz2|*.txz|*.tar.xz)
+                tar tvzf "$1" |
+               awk -v fsize=${fsize} -v bsize=${bsize} '
+               {
+                       if ($5 == 0)
+                               tot += bsize;
+                       else
+                               tot += ((int)(($5 + fsize - 1) / fsize)) * fsize;
+               }
+               END {
+                       printf("%d\n", tot);
+               }'
                 ;;
         *)
                 echo 0
@@ -96,12 +105,70 @@
 exit 1
 }
 
+# Return the usable filesystem size in bytes, given the total size in bytes,
+# and optionally block and fragment sizes
+getffssize() {
+       local bytes="$1"
+       local barg
+       local farg
+       local overhead
+
+       if [ -n "$2" ]
+       then
+               barg="-b $2"
+               if [ -n "$3" ]
+               then
+                       farg="-f $3"
+               fi
+       fi
+
+       overhead=$(newfs -N ${farg} ${barg} -s "${bytes}b" -F /dev/null |
+           awk '/using/ {
+               printf("%d\n", substr($6, 1, length($6) - 3) * 1024 * 1024);
+           }'
+       )
+       echo $(( ${bytes} - ${overhead} ))
+}
+
+# Compute the size of an ffs filesystem that can fit x bytes.
+# Instead of duplicating the newfs calculations here we let
+# it do the job, using binary search.
+makeffssize() {
+       local bytes=$1
+       local bsize=$2
+       local fsize=$3
+       local max=$(( 2 * ${bytes} ))
+       local min="${bytes}"
+       local cur
+       local res
+       while true; do
+               cur="$(( ( ${max} + ${min} ) / 2 ))"
+               res="$(getffssize "${cur}" ${bsize} ${fsize})"
+#              echo ${min} ${cur} ${max} ${res} ${bytes} 1>&2
+               if [ "${res}" -eq "${bytes}" ]
+               then
+                   break
+               elif [ "$(( ${min} + 1 ))" -ge "${max}" ]
+               then
+                   break
+               elif [ "${res}" -lt "${bytes}" ]
+               then
+                   min="${cur}"
+               elif [ "${res}" -gt "${bytes}" ]
+               then
+                   max="${cur}"
+               fi
+       done
+       echo "${cur}"
+}
+
 finish() {
     cleanup
     ${sudo} umount ${mnt}
     ${sudo} vnconfig -u ${vnddev}
 }
 
+
 DIR="$(dirname "$0")"
 PROG="$(basename "$0")"
 bar="==="
@@ -110,6 +177,11 @@
 src="/usr/src"
 obj="/usr/obj"
 
+
+# Presumable block and fragment size.
+bsize=16384
+fsize=2048
+
 # First pass for options to get the host
 OPTS="S:c:h:s:x"
 while getopts "$OPTS" f
@@ -155,17 +227,35 @@
        shift
 fi
 
-total=0
+# calculate the set bytes
+setbytes=0
+echo -n "${bar} computing set sizes ("
 for s in ${sets}; do
        one="$(sizeone ${setsdir}/${s}.tgz)"
-       total=$(( ${total} +  ${one} ))
+       echo -n " $s=$(( ${one} / 1024 / 1024 ))MB"
+       setbytes=$(( ${setbytes} +  ${one} ))
 done
+echo "): $(( ${setbytes} / 1024 / 1024 ))MB ${bar}"
+
 # calculate size of custom files
-custsize=0
+custbytes=0
 if [ -d "${custom}" ]; then
-       custsize=$(ls -lR "${custom}" | awk 'NF == 9 { tot += $5 } END { print tot }')
+       custbytes=$(ls -lR "${custom}" | 
+           awk 'NF == 9 { tot += $5 } END { print tot }')
 fi
-total=$(( ( ( ${total} + ${custsize} ) / 1000000 ) + ${overhead} ))
+echo "${bar} computing custom sizes: $(( ${custbytes} / 1024 / 1024 ))MB ${bar}"
+
+# how many bytes
+rawbytes="$(( ${setbytes} + ${custbytes} ))"
+echo -n "${bar} computing ffs filesystem size for $(( ${rawbytes} / 1024 / 1024 ))MB: "
+ffsbytes="$(makeffssize "${rawbytes}")"
+ffsmb=$(( ${ffsbytes} / 1024 / 1024 ))
+echo " ${ffsmb}MB ${bar}"
+
+# total in MB
+total=$(( ${ffsmb} + ${overhead} ))
+echo "${bar} overhead: ${overhead}MB ${bar}"
+
 if [ $size -eq 0 ]; then
         # auto-size the pkgs fs
         newsize=${total}