tech-userlevel archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

Re: Importing the apropos GSoC code and switching to mandoc(1) by default



On Thu, Feb 02, 2012 at 12:46:54AM +0100, Joerg Sonnenberger wrote:
> attached is the new apropos created by Abhinav Upadhyay during last
> year's Summer of Code. I've added a small whatis(1) implementation to
> round it up. This should be a functional replacement for apropos(1) and
> whatis(1). It obsoletes makewhatis(8).

Attached is the patch to integrate this into base and hook it into
daily/weekly and the boot process. Old apropos/whatis can be obtained
for now by using MKMAKEMANDB=no.

This also fixes a bug found in to makemandb code when using it with
certain packages.

Joerg
Index: distrib/sets/sets.subr
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/sets.subr,v
retrieving revision 1.134
diff -u -p -r1.134 sets.subr
--- distrib/sets/sets.subr      3 Feb 2012 22:18:04 -0000       1.134
+++ distrib/sets/sets.subr      3 Feb 2012 23:44:42 -0000
@@ -228,6 +228,7 @@ SUBST="${SUBST};s#@MACHINE@#${MACHINE}#g
 #      lint                    ${MKLINT} != no
 #      llvm                    ${MKLLVM} != no
 #      lvm                     ${MKLVM} != no
+#      makemandb               ${MKMAKEMANDB} != no
 #      man                     ${MKMAN} != no
 #      manpages                ${MKMANPAGES} != no
 #      manz                    ${MKMANZ} != no
Index: distrib/sets/lists/base/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/base/mi,v
retrieving revision 1.979
diff -u -p -r1.979 mi
--- distrib/sets/lists/base/mi  3 Feb 2012 22:18:04 -0000       1.979
+++ distrib/sets/lists/base/mi  3 Feb 2012 23:44:42 -0000
@@ -1023,7 +1023,7 @@
 ./usr/libexec/fingerd                          base-netutil-bin
 ./usr/libexec/ftp-proxy                                base-obsolete           
obsolete
 ./usr/libexec/ftpd                             base-netutil-bin
-./usr/libexec/getNAME                          base-man-bin
+./usr/libexec/getNAME                          base-man-bin            
!makemandb
 ./usr/libexec/getty                            base-sysutil-bin
 ./usr/libexec/hprop                            base-obsolete           obsolete
 ./usr/libexec/hpropd                           base-krb5-bin           kerberos
@@ -1044,7 +1044,7 @@
 ./usr/libexec/lpr/pjlfilter                    base-lpr-bin
 ./usr/libexec/mail.local                       base-mail-bin
 ./usr/libexec/makekey                          base-crypto-bin
-./usr/libexec/makewhatis                       base-man-bin
+./usr/libexec/makewhatis                       base-man-bin            
!makemandb
 ./usr/libexec/named-xfer                       base-obsolete           obsolete
 ./usr/libexec/ntalkd                           base-netutil-bin
 ./usr/libexec/pfspamd                          base-obsolete           obsolete
@@ -3614,7 +3614,7 @@
 ./usr/share/man/html9/vax                      base-man-share
 ./usr/share/man/html9/x68k                     base-man-share
 ./usr/share/man/html9/x86                      base-man-share
-./usr/share/man/makewhatis.sed                 base-man-share          share
+./usr/share/man/makewhatis.sed                 base-man-share          
!makemandb,share
 ./usr/share/man/man1                           base-man-share
 ./usr/share/man/man1/atari                     base-man-share
 ./usr/share/man/man1/hp300                     base-man-share
@@ -3742,7 +3742,7 @@
 ./usr/share/man/man9/vax                       base-man-share
 ./usr/share/man/man9/x68k                      base-man-share
 ./usr/share/man/man9/x86                       base-man-share
-./usr/share/man/whatis.db                      base-man-share          man
+./usr/share/man/whatis.db                      base-man-share          
man,!makemandb
 ./usr/share/me                                 base-groff-share
 ./usr/share/misc                               base-terminfo-share
 ./usr/share/misc/acronyms                      base-reference-share    share
Index: distrib/sets/lists/comp/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/comp/mi,v
retrieving revision 1.1729
diff -u -p -r1.1729 mi
--- distrib/sets/lists/comp/mi  1 Feb 2012 05:34:38 -0000       1.1729
+++ distrib/sets/lists/comp/mi  3 Feb 2012 17:39:46 -0000
@@ -3971,7 +3971,7 @@
 ./usr/libdata/debug/usr/libexec/fingerd.debug  comp-netutil-debug      debug
 ./usr/libdata/debug/usr/libexec/ftp-proxy.debug        comp-obsolete           
obsolete
 ./usr/libdata/debug/usr/libexec/ftpd.debug     comp-netutil-debug      debug
-./usr/libdata/debug/usr/libexec/getNAME.debug  comp-man-debug          debug
+./usr/libdata/debug/usr/libexec/getNAME.debug  comp-man-debug          
!makemandb,debug
 ./usr/libdata/debug/usr/libexec/getty.debug    comp-sysutil-debug      debug
 ./usr/libdata/debug/usr/libexec/hpropd.debug   comp-krb5-debug         
kerberos,debug
 ./usr/libdata/debug/usr/libexec/httpd.debug    comp-netutil-debug      debug
@@ -3988,7 +3988,7 @@
 ./usr/libdata/debug/usr/libexec/lpr/lpf.debug  comp-lpr-debug          debug
 ./usr/libdata/debug/usr/libexec/mail.local.debug       comp-mail-debug         
debug
 ./usr/libdata/debug/usr/libexec/makekey.debug  comp-crypto-debug       debug
-./usr/libdata/debug/usr/libexec/makewhatis.debug       comp-man-debug          
debug
+./usr/libdata/debug/usr/libexec/makewhatis.debug       comp-man-debug          
!makemandb,debug
 ./usr/libdata/debug/usr/libexec/ntalkd.debug   comp-netutil-debug      debug
 ./usr/libdata/debug/usr/libexec/pcpp.debug     comp-c-debug            
pcc,debug
 ./usr/libdata/debug/usr/libexec/postfix/anvil.debug    comp-postfix-debug      
postfix,debug
@@ -4148,6 +4148,7 @@
 ./usr/libdata/debug/usr/sbin/lpd.debug         comp-lpr-debug          debug
 ./usr/libdata/debug/usr/sbin/lptest.debug      comp-lpr-debug          debug
 ./usr/libdata/debug/usr/sbin/mailwrapper.debug comp-mailwrapper-debug  debug
+./usr/libdata/debug/usr/sbin//makemandb.debug  comp-makemandb-debug    
makemandb,debug
 ./usr/libdata/debug/usr/sbin/makedbm.debug     comp-nis-debug          yp,debug
 ./usr/libdata/debug/usr/sbin/makefs.debug      comp-sysutil-debug      debug
 ./usr/libdata/debug/usr/sbin/map-mbone.debug   comp-netutil-debug      debug
Index: distrib/sets/lists/etc/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/etc/mi,v
retrieving revision 1.227
diff -u -p -r1.227 mi
--- distrib/sets/lists/etc/mi   17 Dec 2011 13:37:49 -0000      1.227
+++ distrib/sets/lists/etc/mi   4 Feb 2012 18:20:52 -0000
@@ -223,6 +223,7 @@
 ./etc/rc.d/local                               etc-sys-rc
 ./etc/rc.d/lpd                                 etc-lpr-rc
 ./etc/rc.d/lvm                                 etc-sys-rc
+./etc/rc.d/makemandb                           etc-man-rc
 ./etc/rc.d/mdnsd                               etc-mdns-rc
 ./etc/rc.d/mixerctl                            etc-audio-rc
 ./etc/rc.d/mopd                                        etc-bootserver-rc
Index: distrib/sets/lists/man/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/man/mi,v
retrieving revision 1.1375
diff -u -p -r1.1375 mi
--- distrib/sets/lists/man/mi   3 Feb 2012 22:18:05 -0000       1.1375
+++ distrib/sets/lists/man/mi   3 Feb 2012 23:44:42 -0000
@@ -2313,7 +2313,7 @@
 ./usr/share/man/cat8/fusermount.0              man-refuse-catman       .cat
 ./usr/share/man/cat8/fwctl.0                   man-sysutil-catman      .cat
 ./usr/share/man/cat8/genassym.sh.0             man-obsolete            obsolete
-./usr/share/man/cat8/getNAME.0                 man-man-catman          .cat
+./usr/share/man/cat8/getNAME.0                 man-man-catman          
!makemandb, .cat
 ./usr/share/man/cat8/getencstat.0              man-sysutil-catman      .cat
 ./usr/share/man/cat8/gettable.0                        man-obsolete            
obsolete
 ./usr/share/man/cat8/getty.0                   man-sysutil-catman      .cat
@@ -2477,8 +2477,9 @@
 ./usr/share/man/cat8/makedev.local.0           man-sysutil-catman      .cat
 ./usr/share/man/cat8/makefs.0                  man-sysutil-catman      .cat
 ./usr/share/man/cat8/makekey.0                 man-crypto-catman       .cat
+./usr/share/man/cat8/makemandb.0               man-man-catman          
makemandb,.cat
 ./usr/share/man/cat8/makemap.0                 man-obsolete            obsolete
-./usr/share/man/cat8/makewhatis.0              man-man-catman          .cat
+./usr/share/man/cat8/makewhatis.0              man-man-catman          
!makemandb,.cat
 ./usr/share/man/cat8/map-mbone.0               man-netutil-catman      .cat
 ./usr/share/man/cat8/master.0                  man-postfix-catman      
postfix,.cat
 ./usr/share/man/cat8/mbr.0                     man-obsolete            obsolete
@@ -5015,7 +5016,7 @@
 ./usr/share/man/html8/ftpd.html                        man-netutil-htmlman     
html
 ./usr/share/man/html8/fusermount.html          man-refuse-htmlman      html
 ./usr/share/man/html8/fwctl.html               man-sysutil-htmlman     html
-./usr/share/man/html8/getNAME.html             man-man-htmlman         html
+./usr/share/man/html8/getNAME.html             man-man-htmlman         
!makemandb,html
 ./usr/share/man/html8/getencstat.html          man-sysutil-htmlman     html
 ./usr/share/man/html8/getty.html               man-sysutil-htmlman     html
 ./usr/share/man/html8/gpioctl.html             man-sysutil-htmlman     html
@@ -5141,7 +5142,8 @@
 ./usr/share/man/html8/makedev.local.html       man-sysutil-htmlman     html
 ./usr/share/man/html8/makefs.html              man-sysutil-htmlman     html
 ./usr/share/man/html8/makekey.html             man-crypto-htmlman      html
-./usr/share/man/html8/makewhatis.html          man-man-htmlman         html
+./usr/share/man/html8/makemandb.html           man-man-htmlman         
makemandb,html
+./usr/share/man/html8/makewhatis.html          man-man-htmlman         
!makemandb,html
 ./usr/share/man/html8/map-mbone.html           man-netutil-htmlman     html
 ./usr/share/man/html8/master.html              man-postfix-htmlman     
postfix,html
 ./usr/share/man/html8/mbrlabel.html            man-sysutil-htmlman     html
@@ -7811,7 +7813,7 @@
 ./usr/share/man/man8/fusermount.8              man-refuse-man          .man
 ./usr/share/man/man8/fwctl.8                   man-sysutil-man         .man
 ./usr/share/man/man8/genassym.sh.8             man-obsolete            obsolete
-./usr/share/man/man8/getNAME.8                 man-man-man             .man
+./usr/share/man/man8/getNAME.8                 man-man-man             
!makemandb,.man
 ./usr/share/man/man8/getencstat.8              man-sysutil-man         .man
 ./usr/share/man/man8/gettable.8                        man-obsolete            
obsolete
 ./usr/share/man/man8/getty.8                   man-sysutil-man         .man
@@ -7976,8 +7978,9 @@
 ./usr/share/man/man8/makedev.local.8           man-sysutil-man         .man
 ./usr/share/man/man8/makefs.8                  man-sysutil-man         .man
 ./usr/share/man/man8/makekey.8                 man-crypto-man          .man
+./usr/share/man/man8/makemandb.8               man-man-man             
makemandb,.man
 ./usr/share/man/man8/makemap.8                 man-obsolete            obsolete
-./usr/share/man/man8/makewhatis.8              man-man-man             .man
+./usr/share/man/man8/makewhatis.8              man-man-man             
!makemandb,.man
 ./usr/share/man/man8/map-mbone.8               man-netutil-man         .man
 ./usr/share/man/man8/master.8                  man-postfix-man         
postfix,.man
 ./usr/share/man/man8/mbr.8                     man-obsolete            obsolete
Index: distrib/sets/lists/text/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/text/mi,v
retrieving revision 1.41
diff -u -p -r1.41 mi
--- distrib/sets/lists/text/mi  11 Oct 2011 19:20:13 -0000      1.41
+++ distrib/sets/lists/text/mi  3 Feb 2012 17:27:53 -0000
@@ -40,6 +40,7 @@
 ./usr/bin/troff                                        text-groff-bin          
groff
 ./usr/bin/vgrind                               text-groff-bin
 ./usr/libexec/vfontedpr                                text-groff-bin
+./usr/sbin/makemandb                           base-man-bin            
makemandb
 ./usr/share/doc/groff                          text-groff-doc
 ./usr/share/doc/groff/mom                      text-groff-doc
 ./usr/share/doc/groff/mom/README.txt           text-groff-doc          
share,groff
Index: distrib/sets/lists/xbase/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/xbase/mi,v
retrieving revision 1.111
diff -u -p -r1.111 mi
--- distrib/sets/lists/xbase/mi 22 Oct 2011 22:23:50 -0000      1.111
+++ distrib/sets/lists/xbase/mi 3 Feb 2012 17:47:28 -0000
@@ -1340,7 +1340,7 @@
 ./usr/X11R6/man/man7/XProjectTeam.7                    xbase-obsolete  obsolete
 ./usr/X11R6/man/man7/XStandards.7                      -unknown-       .man,x11
 ./usr/X11R6/man/man7/Xsecurity.7                       -unknown-       .man,x11
-./usr/X11R6/man/whatis.db                              -unknown-       man,x11
+./usr/X11R6/man/whatis.db                              -unknown-       
man,x11,!makemandb
 ./usr/X11R6/share                                      base-x11-root   x11
 ./usr/X11R6/share/examples                             base-x11-root   x11
 ./usr/X11R6/share/examples/ssh-askpass                 base-x11-root   x11
@@ -2888,7 +2888,7 @@
 ./usr/X11R7/man/man7/XOrgFoundation.7                  -unknown-       
.man,xorg
 ./usr/X11R7/man/man7/XProjectTeam.7                    -unknown-       
.man,xorg
 ./usr/X11R7/man/man7/Xsecurity.7                       -unknown-       
.man,xorg
-./usr/X11R7/man/whatis.db                              -unknown-       man,xorg
+./usr/X11R7/man/whatis.db                              -unknown-       
man,xorg,!makemandb
 ./usr/X11R7/share                                      base-x11-root   xorg
 ./usr/X11R7/share/aclocal                              base-x11-root   xorg
 ./usr/X11R7/share/examples                             base-x11-root   xorg
Index: etc/daily
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/daily,v
retrieving revision 1.78
diff -u -p -r1.78 daily
--- etc/daily   17 Dec 2011 12:27:42 -0000      1.78
+++ etc/daily   4 Feb 2012 21:37:42 -0000
@@ -290,6 +290,16 @@ if checkyesno run_skeyaudit; then
        fi
 fi
 
+run_makemandb=YES
+
+if checkyesno run_makemandb; then
+       if [ -f /etc/man.conf -a -x /usr/sbin/makemandb ]; then
+               echo ""
+               echo "Updating man page index:"
+               nice -n 5 /usr/sbin/makemandb
+       fi
+fi
+
 if [ -f /etc/daily.local ]; then
        ( . /etc/daily.local ) > $TMP 2>&1
        if [ -s $TMP ] ; then
Index: etc/weekly
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/weekly,v
retrieving revision 1.24
diff -u -p -r1.24 weekly
--- etc/weekly  27 Dec 2010 03:38:52 -0000      1.24
+++ etc/weekly  4 Feb 2012 21:37:47 -0000
@@ -81,7 +81,8 @@ fi
 
 if checkyesno rebuild_whatisdb; then
        echo ""
-       if [ -f /etc/man.conf -o -d /usr/share/man ]; then
+       if [ -x /usr/libexec/makewhatis -a \( -f /etc/man.conf -o \
+            -o -d /usr/share/man \) ]; then
                echo "Rebuilding whatis databases:"
                nice -n 5 /usr/libexec/makewhatis -f
        else
@@ -89,6 +90,16 @@ if checkyesno rebuild_whatisdb; then
        fi
 fi
 
+if checkyesno rebuild_mandb; then
+       echo ""
+       if [ -f /etc/man.conf -a -x /usr/sbin/makemandb ]; then
+               echo "Rebuilding man page index:"
+               nice -n 5 /usr/sbin/makemandb -f
+       else
+               echo "Not rebuilding man page index"
+       fi
+fi
+
 if [ -f /etc/weekly.local ]; then
        . /etc/weekly.local > $TMP 2>&1
        if [ -s $TMP ] ; then
Index: etc/defaults/daily.conf
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/defaults/daily.conf,v
retrieving revision 1.15
diff -u -p -r1.15 daily.conf
--- etc/defaults/daily.conf     17 Dec 2011 12:27:42 -0000      1.15
+++ etc/defaults/daily.conf     4 Feb 2012 21:34:45 -0000
@@ -25,6 +25,7 @@ check_network=YES
 full_netstat=NO
 run_fsck=NO
 run_fsck_flags=""
+run_makemandb=YES
 run_rdist=YES
 run_security=YES
 separate_security_email=YES
Index: etc/defaults/rc.conf
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/defaults/rc.conf,v
retrieving revision 1.118
diff -u -p -r1.118 rc.conf
--- etc/defaults/rc.conf        23 Dec 2011 05:05:04 -0000      1.118
+++ etc/defaults/rc.conf        4 Feb 2012 18:20:10 -0000
@@ -366,3 +366,6 @@ veriexec_flags="-k"
 # Entropy load/save to/from /dev/random at startup/shutdown
 #
 random_seed=YES
+
+# Creating / updating of man page index on boot
+makemandb=YES
Index: etc/defaults/weekly.conf
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/defaults/weekly.conf,v
retrieving revision 1.3
diff -u -p -r1.3 weekly.conf
--- etc/defaults/weekly.conf    15 Feb 2007 23:04:10 -0000      1.3
+++ etc/defaults/weekly.conf    4 Feb 2012 18:15:01 -0000
@@ -11,3 +11,4 @@
 
 rebuild_locatedb=YES
 rebuild_whatisdb=NO
+rebuild_mandb=YES
Index: etc/rc.d/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/rc.d/Makefile,v
retrieving revision 1.87
diff -u -p -r1.87 Makefile
--- etc/rc.d/Makefile   17 Dec 2011 13:37:49 -0000      1.87
+++ etc/rc.d/Makefile   4 Feb 2012 18:20:28 -0000
@@ -26,7 +26,7 @@ CONFIGFILES=\
                irdaattach iscsi_target isdnd isibootd \
                kdc \
                ldconfig ldpd local lpd lvm \
-               mdnsd mixerctl mopd motd mountall mountcritlocal \
+               makemandb mdnsd mixerctl mopd motd mountall mountcritlocal \
                mountcritremote mountd moused mrouted \
                named ndbootd network newsyslog nfsd nfslocking npf ntpd 
ntpdate \
                perusertmp pf pf_boot pflogd postfix powerd ppp pwcheck \
Index: etc/rc.d/makemandb
===================================================================
RCS file: etc/rc.d/makemandb
diff -N etc/rc.d/makemandb
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ etc/rc.d/makemandb  4 Feb 2012 21:38:06 -0000
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# $NetBSD$
+#
+
+# PROVIDE: makemandb
+# REQUIRE: DAEMON
+# BEFORE:  LOGIN
+
+$_rc_subr_loaded . /etc/rc.subr
+
+name="makemandb"
+required_files="/etc/man.conf /usr/sbin/makemandb"
+start_cmd="makemandb_start"
+stop_cmd=":"
+
+makemandb_start()
+{
+       # Initiate update build in the background
+       nice 5 /usr/sbin/makemandb -q > /dev/null 2>&1 &
+}
+
+load_rc_config $name
+run_rc_command "$1"
Index: lib/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/lib/Makefile,v
retrieving revision 1.180
diff -u -p -r1.180 Makefile
--- lib/Makefile        5 Nov 2011 22:40:01 -0000       1.180
+++ lib/Makefile        3 Feb 2012 16:31:22 -0000
@@ -179,6 +179,8 @@ SUBDIR+=    libpam          # depends on heimdal
 SUBDIR+=       ../crypto/external/bsd/libsaslc # depends on heimdal, openssl
 .endif
 
+SUBDIR+=       ../external/bsd/mdocml/lib
+
 #==================== 4th library dependency barrier ====================
 SUBDIR+=       .WAIT
 
Index: libexec/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/libexec/Makefile,v
retrieving revision 1.60
diff -u -p -r1.60 Makefile
--- libexec/Makefile    15 Apr 2011 18:05:45 -0000      1.60
+++ libexec/Makefile    3 Feb 2012 16:19:37 -0000
@@ -4,9 +4,13 @@
 .include <bsd.own.mk>
 
 SUBDIR= atrun \
-       comsat fingerd ftpd getNAME getty httpd identd ld.aout_so \
-       ld.elf_so lfs_cleanerd mail.local makekey makewhatis rexecd \
+       comsat fingerd ftpd getty httpd identd ld.aout_so \
+       ld.elf_so lfs_cleanerd mail.local makekey rexecd \
        rlogind rshd rpc.rquotad rpc.rstatd rpc.rusersd rpc.rwalld \
        rpc.sprayd talkd telnetd tftpd utmp_update
 
+.if ${MKMAKEMANDB} == "no"
+SUBDIR+= getNAME makewhatis
+.endif
+
 .include <bsd.subdir.mk>
Index: share/man/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/share/man/Makefile,v
retrieving revision 1.29
diff -u -p -r1.29 Makefile
--- share/man/Makefile  9 Feb 2004 07:58:01 -0000       1.29
+++ share/man/Makefile  3 Feb 2012 16:28:50 -0000
@@ -8,6 +8,7 @@ SUBDIR= man0 man1 man2 man3 man4 man5 ma
 
 WHATISDBDIR?=  /usr/share/man
 
+.if ${MKMAKEMANDB} == "no"
 makedb:
        ${TOOL_MAKEWHATIS} ${DESTDIR}${WHATISDBDIR}
 .if ${MKUNPRIVED} != "no"
@@ -18,5 +19,9 @@ makedb:
 makedb:
 .endif
 
+.else
+makedb:
+.endif
+
 .include <bsd.subdir.mk>
 .include <bsd.sys.mk>
Index: share/man/man0/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/share/man/man0/Makefile,v
retrieving revision 1.16
diff -u -p -r1.16 Makefile
--- share/man/man0/Makefile     25 Oct 2008 22:27:36 -0000      1.16
+++ share/man/man0/Makefile     3 Feb 2012 16:59:52 -0000
@@ -5,7 +5,7 @@ NOOBJ=          # defined
 
 .include <bsd.own.mk>
 
-.if ${MKSHARE} != "no"
+.if ${MKSHARE} != "no" && ${MKMAKEMANDB} == "no"
 FILES=         makewhatis.sed
 FILESDIR=      /usr/share/man
 .endif
Index: share/mk/bsd.own.mk
===================================================================
RCS file: /home/joerg/repo/netbsd/src/share/mk/bsd.own.mk,v
retrieving revision 1.693
diff -u -p -r1.693 bsd.own.mk
--- share/mk/bsd.own.mk 25 Nov 2011 22:03:42 -0000      1.693
+++ share/mk/bsd.own.mk 3 Feb 2012 16:17:35 -0000
@@ -815,6 +815,7 @@ _MKVARS.yes= \
        MKLDAP MKLINKLIB MKLINT MKLVM \
        MKMAN \
        MKMDNS \
+       MKMAKEMANDB \
        MKNLS \
        MKNPF \
        MKOBJ \
Index: usr.bin/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/usr.bin/Makefile,v
retrieving revision 1.206
diff -u -p -r1.206 Makefile
--- usr.bin/Makefile    4 Jan 2012 16:25:13 -0000       1.206
+++ usr.bin/Makefile    3 Feb 2012 16:18:42 -0000
@@ -3,7 +3,7 @@
 
 .include <bsd.own.mk>
 
-SUBDIR= apply apropos asa at audio audiocfg \
+SUBDIR= apply asa at audio audiocfg \
        banner basename biff bthset btkey btpin \
        bzip2 bzip2recover cal calendar cap_mkdb cdplay \
        checknr chflags chpass cksum cmp cleantags col colcrt colrm \
@@ -31,11 +31,15 @@ SUBDIR= apply apropos asa at audio audio
        uniq units unvis unzip usbhidaction usbhidctl users utoppya \
        uudecode uuencode uuidgen vacation vgrind videoctl vis \
        vmstat vndcompress w \
-       wall wc what whatis whereis who whois window \
+       wall wc what whereis who whois window \
        write xargs xinstall xlint xstr yes
 
 SUBDIR+= ../external/zlib/pigz/bin/pigz
 
+.if MKMAKEMANDB == "no"
+SUBDIR+= apropos whatis
+.endif
+
 .if (${MKBSDGREP} != "no")
 SUBDIR+= grep
 .endif
Index: usr.sbin/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/usr.sbin/Makefile,v
retrieving revision 1.260
diff -u -p -r1.260 Makefile
--- usr.sbin/Makefile   4 Jan 2012 16:25:14 -0000       1.260
+++ usr.sbin/Makefile   3 Feb 2012 16:18:00 -0000
@@ -32,6 +32,10 @@ SUBDIR=      ac accton acpitools altq apm apm
        wake wiconfig wlanctl wsconscfg wsfontload wsmoused wsmuxctl \
        zdump zic
 
+.if ${MKMAKEMANDB} != "no"
+SUBDIR+= makemandb
+.endif
+
 .if (${MKYP} != "no")
 SUBDIR+= rpc.yppasswdd ypbind yppoll ypserv ypset
 .endif
Index: usr.sbin/makemandb/DBSCHEMA
===================================================================
RCS file: usr.sbin/makemandb/DBSCHEMA
diff -N usr.sbin/makemandb/DBSCHEMA
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/DBSCHEMA 4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,52 @@
+There are three tables in the database at present:
+
+(1) mandb:
+    This is the main FTS table which contains all the content from 
+    man pages
+
+  COLUMN NAME       DESCRIPTION
+  1. section        Section number of the page  
+  2. name           The name of the page  
+  3. name_desc      The one line description from the NAME section  
+  4. desc           Contains the content from DESCRIPTION 
+                    section and any other section which 
+                    is not indexed in a separate column.
+  5. lib            The LIBRARY section
+  6. return_vals    RETURN VALUES section
+  7. env            ENVIRONMENT section
+  8. files          FILES section
+  9. exit_status    EXIT STATUS section
+  10. diagnostics   DIAGNOSTICS section
+  11. errors        ERRORS section
+  12. md5_hash      md5 hash of the man page (UNIQUE)
+  13. machine       The machine architecture (if any) for which this page is
+                    relevant.
+
+(2) mandb_meta:
+    This table contains md5 hashes of all the indexed man 
+    pages. This is there to make sure we do not index 
+    the same man page twice, i.e. to prevent indexing 
+    hardlinks.
+
+  COLUMN NAME       DESCRIPTION
+  1. device         (dev_t)Logical device number from stat(2)
+  2. inode          (ino_t)Inode number from stat(2)
+  3. mtime          Last modification time from stat(2)
+  4. file           Absolute path name (UNIQUE constraint)
+  5. md5_hash       MD5 Hash of the man page (UNIQUE)
+  6. id             A unique integer ID for the page, which
+                    refers to the docid column in mandb (Implicit foreign 
+                    key?) PRIMARY KEY
+  {device, inode} form a UNIQUE index
+
+(3) mandb_links:
+    This is an index of all the hard/soft links of the man 
+    pages. The list of the linked pages is usually obtained 
+    from the multiple .Nm entries in the page.
+
+  COLUMN NAME       DESCRIPTION
+  1. link           The name of the hard/soft link (index) 
+  2. target         Name of the target page to which it points  
+  3. section        The section number  
+  4. machine        The machine architecture (if any) for which 
+                    the page is relevant
Index: usr.sbin/makemandb/Makefile
===================================================================
RCS file: usr.sbin/makemandb/Makefile
diff -N usr.sbin/makemandb/Makefile
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/Makefile 4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,40 @@
+# $NetBSD$
+
+.include <bsd.own.mk>
+
+MDIST= ${NETBSDSRCDIR}/external/bsd/mdocml/dist
+MDOCDIR=${NETBSDSRCDIR}/external/bsd/mdocml
+
+PROGS=                 makemandb apropos whatis
+SRCS.makemandb=                makemandb.c apropos-utils.c
+SRCS.apropos=  apropos.c apropos-utils.c
+SRCS.whatis=   whatis.c apropos-utils.c
+MAN.makemandb= makemandb.8
+MAN.apropos=   apropos.1
+MAN.whatis=    whatis.1
+
+BINDIR.apropos=                /usr/bin
+BINDIR.makemandb=      /usr/sbin
+BINDIR.whatis=         /usr/bin
+
+CPPFLAGS+=-I${MDIST} -I${.OBJDIR}
+
+MDOCMLOBJDIR!= cd ${MDOCDIR}/lib/libmandoc && ${PRINTOBJDIR}
+MDOCMLLIB=     ${MDOCMLOBJDIR}/libmandoc.a
+
+DPADD.makemandb+=      ${MDOCMLLIB}
+LDADD.makemandb+=      -L${MDOCMLOBJDIR} -lmandoc
+DPADD+=                ${LIBSQLITE3} ${LIBM} ${LIBZ} ${LIBUTIL}
+LDADD+=                -lsqlite3 -lm -lz -lutil
+
+stopwords.c: stopwords.txt
+       ( set -e; ${TOOL_NBPERF} -n stopwords_hash -s -p ${.ALLSRC};    \
+       echo 'static const char *stopwords[] = {';                      \
+       ${TOOL_SED} -e 's|^\(.*\)$$|    "\1",|' ${.ALLSRC};             \
+       echo '};'                                                       \
+       ) > ${.TARGET}
+
+DPSRCS+=       stopwords.c
+CLEANFILES+=   stopwords.c
+
+.include <bsd.prog.mk>
Index: usr.sbin/makemandb/apropos-utils.3
===================================================================
RCS file: usr.sbin/makemandb/apropos-utils.3
diff -N usr.sbin/makemandb/apropos-utils.3
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos-utils.3  4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,67 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt APROPOS-UTILS 3
+.Os
+.Sh NAME
+.Nm apropos-utils
+.Nd support functions for querying the man page index
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft sqlite3 *
+.Fn init_db "int db_flag"
+.Ft void
+.Fn close_db "sqlite3 *db"
+.Ft int
+.Fn run_query "sqlite3 *db" "const char *snippet_args[3]" "query_args *args"
+.Ft int
+.Fn run_query_html "sqlite3 *db" "query_args *args"
+.Ft int
+.Fn run_query_pager "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+These functions all live in the
+.Pa apropos-utils.h
+header file.
+They operate on
+.Pa /var/db/man.db
+which is an Sqlite database containing a full text search index of the manual
+pages.
+The functions provide an easy to use interface to query the database and
+develop applications on top of it.
+.Sh SEE ALSO
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/apropos-utils.c
===================================================================
RCS file: usr.sbin/makemandb/apropos-utils.c
diff -N usr.sbin/makemandb/apropos-utils.c
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos-utils.c  4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,755 @@
+/*     $NetBSD */
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD$");
+
+#include <sys/stat.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <err.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <util.h>
+#include <zlib.h>
+
+#include "apropos-utils.h"
+#include "mandoc.h"
+#include "sqlite3.h"
+
+typedef struct orig_callback_data {
+       void *data;
+       int (*callback) (void *, const char *, const char *, const char *,
+               const char *, size_t);
+} orig_callback_data;
+
+typedef struct inverse_document_frequency {
+       double value;
+       int status;
+} inverse_document_frequency;
+
+/* weights for individual columns */
+static const double col_weights[] = {
+       2.0,    // NAME
+       2.00,   // Name-description
+       0.55,   // DESCRIPTION
+       0.10,   // LIBRARY
+       0.001,  //RETURN VALUES
+       0.20,   //ENVIRONMENT
+       0.01,   //FILES
+       0.001,  //EXIT STATUS
+       2.00,   //DIAGNOSTICS
+       0.05,   //ERRORS
+       0.00,   //md5_hash
+       1.00    //machine
+};
+
+/*
+ * lower --
+ *  Converts the string str to lower case
+ */
+char *
+lower(char *str)
+{
+       assert(str);
+       int i = 0;
+       char c;
+       while (str[i] != '\0') {
+               c = tolower((unsigned char) str[i]);
+               str[i++] = c;
+       }
+       return str;
+}
+
+/*
+* concat--
+*  Utility function. Concatenates together: dst, a space character and src. 
+* dst + " " + src 
+*/
+void
+concat(char **dst, const char *src)
+{
+       concat2(dst, src, strlen(src));
+}
+
+void
+concat2(char **dst, const char *src, size_t srclen)
+{
+       size_t total_len, dst_len;
+       assert(src != NULL);
+
+       /* If destination buffer dst is NULL, then simply strdup the source 
buffer */
+       if (*dst == NULL) {
+               *dst = estrdup(src);
+               return;
+       }
+
+       dst_len = strlen(*dst);
+       /*
+        * NUL Byte and separator space
+        */
+       total_len = dst_len + srclen + 2;
+
+       *dst = erealloc(*dst, total_len);
+
+       /* Append a space at the end of dst */
+       (*dst)[dst_len++] = ' ';
+
+       /* Now, copy src at the end of dst */   
+       memcpy(*dst + dst_len, src, srclen + 1);
+}
+
+void
+close_db(sqlite3 *db)
+{
+       sqlite3_close(db);
+       sqlite3_shutdown();
+}
+
+/*
+ * create_db --
+ *  Creates the database schema.
+ */
+static int
+create_db(sqlite3 *db)
+{
+       const char *sqlstr = NULL;
+       char *schemasql;
+       char *errmsg = NULL;
+       
+/*------------------------ Create the tables------------------------------*/
+
+#if NOTYET
+       sqlite3_exec(db, "PRAGMA journal_mode = WAL", NULL, NULL, NULL);
+#else
+       sqlite3_exec(db, "PRAGMA journal_mode = DELETE", NULL, NULL, NULL);
+#endif
+
+       schemasql = sqlite3_mprintf("PRAGMA user_version = %d",
+           APROPOS_SCHEMA_VERSION);
+       sqlite3_exec(db, schemasql, NULL, NULL, &errmsg);
+       if (errmsg != NULL)
+               goto out;
+       sqlite3_free(schemasql);
+
+       sqlstr = "CREATE VIRTUAL TABLE mandb USING fts4(section, name, "
+                           "name_desc, desc, lib, return_vals, env, files, "
+                           "exit_status, diagnostics, errors, md5_hash UNIQUE, 
machine, "
+                           "compress=zip, uncompress=unzip, tokenize=porter); 
"        //mandb
+                       "CREATE TABLE IF NOT EXISTS mandb_meta(device, inode, 
mtime, "
+                           "file UNIQUE, md5_hash UNIQUE, id  INTEGER PRIMARY 
KEY); "
+                               //mandb_meta
+                       "CREATE TABLE IF NOT EXISTS mandb_links(link, target, 
section, "
+                           "machine); ";       //mandb_links
+
+       sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+       if (errmsg != NULL)
+               goto out;
+
+       sqlstr = "CREATE INDEX IF NOT EXISTS index_mandb_links ON mandb_links "
+                       "(link); "
+                       "CREATE INDEX IF NOT EXISTS index_mandb_meta_dev ON 
mandb_meta "
+                       "(device, inode)";
+       sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+       if (errmsg != NULL)
+               goto out;
+       return 0;
+
+out:
+       warnx("%s", errmsg);
+       free(errmsg);
+       sqlite3_close(db);
+       sqlite3_shutdown();
+       return -1;
+}
+
+/*
+ * zip --
+ *  User defined Sqlite function to compress the FTS table
+ */
+static void
+zip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{      
+       int nin;
+       long int nout;
+       const unsigned char * inbuf;
+       unsigned char *outbuf;
+
+       assert(nval == 1);
+       nin = sqlite3_value_bytes(apval[0]);
+       inbuf = (const unsigned char *) sqlite3_value_blob(apval[0]);
+       nout = nin + 13 + (nin + 999) / 1000;
+       outbuf = emalloc(nout);
+       compress(outbuf, (unsigned long *) &nout, inbuf, nin);
+       sqlite3_result_blob(pctx, outbuf, nout, free);
+}
+
+/*
+ * unzip --
+ *  User defined Sqlite function to uncompress the FTS table.
+ */
+static void
+unzip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{
+       unsigned int rc;
+       unsigned char *outbuf;
+       z_stream stream;
+
+       assert(nval == 1);
+       stream.next_in = __UNCONST(sqlite3_value_blob(apval[0]));
+       stream.avail_in = sqlite3_value_bytes(apval[0]);
+       stream.avail_out = stream.avail_in * 2 + 100;
+       stream.next_out = outbuf = emalloc(stream.avail_out);
+       stream.zalloc = NULL;
+       stream.zfree = NULL;
+
+       if (inflateInit(&stream) != Z_OK) {
+               free(outbuf);
+               return;
+       }
+
+       while ((rc = inflate(&stream, Z_SYNC_FLUSH)) != Z_STREAM_END) {
+               if (rc != Z_OK ||
+                   (stream.avail_out != 0 && stream.avail_in == 0)) {
+                       free(outbuf);
+                       return;
+               }
+               outbuf = erealloc(outbuf, stream.total_out * 2);
+               stream.next_out = outbuf + stream.total_out;
+               stream.avail_out = stream.total_out;
+       }
+       if (inflateEnd(&stream) != Z_OK) {
+               free(outbuf);
+               return;
+       }
+       outbuf = erealloc(outbuf, stream.total_out);
+       sqlite3_result_text(pctx, (const char *) outbuf, stream.total_out, 
free);
+}
+
+/* init_db --
+ *   Prepare the database. Register the compress/uncompress functions and the
+ *   stopword tokenizer.
+ *      db_flag specifies the mode in which to open the database. 3 options 
are 
+ *   available:
+ *     1. DB_READONLY: Open in READONLY mode. An error if db does not exist.
+ *     2. DB_READWRITE: Open in read-write mode. An error if db does not exist.
+ *     3. DB_CREATE: Open in read-write mode. It will try to create the db if
+ *                     it does not exist already.
+ *  RETURN VALUES:
+ *             The function will return NULL in case the db does not exist and 
DB_CREATE 
+ *     was not specified. And in case DB_CREATE was specified and yet NULL is 
+ *     returned, then there was some other error.
+ *     In normal cases the function should return a handle to the db.
+ */
+sqlite3 *
+init_db(int db_flag)
+{
+       sqlite3 *db = NULL;
+       sqlite3_stmt *stmt;
+       struct stat sb;
+       int rc;
+       int create_db_flag = 0;
+
+       /* Check if the database exists or not */
+       if (!(stat(DBPATH, &sb) == 0 && S_ISREG(sb.st_mode))) {
+               /* Database does not exist, check if DB_CREATE was specified, 
and set
+                * flag to create the database schema
+                */
+               if (db_flag != (MANDB_CREATE)) {
+                       warnx("Missing apropos database. "
+                             "Please run makemandb to create it.");
+                       return NULL;
+               }
+               create_db_flag = 1;
+       }
+
+       /* Now initialize the database connection */
+       sqlite3_initialize();
+       rc = sqlite3_open_v2(DBPATH, &db, db_flag, NULL);
+       
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               sqlite3_shutdown();
+               return NULL;
+       }
+
+       if (create_db_flag && create_db(db) < 0) {
+               warnx("%s", "Unable to create database schema");
+               goto error;
+       }
+
+       rc = sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &stmt, NULL);
+       if (rc != SQLITE_OK) {
+               warnx("Unable to query schema version");
+               goto error;
+       }
+       if (sqlite3_step(stmt) != SQLITE_ROW) {
+               sqlite3_finalize(stmt);
+               warnx("Unable to query schema version");
+               goto error;
+       }
+       if (sqlite3_column_int(stmt, 0) != APROPOS_SCHEMA_VERSION) {
+               sqlite3_finalize(stmt);
+               warnx("Incorrect schema version found. "
+                     "Please run makemandb -f.");
+               goto error;
+       }
+       sqlite3_finalize(stmt);
+
+       sqlite3_extended_result_codes(db, 1);
+       
+       /* Register the zip and unzip functions for FTS compression */
+       rc = sqlite3_create_function(db, "zip", 1, SQLITE_ANY, NULL, zip, NULL, 
NULL);
+       if (rc != SQLITE_OK) {
+               warnx("Unable to register function: compress");
+               goto error;
+       }
+
+       rc = sqlite3_create_function(db, "unzip", 1, SQLITE_ANY, NULL, 
+                                 unzip, NULL, NULL);
+       if (rc != SQLITE_OK) {
+               warnx("Unable to register function: uncompress");
+               goto error;
+       }
+       return db;
+error:
+       sqlite3_close(db);
+       sqlite3_shutdown();
+       return NULL;
+}
+
+/*
+ * rank_func --
+ *  Sqlite user defined function for ranking the documents.
+ *  For each phrase of the query, it computes the tf and idf and adds them 
over.
+ *  It computes the final rank, by multiplying tf and idf together.
+ *  Weight of term t for document d = (term frequency of t in d * 
+ *                                      inverse document frequency of t) 
+ *
+ *  Term Frequency of term t in document d = Number of times t occurs in d / 
+ *                                             Number of times t appears in 
all 
+ *                                                                             
        documents
+ *
+ *  Inverse document frequency of t = log(Total number of documents / 
+ *                                                                             
Number of documents in which t occurs)
+ */
+static void
+rank_func(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{
+       inverse_document_frequency *idf = sqlite3_user_data(pctx);
+       double tf = 0.0;
+       const unsigned int *matchinfo;
+       int ncol;
+       int nphrase;
+       int iphrase;
+       int ndoc;
+       int doclen = 0;
+       const double k = 3.75;
+       /* Check that the number of arguments passed to this function is 
correct. */
+       assert(nval == 1);
+
+       matchinfo = (const unsigned int *) sqlite3_value_blob(apval[0]);
+       nphrase = matchinfo[0];
+       ncol = matchinfo[1];
+       ndoc = matchinfo[2 + 3 * ncol * nphrase + ncol];
+       for (iphrase = 0; iphrase < nphrase; iphrase++) {
+               int icol;
+               const unsigned int *phraseinfo = &matchinfo[2 + ncol+ iphrase * 
ncol * 3];
+               for(icol = 1; icol < ncol; icol++) {
+                       
+                       /* nhitcount: number of times the current phrase occurs 
in the current
+                        *            column in the current document.
+                        * nglobalhitcount: number of times current phrase 
occurs in the current
+                        *                  column in all documents.
+                        * ndocshitcount:   number of documents in which the 
current phrase 
+                        *                  occurs in the current column at 
least once.
+                        */
+                       int nhitcount = phraseinfo[3 * icol];
+                       int nglobalhitcount = phraseinfo[3 * icol + 1];
+                       int ndocshitcount = phraseinfo[3 * icol + 2];
+                       doclen = matchinfo[2 + icol ];
+                       double weight = col_weights[icol - 1];
+                       if (idf->status == 0 && ndocshitcount)
+                               idf->value += log(((double)ndoc / 
ndocshitcount))* weight;
+
+                       /* Dividing the tf by document length to normalize the 
effect of 
+                        * longer documents.
+                        */
+                       if (nglobalhitcount > 0 && nhitcount)
+                               tf += (((double)nhitcount  * weight) / 
(nglobalhitcount * doclen));
+               }
+       }
+       idf->status = 1;
+       
+       /* Final score = (tf * idf)/ ( k + tf)
+        *      Dividing by k+ tf further normalizes the weight leading to 
better 
+        *  results.
+        *  The value of k is experimental
+        */
+       double score = (tf * idf->value/ ( k + tf)) ;
+       sqlite3_result_double(pctx, score);
+       return;
+}
+
+/*
+ *  run_query --
+ *  Performs the searches for the keywords entered by the user.
+ *  The 2nd param: snippet_args is an array of strings providing values for the
+ *  last three parameters to the snippet function of sqlite. (Look at the 
docs).
+ *  The 3rd param: args contains rest of the search parameters. Look at 
+ *  arpopos-utils.h for the description of individual fields.
+ *  
+ */
+int
+run_query(sqlite3 *db, const char *snippet_args[3], query_args *args)
+{
+       const char *default_snippet_args[3];
+       char *section_clause = NULL;
+       char *limit_clause = NULL;
+       char *machine_clause = NULL;
+       char *query;
+       const char *section;
+       char *name;
+       const char *name_desc;
+       const char *machine;
+       const char *snippet;
+       char *m = NULL;
+       int rc;
+       inverse_document_frequency idf = {0, 0};
+       sqlite3_stmt *stmt;
+
+       if (args->machine)
+               easprintf(&machine_clause, "AND machine = \'%s\' ", 
args->machine);
+
+       /* Register the rank function */
+       rc = sqlite3_create_function(db, "rank_func", 1, SQLITE_ANY, (void 
*)&idf, 
+                                    rank_func, NULL, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_close(db);
+               sqlite3_shutdown();
+               errx(EXIT_FAILURE, "Unable to register the ranking function");
+       }
+       
+       /* We want to build a query of the form: "select x,y,z from mandb where
+        * mandb match :query [AND (section LIKE '1' OR section LIKE '2' OR...)]
+        * ORDER BY rank DESC..."
+        * NOTES: 1. The portion in square brackets is optional, it will be 
there 
+        * only if the user has specified an option on the command line to 
search in 
+        * one or more specific sections.
+        * 2. I am using LIKE operator because '=' or IN operators do not seem 
to be
+        * working with the compression option enabled.
+        */
+
+       if (args->sec_nums) {
+               char *temp;
+               int i;
+
+               for (i = 0; i < SECMAX; i++) {
+                       if (args->sec_nums[i] == 0)
+                               continue;
+                       easprintf(&temp, " OR section = \'%d\'", i + 1);
+                       if (section_clause) {
+                               concat(&section_clause, temp);
+                               free(temp);
+                       } else {
+                               section_clause = temp;
+                       }
+               }
+               if (section_clause) {
+                       /*
+                        * At least one section requested, add glue for query.
+                        */
+                       temp = section_clause;
+                       /* Skip " OR " before first term. */
+                       easprintf(&section_clause, " AND (%s)", temp + 4);
+                       free(temp);
+               }
+       }
+       if (args->nrec >= 0) {
+               /* Use the provided number of records and offset */
+               easprintf(&limit_clause, " LIMIT %d OFFSET %d",
+                   args->nrec, args->offset);
+       }
+
+       if (snippet_args == NULL) {
+               default_snippet_args[0] = "";
+               default_snippet_args[1] = "";
+               default_snippet_args[2] = "...";
+               snippet_args = default_snippet_args;
+       }
+       query = sqlite3_mprintf("SELECT section, name, name_desc, machine,"
+           " snippet(mandb, %Q, %Q, %Q, -1, 40 ),"
+           " rank_func(matchinfo(mandb, \"pclxn\")) AS rank"
+           " FROM mandb"
+           " WHERE mandb MATCH %Q %s "
+           "%s"
+           " ORDER BY rank DESC"
+           "%s",
+           snippet_args[0], snippet_args[1], snippet_args[2], args->search_str,
+           machine_clause ? machine_clause : "",
+           section_clause ? section_clause : "",
+           limit_clause ? limit_clause : "");
+
+       free(machine_clause);
+       free(section_clause);
+       free(limit_clause);
+
+       if (query == NULL) {
+               *args->errmsg = estrdup("malloc failed");
+               return -1;
+       }
+       rc = sqlite3_prepare_v2(db, query, -1, &stmt, NULL);
+       if (rc == SQLITE_IOERR) {
+               warnx("Corrupt database. Please rerun makemandb");
+               sqlite3_free(query);
+               return -1;
+       } else if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               sqlite3_free(query);
+               return -1;
+       }
+
+       while (sqlite3_step(stmt) == SQLITE_ROW) {
+               section = (const char *) sqlite3_column_text(stmt, 0);
+               name_desc = (const char *) sqlite3_column_text(stmt, 2);
+               machine = (const char *) sqlite3_column_text(stmt, 3);
+               snippet = (const char *) sqlite3_column_text(stmt, 4);
+               if (machine && machine[0]) {
+                       m = estrdup(machine);
+                       easprintf(&name, "%s/%s", lower(m),
+                               sqlite3_column_text(stmt, 1));
+                       free(m);
+               } else {
+                       name = estrdup((const char *) sqlite3_column_text(stmt, 
1));
+               }
+
+               (args->callback)(args->callback_data, section, name, name_desc, 
snippet,
+                       strlen(snippet));
+
+               free(name);
+       }
+
+       sqlite3_finalize(stmt);
+       sqlite3_free(query);
+       return *(args->errmsg) == NULL ? 0 : -1;
+}
+
+/*
+ * callback_html --
+ *  Callback function for run_query_html. It builds the html output and then
+ *  calls the actual user supplied callback function.
+ */
+static int
+callback_html(void *data, const char *section, const char *name,
+       const char *name_desc, const char *snippet, size_t snippet_length)
+{
+       const char *temp = snippet;
+       int i = 0;
+       size_t sz = 0;
+       int count = 0;
+       struct orig_callback_data *orig_data = (struct orig_callback_data *) 
data;
+       int (*callback) (void *, const char *, const char *, const char *, 
+               const char *, size_t) = orig_data->callback;
+
+       /* First scan the snippet to find out the number of occurrences of 
{'>', '<'
+        * '"', '&'}.
+        * Then allocate a new buffer with sufficient space to be able to store 
the
+        * quoted versions of the special characters {&gt;, &lt;, &quot;, 
&amp;}.
+        * Copy over the characters from the original snippet to this buffer 
while
+        * replacing the special characters with their quoted versions.
+        */
+
+       while (*temp) {
+               sz = strcspn(temp, "<>\"&\002\003");
+               temp += sz + 1;
+               count++;
+       }
+       size_t qsnippet_length = snippet_length + count * 5;
+       char *qsnippet = emalloc(qsnippet_length + 1);
+       sz = 0;
+       while (*snippet) {
+               sz = strcspn(snippet, "<>\"&\002\003");
+               if (sz) {
+                       memcpy(&qsnippet[i], snippet, sz);
+                       snippet += sz;
+                       i += sz;
+               }
+
+               switch (*snippet++) {
+               case '<':
+                       memcpy(&qsnippet[i], "&lt;", 4);
+                       i += 4;
+                       break;
+               case '>':
+                       memcpy(&qsnippet[i], "&gt;", 4);
+                       i += 4;
+                       break;
+               case '\"':
+                       memcpy(&qsnippet[i], "&quot;", 6);
+                       i += 6;
+                       break;
+               case '&':
+                       /* Don't perform the quoting if this & is part of an 
mdoc escape
+                        * sequence, e.g. \&
+                        */
+                       if (i && *(snippet - 2) != '\\') {
+                               memcpy(&qsnippet[i], "&amp;", 5);
+                               i += 5;
+                       } else {
+                               qsnippet[i++] = '&';
+                       }
+                       break;
+               case '\002':
+                       memcpy(&qsnippet[i], "<b>", 3);
+                       i += 3;
+                       break;
+               case '\003':
+                       memcpy(&qsnippet[i], "</b>", 4);
+                       i += 4;
+                       break;
+               default:
+                       break;
+               }
+       }
+       qsnippet[++i] = 0;
+       (*callback)(orig_data->data, section, name, name_desc,
+               (const char *)qsnippet, qsnippet_length);
+       free(qsnippet);
+       return 0;
+}
+
+/*
+ * run_query_html --
+ *  Utility function to output query result in HTML format.
+ *  It internally calls run_query only, but it first passes the output to it's 
+ *  own custom callback function, which preprocess the snippet for quoting
+ *  inline HTML fragments.
+ *  After that it delegates the call the actual user supplied callback 
function.
+ */
+int
+run_query_html(sqlite3 *db, query_args *args)
+{
+       struct orig_callback_data orig_data;
+       orig_data.callback = args->callback;
+       orig_data.data = args->callback_data;
+       const char *snippet_args[] = {"\002", "\003", "..."};
+       args->callback = &callback_html;
+       args->callback_data = (void *) &orig_data;
+       return run_query(db, snippet_args, args);
+}
+
+/*
+ * callback_pager --
+ *  A callback similar to callback_html. It overstrikes the matching text in
+ *  the snippet so that it appears emboldened when viewed using a pager like
+ *  more or less.
+ */
+static int
+callback_pager(void *data, const char *section, const char *name, 
+       const char *name_desc, const char *snippet, size_t snippet_length)
+{
+       struct orig_callback_data *orig_data = (struct orig_callback_data *) 
data;
+       char *psnippet;
+       const char *temp = snippet;
+       int count = 0;
+       int i = 0;
+       size_t sz = 0;
+       size_t psnippet_length;
+
+       /* Count the number of bytes of matching text. For each of these bytes 
we
+        * will use 2 extra bytes to overstrike it so that it appears bold when
+        * viewed using a pager.
+        */
+       while (*temp) {
+               sz = strcspn(temp, "\002\003");
+               temp += sz;
+               if (*temp == '\003') {
+                       count += 2 * (sz);
+               }
+               temp++;
+       }
+
+       psnippet_length = snippet_length + count;
+       psnippet = emalloc(psnippet_length + 1);
+
+       /* Copy the bytes from snippet to psnippet:
+        * 1. Copy the bytes before \002 as it is.
+        * 2. The bytes after \002 need to be overstriked till we encounter 
\003.
+        * 3. To overstrike a byte 'A' we need to write 'A\bA'
+        */
+       while (*snippet) {
+               sz = strcspn(snippet, "\002");
+               memcpy(&psnippet[i], snippet, sz);
+               snippet += sz;
+               i += sz;
+
+               /* Don't change this. Advancing the pointer without reading the 
byte
+                * is causing strange behavior.
+                */
+               if (*snippet == '\002')
+                       snippet++;
+               while (*snippet && *snippet != '\003') {
+                       psnippet[i++] = *snippet;
+                       psnippet[i++] = '\b';
+                       psnippet[i++] = *snippet++;
+               }
+               if (*snippet)
+                       snippet++;
+       }
+
+       psnippet[i] = 0;
+       (orig_data->callback)(orig_data->data, section, name, name_desc, 
psnippet,
+               psnippet_length);
+       free(psnippet);
+       return 0;
+}
+
+/*
+ * run_query_pager --
+ *  Utility function similar to run_query_html. This function tries to
+ *  pre-process the result assuming it will be piped to a pager.
+ *  For this purpose it first calls it's own callback function callback_pager
+ *  which then delegates the call to the user supplied callback.
+ */
+int run_query_pager(sqlite3 *db, query_args *args)
+{
+       struct orig_callback_data orig_data;
+       orig_data.callback = args->callback;
+       orig_data.data = args->callback_data;
+       const char *snippet_args[] = {"\002", "\003", "..."};
+       args->callback = &callback_pager;
+       args->callback_data = (void *) &orig_data;
+       return run_query(db, snippet_args, args);
+}
Index: usr.sbin/makemandb/apropos-utils.h
===================================================================
RCS file: usr.sbin/makemandb/apropos-utils.h
diff -N usr.sbin/makemandb/apropos-utils.h
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos-utils.h  4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,92 @@
+/*     $NetBSD */
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef APROPOS_UTILS_H
+#define APROPOS_UTILS_H
+
+#include "sqlite3.h"
+
+#define DBPATH "/var/db/man.db"
+#define SECMAX 9
+
+/* Flags for opening the database */
+#define MANDB_READONLY SQLITE_OPEN_READONLY
+#define MANDB_WRITE SQLITE_OPEN_READWRITE
+#define MANDB_CREATE SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE
+
+#define APROPOS_SCHEMA_VERSION 20120130
+
+/*
+ * Used to identify the section of a man(7) page.
+ * This is similar to the enum mdoc_sec defined in mdoc.h from mdocml project.
+ */
+enum man_sec {
+       MANSEC_NAME = 0,
+       MANSEC_SYNOPSIS,
+       MANSEC_LIBRARY,
+       MANSEC_ERRORS,
+       MANSEC_FILES,
+       MANSEC_RETURN_VALUES,
+       MANSEC_EXIT_STATUS,
+       MANSEC_DESCRIPTION,
+       MANSEC_ENVIRONMENT,
+       MANSEC_DIAGNOSTICS,
+       MANSEC_EXAMPLES,
+       MANSEC_STANDARDS,
+       MANSEC_HISTORY,
+       MANSEC_BUGS,
+       MANSEC_AUTHORS,
+       MANSEC_COPYRIGHT,
+       MANSEC_NONE
+};
+
+typedef struct query_args {
+       const char *search_str;         // user query
+       int *sec_nums;          // Section in which to do the search
+       int nrec;                       // number of records to fetch
+       int offset;             //From which position to start processing the 
records
+       const char *machine;
+       int (*callback) (void *, const char *, const char *, const char *,
+               const char *, size_t);  // The callback function
+       void *callback_data;    // data to pass to the callback function
+       char **errmsg;          // buffer for storing the error msg
+} query_args;
+
+char *lower(char *);
+void concat(char **, const char *);
+void concat2(char **, const char *, size_t);
+sqlite3 *init_db(int);
+void close_db(sqlite3 *);
+int run_query(sqlite3 *, const char *[3], query_args *);
+int run_query_html(sqlite3 *, query_args *);
+int run_query_pager(sqlite3 *, query_args *);
+#endif 
Index: usr.sbin/makemandb/apropos.1
===================================================================
RCS file: usr.sbin/makemandb/apropos.1
diff -N usr.sbin/makemandb/apropos.1
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos.1        4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,106 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd January 22, 2012
+.Dt APROPOS 1
+.Os
+.Sh NAME
+.Nm apropos
+.Nd search the complete content of all man pages
+.Sh SYNOPSIS
+.Nm
+.Op Fl 123456789p
+.Op Fl n Ar Number of results
+.Op Fl S Ar machine
+.Ar query
+.Sh DESCRIPTION
+The
+.Nm
+utility performs a full text search over the complete content of all man pages.
+It uses the FTS engine of Sqlite to perform the search.
+The database is created with the help of
+.Xr makemandb 1
+utility.
+.Pp
+This implementation of the
+.Nm
+utility is more sophisticated than the classical version of
+.Nm .
+Like modern search applications, it uses advanced techniques like stemming
+and term weighting to rank the matching results in decreasing order of
+relevance.
+By default
+.Nm
+will only display the top 10 matches in the output.
+.Pp
+It supports the following options:
+.Bl -tag -width indent
+.It Fl 1
+Search only within section 1 manual pages.
+.It Fl 2
+Search only within section 2 manual pages.
+.It Fl 3
+Search only within section 3 manual pages.
+.It Fl 4
+Search only within section 4 manual pages.
+.It Fl 5
+Search only within section 5 manual pages.
+.It Fl 6
+Search only within section 6 manual pages.
+.It Fl 7
+Search only within section 7 manual pages.
+.It Fl 8
+Search only within section 8 manual pages.
+.It Fl 9
+Search only within section 9 manual pages.
+.It Fl n
+Output only the specified number of search results.
+This option can be used to override the default limit of maximum 10 results per
+query.
+.It Fl p
+Display all matching results (instead of the default of 10)
+and pipe them through a pager (defaulting to
+.Xr more 1 ) .
+.It Fl S
+Limit the search to the pages for the specified machine architecture.
+By default pages for all architectures are shown in the search results.
+.El
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr makemandb 1 ,
+.Xr man 1
+.Sh AUTHORS
+.An Abhinav Upadhyay
+
Index: usr.sbin/makemandb/apropos.c
===================================================================
RCS file: usr.sbin/makemandb/apropos.c
diff -N usr.sbin/makemandb/apropos.c
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos.c        4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,259 @@
+/*     $NetBSD */
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD$");
+
+#include <err.h>
+#include <search.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <util.h>
+
+#include "apropos-utils.h"
+#include "sqlite3.h"
+
+typedef struct apropos_flags {
+       int sec_nums[SECMAX];
+       int nresults;
+       int pager;
+       const char *machine;
+} apropos_flags;
+
+typedef struct callback_data {
+       int count;
+       FILE *out;
+} callback_data;
+
+static char *remove_stopwords(const char *);
+static int query_callback(void *, const char * , const char *, const char *,
+       const char *, size_t);
+__dead static void usage(void);
+
+int
+main(int argc, char *argv[])
+{
+       char *query = NULL;     // the user query
+       char ch;
+       char *errmsg = NULL;
+       char *str;
+       int rc = 0;
+       callback_data cbdata;
+       const char *snippet_args[] = {"\033[1m", "\033[0m", "..."};
+       cbdata.out = stdout;            // the default output stream
+       cbdata.count = 0;
+       apropos_flags aflags;
+       sqlite3 *db;
+       setprogname(argv[0]);
+       if (argc < 2)
+               usage();
+
+       memset(&aflags, 0, sizeof(aflags));
+       
+       /*If the user specifies a section number as an option, the 
corresponding 
+        * index element in sec_nums is set to the string representing that 
+        * section number.
+        */
+       while ((ch = getopt(argc, argv, "123456789n:pS:")) != -1) {
+               switch (ch) {
+               case '1':
+               case '2':
+               case '3':
+               case '4':
+               case '5':
+               case '6':
+               case '7':
+               case '8':
+               case '9':
+                       aflags.sec_nums[atoi(&ch) - 1] = 1;
+                       break;
+               case 'n':
+                       aflags.nresults = atoi(optarg);
+                       break;
+               case 'p':       //user wants to view more than 10 results and 
page them
+                       aflags.pager = 1;
+                       aflags.nresults = -1;   // Fetch all records
+                       break;
+               case 'S':
+                       aflags.machine = optarg;
+                       break;
+               case '?':
+               default:
+                       usage();
+               }
+       }
+       
+       argc -= optind;
+       argv += optind;
+       
+       if (!argc)
+               usage();
+
+       str = NULL;
+       while (argc--)
+               concat(&str, *argv++);
+       /* Eliminate any stopwords from the query */
+       query = remove_stopwords(lower(str));
+       free(str);
+
+       /* if any error occured in remove_stopwords, exit */
+       if (query == NULL)
+               errx(EXIT_FAILURE, "Try using more relevant keywords");
+
+       if ((db = init_db(MANDB_READONLY)) == NULL)
+               exit(EXIT_FAILURE);
+
+       /* If user wants to page the output, then set some settings */
+       if (aflags.pager) {
+               /* Open a pipe to the pager */
+               if ((cbdata.out = popen("more", "w")) == NULL) {
+                       close_db(db);
+                       err(EXIT_FAILURE, "pipe failed");
+               }
+       }
+
+       query_args args;
+       args.search_str = query;
+       args.sec_nums = aflags.sec_nums;
+       args.nrec = aflags.nresults ? aflags.nresults : 10;
+       args.offset = 0;
+       args.machine = aflags.machine;
+       args.callback = &query_callback;
+       args.callback_data = &cbdata;
+       args.errmsg = &errmsg;
+
+       if (aflags.pager) {
+               rc = run_query_pager(db, &args);
+               pclose(cbdata.out);
+       } else {
+               rc = run_query(db, snippet_args, &args);
+       }
+
+       free(query);
+       close_db(db);
+       if (errmsg) {
+               warnx("%s", errmsg);
+               free(errmsg);
+               exit(EXIT_FAILURE);
+       }
+
+       if (rc < 0) {
+               /* Something wrong with the database. Exit */
+               exit(EXIT_FAILURE);
+       }
+       
+       if (cbdata.count == 0) {
+               warnx("No relevant results obtained.\n"
+                         "Please make sure that you spelled all the terms 
correctly\n"
+                         "Or try using better keywords.");
+       }
+       return 0;
+}
+
+/*
+ * query_callback --
+ *  Callback function for run_query.
+ *  It simply outputs the results from do_query. If the user specified the -p
+ *  option, then the output is sent to a pager, otherwise stdout is the default
+ *  output stream.
+ */
+static int
+query_callback(void *data, const char *section, const char *name,
+       const char *name_desc, const char *snippet, size_t snippet_length)
+{
+       callback_data *cbdata = (callback_data *) data;
+       FILE *out = cbdata->out;
+       cbdata->count++;
+       fprintf(out, "%s(%s)\t%s\n%s\n\n", name, section, name_desc, 
+                       snippet);
+       return 0;
+}
+
+#include "stopwords.c"
+
+/*
+ * remove_stopwords--
+ *  Scans the query and removes any stop words from it.
+ *  Returns the modified query or NULL, if it contained only stop words.
+ */
+
+static char *
+remove_stopwords(const char *query)
+{
+       size_t len, idx;
+       char *output, *buf;
+       const char *sep, *next;
+
+       output = buf = emalloc(strlen(query) + 1);
+
+       for (; query[0] != '\0'; query = next) {
+               sep = strchr(query, ' ');
+               if (sep == NULL) {
+                       len = strlen(query);
+                       next = query + len;
+               } else {
+                       len = sep - query;
+                       next = sep + 1;
+               }
+               if (len == 0)
+                       continue;
+               idx = stopwords_hash(query, len);
+               if (memcmp(stopwords[idx], query, len) == 0 &&
+                   stopwords[idx][len] == '\0')
+                       continue;
+               memcpy(buf, query, len);
+               buf += len;
+               *buf++ = ' ';
+       }
+
+       if (output == buf) {
+               free(output);
+               return NULL;
+       }
+       buf[-1] = '\0';
+       return output;
+}
+
+/*
+ * usage --
+ *     print usage message and die
+ */
+static void
+usage(void)
+{
+       fprintf(stderr,
+               "Usage: %s [-n Number of records] [-p] [-123456789] [-S 
machine] query\n",
+               getprogname());
+       exit(1);
+}
Index: usr.sbin/makemandb/close_db.3
===================================================================
RCS file: usr.sbin/makemandb/close_db.3
diff -N usr.sbin/makemandb/close_db.3
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/close_db.3       4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,62 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt CLOSE_DB 3
+.Os
+.Sh NAME
+.Nm close_db
+.Nd close apropos database connection
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft void
+.Fn close_db "sqlite3 *db"
+.Sh DESCRIPTION
+The
+.Fn close_db
+function will close the connection to
+.Pa /var/db/man.db
+referenced by
+.Fa db
+and release any resources being used by it safely.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/init_db.3
===================================================================
RCS file: usr.sbin/makemandb/init_db.3
diff -N usr.sbin/makemandb/init_db.3
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/init_db.3        4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,92 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt INIT_DB 3
+.Os
+.Sh NAME
+.Nm init_db
+.Nd open apropos database connection
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft sqlite3 *
+.Fn init_db "int db_flag"
+.Sh DESCRIPTION
+The
+.Fn init_db
+function will try to initialize a connection to
+.Pa man.db
+which is an
+Sqlite database and return a pointer to a sqlite3 structure.
+The connection should subsequentially be closed by calling
+.Fn close_db
+which will also release any resources being used by it.
+.Pp
+The argument
+.Fa db_flag
+can be one of the following:
+.Bl -hang -width -compact
+.It Dv MANDB_READONLY
+This will open the database in read-only mode.
+Use this when you only want to query the database.
+.It Dv MANDB_WRITE
+This will open the database in read/write mode.
+.It Dv MANDB_CREATE
+This will open the database in read/write mode, and will also create
+the database schema if it does not exist already.
+.El
+.Sh RETURN VALUES
+On successful execution the
+.Fn init_db
+function will return a pointer to a sqlite3 structure which represents
+a connection to the database.
+.Pp
+In case the file
+.Pa /var/db/man.db
+does not exist and
+.Dv DB_CREATE
+is not used as a value of
+.Fa db_flag ,
+.Dv NULL
+will be returned.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/makemandb.8
===================================================================
RCS file: usr.sbin/makemandb/makemandb.8
diff -N usr.sbin/makemandb/makemandb.8
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/makemandb.8      4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,109 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd February 4, 2012
+.Dt MAKEMANDB 8
+.Os
+.Sh NAME
+.Nm makemandb
+.Nd parse the manual pages and build a search index over them
+.Sh SYNOPSIS
+.Nm
+.Op Fl C Ar path
+.Op Fl floqv
+.Sh DESCRIPTION
+The
+.Nm
+utility traverses the directories containing man pages, parses the manual
+pages with the help of libmandoc and builds an Sqlite database
+.Pa man.db
+to support full text searches.
+It obtains the list of directories to traverse using the
+.Fl Ar path
+and
+.Fl p
+options of
+.Xr man 1 .
+.Pp
+It supports the following options:
+.Bl -tag -width indent
+.It Fl C Ar path
+Use different
+.Xr man 1
+configuration file than the default,
+.Pa /etc/man.conf .
+.It Fl f
+Force rebuilding the index from scratch, pruning the existing one.
+.It Fl l
+Limit the parsing to only the NAME section of the pages.
+This option can be used to mimic the behavior of the classic
+.Xr apropos 1
+and also to substantially save disk space.
+.It Fl o
+Use this option to optimize the index for speed and also
+to significantly reduce disk space usage.
+This is a somewhat expensive operation.
+.It Fl q
+Print only error messages and no status updates.
+.It Fl v
+Enable verbose output.
+This prints the name of every file being parsed.
+.El
+.Pp
+As the database file is stored under
+.Pa /var/db ,
+root privileges are required to run
+.Nm .
+.Ss DATABASE SCHEMA
+The name of the FTS table is mandb and its schema is as follows:
+.Bl -column -offset indent "Column Name" "Column Description"
+.It Li section Ta The section number of the page
+.It Li name Ta The name of the page from the NAME section.
+.It Li name_desc Ta The one line description from the NAME section.
+.It Li desc Ta The DESCRIPTION section.
+.It Li lib Ta The LIBRARY section.
+.It Li return_vals Ta The RETURN VALUES section.
+.It Li env Ta The ENVIRONMENT section.
+.It Li files Ta The FILES section.
+.It Li exit_status Ta The EXIT STATUS section.
+.It Li errors Ta The ERRORS section.
+.El
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos 1 ,
+.Xr man 1 ,
+.Xr man.conf 5
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/makemandb.c
===================================================================
RCS file: usr.sbin/makemandb/makemandb.c
diff -N usr.sbin/makemandb/makemandb.c
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/makemandb.c      4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,1934 @@
+/*     $NetBSD */
+/*
+ * Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+ * Copyright (c) 2011 Kristaps Dzonsons <kristaps%bsd.lv@localhost>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD$");
+
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <err.h>
+#include <md5.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <util.h>
+
+#include "apropos-utils.h"
+#include "man.h"
+#include "mandoc.h"
+#include "mdoc.h"
+#include "sqlite3.h"
+
+#define BUFLEN 1024
+#define MDOC 0 //If the page is of mdoc(7) type
+#define MAN 1  //If the page  is of man(7) type
+
+/*
+ * A data structure for holding section specific data.
+ */
+typedef struct secbuff {
+       char *data;
+       size_t buflen;  //Total length of buffer allocated initially
+       size_t offset;  // Current offset in the buffer.
+} secbuff;
+
+typedef struct makemandb_flags {
+       int optimize;
+       int limit;      // limit the indexing to only NAME section
+       int recreate;   // Database was created from scratch
+       int verbosity;  // 0: quiet, 1: default, 2: verbose
+} makemandb_flags;
+
+typedef struct mandb_rec {
+       /* Fields for mandb table */
+       char *name;     // for storing the name of the man page
+       char *name_desc; // for storing the one line description (.Nd)
+       secbuff desc; // for storing the DESCRIPTION section
+       secbuff lib; // for the LIBRARY section
+       secbuff return_vals; // RETURN VALUES
+       secbuff env; // ENVIRONMENT
+       secbuff files; // FILES
+       secbuff exit_status; // EXIT STATUS
+       secbuff diagnostics; // DIAGNOSTICS
+       secbuff errors; // ERRORS
+       char section[2];
+
+       int xr_found;
+
+       /* Fields for mandb_meta table */
+       char *md5_hash;
+       dev_t device;
+       ino_t inode;
+       time_t mtime;
+
+       /* Fields for mandb_links table */
+       char *machine;
+       char *links; //all the links to a page in a space separated form
+       char *file_path;
+
+       /* Non-db fields */
+       int page_type; //Indicates the type of page: mdoc or man
+} mandb_rec;
+
+static void append(secbuff *sbuff, const char *src);
+static void init_secbuffs(mandb_rec *);
+static void free_secbuffs(mandb_rec *);
+static int check_md5(const char *, sqlite3 *, const char *, char **);
+static void cleanup(mandb_rec *);
+static void set_section(const struct mdoc *, const struct man *, mandb_rec *);
+static void set_machine(const struct mdoc *, mandb_rec *);
+static int insert_into_db(sqlite3 *, mandb_rec *);
+static void begin_parse(const char *, struct mparse *, mandb_rec *);
+static void pmdoc_node(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Nm(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Nd(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Sh(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Xr(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Pp(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_macro_handler(const struct mdoc_node *, mandb_rec *, 
+                               enum mdoct);
+static void pman_node(const struct man_node *n, mandb_rec *);
+static void pman_parse_node(const struct man_node *, secbuff *);
+static void pman_parse_name(const struct man_node *, mandb_rec *);
+static void pman_sh(const struct man_node *, mandb_rec *);
+static void pman_block(const struct man_node *, mandb_rec *);
+static void traversedir(const char *, sqlite3 *, struct mparse *);
+static void mdoc_parse_section(enum mdoc_sec, const char *, mandb_rec *);
+static void man_parse_section(enum man_sec, const struct man_node *, mandb_rec 
*);
+static void build_file_cache(sqlite3 *, const char *, struct stat *);
+static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
+__dead static void usage(void);
+static void optimize(sqlite3 *);
+static char *parse_escape(const char *);
+static makemandb_flags mflags = { .verbosity = 1 };
+
+typedef        void (*pman_nf)(const struct man_node *n, mandb_rec *);
+typedef        void (*pmdoc_nf)(const struct mdoc_node *n, mandb_rec *);
+static const pmdoc_nf mdocs[MDOC_MAX] = {
+       NULL, /* Ap */
+       NULL, /* Dd */
+       NULL, /* Dt */
+       NULL, /* Os */
+       pmdoc_Sh, /* Sh */
+       NULL, /* Ss */
+       pmdoc_Pp, /* Pp */
+       NULL, /* D1 */
+       NULL, /* Dl */
+       NULL, /* Bd */
+       NULL, /* Ed */
+       NULL, /* Bl */
+       NULL, /* El */
+       NULL, /* It */
+       NULL, /* Ad */
+       NULL, /* An */
+       NULL, /* Ar */
+       NULL, /* Cd */
+       NULL, /* Cm */
+       NULL, /* Dv */
+       NULL, /* Er */
+       NULL, /* Ev */
+       NULL, /* Ex */
+       NULL, /* Fa */
+       NULL, /* Fd */
+       NULL, /* Fl */
+       NULL, /* Fn */
+       NULL, /* Ft */
+       NULL, /* Ic */
+       NULL, /* In */
+       NULL, /* Li */
+       pmdoc_Nd, /* Nd */
+       pmdoc_Nm, /* Nm */
+       NULL, /* Op */
+       NULL, /* Ot */
+       NULL, /* Pa */
+       NULL, /* Rv */
+       NULL, /* St */
+       NULL, /* Va */
+       NULL, /* Vt */
+       pmdoc_Xr, /* Xr */
+       NULL, /* %A */
+       NULL, /* %B */
+       NULL, /* %D */
+       NULL, /* %I */
+       NULL, /* %J */
+       NULL, /* %N */
+       NULL, /* %O */
+       NULL, /* %P */
+       NULL, /* %R */
+       NULL, /* %T */
+       NULL, /* %V */
+       NULL, /* Ac */
+       NULL, /* Ao */
+       NULL, /* Aq */
+       NULL, /* At */
+       NULL, /* Bc */
+       NULL, /* Bf */
+       NULL, /* Bo */
+       NULL, /* Bq */
+       NULL, /* Bsx */
+       NULL, /* Bx */
+       NULL, /* Db */
+       NULL, /* Dc */
+       NULL, /* Do */
+       NULL, /* Dq */
+       NULL, /* Ec */
+       NULL, /* Ef */
+       NULL, /* Em */
+       NULL, /* Eo */
+       NULL, /* Fx */
+       NULL, /* Ms */
+       NULL, /* No */
+       NULL, /* Ns */
+       NULL, /* Nx */
+       NULL, /* Ox */
+       NULL, /* Pc */
+       NULL, /* Pf */
+       NULL, /* Po */
+       NULL, /* Pq */
+       NULL, /* Qc */
+       NULL, /* Ql */
+       NULL, /* Qo */
+       NULL, /* Qq */
+       NULL, /* Re */
+       NULL, /* Rs */
+       NULL, /* Sc */
+       NULL, /* So */
+       NULL, /* Sq */
+       NULL, /* Sm */
+       NULL, /* Sx */
+       NULL, /* Sy */
+       NULL, /* Tn */
+       NULL, /* Ux */
+       NULL, /* Xc */
+       NULL, /* Xo */
+       NULL, /* Fo */
+       NULL, /* Fc */
+       NULL, /* Oo */
+       NULL, /* Oc */
+       NULL, /* Bk */
+       NULL, /* Ek */
+       NULL, /* Bt */
+       NULL, /* Hf */
+       NULL, /* Fr */
+       NULL, /* Ud */
+       NULL, /* Lb */
+       NULL, /* Lp */
+       NULL, /* Lk */
+       NULL, /* Mt */
+       NULL, /* Brq */
+       NULL, /* Bro */
+       NULL, /* Brc */
+       NULL, /* %C */
+       NULL, /* Es */
+       NULL, /* En */
+       NULL, /* Dx */
+       NULL, /* %Q */
+       NULL, /* br */
+       NULL, /* sp */
+       NULL, /* %U */
+       NULL, /* Ta */
+};
+
+static const pman_nf mans[MAN_MAX] = {
+       NULL,   //br
+       NULL,   //TH
+       pman_sh, //SH
+       NULL,   //SS
+       NULL,   //TP
+       NULL,   //LP
+       NULL,   //PP
+       NULL,   //P
+       NULL,   //IP
+       NULL,   //HP
+       NULL,   //SM
+       NULL,   //SB
+       NULL,   //BI
+       NULL,   //IB
+       NULL,   //BR
+       NULL,   //RB
+       NULL,   //R
+       pman_block,     //B
+       NULL,   //I
+       NULL,   //IR
+       NULL,   //RI
+       NULL,   //na
+       NULL,   //sp
+       NULL,   //nf
+       NULL,   //fi
+       NULL,   //RE
+       NULL,   //RS
+       NULL,   //DT
+       NULL,   //UC
+       NULL,   //PD
+       NULL,   //AT
+       NULL,   //in
+       NULL,   //ft
+};
+
+
+int
+main(int argc, char *argv[])
+{
+       FILE *file;
+       const char *sqlstr, *manconf = NULL;
+       char *line, *command;
+       char *errmsg;
+       int ch;
+       struct mparse *mp;
+       sqlite3 *db;
+       ssize_t len;
+       size_t linesize;
+       struct mandb_rec rec;
+
+       while ((ch = getopt(argc, argv, "C:floqv")) != -1) {
+               switch (ch) {
+               case 'C':
+                       manconf = optarg;
+                       break;
+               case 'f':
+                       remove(DBPATH);
+                       mflags.recreate = 1;
+                       break;
+               case 'l':
+                       mflags.limit = 1;
+                       break;
+               case 'o':
+                       mflags.optimize = 1;
+                       break;
+               case 'q':
+                       mflags.verbosity = 0;
+                       break;
+               case 'v':
+                       mflags.verbosity = 2;
+                       break;
+               default:
+                       usage();
+               }
+       }
+
+       memset(&rec, 0, sizeof(rec));
+
+       init_secbuffs(&rec);
+       mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL);
+
+       if ((db = init_db(MANDB_CREATE)) == NULL)
+               exit(EXIT_FAILURE);
+
+       sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL,  &errmsg);
+       if (errmsg != NULL) {
+               warnx("%s", errmsg);
+               free(errmsg);
+               close_db(db);
+               exit(EXIT_FAILURE);
+       }
+
+       sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
+           &errmsg);
+       if (errmsg != NULL) {
+               warnx("%s", errmsg);
+               free(errmsg);
+               close_db(db);
+               exit(EXIT_FAILURE);
+       }
+
+       if (manconf) {
+               char *arg;
+               size_t command_len = shquote(manconf, NULL, 0) + 1;
+               arg = malloc(command_len );
+               shquote(manconf, arg, command_len);
+               easprintf(&command, "man -p -C %s", arg);
+               free(arg);
+       } else {
+               command = estrdup("man -p");
+       }
+
+       /* Call man -p to get the list of man page dirs */
+       if ((file = popen(command, "r")) == NULL) {
+               close_db(db);
+               err(EXIT_FAILURE, "fopen failed");
+       }
+       free(command);
+
+       /* Begin the transaction for indexing the pages */
+       sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
+       if (errmsg != NULL) {
+               warnx("%s", errmsg);
+               free(errmsg);
+               exit(EXIT_FAILURE);
+       }
+               
+       sqlstr = "CREATE TABLE IF NOT EXISTS metadb.file_cache(device, inode,"
+                " mtime, file PRIMARY KEY);"
+                "CREATE UNIQUE INDEX IF NOT EXISTS metadb.index_file_cache_dev"
+                " ON file_cache (device, inode)";
+
+       sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+       if (errmsg != NULL) {
+               warnx("%s", errmsg);
+               free(errmsg);
+               close_db(db);
+               exit(EXIT_FAILURE);
+       }
+
+       if (mflags.verbosity)
+               printf("Building temporary file cache\n");
+       line = NULL;
+       linesize = 0;
+       while ((len = getline(&line, &linesize, file)) != -1) {
+               /* Replace the new line character at the end of string with 
'\0' */
+               line[len - 1] = '\0';
+               /* Traverse the man page directories and parse the pages */
+               traversedir(line, db, mp);
+       }
+       free(line);
+
+       if (pclose(file) == -1) {
+               close_db(db);
+               cleanup(&rec);
+               free_secbuffs(&rec);
+               err(EXIT_FAILURE, "pclose error");
+       }
+
+       update_db(db, mp, &rec);
+       mparse_free(mp);
+       free_secbuffs(&rec);
+
+       /* Commit the transaction */
+       sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
+       if (errmsg != NULL) {
+               warnx("%s", errmsg);
+               free(errmsg);
+               exit(EXIT_FAILURE);
+       }
+
+       if (mflags.optimize)
+               optimize(db);
+
+       close_db(db);
+       return 0;
+}
+
+/*
+ * traversedir --
+ *  Traverses the given directory recursively and passes all the man page files
+ *  in the way to build_file_cache()
+ */
+static void
+traversedir(const char *file, sqlite3 *db, struct mparse *mp)
+{
+       struct stat sb;
+       struct dirent *dirp;
+       DIR *dp;
+       char *buf;
+
+       if (stat(file, &sb) < 0) {
+               warn("stat failed: %s", file);
+               return;
+       }
+       
+       /* If it is a regular file or a symlink, pass it to build_cache() */
+       if (S_ISREG(sb.st_mode) || S_ISLNK(sb.st_mode)) {
+               build_file_cache(db, file, &sb);
+               return;
+       }
+       
+       /* If it is a directory, traverse it recursively */
+       if (S_ISDIR(sb.st_mode)) {
+               if ((dp = opendir(file)) == NULL) {
+                       warn("opendir error: %s", file);
+                       return;
+               }
+               
+               while ((dirp = readdir(dp)) != NULL) {
+                       /* Avoid . and .. entries in a directory */
+                       if (strncmp(dirp->d_name, ".", 1)) {
+                               easprintf(&buf, "%s/%s", file, dirp->d_name);
+                               traversedir(buf, db, mp);
+                               free(buf);
+                       }
+               }
+               closedir(dp);
+       }
+}
+
+/* build_file_cache --
+ *   This function generates an md5 hash of the file passed as it's 2nd 
parameter
+ *   and stores it in a temporary table file_cache along with the full file 
path.
+ *   This is done to support incremental updation of the database.
+ *   The temporary table file_cache is dropped thereafter in the function
+ *   update_db(), once the database has been updated.
+ */
+static void
+build_file_cache(sqlite3 *db, const char *file, struct stat *sb)
+{
+       const char *sqlstr;
+       sqlite3_stmt *stmt = NULL;
+       int rc, idx;
+       assert(file != NULL);
+       dev_t device_cache = sb->st_dev;
+       ino_t inode_cache = sb->st_ino;
+       time_t mtime_cache = sb->st_mtime;
+
+       sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
+                " :mtime, :file)";
+       rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               return;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":device");
+       rc = sqlite3_bind_int64(stmt, idx, device_cache);
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               sqlite3_finalize(stmt);
+               return;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":inode");
+       rc = sqlite3_bind_int64(stmt, idx, inode_cache);
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               sqlite3_finalize(stmt);
+               return;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+       rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               sqlite3_finalize(stmt);
+               return;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":file");
+       rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               sqlite3_finalize(stmt);
+               return;
+       }
+
+       sqlite3_step(stmt);
+       sqlite3_finalize(stmt);
+}
+
+static void
+update_existing_entry(sqlite3 *db, const char *file, const char *hash,
+    mandb_rec *rec, int *new_count, int *link_count, int *err_count)
+{
+       int update_count, rc, idx;
+       const char *inner_sqlstr;
+       sqlite3_stmt *inner_stmt;
+
+       update_count = sqlite3_total_changes(db);
+       inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
+                      " inode = :inode, mtime = :mtime WHERE"
+                      " md5_hash = :md5 AND file = :file AND"
+                      " (device <> :device2 OR inode <> "
+                      "  :inode2 OR mtime <> :mtime2)";
+       rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               return;
+       }
+       idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
+       sqlite3_bind_int64(inner_stmt, idx, rec->device);
+       idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
+       sqlite3_bind_int64(inner_stmt, idx, rec->inode);
+       idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
+       sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
+       idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
+       sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
+       idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
+       sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
+       idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
+       sqlite3_bind_int64(inner_stmt, idx, rec->device);
+       idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
+       sqlite3_bind_int64(inner_stmt, idx, rec->inode);
+       idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
+       sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
+
+       rc = sqlite3_step(inner_stmt);
+       if (rc == SQLITE_DONE) {
+               /* Check if an update has been performed. */
+               if (update_count != sqlite3_total_changes(db)) {
+                       if (mflags.verbosity)
+                               printf("Updated %s\n", file);
+                       (*new_count)++;
+               } else {
+                       /* Otherwise it was a hardlink. */
+                       (*link_count)++;
+               }
+       } else {
+               warnx("Could not update the meta data for %s", file);
+               (*err_count)++;
+       }
+       sqlite3_finalize(inner_stmt);
+}
+
+/* update_db --
+ *     Does an incremental updation of the database by checking the file_cache.
+ *     It parses and adds the pages which are present in file_cache,
+ *     but not in the database.
+ *     It also removes the pages which are present in the databse,
+ *     but not in the file_cache.
+ */
+static void
+update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
+{
+       const char *sqlstr;
+       sqlite3_stmt *stmt = NULL;
+       const char *file;
+       char *errmsg = NULL;
+       char *buf = NULL;
+       int new_count = 0;      /* Counter for newly indexed/updated pages */
+       int total_count = 0;    /* Counter for total number of pages */
+       int err_count = 0;      /* Counter for number of failed pages */
+       int link_count = 0;     /* Counter for number of hard/sym links */
+       int md5_status;
+       int rc;
+
+       sqlstr = "SELECT device, inode, mtime, file FROM metadb.file_cache"
+                " EXCEPT SELECT device, inode, mtime, file from mandb_meta";
+
+       rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               close_db(db);
+               errx(EXIT_FAILURE, "Could not query file cache");
+       }
+
+       while (sqlite3_step(stmt) == SQLITE_ROW) {
+               total_count++;
+               rec->device = sqlite3_column_int64(stmt, 0);
+               rec->inode = sqlite3_column_int64(stmt, 1);
+               rec->mtime = sqlite3_column_int64(stmt, 2);
+               file = (const char *) sqlite3_column_text(stmt, 3);
+               md5_status = check_md5(file, db, "mandb_meta", &buf);
+               assert(buf != NULL);
+               if (md5_status == -1) {
+                       warnx("An error occurred in checking md5 value"
+                             " for file %s", file);
+                       err_count++;
+                       continue;
+               }
+
+               if (md5_status == 0) {
+                       /*
+                        * The MD5 hash is already present in the database,
+                        * so simply update the metadata, ignoring symlinks.
+                        */
+                       struct stat sb;
+                       stat(file, &sb);
+                       if (S_ISLNK(sb.st_mode)) {
+                               free(buf);
+                               link_count++;
+                               continue;
+                       }
+                       update_existing_entry(db, file, buf, rec,
+                           &new_count, &link_count, &err_count);
+                       free(buf);
+                       continue;
+               }
+
+               if (md5_status == 1) {
+                       /*
+                        * The MD5 hash was not present in the database.
+                        * This means is either a new file or an updated file.
+                        * We should go ahead with parsing.
+                        */
+                       if (mflags.verbosity > 1)
+                               printf("Parsing: %s\n", file);
+                       rec->md5_hash = buf;
+                       rec->file_path = estrdup(file);
+                       // file_path is freed by insert_into_db itself.
+                       begin_parse(file, mp, rec);
+                       if (insert_into_db(db, rec) < 0) {
+                               warnx("Error in indexing %s", file);
+                               err_count++;
+                       } else {
+                               new_count++;
+                       }
+               }
+       }
+       
+       sqlite3_finalize(stmt);
+       
+       if (mflags.verbosity) {
+               printf("Total Number of new or updated pages enountered = %d\n"
+                       "Total number of pages that were successfully"
+                       " indexed/updated = %d\n"
+                       "Total number of (hard or symbolic) links found = %d\n"
+                       "Total number of pages that could not be indexed"
+                       " due to errors = %d\n",
+                       total_count, new_count, link_count, err_count);
+       }
+
+       if (mflags.recreate == 0)
+               return;
+
+       if (mflags.verbosity)
+               printf("Deleting stale index entries\n");
+
+       sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
+                " (SELECT file FROM metadb.file_cache);"
+                "DROP TABLE metadb.file_cache;"
+                "DELETE FROM mandb WHERE rowid NOT IN"
+                " (SELECT id FROM mandb_meta);";
+
+       sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+       if (errmsg != NULL) {
+               warnx("Removing old entries failed: %s", errmsg);
+               warnx("Please rebuild database from scratch with -f.");
+               free(errmsg);
+               return;
+       }
+}
+
+/*
+ * begin_parse --
+ *  parses the man page using libmandoc
+ */
+static void
+begin_parse(const char *file, struct mparse *mp, mandb_rec *rec)
+{
+       struct mdoc *mdoc;
+       struct man *man;
+       mparse_reset(mp);
+
+       rec->xr_found = 0;
+
+       if (mparse_readfd(mp, -1, file) >= MANDOCLEVEL_FATAL) {
+               warnx("%s: Parse failure", file);
+               return;
+       }
+
+       mparse_result(mp, &mdoc, &man);
+       if (mdoc == NULL && man == NULL) {
+               warnx("Not a man(7) or mdoc(7) page");
+               return;
+       }
+
+       set_machine(mdoc, rec);
+       set_section(mdoc, man, rec);
+       if (mdoc) {
+               rec->page_type = MDOC;
+               pmdoc_node(mdoc_node(mdoc), rec);
+       } else {
+               rec->page_type = MAN;
+               pman_node(man_node(man), rec);
+       }
+}
+
+/*
+ * set_section --
+ *  Extracts the section number and normalizes it to only the numeric part
+ *  (Which should be the first character of the string).
+ */
+static void
+set_section(const struct mdoc *md, const struct man *m, mandb_rec *rec)
+{
+       if (md) {
+               const struct mdoc_meta *md_meta = mdoc_meta(md);
+               rec->section[0] = md_meta->msec[0];
+       } else if (m) {
+               const struct man_meta *m_meta = man_meta(m);
+               rec->section[0] = m_meta->msec[0];
+       }
+}
+
+/*
+ * get_machine --
+ *  Extracts the machine architecture information if available.
+ */
+static void
+set_machine(const struct mdoc *md, mandb_rec *rec)
+{
+       if (md == NULL)
+               return;
+       const struct mdoc_meta *md_meta = mdoc_meta(md);
+       if (md_meta->arch)
+               rec->machine = estrdup(md_meta->arch);
+}
+
+static void
+pmdoc_node(const struct mdoc_node *n, mandb_rec *rec)
+{
+
+       if (n == NULL)
+               return;
+
+       switch (n->type) {
+       case (MDOC_BODY):
+               /* FALLTHROUGH */
+       case (MDOC_TAIL):
+               /* FALLTHROUGH */
+       case (MDOC_ELEM):
+               if (mdocs[n->tok] == NULL)
+                       break;
+               (*mdocs[n->tok])(n, rec);
+               break;
+       default:
+               break;
+       }
+
+       pmdoc_node(n->child, rec);
+       pmdoc_node(n->next, rec);
+}
+
+/*
+ * pmdoc_Nm --
+ *  Extracts the Name of the manual page from the .Nm macro
+ */
+static void
+pmdoc_Nm(const struct mdoc_node *n, mandb_rec *rec)
+{
+       if (n->sec != SEC_NAME)
+               return;
+
+       for (n = n->child; n; n = n->next) {
+               if (n->type == MDOC_TEXT) {
+                       concat(&rec->name, n->string);
+               }
+       }
+}
+
+/*
+ * pmdoc_Nd --
+ *  Extracts the one line description of the man page from the .Nd macro
+ */
+static void
+pmdoc_Nd(const struct mdoc_node *n, mandb_rec *rec)
+{
+       /*
+        * A static variable for keeping track of whether a Xr macro was seen
+        * previously.
+        */
+       char *buf = NULL;
+       char *temp;
+
+       if (n == NULL)
+               return;
+
+       if (n->type == MDOC_TEXT) {
+               if (rec->xr_found && n->next) {
+                       /*
+                        * An Xr macro was seen previously, so parse this
+                        * and the next node.
+                        */
+                       temp = estrdup(n->string);
+                       n = n->next;
+                       easprintf(&buf, "%s(%s)", temp, n->string);
+                       concat(&rec->name_desc, buf);
+                       free(buf);
+                       free(temp);
+               } else {
+                       concat(&rec->name_desc, n->string);
+               }
+               rec->xr_found = 0;
+       } else if (mdocs[n->tok] == pmdoc_Xr) {
+               /* Remember that we have encountered an Xr macro */
+               rec->xr_found = 1;
+       }
+
+       if (n->child)
+               pmdoc_Nd(n->child, rec);
+
+       if(n->next)
+               pmdoc_Nd(n->next, rec);
+}
+
+/*
+ * pmdoc_macro_handler--
+ *  This function is a single point of handling all the special macros that we
+ *  want to handle especially. For example the .Xr macro for properly parsing
+ *  the referenced page name along with the section number, or the .Pp macro
+ *  for adding a new line whenever we encounter it.
+ */
+static void
+pmdoc_macro_handler(const struct mdoc_node *n, mandb_rec *rec, enum mdoct doct)
+{
+       const struct mdoc_node *sn;
+       assert(n);
+
+       switch (doct) {
+       /*  Parse the man page references.
+        * Basically the .Xr macros are used like:
+        *  .Xr ls 1
+        *  and formatted like this:
+        *  ls(1)
+        *  Prepare a buffer to format the data like the above example and call
+        *  pmdoc_parse_section to append it.
+        */
+       case MDOC_Xr:
+               n = n->child;
+               while (n->type != MDOC_TEXT && n->next)
+                       n = n->next;
+
+               if (n && n->type != MDOC_TEXT)
+                       return;
+               sn = n;
+               if (n->next)
+                       n = n->next;
+
+               while (n->type != MDOC_TEXT && n->next)
+                       n = n->next;
+
+               if (n && n->type == MDOC_TEXT) {
+                       size_t len = strlen(sn->string);
+                       char *buf = emalloc(len + 4);
+                       memcpy(buf, sn->string, len);
+                       buf[len] = '(';
+                       buf[len + 1] = n->string[0];
+                       buf[len + 2] = ')';
+                       buf[len + 3] = 0;
+                       mdoc_parse_section(n->sec, buf, rec);
+                       free(buf);
+               }
+
+               break;
+
+       /* Parse the .Pp macro to add a new line */
+       case MDOC_Pp:
+               if (n->type == MDOC_TEXT)
+                       mdoc_parse_section(n->sec, "\n", rec);
+               break;
+       default:
+               break;
+       }
+
+}
+
+/*
+ * pmdoc_Xr, pmdoc_Pp--
+ *  Empty stubs.
+ *  The parser calls these functions each time it encounters
+ *  a .Xr or .Pp macro. We are parsing all the data from
+ *  the pmdoc_Sh function, so don't do anything here.
+ *  (See if else blocks in pmdoc_Sh.)
+ */
+static void
+pmdoc_Xr(const struct mdoc_node *n, mandb_rec *rec)
+{
+}
+
+static void
+pmdoc_Pp(const struct mdoc_node *n, mandb_rec *rec)
+{
+}
+
+/*
+ * pmdoc_Sh --
+ *  Called when a .Sh macro is encountered and loops through its body, calling
+ *  mdoc_parse_section to append the data to the section specific buffer.
+ *  Two special macros which may occur inside the body of Sh are .Nm and .Xr 
and
+ *  they need special handling, thus the separate if branches for them.
+ */
+static void
+pmdoc_Sh(const struct mdoc_node *n, mandb_rec *rec)
+{
+       if (n == NULL)
+               return;
+       int xr_found = 0;
+
+       if (n->type == MDOC_TEXT) {
+               mdoc_parse_section(n->sec, n->string, rec);
+       } else if (mdocs[n->tok] == pmdoc_Nm && rec->name != NULL) {
+               /*
+                * When encountering a .Nm macro, substitute it
+                * with its previously cached value of the argument.
+                */
+               mdoc_parse_section(n->sec, rec->name, rec);
+       } else if (mdocs[n->tok] == pmdoc_Xr) {
+               /*
+                * When encountering other inline macros,
+                * call pmdoc_macro_handler.
+                */
+               pmdoc_macro_handler(n, rec, MDOC_Xr);
+               xr_found = 1;
+       } else if (mdocs[n->tok] == pmdoc_Pp) {
+               pmdoc_macro_handler(n, rec, MDOC_Pp);
+       }
+
+       /*
+        * If an Xr macro was encountered then the child node has
+        * already been explored by pmdoc_macro_handler.
+        */
+       if (xr_found == 0)
+               pmdoc_Sh(n->child, rec);
+       pmdoc_Sh(n->next, rec);
+}
+
+/*
+ * mdoc_parse_section--
+ *  Utility function for parsing sections of the mdoc type pages.
+ *  Takes two params:
+ *   1. sec is an enum which indicates the section in which we are present
+ *   2. string is the string which we need to append to the secbuff for this
+ *      particular section.
+ *  The function appends string to the global section buffer and returns.
+ */
+static void
+mdoc_parse_section(enum mdoc_sec sec, const char *string, mandb_rec *rec)
+{
+       /*
+        * If the user specified the 'l' flag, then parse and store only the
+        * NAME section. Ignore the rest.
+        */
+       if (mflags.limit)
+               return;
+
+       switch (sec) {
+       case SEC_LIBRARY:
+               append(&rec->lib, string);
+               break;
+       case SEC_RETURN_VALUES:
+               append(&rec->return_vals, string);
+               break;
+       case SEC_ENVIRONMENT:
+               append(&rec->env, string);
+               break;
+       case SEC_FILES:
+               append(&rec->files, string);
+               break;
+       case SEC_EXIT_STATUS:
+               append(&rec->exit_status, string);
+               break;
+       case SEC_DIAGNOSTICS:
+               append(&rec->diagnostics, string);
+               break;
+       case SEC_ERRORS:
+               append(&rec->errors, string);
+               break;
+       case SEC_NAME:
+       case SEC_SYNOPSIS:
+       case SEC_EXAMPLES:
+       case SEC_STANDARDS:
+       case SEC_HISTORY:
+       case SEC_AUTHORS:
+       case SEC_BUGS:
+               break;
+       default:
+               append(&rec->desc, string);
+               break;
+       }
+}
+
+static void
+pman_node(const struct man_node *n, mandb_rec *rec)
+{
+       if (n == NULL)
+               return;
+
+       switch (n->type) {
+       case (MAN_BODY):
+               /* FALLTHROUGH */
+       case (MAN_TAIL):
+               /* FALLTHROUGH */
+       case (MAN_BLOCK):
+               /* FALLTHROUGH */
+       case (MAN_ELEM):
+               if (mans[n->tok] != NULL)
+                       (*mans[n->tok])(n, rec);
+               break;
+       default:
+               break;
+       }
+
+       pman_node(n->child, rec);
+       pman_node(n->next, rec);
+}
+
+/* 
+ * pman_parse_name --
+ *  Parses the NAME section and puts the complete content in the name_desc
+ *  variable.
+ */
+static void
+pman_parse_name(const struct man_node *n, mandb_rec *rec)
+{
+       if (n == NULL)
+               return;
+
+       if (n->type == MAN_TEXT) 
+               concat(&rec->name_desc, n->string);
+       
+       if (n->child)
+               pman_parse_name(n->child, rec);
+
+       if(n->next)
+               pman_parse_name(n->next, rec);
+}
+
+/*
+ * A stub function to be able to parse the macros like .B embedded inside
+ * a section.
+ */
+static void
+pman_block(const struct man_node *n, mandb_rec *rec)
+{
+}
+
+/* 
+ * pman_sh --
+ * This function does one of the two things:
+ *  1. If the present section is NAME, then it will:
+ *    (a) Extract the name of the page (in case of multiple comma separated
+ *        names, it will pick up the first one).
+ *    (b) Build a space spearated list of all the symlinks/hardlinks to
+ *        this page and store in the buffer 'links'. These are extracted from
+ *        the comma separated list of names in the NAME section as well.
+ *    (c) Move on to the one line description section, which is after the list
+ *        of names in the NAME section.
+ *  2. Otherwise, it will check the section name and call the man_parse_section
+ *     function, passing the enum corresponding that section.
+ */
+static void
+pman_sh(const struct man_node *n, mandb_rec *rec)
+{
+       static const struct {
+               enum man_sec section;
+               const char *header;
+       } mapping[] = {
+           { MANSEC_DESCRIPTION, "DESCRIPTION" },
+           { MANSEC_SYNOPSIS, "SYNOPSIS" },
+           { MANSEC_LIBRARY, "LIBRARY" },
+           { MANSEC_ERRORS, "ERRORS" },
+           { MANSEC_FILES, "FILES" },
+           { MANSEC_RETURN_VALUES, "RETURN VALUE" },
+           { MANSEC_RETURN_VALUES, "RETURN VALUES" },
+           { MANSEC_EXIT_STATUS, "EXIT STATUS" },
+           { MANSEC_EXAMPLES, "EXAMPLES" },
+           { MANSEC_EXAMPLES, "EXAMPLE" },
+           { MANSEC_STANDARDS, "STANDARDS" },
+           { MANSEC_HISTORY, "HISTORY" },
+           { MANSEC_BUGS, "BUGS" },
+           { MANSEC_AUTHORS, "AUTHORS" },
+           { MANSEC_COPYRIGHT, "COPYRIGHT" },
+       };
+       const struct man_node *head;
+       char *name_desc;
+       int sz;
+       size_t i;
+
+       if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
+           head->type != MAN_TEXT)
+               return;
+
+       /*
+        * Check if this section should be extracted and
+        * where it should be stored. Handled the trival cases first.
+        */
+       for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
+               if (strcmp(head->string, mapping[i].header) == 0) {
+                       man_parse_section(mapping[i].section, n, rec);
+                       return;
+               }
+       }
+
+       if (strcmp(head->string, "NAME") == 0) {
+               /*
+                * We are in the NAME section.
+                * pman_parse_name will put the complete content in name_desc.
+                */
+               pman_parse_name(n, rec);
+
+               name_desc = rec->name_desc;
+
+               /* Remove any leading spaces. */
+               while (name_desc[0] == ' ')
+                       name_desc++;
+                       
+               /* If the line begins with a "\&", avoid those */
+               if (name_desc[0] == '\\' && name_desc[1] == '&')
+                       name_desc += 2;
+
+               /* Now name_desc should be left with a comma-space
+                * separated list of names and the one line description
+                * of the page:
+                *     "a, b, c \- sample description"
+                * Take out the first name, before the first comma
+                * (or space) and store it in rec->name.
+                * If the page has aliases then they should be
+                * in the form of a comma separated list.
+                * Keep looping while there is a comma in name_desc,
+                * extract the alias name and store in rec->links.
+                * When there are no more commas left, break out.
+                */
+               int has_alias = 0;      // Any more aliases left?
+               while (*name_desc) {
+                       /* Remove any leading spaces. */
+                       if (name_desc[0] == ' ') {
+                               name_desc++;
+                               continue;
+                       }
+                       sz = strcspn(name_desc, ", ");
+
+                       /* Extract the first term and store it in rec->name. */
+                       if (rec->name == NULL) {
+                               if (name_desc[sz] == ',')
+                                       has_alias = 1;
+                               name_desc[sz] = 0;
+                               rec->name = emalloc(sz + 1);
+                               memcpy(rec->name, name_desc, sz + 1);
+                               name_desc += sz + 1;
+                               continue;
+                       }
+
+                       /*
+                        * Once rec->name is set, rest of the names
+                        * are to be treated as links or aliases.
+                        */
+                       if (rec->name && has_alias) {
+                               if (name_desc[sz] != ',') {
+                                       /* No more commas left -->
+                                        * no more aliases to take out
+                                        */
+                                       has_alias = 0;
+                               }
+                               name_desc[sz] = 0;
+                               concat2(&rec->links, name_desc, sz);
+                               name_desc += sz + 1;
+                               continue;
+                       }
+                       break;
+               }
+
+               /* Parse any escape sequences that might be there */
+               char *temp = parse_escape(name_desc);
+               free(rec->name_desc);
+               rec->name_desc = temp;
+               temp = parse_escape(rec->name);
+               free(rec->name);
+               rec->name = temp;
+               return;
+       }
+
+       /* The RETURN VALUE section might be specified in multiple ways */
+       if (strcmp(head->string, "RETURN") == 0 &&
+           head->next != NULL && head->next->type == MAN_TEXT &&
+           (strcmp(head->next->string, "VALUE") == 0 ||
+           strcmp(head->next->string, "VALUES") == 0)) {
+               man_parse_section(MANSEC_RETURN_VALUES, n, rec);
+               return;
+       }
+
+       /*
+        * EXIT STATUS section can also be specified all on one line or on two
+        * separate lines.
+        */
+       if (strcmp(head->string, "EXIT") == 0 &&
+           head->next != NULL && head->next->type == MAN_TEXT &&
+           strcmp(head->next->string, "STATUS") == 0) {
+               man_parse_section(MANSEC_EXIT_STATUS, n, rec);
+               return;
+       }
+
+       /* Store the rest of the content in desc. */
+       man_parse_section(MANSEC_NONE, n, rec);
+}
+
+/*
+ * pman_parse_node --
+ *  Generic function to iterate through a node. Usually called from 
+ *  man_parse_section to parse a particular section of the man page.
+ */
+static void
+pman_parse_node(const struct man_node *n, secbuff *s)
+{
+       if (n == NULL)
+               return;
+
+       if (n->type == MAN_TEXT)
+               append(s, n->string);
+               
+       pman_parse_node(n->child, s);
+       pman_parse_node(n->next, s);
+}
+
+/*
+ * man_parse_section --
+ *  Takes two parameters: 
+ *   sec: Tells which section we are present in
+ *   n: Is the present node of the AST.
+ * Depending on the section, we call pman_parse_node to parse that section and
+ * concatenate the content from that section into the buffer for that section.
+ */
+static void
+man_parse_section(enum man_sec sec, const struct man_node *n, mandb_rec *rec)
+{
+       /*
+        * If the user sepecified the 'l' flag then just parse
+        * the NAME section, ignore the rest.
+        */
+       if (mflags.limit)
+               return;
+
+       switch (sec) {
+       case MANSEC_LIBRARY:
+               pman_parse_node(n, &rec->lib);
+               break;
+       case MANSEC_RETURN_VALUES:
+               pman_parse_node(n, &rec->return_vals);
+               break;
+       case MANSEC_ENVIRONMENT:
+               pman_parse_node(n, &rec->env);
+               break;
+       case MANSEC_FILES:
+               pman_parse_node(n, &rec->files);
+               break;
+       case MANSEC_EXIT_STATUS:
+               pman_parse_node(n, &rec->exit_status);
+               break;
+       case MANSEC_DIAGNOSTICS:
+               pman_parse_node(n, &rec->diagnostics);
+               break;
+       case MANSEC_ERRORS:
+               pman_parse_node(n, &rec->errors);
+               break;
+       case MANSEC_NAME:
+       case MANSEC_SYNOPSIS:
+       case MANSEC_EXAMPLES:
+       case MANSEC_STANDARDS:
+       case MANSEC_HISTORY:
+       case MANSEC_BUGS:
+       case MANSEC_AUTHORS:
+       case MANSEC_COPYRIGHT:
+               break;
+       default:
+               pman_parse_node(n, &rec->desc);
+               break;
+       }
+
+}
+
+/*
+ * insert_into_db --
+ *  Inserts the parsed data of the man page in the Sqlite databse.
+ *  If any of the values is NULL, then we cleanup and return -1 indicating
+ *  an error.
+ *  Otherwise, store the data in the database and return 0.
+ */
+static int
+insert_into_db(sqlite3 *db, mandb_rec *rec)
+{
+       int rc = 0;
+       int idx = -1;
+       const char *sqlstr = NULL;
+       sqlite3_stmt *stmt = NULL;
+       char *ln = NULL;
+       char *errmsg = NULL;
+       long int mandb_rowid;
+       
+       /*
+        * At the very minimum we want to make sure that we store
+        * the following data:
+        *   Name, one line description, and the MD5 hash
+        */             
+       if (rec->name == NULL || rec->name_desc == NULL ||
+           rec->md5_hash == NULL) {
+               cleanup(rec);
+               return -1;
+       }
+
+       /* Write null byte at the end of all the sec_buffs */
+       rec->desc.data[rec->desc.offset] = 0;
+       rec->lib.data[rec->lib.offset] = 0;
+       rec->env.data[rec->env.offset] = 0;
+       rec->return_vals.data[rec->return_vals.offset] = 0;
+       rec->exit_status.data[rec->exit_status.offset] = 0;
+       rec->files.data[rec->files.offset] = 0;
+       rec->diagnostics.data[rec->diagnostics.offset] = 0;
+       rec->errors.data[rec->errors.offset] = 0;
+
+       /*
+        * In case of a mdoc page: (sorry, no better place to put this code)
+        * parse the comma separated list of names of man pages,
+        * the first name will be stored in the mandb table, rest will be
+        * treated as links and put in the mandb_links table.
+        */
+       if (rec->page_type == MDOC) {
+               char *tmp;
+               rec->links = estrdup(rec->name);
+               free(rec->name);
+               int sz = strcspn(rec->links, " \0");
+               rec->name = emalloc(sz + 1);
+               memcpy(rec->name, rec->links, sz);
+               if(rec->name[sz - 1] == ',')
+                       rec->name[sz - 1] = 0;
+               else
+                       rec->name[sz] = 0;
+               while (rec->links[sz] == ' ')
+                       ++sz;
+               tmp = estrdup(rec->links + sz);
+               free(rec->links);
+               rec->links = tmp;
+       }
+
+/*------------------------ Populate the mandb 
table---------------------------*/
+       sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
+                " :lib, :return_vals, :env, :files, :exit_status,"
+                " :diagnostics, :errors, :md5_hash, :machine)";
+
+       rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+       if (rc != SQLITE_OK)
+               goto Out;
+
+       idx = sqlite3_bind_parameter_index(stmt, ":name");
+       rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":section");
+       rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
+       rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":desc");
+       rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
+                              rec->desc.offset + 1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":lib");
+       rc = sqlite3_bind_text(stmt, idx, rec->lib.data, rec->lib.offset + 1, 
NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
+       rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
+                             rec->return_vals.offset + 1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":env");
+       rc = sqlite3_bind_text(stmt, idx, rec->env.data, rec->env.offset + 1, 
NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":files");
+       rc = sqlite3_bind_text(stmt, idx, rec->files.data,
+                              rec->files.offset + 1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
+       rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
+                              rec->exit_status.offset + 1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
+       rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
+                              rec->diagnostics.offset + 1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":errors");
+       rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
+                              rec->errors.offset + 1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+       
+       idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+       rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+       
+       idx = sqlite3_bind_parameter_index(stmt, ":machine");
+       if (rec->machine)
+               rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
+       else
+               rc = sqlite3_bind_null(stmt, idx);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       rc = sqlite3_step(stmt);
+       if (rc != SQLITE_DONE) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       sqlite3_finalize(stmt);
+       
+       /* Get the row id of the last inserted row */
+       mandb_rowid = sqlite3_last_insert_rowid(db);
+               
+/*------------------------Populate the mandb_meta 
table-----------------------*/
+       sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
+                " :file, :md5_hash, :id)";
+       rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+       if (rc != SQLITE_OK)
+               goto Out;
+
+       idx = sqlite3_bind_parameter_index(stmt, ":device");
+       rc = sqlite3_bind_int64(stmt, idx, rec->device);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":inode");
+       rc = sqlite3_bind_int64(stmt, idx, rec->inode);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+       rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":file");
+       rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+       rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":id");
+       rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
+       if (rc != SQLITE_OK) {
+               sqlite3_finalize(stmt);
+               goto Out;
+       }
+
+       rc = sqlite3_step(stmt);
+       sqlite3_finalize(stmt);
+       if (rc == SQLITE_CONSTRAINT) {
+               /* The *most* probable reason for reaching here is that
+                * the UNIQUE contraint on the file column of the mandb_meta
+                * table was violated.
+                * This can happen when a file was updated/modified.
+                * To fix this we need to do two things:
+                * 1. Delete the row for the older version of this file
+                *    from mandb table.
+                * 2. Run an UPDATE query to update the row for this file
+                *    in the mandb_meta table.
+                */
+               warnx("Trying to update index for %s", rec->file_path);
+               char *sql = sqlite3_mprintf("DELETE FROM mandb "
+                                           "WHERE rowid = (SELECT id"
+                                           "  FROM mandb_meta"
+                                           "  WHERE file = %Q)",
+                                           rec->file_path);
+               sqlite3_exec(db, sql, NULL, NULL, &errmsg);
+               sqlite3_free(sql);
+               if (errmsg != NULL) {
+                       warnx("%s", errmsg);
+                       free(errmsg);
+               }
+               sqlstr = "UPDATE mandb_meta SET device = :device,"
+                        " inode = :inode, mtime = :mtime, id = :id,"
+                        " md5_hash = :md5 WHERE file = :file";
+               rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+               if (rc != SQLITE_OK) {
+                       warnx("Update failed with error: %s",
+                           sqlite3_errmsg(db));
+                       close_db(db);
+                       cleanup(rec);
+                       errx(EXIT_FAILURE,
+                           "Consider running makemandb with -f option");
+               }
+
+               idx = sqlite3_bind_parameter_index(stmt, ":device");
+               sqlite3_bind_int64(stmt, idx, rec->device);
+               idx = sqlite3_bind_parameter_index(stmt, ":inode");
+               sqlite3_bind_int64(stmt, idx, rec->inode);
+               idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+               sqlite3_bind_int64(stmt, idx, rec->mtime);
+               idx = sqlite3_bind_parameter_index(stmt, ":id");
+               sqlite3_bind_int64(stmt, idx, mandb_rowid);
+               idx = sqlite3_bind_parameter_index(stmt, ":md5");
+               sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+               idx = sqlite3_bind_parameter_index(stmt, ":file");
+               sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
+               rc = sqlite3_step(stmt);
+               sqlite3_finalize(stmt);
+
+               if (rc != SQLITE_DONE) {
+                       warnx("%s", sqlite3_errmsg(db));
+                       close_db(db);
+                       cleanup(rec);
+                       errx(EXIT_FAILURE,
+                           "Consider running makemandb with -f option");
+               }
+       } else if (rc != SQLITE_DONE) {
+               /* Otherwise make this error fatal */
+               warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
+               cleanup(rec);
+               close_db(db);
+               exit(EXIT_FAILURE);
+       }
+
+/*------------------------ Populate the mandb_links 
table---------------------*/
+       char *str = NULL;
+       char *links;    
+       if (rec->links && strlen(rec->links)) {
+               links = rec->links;
+               for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
+                       if (ln[0] == ',')
+                               ln++;
+                       if(ln[strlen(ln) - 1] == ',')
+                               ln[strlen(ln) - 1] = 0;
+                       
+                       str = sqlite3_mprintf("INSERT INTO mandb_links"
+                                             " VALUES (%Q, %Q, %Q, %Q)",
+                                             ln, rec->name, rec->section,
+                                             rec->machine);
+                       sqlite3_exec(db, str, NULL, NULL, &errmsg);
+                       sqlite3_free(str);
+                       if (errmsg != NULL) {
+                               warnx("%s", errmsg);
+                               cleanup(rec);
+                               free(errmsg);
+                               return -1;
+                       }
+               }
+       }
+
+       cleanup(rec);
+       return 0;
+
+  Out:
+       warnx("%s", sqlite3_errmsg(db));
+       cleanup(rec);
+       return -1;
+}
+
+/*
+ * check_md5--
+ *  Generates the md5 hash of the file and checks if it already doesn't exist
+ *  in the table (passed as the 3rd parameter).
+ *  This function is being used to avoid hardlinks.
+ *  On successful completion it will also set the value of the fourth parameter
+ *  to the md5 hash of the file (computed previously). It is the responsibility
+ *  of the caller to free this buffer.
+ *  Return values:
+ *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
+ *  0: If the md5 hash does not exist in the table.
+ *  1: If the hash exists in the database.
+ */
+static int
+check_md5(const char *file, sqlite3 *db, const char *table, char **buf)
+{
+       int rc = 0;
+       int idx = -1;
+       char *sqlstr = NULL;
+       sqlite3_stmt *stmt = NULL;
+
+       assert(file != NULL);
+       *buf = MD5File(file, NULL);
+       if (*buf == NULL) {
+               warn("md5 failed: %s", file);
+               return -1;
+       }
+
+       easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash",
+           table);
+       rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+       if (rc != SQLITE_OK) {
+               free(sqlstr);
+               free(*buf);
+               *buf = NULL;
+               return -1;
+       }
+
+       idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+       rc = sqlite3_bind_text(stmt, idx, *buf, -1, NULL);
+       if (rc != SQLITE_OK) {
+               warnx("%s", sqlite3_errmsg(db));
+               sqlite3_finalize(stmt);
+               free(sqlstr);
+               free(*buf);
+               *buf = NULL;
+               return -1;
+       }
+
+       if (sqlite3_step(stmt) == SQLITE_ROW) {
+               sqlite3_finalize(stmt); 
+               free(sqlstr);
+               return 0;
+       }
+
+       sqlite3_finalize(stmt);
+       free(sqlstr);
+       return 1;
+}
+
+/* Optimize the index for faster search */
+static void
+optimize(sqlite3 *db)
+{
+       const char *sqlstr;
+       char *errmsg = NULL;
+
+       if (mflags.verbosity)
+               printf("Optimizing the database index\n");
+       sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
+                "VACUUM";
+       sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+       if (errmsg != NULL) {
+               warnx("%s", errmsg);
+               free(errmsg);
+               return;
+       }
+}
+
+/* 
+ * cleanup --
+ *  cleans up the global buffers
+ */
+static void
+cleanup(mandb_rec *rec)
+{
+       rec->desc.offset = 0;
+       rec->lib.offset = 0;
+       rec->return_vals.offset = 0;
+       rec->env.offset = 0;
+       rec->exit_status.offset = 0;
+       rec->diagnostics.offset = 0;
+       rec->errors.offset = 0;
+       rec->files.offset = 0;
+
+       free(rec->machine);
+       rec->machine = NULL;
+
+       free(rec->links);
+       rec->links = NULL;
+
+       free(rec->file_path);
+       rec->file_path = NULL;
+
+       free(rec->name);
+       rec->name = NULL;
+
+       free(rec->name_desc);
+       rec->name_desc = NULL;
+
+       free(rec->md5_hash);
+       rec->md5_hash = NULL;
+}
+
+/*
+ * init_secbuffs--
+ *  Sets the value of buflen for all the sec_buff field of rec. And then
+ *  allocate memory to each sec_buff member of rec.
+ */
+static void
+init_secbuffs(mandb_rec *rec)
+{
+       /*
+        * Some sec_buff might need more memory, for example desc,
+        * which stores the data of the DESCRIPTION section,
+        * while some might need very small amount of memory.
+        * Therefore explicitly setting the value of buflen field for
+        * each sec_buff.
+        */
+       rec->desc.buflen = 10 * BUFLEN;
+       rec->desc.data = emalloc(rec->desc.buflen);
+       rec->desc.offset = 0;
+
+       rec->lib.buflen = BUFLEN / 2;
+       rec->lib.data = emalloc(rec->lib.buflen);
+       rec->lib.offset = 0;
+
+       rec->return_vals.buflen = BUFLEN;
+       rec->return_vals.data = emalloc(rec->return_vals.buflen);
+       rec->return_vals.offset = 0;
+
+       rec->exit_status.buflen = BUFLEN;
+       rec->exit_status.data = emalloc(rec->exit_status.buflen);
+       rec->exit_status.offset = 0;
+
+       rec->env.buflen = BUFLEN;
+       rec->env.data = emalloc(rec->env.buflen);
+       rec->env.offset = 0;
+
+       rec->files.buflen = BUFLEN;
+       rec->files.data = emalloc(rec->files.buflen);
+       rec->files.offset = 0;
+
+       rec->diagnostics.buflen = BUFLEN;
+       rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
+       rec->diagnostics.offset = 0;
+
+       rec->errors.buflen = BUFLEN;
+       rec->errors.data = emalloc(rec->errors.buflen);
+       rec->errors.offset = 0;
+}
+
+/*
+ * free_secbuffs--
+ *  This function should be called at the end, when all the pages have been
+ *  parsed.
+ *  It frees the memory allocated to sec_buffs by init_secbuffs in the 
starting.
+ */
+static void
+free_secbuffs(mandb_rec *rec)
+{
+       free(rec->desc.data);
+       free(rec->lib.data);
+       free(rec->return_vals.data);
+       free(rec->exit_status.data);
+       free(rec->env.data);
+       free(rec->files.data);
+       free(rec->diagnostics.data);
+       free(rec->errors.data);
+}
+
+static char *
+parse_escape(const char *str)
+{
+       const char *backslash, *last_backslash;
+       char *result, *iter;
+       size_t len;
+
+       assert(str);
+
+       last_backslash = str;
+       backslash = strchr(str, '\\');
+       if (backslash == NULL)
+               return estrdup(str);
+
+       result = emalloc(strlen(str) + 1);
+       iter = result;
+
+       do {
+               len = backslash - last_backslash;
+               memcpy(iter, last_backslash, len);
+               iter += len;
+               if (backslash[1] == '-' || backslash[1] == ' ') {
+                       *iter++ = backslash[1];
+                       last_backslash = backslash + 2;
+                       backslash = strchr(backslash + 2, '\\');
+               } else {
+                       ++backslash;
+                       mandoc_escape(&backslash, NULL, NULL);
+                       last_backslash = backslash;
+                       if (backslash == NULL)
+                               break;
+                       backslash = strchr(last_backslash, '\\');
+               }
+       } while (backslash != NULL);
+       if (last_backslash != NULL)
+               strcpy(iter, last_backslash);
+       return result;
+}
+
+/*
+ * append--
+ *  Concatenates a space and src at the end of sbuff->data (much like concat in
+ *  apropos-utils.c).
+ *  Rather than reallocating space for writing data, it uses the value of the
+ *  offset field of sec_buff to write new data at the free space left in the
+ *  buffer.
+ *  In case the size of the data to be appended exceeds the number of bytes 
left
+ *  in the buffer, it reallocates buflen number of bytes and then continues.
+ *  Value of offset field should be adjusted as new data is written.
+ *
+ *  NOTE: This function does not write the null byte at the end of the buffers,
+ *  write a null byte at the position pointed to by offset before inserting 
data
+ *  in the db.
+ */
+static void
+append(secbuff *sbuff, const char *src)
+{
+       short flag = 0;
+       size_t srclen, newlen;
+       char *temp;
+
+       assert(src != NULL);
+       temp = parse_escape(src);
+       srclen = strlen(temp);
+
+       if (sbuff->data == NULL) {
+               sbuff->data = emalloc(sbuff->buflen);
+               sbuff->offset = 0;
+       }
+
+       newlen = sbuff->offset + srclen + 2;
+       if (newlen >= sbuff->buflen) {
+               while (sbuff->buflen < newlen)
+                       sbuff->buflen += sbuff->buflen;
+               sbuff->data = erealloc(sbuff->data, sbuff->buflen);
+               flag = 1;
+       }
+
+       /* Append a space at the end of the buffer. */
+       if (sbuff->offset || flag)
+               sbuff->data[sbuff->offset++] = ' ';
+       /* Now, copy src at the end of the buffer. */
+       memcpy(sbuff->data + sbuff->offset, temp, srclen);
+       sbuff->offset += srclen;
+       free(temp);
+}
+
+static void
+usage(void)
+{
+       fprintf(stderr, "Usage: %s [-flo]\n", getprogname());
+       exit(1);
+}
Index: usr.sbin/makemandb/run_query.3
===================================================================
RCS file: usr.sbin/makemandb/run_query.3
diff -N usr.sbin/makemandb/run_query.3
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/run_query.3      4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,223 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY 3
+.Os
+.Sh NAME
+.Nm run_query
+.Nd run a query against
+.Pa /var/db/man.db
+and process the results in a callback function.
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query "sqlite3 *db" "const char **snippet_args" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query
+function prepares the user supplied query in a form suitable to be run
+against
+.Pa /var/db/man.db
+and executes the query.
+For each row obtained in the result set,
+.Fn run_query
+will call the user supplied callback function, which should contain the
+logic for processing the data thus obtained.
+.Pp
+The
+.Fn run_query
+function takes following arguments:
+.Bl -tag -width 8n
+.It Fa sqlite3 *db
+Handle to the database connection which can be obtained by calling
+.Fn init_db .
+.It Fa const char *snippet_args
+An array of strings which specify the
+delimiters to the matching text in snippet.
+It is an optional argument and caller can supply a
+.Dv NULL
+value for it, in which case, a default value of
+.Brq \&"\&", \&"\&", \&"...\&"
+will be used.
+The 3 members of the array specify the following values:
+.Bl -enum -offset indent
+.It
+The first element marks the beginning of each matching token in the snippet.
+.It
+The second element the end of each matching token in the snippet.
+.It
+The third element is used to delimit the beginning and end of the snippet.
+.El
+For example for highlighting matching tokens in HTML style mark-up, use this
+value:
+.Bd -literal -offset indent
+ const char **snippet_args = {
+       "<b>",
+       "</b>",
+       "..."
+ };
+.Ed
+.It Fa query_args *args
+.Ft query_args
+is a struct which is defined in
+.Pa apropos-utils.h .
+It has the following fields:
+.Bl -tag -width 8n -offset indent
+.It Li const char *search_str
+This is the query as entered by the user.
+You may want to pre-process it to do sanitization etc.
+.It Li int *sec_nums
+This is an array of
+.Ft int
+wherein each index element represents the
+section number in which search should be performed.
+The sections whose corresponding index element in this array is set to
+.Dv NULL
+are not searched.
+If all the elements in the array are
+.Dv 0
+then all the sections are searched.
+Alternatively pass
+.Dv NULL
+as to search in all the sections.
+.It Li int nrec
+This specifies the number of matching rows to fetch from the database.
+Specifiy a negative value to fetch all the matching rows.
+.It Li int offset
+This specifies the offset within the result-set. Use it to specify the position
+from where to start processing the result-set.
+For example if the value of nrec is m and value of offset is n, then the first
+n records from the result-set will be omitted and rest m-n (or less) records 
will
+be available for processing inside the callback.
+Use a negative value if you don't wish to offset any records.
+.It Li const char *machine
+The machine architecture to which the searches should be restricted.
+Specify NULL if the search should not be restricted a particular machine 
architecture.
+.It Li int (*callback) (void *, const char *, const char *, const char *,\
+const char *, size_t)
+This is the callback function which will
+be called for each row retrieved from the database.
+The function should return a value of 0 on successful execution.
+A non-zero return value will indicate a failure and
+.Fn run_query
+will return.
+The interface of the callback function is described later in this section.
+.It Li void *callback_data
+Use this argument to pass any data to the callback function.
+It can be retrieved inside the callback function from its first argument.
+.It Li char **errmsg
+If an error occurs while fetching the data from the database,
+a human readable error message will be stored in
+.Fa errmsg .
+If no error occurs then
+.Fa errmsg
+will be set to
+.Dv NULL .
+In case
+.Fa errmsg
+is not
+.Dv NULL ,
+then the caller should make sure to free it.
+.El
+.El
+.Pp
+The interface of the callback function is
+.Ft int
+.Fn (*callback) "void *callback_data" "const char *section" "const char *name"\
+"const char *name_desc" "const char *snippet" "size_t snippet_length"
+.Pp
+Its arguments are:
+.Bl -column -offset indent "Function" "Argument Description"
+.It Li void *callback_data Ta This is the caller supplied data.
+.It Li const char *section Ta Ta \&It is the section number to which this man
+page belongs.
+.It Li const char *name Ta This is the name of the man page
+.It Li const char *name_desc Ta This is the one line description of this man
+page obtained from it's NAME section.
+.It Li const char *snippet Ta This is a snippet of the matching text from this
+man page.
+.It Li size_t snippet_length Ta This is the length of the snippet.
+.El
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Pa apropos.c
+uses
+.Fn run_query .
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = 10;
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = NULL;
+args.errmsg = &errmsg;
+if (run_query(db, NULL, &args) < 0)
+               errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+       const char *name_desc, const char *snippet, size_t snippet_length )
+{
+       /* The user supplied data could be obtained as follows */
+       // my_data *buf = (my_data *) data;
+
+       fprintf(stdout, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+               snippet);
+       return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/run_query_html.3
===================================================================
RCS file: usr.sbin/makemandb/run_query_html.3
diff -N usr.sbin/makemandb/run_query_html.3
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/run_query_html.3 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,116 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY_HTML 3
+.Os
+.Sh NAME
+.Nm run_query_html
+.Nd run a query against
+.Pa /var/db/man.db
+and returns the results in HTML format
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query_html "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query_html
+is very much similar to
+.Fn run_query
+but with the following three differences.
+.Bl -enum -offset indent
+.It
+.Nm
+takes only two arguments.
+.It
+The snippet returned by
+.Nm
+uses <b> and </b> tags to highlight the matching pieces of text so that it
+appears bold in a web browser.
+.It
+.Nm
+will do limited quoting of the inline HTML fragments in the snippet.
+Concretely, it will replace <, >, ", & with &lt; &gt; &quot; and &amp;
+respectively.
+.El
+For more details, refer to the manual page of
+.Xr run_query 3 .
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query_html
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Nm
+can be used.
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = 10;
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = NULL;
+args.errmsg = &errmsg;
+if (run_query(db, &args) < 0)
+               errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+       const char *name_desc, const char *snippet, size_t snippet_length)
+{
+       fprintf(stdout, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+               snippet);
+       return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/run_query_pager.3
===================================================================
RCS file: usr.sbin/makemandb/run_query_pager.3
diff -N usr.sbin/makemandb/run_query_pager.3
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/run_query_pager.3        4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,106 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay 
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY_PAGER 3
+.Os
+.Sh NAME
+.Nm run_query_pager
+.Nd run a query against
+.Pa /var/db/man.db
+and returns the results in a format which can be piped to a pager
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query_pager "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query_pager
+is very similar to
+.Xr run_query_html 3
+with the only difference that the snippet returned is formatted in a manner so
+that the matching text in it appears highlighted when viewed using a pager.
+.Pp
+For more details, refer to the manual page of
+.Xr run_query 3 .
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query_pager
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Nm
+can be used.
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+FILE *pager = popen("more", "w");
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = -1;        //A negative value means fetch all matching records
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = (void *)pager;
+args.errmsg = &errmsg;
+if (run_query(db, &args) < 0)
+               errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+pclose(pager);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+       const char *name_desc, const char *snippet, size_t snippet_length)
+{
+       FILE *out = (FILE *) data;
+       fprintf(out, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+               snippet);
+       return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/stopwords.txt
===================================================================
RCS file: usr.sbin/makemandb/stopwords.txt
diff -N usr.sbin/makemandb/stopwords.txt
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/stopwords.txt    4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,251 @@
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+about
+again
+all
+also
+always
+an
+and
+another
+any
+are
+around
+as
+ask
+at
+b
+back
+be
+because
+been
+before
+below
+between
+but
+by
+bye
+can
+case
+consist
+could
+d
+did
+does
+down
+e
+each
+early
+either
+end
+enough
+even
+every
+f
+fact
+far
+few
+follow
+four
+from
+full
+further
+g
+general
+get
+give
+given
+good
+got
+great
+h
+had
+has
+have
+having
+here
+high
+him
+his
+how
+however
+i
+if
+important
+in
+interest
+into
+is
+it
+j
+just
+k
+keep
+keeps
+kind
+knew
+know
+l
+large
+larger
+last
+later
+latest
+latter
+least
+let
+like
+likely
+long
+longer
+m
+made
+many
+may
+me
+might
+most
+mostly
+much
+must
+my
+n
+names
+necessary
+need
+needs
+never
+new
+next
+no
+non
+noone
+not
+nothing
+o
+of
+off
+often
+old
+older
+on
+once
+only
+or
+order
+our
+out
+over
+p
+part
+per
+perhaps
+possible
+present
+problem
+q
+quite
+r
+rather
+really
+right
+room
+s
+said
+same
+saw
+say
+says
+second
+see
+seem
+seemed
+seems
+sees
+several
+shall
+should
+side
+sides
+small
+smaller
+so
+some
+something
+state
+states
+still
+such
+sure
+t
+take
+taken
+that
+the
+their
+them
+then
+there
+therefore
+these
+thing
+think
+thinks
+this
+those
+though
+three
+thus
+to
+together
+too
+took
+toward
+turn
+two
+u
+until
+up
+upon
+us
+use
+used
+uses
+v
+very
+w
+want
+wanted
+wants
+was
+way
+ways
+we
+well
+went
+were
+what
+when
+whether
+why
+will
+willing
+with
+within
+without
+work
+would
+x
+y
+year
+yet
+you
+z
Index: usr.sbin/makemandb/whatis.1
===================================================================
RCS file: usr.sbin/makemandb/whatis.1
diff -N usr.sbin/makemandb/whatis.1
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/whatis.1 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,58 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2012 Joerg Sonnenberger <joerg%NetBSD.org@localhost>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd February 1, 2012
+.Dt WHATIS 1
+.Os
+.Sh NAME
+.Nm whatis
+.Nd describe what a command is
+.Sh SYNOPSIS
+.Nm
+.Ar command Ar ...
+.Sh DESCRIPTION
+The
+.Nm
+program queries the database
+.Pa /var/db/man.db
+for manual pages with name
+.Ar command
+and prints the name of the manual page, the section and the description from
+the section NAME.
+.Sh SEE ALSO
+.Xr apropos 1 ,
+.Xr makemandb 8
+.Sh HISTORY
+The
+.Nm
+command appeared in 3.0BSD.
+It was rewritten in
+.Nx 6.0
+to use the SQLite index created by
+.Xr makemandb 8 .
Index: usr.sbin/makemandb/whatis.c
===================================================================
RCS file: usr.sbin/makemandb/whatis.c
diff -N usr.sbin/makemandb/whatis.c
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/whatis.c 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,101 @@
+/*     $NetBSD */
+/*-
+ * Copyright (c) 2012 Joerg Sonnenberger <joerg%NetBSD.org@localhost>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD$");
+
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "apropos-utils.h"
+
+__dead static void
+usage(void)
+{
+       fprintf(stderr, "%s ...\n", "whatis");
+       exit(EXIT_FAILURE);
+}
+
+static int
+whatis(sqlite3 *db, const char *cmd)
+{
+       static const char sqlstr[] = "SELECT name, section, name_desc"
+                                    " FROM mandb WHERE name=?"
+                                    " ORDER BY section, name";
+       sqlite3_stmt *stmt = NULL;
+       int retval;
+
+       if (sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL) != SQLITE_OK)
+               errx(EXIT_FAILURE, "Unable to query database");
+       if (sqlite3_bind_text(stmt, 1, cmd, -1, NULL) != SQLITE_OK)
+               errx(EXIT_FAILURE, "Unable to query database");
+       retval = 1;
+       while (sqlite3_step(stmt) == SQLITE_ROW) {
+               printf("%s(%s) - %s\n", sqlite3_column_text(stmt, 0),
+                   sqlite3_column_text(stmt, 1),
+                   sqlite3_column_text(stmt, 2));
+               retval = 0;
+       }
+       sqlite3_finalize(stmt);
+       if (retval)
+               fprintf(stderr, "%s: not found\n", cmd);
+       return retval;
+}
+
+int
+main(int argc, char *argv[])
+{
+       sqlite3 *db;
+       int ch, retval;
+
+       while ((ch = getopt(argc, argv, "")) != -1) {
+               switch (ch) {
+               default:
+                       usage();
+               }
+       }
+       argc -= optind;
+       argv += optind;
+
+       if (argc == 0)
+               usage();
+
+       if ((db = init_db(MANDB_READONLY)) == NULL)
+               exit(EXIT_FAILURE);
+
+       retval = 0;
+       while (argc--)
+               retval |= whatis(db, *argv++);
+
+       close_db(db);
+       return retval;
+}


Home | Main Index | Thread Index | Old Index