tech-userlevel archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
Re: Importing the apropos GSoC code and switching to mandoc(1) by default
On Thu, Feb 02, 2012 at 12:46:54AM +0100, Joerg Sonnenberger wrote:
> attached is the new apropos created by Abhinav Upadhyay during last
> year's Summer of Code. I've added a small whatis(1) implementation to
> round it up. This should be a functional replacement for apropos(1) and
> whatis(1). It obsoletes makewhatis(8).
Attached is the patch to integrate this into base and hook it into
daily/weekly and the boot process. Old apropos/whatis can be obtained
for now by using MKMAKEMANDB=no.
This also fixes a bug found in to makemandb code when using it with
certain packages.
Joerg
Index: distrib/sets/sets.subr
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/sets.subr,v
retrieving revision 1.134
diff -u -p -r1.134 sets.subr
--- distrib/sets/sets.subr 3 Feb 2012 22:18:04 -0000 1.134
+++ distrib/sets/sets.subr 3 Feb 2012 23:44:42 -0000
@@ -228,6 +228,7 @@ SUBST="${SUBST};s#@MACHINE@#${MACHINE}#g
# lint ${MKLINT} != no
# llvm ${MKLLVM} != no
# lvm ${MKLVM} != no
+# makemandb ${MKMAKEMANDB} != no
# man ${MKMAN} != no
# manpages ${MKMANPAGES} != no
# manz ${MKMANZ} != no
Index: distrib/sets/lists/base/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/base/mi,v
retrieving revision 1.979
diff -u -p -r1.979 mi
--- distrib/sets/lists/base/mi 3 Feb 2012 22:18:04 -0000 1.979
+++ distrib/sets/lists/base/mi 3 Feb 2012 23:44:42 -0000
@@ -1023,7 +1023,7 @@
./usr/libexec/fingerd base-netutil-bin
./usr/libexec/ftp-proxy base-obsolete
obsolete
./usr/libexec/ftpd base-netutil-bin
-./usr/libexec/getNAME base-man-bin
+./usr/libexec/getNAME base-man-bin
!makemandb
./usr/libexec/getty base-sysutil-bin
./usr/libexec/hprop base-obsolete obsolete
./usr/libexec/hpropd base-krb5-bin kerberos
@@ -1044,7 +1044,7 @@
./usr/libexec/lpr/pjlfilter base-lpr-bin
./usr/libexec/mail.local base-mail-bin
./usr/libexec/makekey base-crypto-bin
-./usr/libexec/makewhatis base-man-bin
+./usr/libexec/makewhatis base-man-bin
!makemandb
./usr/libexec/named-xfer base-obsolete obsolete
./usr/libexec/ntalkd base-netutil-bin
./usr/libexec/pfspamd base-obsolete obsolete
@@ -3614,7 +3614,7 @@
./usr/share/man/html9/vax base-man-share
./usr/share/man/html9/x68k base-man-share
./usr/share/man/html9/x86 base-man-share
-./usr/share/man/makewhatis.sed base-man-share share
+./usr/share/man/makewhatis.sed base-man-share
!makemandb,share
./usr/share/man/man1 base-man-share
./usr/share/man/man1/atari base-man-share
./usr/share/man/man1/hp300 base-man-share
@@ -3742,7 +3742,7 @@
./usr/share/man/man9/vax base-man-share
./usr/share/man/man9/x68k base-man-share
./usr/share/man/man9/x86 base-man-share
-./usr/share/man/whatis.db base-man-share man
+./usr/share/man/whatis.db base-man-share
man,!makemandb
./usr/share/me base-groff-share
./usr/share/misc base-terminfo-share
./usr/share/misc/acronyms base-reference-share share
Index: distrib/sets/lists/comp/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/comp/mi,v
retrieving revision 1.1729
diff -u -p -r1.1729 mi
--- distrib/sets/lists/comp/mi 1 Feb 2012 05:34:38 -0000 1.1729
+++ distrib/sets/lists/comp/mi 3 Feb 2012 17:39:46 -0000
@@ -3971,7 +3971,7 @@
./usr/libdata/debug/usr/libexec/fingerd.debug comp-netutil-debug debug
./usr/libdata/debug/usr/libexec/ftp-proxy.debug comp-obsolete
obsolete
./usr/libdata/debug/usr/libexec/ftpd.debug comp-netutil-debug debug
-./usr/libdata/debug/usr/libexec/getNAME.debug comp-man-debug debug
+./usr/libdata/debug/usr/libexec/getNAME.debug comp-man-debug
!makemandb,debug
./usr/libdata/debug/usr/libexec/getty.debug comp-sysutil-debug debug
./usr/libdata/debug/usr/libexec/hpropd.debug comp-krb5-debug
kerberos,debug
./usr/libdata/debug/usr/libexec/httpd.debug comp-netutil-debug debug
@@ -3988,7 +3988,7 @@
./usr/libdata/debug/usr/libexec/lpr/lpf.debug comp-lpr-debug debug
./usr/libdata/debug/usr/libexec/mail.local.debug comp-mail-debug
debug
./usr/libdata/debug/usr/libexec/makekey.debug comp-crypto-debug debug
-./usr/libdata/debug/usr/libexec/makewhatis.debug comp-man-debug
debug
+./usr/libdata/debug/usr/libexec/makewhatis.debug comp-man-debug
!makemandb,debug
./usr/libdata/debug/usr/libexec/ntalkd.debug comp-netutil-debug debug
./usr/libdata/debug/usr/libexec/pcpp.debug comp-c-debug
pcc,debug
./usr/libdata/debug/usr/libexec/postfix/anvil.debug comp-postfix-debug
postfix,debug
@@ -4148,6 +4148,7 @@
./usr/libdata/debug/usr/sbin/lpd.debug comp-lpr-debug debug
./usr/libdata/debug/usr/sbin/lptest.debug comp-lpr-debug debug
./usr/libdata/debug/usr/sbin/mailwrapper.debug comp-mailwrapper-debug debug
+./usr/libdata/debug/usr/sbin//makemandb.debug comp-makemandb-debug
makemandb,debug
./usr/libdata/debug/usr/sbin/makedbm.debug comp-nis-debug yp,debug
./usr/libdata/debug/usr/sbin/makefs.debug comp-sysutil-debug debug
./usr/libdata/debug/usr/sbin/map-mbone.debug comp-netutil-debug debug
Index: distrib/sets/lists/etc/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/etc/mi,v
retrieving revision 1.227
diff -u -p -r1.227 mi
--- distrib/sets/lists/etc/mi 17 Dec 2011 13:37:49 -0000 1.227
+++ distrib/sets/lists/etc/mi 4 Feb 2012 18:20:52 -0000
@@ -223,6 +223,7 @@
./etc/rc.d/local etc-sys-rc
./etc/rc.d/lpd etc-lpr-rc
./etc/rc.d/lvm etc-sys-rc
+./etc/rc.d/makemandb etc-man-rc
./etc/rc.d/mdnsd etc-mdns-rc
./etc/rc.d/mixerctl etc-audio-rc
./etc/rc.d/mopd etc-bootserver-rc
Index: distrib/sets/lists/man/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/man/mi,v
retrieving revision 1.1375
diff -u -p -r1.1375 mi
--- distrib/sets/lists/man/mi 3 Feb 2012 22:18:05 -0000 1.1375
+++ distrib/sets/lists/man/mi 3 Feb 2012 23:44:42 -0000
@@ -2313,7 +2313,7 @@
./usr/share/man/cat8/fusermount.0 man-refuse-catman .cat
./usr/share/man/cat8/fwctl.0 man-sysutil-catman .cat
./usr/share/man/cat8/genassym.sh.0 man-obsolete obsolete
-./usr/share/man/cat8/getNAME.0 man-man-catman .cat
+./usr/share/man/cat8/getNAME.0 man-man-catman
!makemandb, .cat
./usr/share/man/cat8/getencstat.0 man-sysutil-catman .cat
./usr/share/man/cat8/gettable.0 man-obsolete
obsolete
./usr/share/man/cat8/getty.0 man-sysutil-catman .cat
@@ -2477,8 +2477,9 @@
./usr/share/man/cat8/makedev.local.0 man-sysutil-catman .cat
./usr/share/man/cat8/makefs.0 man-sysutil-catman .cat
./usr/share/man/cat8/makekey.0 man-crypto-catman .cat
+./usr/share/man/cat8/makemandb.0 man-man-catman
makemandb,.cat
./usr/share/man/cat8/makemap.0 man-obsolete obsolete
-./usr/share/man/cat8/makewhatis.0 man-man-catman .cat
+./usr/share/man/cat8/makewhatis.0 man-man-catman
!makemandb,.cat
./usr/share/man/cat8/map-mbone.0 man-netutil-catman .cat
./usr/share/man/cat8/master.0 man-postfix-catman
postfix,.cat
./usr/share/man/cat8/mbr.0 man-obsolete obsolete
@@ -5015,7 +5016,7 @@
./usr/share/man/html8/ftpd.html man-netutil-htmlman
html
./usr/share/man/html8/fusermount.html man-refuse-htmlman html
./usr/share/man/html8/fwctl.html man-sysutil-htmlman html
-./usr/share/man/html8/getNAME.html man-man-htmlman html
+./usr/share/man/html8/getNAME.html man-man-htmlman
!makemandb,html
./usr/share/man/html8/getencstat.html man-sysutil-htmlman html
./usr/share/man/html8/getty.html man-sysutil-htmlman html
./usr/share/man/html8/gpioctl.html man-sysutil-htmlman html
@@ -5141,7 +5142,8 @@
./usr/share/man/html8/makedev.local.html man-sysutil-htmlman html
./usr/share/man/html8/makefs.html man-sysutil-htmlman html
./usr/share/man/html8/makekey.html man-crypto-htmlman html
-./usr/share/man/html8/makewhatis.html man-man-htmlman html
+./usr/share/man/html8/makemandb.html man-man-htmlman
makemandb,html
+./usr/share/man/html8/makewhatis.html man-man-htmlman
!makemandb,html
./usr/share/man/html8/map-mbone.html man-netutil-htmlman html
./usr/share/man/html8/master.html man-postfix-htmlman
postfix,html
./usr/share/man/html8/mbrlabel.html man-sysutil-htmlman html
@@ -7811,7 +7813,7 @@
./usr/share/man/man8/fusermount.8 man-refuse-man .man
./usr/share/man/man8/fwctl.8 man-sysutil-man .man
./usr/share/man/man8/genassym.sh.8 man-obsolete obsolete
-./usr/share/man/man8/getNAME.8 man-man-man .man
+./usr/share/man/man8/getNAME.8 man-man-man
!makemandb,.man
./usr/share/man/man8/getencstat.8 man-sysutil-man .man
./usr/share/man/man8/gettable.8 man-obsolete
obsolete
./usr/share/man/man8/getty.8 man-sysutil-man .man
@@ -7976,8 +7978,9 @@
./usr/share/man/man8/makedev.local.8 man-sysutil-man .man
./usr/share/man/man8/makefs.8 man-sysutil-man .man
./usr/share/man/man8/makekey.8 man-crypto-man .man
+./usr/share/man/man8/makemandb.8 man-man-man
makemandb,.man
./usr/share/man/man8/makemap.8 man-obsolete obsolete
-./usr/share/man/man8/makewhatis.8 man-man-man .man
+./usr/share/man/man8/makewhatis.8 man-man-man
!makemandb,.man
./usr/share/man/man8/map-mbone.8 man-netutil-man .man
./usr/share/man/man8/master.8 man-postfix-man
postfix,.man
./usr/share/man/man8/mbr.8 man-obsolete obsolete
Index: distrib/sets/lists/text/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/text/mi,v
retrieving revision 1.41
diff -u -p -r1.41 mi
--- distrib/sets/lists/text/mi 11 Oct 2011 19:20:13 -0000 1.41
+++ distrib/sets/lists/text/mi 3 Feb 2012 17:27:53 -0000
@@ -40,6 +40,7 @@
./usr/bin/troff text-groff-bin
groff
./usr/bin/vgrind text-groff-bin
./usr/libexec/vfontedpr text-groff-bin
+./usr/sbin/makemandb base-man-bin
makemandb
./usr/share/doc/groff text-groff-doc
./usr/share/doc/groff/mom text-groff-doc
./usr/share/doc/groff/mom/README.txt text-groff-doc
share,groff
Index: distrib/sets/lists/xbase/mi
===================================================================
RCS file: /home/joerg/repo/netbsd/src/distrib/sets/lists/xbase/mi,v
retrieving revision 1.111
diff -u -p -r1.111 mi
--- distrib/sets/lists/xbase/mi 22 Oct 2011 22:23:50 -0000 1.111
+++ distrib/sets/lists/xbase/mi 3 Feb 2012 17:47:28 -0000
@@ -1340,7 +1340,7 @@
./usr/X11R6/man/man7/XProjectTeam.7 xbase-obsolete obsolete
./usr/X11R6/man/man7/XStandards.7 -unknown- .man,x11
./usr/X11R6/man/man7/Xsecurity.7 -unknown- .man,x11
-./usr/X11R6/man/whatis.db -unknown- man,x11
+./usr/X11R6/man/whatis.db -unknown-
man,x11,!makemandb
./usr/X11R6/share base-x11-root x11
./usr/X11R6/share/examples base-x11-root x11
./usr/X11R6/share/examples/ssh-askpass base-x11-root x11
@@ -2888,7 +2888,7 @@
./usr/X11R7/man/man7/XOrgFoundation.7 -unknown-
.man,xorg
./usr/X11R7/man/man7/XProjectTeam.7 -unknown-
.man,xorg
./usr/X11R7/man/man7/Xsecurity.7 -unknown-
.man,xorg
-./usr/X11R7/man/whatis.db -unknown- man,xorg
+./usr/X11R7/man/whatis.db -unknown-
man,xorg,!makemandb
./usr/X11R7/share base-x11-root xorg
./usr/X11R7/share/aclocal base-x11-root xorg
./usr/X11R7/share/examples base-x11-root xorg
Index: etc/daily
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/daily,v
retrieving revision 1.78
diff -u -p -r1.78 daily
--- etc/daily 17 Dec 2011 12:27:42 -0000 1.78
+++ etc/daily 4 Feb 2012 21:37:42 -0000
@@ -290,6 +290,16 @@ if checkyesno run_skeyaudit; then
fi
fi
+run_makemandb=YES
+
+if checkyesno run_makemandb; then
+ if [ -f /etc/man.conf -a -x /usr/sbin/makemandb ]; then
+ echo ""
+ echo "Updating man page index:"
+ nice -n 5 /usr/sbin/makemandb
+ fi
+fi
+
if [ -f /etc/daily.local ]; then
( . /etc/daily.local ) > $TMP 2>&1
if [ -s $TMP ] ; then
Index: etc/weekly
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/weekly,v
retrieving revision 1.24
diff -u -p -r1.24 weekly
--- etc/weekly 27 Dec 2010 03:38:52 -0000 1.24
+++ etc/weekly 4 Feb 2012 21:37:47 -0000
@@ -81,7 +81,8 @@ fi
if checkyesno rebuild_whatisdb; then
echo ""
- if [ -f /etc/man.conf -o -d /usr/share/man ]; then
+ if [ -x /usr/libexec/makewhatis -a \( -f /etc/man.conf -o \
+ -o -d /usr/share/man \) ]; then
echo "Rebuilding whatis databases:"
nice -n 5 /usr/libexec/makewhatis -f
else
@@ -89,6 +90,16 @@ if checkyesno rebuild_whatisdb; then
fi
fi
+if checkyesno rebuild_mandb; then
+ echo ""
+ if [ -f /etc/man.conf -a -x /usr/sbin/makemandb ]; then
+ echo "Rebuilding man page index:"
+ nice -n 5 /usr/sbin/makemandb -f
+ else
+ echo "Not rebuilding man page index"
+ fi
+fi
+
if [ -f /etc/weekly.local ]; then
. /etc/weekly.local > $TMP 2>&1
if [ -s $TMP ] ; then
Index: etc/defaults/daily.conf
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/defaults/daily.conf,v
retrieving revision 1.15
diff -u -p -r1.15 daily.conf
--- etc/defaults/daily.conf 17 Dec 2011 12:27:42 -0000 1.15
+++ etc/defaults/daily.conf 4 Feb 2012 21:34:45 -0000
@@ -25,6 +25,7 @@ check_network=YES
full_netstat=NO
run_fsck=NO
run_fsck_flags=""
+run_makemandb=YES
run_rdist=YES
run_security=YES
separate_security_email=YES
Index: etc/defaults/rc.conf
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/defaults/rc.conf,v
retrieving revision 1.118
diff -u -p -r1.118 rc.conf
--- etc/defaults/rc.conf 23 Dec 2011 05:05:04 -0000 1.118
+++ etc/defaults/rc.conf 4 Feb 2012 18:20:10 -0000
@@ -366,3 +366,6 @@ veriexec_flags="-k"
# Entropy load/save to/from /dev/random at startup/shutdown
#
random_seed=YES
+
+# Creating / updating of man page index on boot
+makemandb=YES
Index: etc/defaults/weekly.conf
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/defaults/weekly.conf,v
retrieving revision 1.3
diff -u -p -r1.3 weekly.conf
--- etc/defaults/weekly.conf 15 Feb 2007 23:04:10 -0000 1.3
+++ etc/defaults/weekly.conf 4 Feb 2012 18:15:01 -0000
@@ -11,3 +11,4 @@
rebuild_locatedb=YES
rebuild_whatisdb=NO
+rebuild_mandb=YES
Index: etc/rc.d/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/etc/rc.d/Makefile,v
retrieving revision 1.87
diff -u -p -r1.87 Makefile
--- etc/rc.d/Makefile 17 Dec 2011 13:37:49 -0000 1.87
+++ etc/rc.d/Makefile 4 Feb 2012 18:20:28 -0000
@@ -26,7 +26,7 @@ CONFIGFILES=\
irdaattach iscsi_target isdnd isibootd \
kdc \
ldconfig ldpd local lpd lvm \
- mdnsd mixerctl mopd motd mountall mountcritlocal \
+ makemandb mdnsd mixerctl mopd motd mountall mountcritlocal \
mountcritremote mountd moused mrouted \
named ndbootd network newsyslog nfsd nfslocking npf ntpd
ntpdate \
perusertmp pf pf_boot pflogd postfix powerd ppp pwcheck \
Index: etc/rc.d/makemandb
===================================================================
RCS file: etc/rc.d/makemandb
diff -N etc/rc.d/makemandb
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ etc/rc.d/makemandb 4 Feb 2012 21:38:06 -0000
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# $NetBSD$
+#
+
+# PROVIDE: makemandb
+# REQUIRE: DAEMON
+# BEFORE: LOGIN
+
+$_rc_subr_loaded . /etc/rc.subr
+
+name="makemandb"
+required_files="/etc/man.conf /usr/sbin/makemandb"
+start_cmd="makemandb_start"
+stop_cmd=":"
+
+makemandb_start()
+{
+ # Initiate update build in the background
+ nice 5 /usr/sbin/makemandb -q > /dev/null 2>&1 &
+}
+
+load_rc_config $name
+run_rc_command "$1"
Index: lib/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/lib/Makefile,v
retrieving revision 1.180
diff -u -p -r1.180 Makefile
--- lib/Makefile 5 Nov 2011 22:40:01 -0000 1.180
+++ lib/Makefile 3 Feb 2012 16:31:22 -0000
@@ -179,6 +179,8 @@ SUBDIR+= libpam # depends on heimdal
SUBDIR+= ../crypto/external/bsd/libsaslc # depends on heimdal, openssl
.endif
+SUBDIR+= ../external/bsd/mdocml/lib
+
#==================== 4th library dependency barrier ====================
SUBDIR+= .WAIT
Index: libexec/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/libexec/Makefile,v
retrieving revision 1.60
diff -u -p -r1.60 Makefile
--- libexec/Makefile 15 Apr 2011 18:05:45 -0000 1.60
+++ libexec/Makefile 3 Feb 2012 16:19:37 -0000
@@ -4,9 +4,13 @@
.include <bsd.own.mk>
SUBDIR= atrun \
- comsat fingerd ftpd getNAME getty httpd identd ld.aout_so \
- ld.elf_so lfs_cleanerd mail.local makekey makewhatis rexecd \
+ comsat fingerd ftpd getty httpd identd ld.aout_so \
+ ld.elf_so lfs_cleanerd mail.local makekey rexecd \
rlogind rshd rpc.rquotad rpc.rstatd rpc.rusersd rpc.rwalld \
rpc.sprayd talkd telnetd tftpd utmp_update
+.if ${MKMAKEMANDB} == "no"
+SUBDIR+= getNAME makewhatis
+.endif
+
.include <bsd.subdir.mk>
Index: share/man/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/share/man/Makefile,v
retrieving revision 1.29
diff -u -p -r1.29 Makefile
--- share/man/Makefile 9 Feb 2004 07:58:01 -0000 1.29
+++ share/man/Makefile 3 Feb 2012 16:28:50 -0000
@@ -8,6 +8,7 @@ SUBDIR= man0 man1 man2 man3 man4 man5 ma
WHATISDBDIR?= /usr/share/man
+.if ${MKMAKEMANDB} == "no"
makedb:
${TOOL_MAKEWHATIS} ${DESTDIR}${WHATISDBDIR}
.if ${MKUNPRIVED} != "no"
@@ -18,5 +19,9 @@ makedb:
makedb:
.endif
+.else
+makedb:
+.endif
+
.include <bsd.subdir.mk>
.include <bsd.sys.mk>
Index: share/man/man0/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/share/man/man0/Makefile,v
retrieving revision 1.16
diff -u -p -r1.16 Makefile
--- share/man/man0/Makefile 25 Oct 2008 22:27:36 -0000 1.16
+++ share/man/man0/Makefile 3 Feb 2012 16:59:52 -0000
@@ -5,7 +5,7 @@ NOOBJ= # defined
.include <bsd.own.mk>
-.if ${MKSHARE} != "no"
+.if ${MKSHARE} != "no" && ${MKMAKEMANDB} == "no"
FILES= makewhatis.sed
FILESDIR= /usr/share/man
.endif
Index: share/mk/bsd.own.mk
===================================================================
RCS file: /home/joerg/repo/netbsd/src/share/mk/bsd.own.mk,v
retrieving revision 1.693
diff -u -p -r1.693 bsd.own.mk
--- share/mk/bsd.own.mk 25 Nov 2011 22:03:42 -0000 1.693
+++ share/mk/bsd.own.mk 3 Feb 2012 16:17:35 -0000
@@ -815,6 +815,7 @@ _MKVARS.yes= \
MKLDAP MKLINKLIB MKLINT MKLVM \
MKMAN \
MKMDNS \
+ MKMAKEMANDB \
MKNLS \
MKNPF \
MKOBJ \
Index: usr.bin/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/usr.bin/Makefile,v
retrieving revision 1.206
diff -u -p -r1.206 Makefile
--- usr.bin/Makefile 4 Jan 2012 16:25:13 -0000 1.206
+++ usr.bin/Makefile 3 Feb 2012 16:18:42 -0000
@@ -3,7 +3,7 @@
.include <bsd.own.mk>
-SUBDIR= apply apropos asa at audio audiocfg \
+SUBDIR= apply asa at audio audiocfg \
banner basename biff bthset btkey btpin \
bzip2 bzip2recover cal calendar cap_mkdb cdplay \
checknr chflags chpass cksum cmp cleantags col colcrt colrm \
@@ -31,11 +31,15 @@ SUBDIR= apply apropos asa at audio audio
uniq units unvis unzip usbhidaction usbhidctl users utoppya \
uudecode uuencode uuidgen vacation vgrind videoctl vis \
vmstat vndcompress w \
- wall wc what whatis whereis who whois window \
+ wall wc what whereis who whois window \
write xargs xinstall xlint xstr yes
SUBDIR+= ../external/zlib/pigz/bin/pigz
+.if MKMAKEMANDB == "no"
+SUBDIR+= apropos whatis
+.endif
+
.if (${MKBSDGREP} != "no")
SUBDIR+= grep
.endif
Index: usr.sbin/Makefile
===================================================================
RCS file: /home/joerg/repo/netbsd/src/usr.sbin/Makefile,v
retrieving revision 1.260
diff -u -p -r1.260 Makefile
--- usr.sbin/Makefile 4 Jan 2012 16:25:14 -0000 1.260
+++ usr.sbin/Makefile 3 Feb 2012 16:18:00 -0000
@@ -32,6 +32,10 @@ SUBDIR= ac accton acpitools altq apm apm
wake wiconfig wlanctl wsconscfg wsfontload wsmoused wsmuxctl \
zdump zic
+.if ${MKMAKEMANDB} != "no"
+SUBDIR+= makemandb
+.endif
+
.if (${MKYP} != "no")
SUBDIR+= rpc.yppasswdd ypbind yppoll ypserv ypset
.endif
Index: usr.sbin/makemandb/DBSCHEMA
===================================================================
RCS file: usr.sbin/makemandb/DBSCHEMA
diff -N usr.sbin/makemandb/DBSCHEMA
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/DBSCHEMA 4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,52 @@
+There are three tables in the database at present:
+
+(1) mandb:
+ This is the main FTS table which contains all the content from
+ man pages
+
+ COLUMN NAME DESCRIPTION
+ 1. section Section number of the page
+ 2. name The name of the page
+ 3. name_desc The one line description from the NAME section
+ 4. desc Contains the content from DESCRIPTION
+ section and any other section which
+ is not indexed in a separate column.
+ 5. lib The LIBRARY section
+ 6. return_vals RETURN VALUES section
+ 7. env ENVIRONMENT section
+ 8. files FILES section
+ 9. exit_status EXIT STATUS section
+ 10. diagnostics DIAGNOSTICS section
+ 11. errors ERRORS section
+ 12. md5_hash md5 hash of the man page (UNIQUE)
+ 13. machine The machine architecture (if any) for which this page is
+ relevant.
+
+(2) mandb_meta:
+ This table contains md5 hashes of all the indexed man
+ pages. This is there to make sure we do not index
+ the same man page twice, i.e. to prevent indexing
+ hardlinks.
+
+ COLUMN NAME DESCRIPTION
+ 1. device (dev_t)Logical device number from stat(2)
+ 2. inode (ino_t)Inode number from stat(2)
+ 3. mtime Last modification time from stat(2)
+ 4. file Absolute path name (UNIQUE constraint)
+ 5. md5_hash MD5 Hash of the man page (UNIQUE)
+ 6. id A unique integer ID for the page, which
+ refers to the docid column in mandb (Implicit foreign
+ key?) PRIMARY KEY
+ {device, inode} form a UNIQUE index
+
+(3) mandb_links:
+ This is an index of all the hard/soft links of the man
+ pages. The list of the linked pages is usually obtained
+ from the multiple .Nm entries in the page.
+
+ COLUMN NAME DESCRIPTION
+ 1. link The name of the hard/soft link (index)
+ 2. target Name of the target page to which it points
+ 3. section The section number
+ 4. machine The machine architecture (if any) for which
+ the page is relevant
Index: usr.sbin/makemandb/Makefile
===================================================================
RCS file: usr.sbin/makemandb/Makefile
diff -N usr.sbin/makemandb/Makefile
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/Makefile 4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,40 @@
+# $NetBSD$
+
+.include <bsd.own.mk>
+
+MDIST= ${NETBSDSRCDIR}/external/bsd/mdocml/dist
+MDOCDIR=${NETBSDSRCDIR}/external/bsd/mdocml
+
+PROGS= makemandb apropos whatis
+SRCS.makemandb= makemandb.c apropos-utils.c
+SRCS.apropos= apropos.c apropos-utils.c
+SRCS.whatis= whatis.c apropos-utils.c
+MAN.makemandb= makemandb.8
+MAN.apropos= apropos.1
+MAN.whatis= whatis.1
+
+BINDIR.apropos= /usr/bin
+BINDIR.makemandb= /usr/sbin
+BINDIR.whatis= /usr/bin
+
+CPPFLAGS+=-I${MDIST} -I${.OBJDIR}
+
+MDOCMLOBJDIR!= cd ${MDOCDIR}/lib/libmandoc && ${PRINTOBJDIR}
+MDOCMLLIB= ${MDOCMLOBJDIR}/libmandoc.a
+
+DPADD.makemandb+= ${MDOCMLLIB}
+LDADD.makemandb+= -L${MDOCMLOBJDIR} -lmandoc
+DPADD+= ${LIBSQLITE3} ${LIBM} ${LIBZ} ${LIBUTIL}
+LDADD+= -lsqlite3 -lm -lz -lutil
+
+stopwords.c: stopwords.txt
+ ( set -e; ${TOOL_NBPERF} -n stopwords_hash -s -p ${.ALLSRC}; \
+ echo 'static const char *stopwords[] = {'; \
+ ${TOOL_SED} -e 's|^\(.*\)$$| "\1",|' ${.ALLSRC}; \
+ echo '};' \
+ ) > ${.TARGET}
+
+DPSRCS+= stopwords.c
+CLEANFILES+= stopwords.c
+
+.include <bsd.prog.mk>
Index: usr.sbin/makemandb/apropos-utils.3
===================================================================
RCS file: usr.sbin/makemandb/apropos-utils.3
diff -N usr.sbin/makemandb/apropos-utils.3
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos-utils.3 4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,67 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt APROPOS-UTILS 3
+.Os
+.Sh NAME
+.Nm apropos-utils
+.Nd support functions for querying the man page index
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft sqlite3 *
+.Fn init_db "int db_flag"
+.Ft void
+.Fn close_db "sqlite3 *db"
+.Ft int
+.Fn run_query "sqlite3 *db" "const char *snippet_args[3]" "query_args *args"
+.Ft int
+.Fn run_query_html "sqlite3 *db" "query_args *args"
+.Ft int
+.Fn run_query_pager "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+These functions all live in the
+.Pa apropos-utils.h
+header file.
+They operate on
+.Pa /var/db/man.db
+which is an Sqlite database containing a full text search index of the manual
+pages.
+The functions provide an easy to use interface to query the database and
+develop applications on top of it.
+.Sh SEE ALSO
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/apropos-utils.c
===================================================================
RCS file: usr.sbin/makemandb/apropos-utils.c
diff -N usr.sbin/makemandb/apropos-utils.c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos-utils.c 4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,755 @@
+/* $NetBSD */
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD$");
+
+#include <sys/stat.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <err.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <util.h>
+#include <zlib.h>
+
+#include "apropos-utils.h"
+#include "mandoc.h"
+#include "sqlite3.h"
+
+typedef struct orig_callback_data {
+ void *data;
+ int (*callback) (void *, const char *, const char *, const char *,
+ const char *, size_t);
+} orig_callback_data;
+
+typedef struct inverse_document_frequency {
+ double value;
+ int status;
+} inverse_document_frequency;
+
+/* weights for individual columns */
+static const double col_weights[] = {
+ 2.0, // NAME
+ 2.00, // Name-description
+ 0.55, // DESCRIPTION
+ 0.10, // LIBRARY
+ 0.001, //RETURN VALUES
+ 0.20, //ENVIRONMENT
+ 0.01, //FILES
+ 0.001, //EXIT STATUS
+ 2.00, //DIAGNOSTICS
+ 0.05, //ERRORS
+ 0.00, //md5_hash
+ 1.00 //machine
+};
+
+/*
+ * lower --
+ * Converts the string str to lower case
+ */
+char *
+lower(char *str)
+{
+ assert(str);
+ int i = 0;
+ char c;
+ while (str[i] != '\0') {
+ c = tolower((unsigned char) str[i]);
+ str[i++] = c;
+ }
+ return str;
+}
+
+/*
+* concat--
+* Utility function. Concatenates together: dst, a space character and src.
+* dst + " " + src
+*/
+void
+concat(char **dst, const char *src)
+{
+ concat2(dst, src, strlen(src));
+}
+
+void
+concat2(char **dst, const char *src, size_t srclen)
+{
+ size_t total_len, dst_len;
+ assert(src != NULL);
+
+ /* If destination buffer dst is NULL, then simply strdup the source
buffer */
+ if (*dst == NULL) {
+ *dst = estrdup(src);
+ return;
+ }
+
+ dst_len = strlen(*dst);
+ /*
+ * NUL Byte and separator space
+ */
+ total_len = dst_len + srclen + 2;
+
+ *dst = erealloc(*dst, total_len);
+
+ /* Append a space at the end of dst */
+ (*dst)[dst_len++] = ' ';
+
+ /* Now, copy src at the end of dst */
+ memcpy(*dst + dst_len, src, srclen + 1);
+}
+
+void
+close_db(sqlite3 *db)
+{
+ sqlite3_close(db);
+ sqlite3_shutdown();
+}
+
+/*
+ * create_db --
+ * Creates the database schema.
+ */
+static int
+create_db(sqlite3 *db)
+{
+ const char *sqlstr = NULL;
+ char *schemasql;
+ char *errmsg = NULL;
+
+/*------------------------ Create the tables------------------------------*/
+
+#if NOTYET
+ sqlite3_exec(db, "PRAGMA journal_mode = WAL", NULL, NULL, NULL);
+#else
+ sqlite3_exec(db, "PRAGMA journal_mode = DELETE", NULL, NULL, NULL);
+#endif
+
+ schemasql = sqlite3_mprintf("PRAGMA user_version = %d",
+ APROPOS_SCHEMA_VERSION);
+ sqlite3_exec(db, schemasql, NULL, NULL, &errmsg);
+ if (errmsg != NULL)
+ goto out;
+ sqlite3_free(schemasql);
+
+ sqlstr = "CREATE VIRTUAL TABLE mandb USING fts4(section, name, "
+ "name_desc, desc, lib, return_vals, env, files, "
+ "exit_status, diagnostics, errors, md5_hash UNIQUE,
machine, "
+ "compress=zip, uncompress=unzip, tokenize=porter);
" //mandb
+ "CREATE TABLE IF NOT EXISTS mandb_meta(device, inode,
mtime, "
+ "file UNIQUE, md5_hash UNIQUE, id INTEGER PRIMARY
KEY); "
+ //mandb_meta
+ "CREATE TABLE IF NOT EXISTS mandb_links(link, target,
section, "
+ "machine); "; //mandb_links
+
+ sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+ if (errmsg != NULL)
+ goto out;
+
+ sqlstr = "CREATE INDEX IF NOT EXISTS index_mandb_links ON mandb_links "
+ "(link); "
+ "CREATE INDEX IF NOT EXISTS index_mandb_meta_dev ON
mandb_meta "
+ "(device, inode)";
+ sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+ if (errmsg != NULL)
+ goto out;
+ return 0;
+
+out:
+ warnx("%s", errmsg);
+ free(errmsg);
+ sqlite3_close(db);
+ sqlite3_shutdown();
+ return -1;
+}
+
+/*
+ * zip --
+ * User defined Sqlite function to compress the FTS table
+ */
+static void
+zip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{
+ int nin;
+ long int nout;
+ const unsigned char * inbuf;
+ unsigned char *outbuf;
+
+ assert(nval == 1);
+ nin = sqlite3_value_bytes(apval[0]);
+ inbuf = (const unsigned char *) sqlite3_value_blob(apval[0]);
+ nout = nin + 13 + (nin + 999) / 1000;
+ outbuf = emalloc(nout);
+ compress(outbuf, (unsigned long *) &nout, inbuf, nin);
+ sqlite3_result_blob(pctx, outbuf, nout, free);
+}
+
+/*
+ * unzip --
+ * User defined Sqlite function to uncompress the FTS table.
+ */
+static void
+unzip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{
+ unsigned int rc;
+ unsigned char *outbuf;
+ z_stream stream;
+
+ assert(nval == 1);
+ stream.next_in = __UNCONST(sqlite3_value_blob(apval[0]));
+ stream.avail_in = sqlite3_value_bytes(apval[0]);
+ stream.avail_out = stream.avail_in * 2 + 100;
+ stream.next_out = outbuf = emalloc(stream.avail_out);
+ stream.zalloc = NULL;
+ stream.zfree = NULL;
+
+ if (inflateInit(&stream) != Z_OK) {
+ free(outbuf);
+ return;
+ }
+
+ while ((rc = inflate(&stream, Z_SYNC_FLUSH)) != Z_STREAM_END) {
+ if (rc != Z_OK ||
+ (stream.avail_out != 0 && stream.avail_in == 0)) {
+ free(outbuf);
+ return;
+ }
+ outbuf = erealloc(outbuf, stream.total_out * 2);
+ stream.next_out = outbuf + stream.total_out;
+ stream.avail_out = stream.total_out;
+ }
+ if (inflateEnd(&stream) != Z_OK) {
+ free(outbuf);
+ return;
+ }
+ outbuf = erealloc(outbuf, stream.total_out);
+ sqlite3_result_text(pctx, (const char *) outbuf, stream.total_out,
free);
+}
+
+/* init_db --
+ * Prepare the database. Register the compress/uncompress functions and the
+ * stopword tokenizer.
+ * db_flag specifies the mode in which to open the database. 3 options
are
+ * available:
+ * 1. DB_READONLY: Open in READONLY mode. An error if db does not exist.
+ * 2. DB_READWRITE: Open in read-write mode. An error if db does not exist.
+ * 3. DB_CREATE: Open in read-write mode. It will try to create the db if
+ * it does not exist already.
+ * RETURN VALUES:
+ * The function will return NULL in case the db does not exist and
DB_CREATE
+ * was not specified. And in case DB_CREATE was specified and yet NULL is
+ * returned, then there was some other error.
+ * In normal cases the function should return a handle to the db.
+ */
+sqlite3 *
+init_db(int db_flag)
+{
+ sqlite3 *db = NULL;
+ sqlite3_stmt *stmt;
+ struct stat sb;
+ int rc;
+ int create_db_flag = 0;
+
+ /* Check if the database exists or not */
+ if (!(stat(DBPATH, &sb) == 0 && S_ISREG(sb.st_mode))) {
+ /* Database does not exist, check if DB_CREATE was specified,
and set
+ * flag to create the database schema
+ */
+ if (db_flag != (MANDB_CREATE)) {
+ warnx("Missing apropos database. "
+ "Please run makemandb to create it.");
+ return NULL;
+ }
+ create_db_flag = 1;
+ }
+
+ /* Now initialize the database connection */
+ sqlite3_initialize();
+ rc = sqlite3_open_v2(DBPATH, &db, db_flag, NULL);
+
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ sqlite3_shutdown();
+ return NULL;
+ }
+
+ if (create_db_flag && create_db(db) < 0) {
+ warnx("%s", "Unable to create database schema");
+ goto error;
+ }
+
+ rc = sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &stmt, NULL);
+ if (rc != SQLITE_OK) {
+ warnx("Unable to query schema version");
+ goto error;
+ }
+ if (sqlite3_step(stmt) != SQLITE_ROW) {
+ sqlite3_finalize(stmt);
+ warnx("Unable to query schema version");
+ goto error;
+ }
+ if (sqlite3_column_int(stmt, 0) != APROPOS_SCHEMA_VERSION) {
+ sqlite3_finalize(stmt);
+ warnx("Incorrect schema version found. "
+ "Please run makemandb -f.");
+ goto error;
+ }
+ sqlite3_finalize(stmt);
+
+ sqlite3_extended_result_codes(db, 1);
+
+ /* Register the zip and unzip functions for FTS compression */
+ rc = sqlite3_create_function(db, "zip", 1, SQLITE_ANY, NULL, zip, NULL,
NULL);
+ if (rc != SQLITE_OK) {
+ warnx("Unable to register function: compress");
+ goto error;
+ }
+
+ rc = sqlite3_create_function(db, "unzip", 1, SQLITE_ANY, NULL,
+ unzip, NULL, NULL);
+ if (rc != SQLITE_OK) {
+ warnx("Unable to register function: uncompress");
+ goto error;
+ }
+ return db;
+error:
+ sqlite3_close(db);
+ sqlite3_shutdown();
+ return NULL;
+}
+
+/*
+ * rank_func --
+ * Sqlite user defined function for ranking the documents.
+ * For each phrase of the query, it computes the tf and idf and adds them
over.
+ * It computes the final rank, by multiplying tf and idf together.
+ * Weight of term t for document d = (term frequency of t in d *
+ * inverse document frequency of t)
+ *
+ * Term Frequency of term t in document d = Number of times t occurs in d /
+ * Number of times t appears in
all
+ *
documents
+ *
+ * Inverse document frequency of t = log(Total number of documents /
+ *
Number of documents in which t occurs)
+ */
+static void
+rank_func(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{
+ inverse_document_frequency *idf = sqlite3_user_data(pctx);
+ double tf = 0.0;
+ const unsigned int *matchinfo;
+ int ncol;
+ int nphrase;
+ int iphrase;
+ int ndoc;
+ int doclen = 0;
+ const double k = 3.75;
+ /* Check that the number of arguments passed to this function is
correct. */
+ assert(nval == 1);
+
+ matchinfo = (const unsigned int *) sqlite3_value_blob(apval[0]);
+ nphrase = matchinfo[0];
+ ncol = matchinfo[1];
+ ndoc = matchinfo[2 + 3 * ncol * nphrase + ncol];
+ for (iphrase = 0; iphrase < nphrase; iphrase++) {
+ int icol;
+ const unsigned int *phraseinfo = &matchinfo[2 + ncol+ iphrase *
ncol * 3];
+ for(icol = 1; icol < ncol; icol++) {
+
+ /* nhitcount: number of times the current phrase occurs
in the current
+ * column in the current document.
+ * nglobalhitcount: number of times current phrase
occurs in the current
+ * column in all documents.
+ * ndocshitcount: number of documents in which the
current phrase
+ * occurs in the current column at
least once.
+ */
+ int nhitcount = phraseinfo[3 * icol];
+ int nglobalhitcount = phraseinfo[3 * icol + 1];
+ int ndocshitcount = phraseinfo[3 * icol + 2];
+ doclen = matchinfo[2 + icol ];
+ double weight = col_weights[icol - 1];
+ if (idf->status == 0 && ndocshitcount)
+ idf->value += log(((double)ndoc /
ndocshitcount))* weight;
+
+ /* Dividing the tf by document length to normalize the
effect of
+ * longer documents.
+ */
+ if (nglobalhitcount > 0 && nhitcount)
+ tf += (((double)nhitcount * weight) /
(nglobalhitcount * doclen));
+ }
+ }
+ idf->status = 1;
+
+ /* Final score = (tf * idf)/ ( k + tf)
+ * Dividing by k+ tf further normalizes the weight leading to
better
+ * results.
+ * The value of k is experimental
+ */
+ double score = (tf * idf->value/ ( k + tf)) ;
+ sqlite3_result_double(pctx, score);
+ return;
+}
+
+/*
+ * run_query --
+ * Performs the searches for the keywords entered by the user.
+ * The 2nd param: snippet_args is an array of strings providing values for the
+ * last three parameters to the snippet function of sqlite. (Look at the
docs).
+ * The 3rd param: args contains rest of the search parameters. Look at
+ * arpopos-utils.h for the description of individual fields.
+ *
+ */
+int
+run_query(sqlite3 *db, const char *snippet_args[3], query_args *args)
+{
+ const char *default_snippet_args[3];
+ char *section_clause = NULL;
+ char *limit_clause = NULL;
+ char *machine_clause = NULL;
+ char *query;
+ const char *section;
+ char *name;
+ const char *name_desc;
+ const char *machine;
+ const char *snippet;
+ char *m = NULL;
+ int rc;
+ inverse_document_frequency idf = {0, 0};
+ sqlite3_stmt *stmt;
+
+ if (args->machine)
+ easprintf(&machine_clause, "AND machine = \'%s\' ",
args->machine);
+
+ /* Register the rank function */
+ rc = sqlite3_create_function(db, "rank_func", 1, SQLITE_ANY, (void
*)&idf,
+ rank_func, NULL, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_close(db);
+ sqlite3_shutdown();
+ errx(EXIT_FAILURE, "Unable to register the ranking function");
+ }
+
+ /* We want to build a query of the form: "select x,y,z from mandb where
+ * mandb match :query [AND (section LIKE '1' OR section LIKE '2' OR...)]
+ * ORDER BY rank DESC..."
+ * NOTES: 1. The portion in square brackets is optional, it will be
there
+ * only if the user has specified an option on the command line to
search in
+ * one or more specific sections.
+ * 2. I am using LIKE operator because '=' or IN operators do not seem
to be
+ * working with the compression option enabled.
+ */
+
+ if (args->sec_nums) {
+ char *temp;
+ int i;
+
+ for (i = 0; i < SECMAX; i++) {
+ if (args->sec_nums[i] == 0)
+ continue;
+ easprintf(&temp, " OR section = \'%d\'", i + 1);
+ if (section_clause) {
+ concat(§ion_clause, temp);
+ free(temp);
+ } else {
+ section_clause = temp;
+ }
+ }
+ if (section_clause) {
+ /*
+ * At least one section requested, add glue for query.
+ */
+ temp = section_clause;
+ /* Skip " OR " before first term. */
+ easprintf(§ion_clause, " AND (%s)", temp + 4);
+ free(temp);
+ }
+ }
+ if (args->nrec >= 0) {
+ /* Use the provided number of records and offset */
+ easprintf(&limit_clause, " LIMIT %d OFFSET %d",
+ args->nrec, args->offset);
+ }
+
+ if (snippet_args == NULL) {
+ default_snippet_args[0] = "";
+ default_snippet_args[1] = "";
+ default_snippet_args[2] = "...";
+ snippet_args = default_snippet_args;
+ }
+ query = sqlite3_mprintf("SELECT section, name, name_desc, machine,"
+ " snippet(mandb, %Q, %Q, %Q, -1, 40 ),"
+ " rank_func(matchinfo(mandb, \"pclxn\")) AS rank"
+ " FROM mandb"
+ " WHERE mandb MATCH %Q %s "
+ "%s"
+ " ORDER BY rank DESC"
+ "%s",
+ snippet_args[0], snippet_args[1], snippet_args[2], args->search_str,
+ machine_clause ? machine_clause : "",
+ section_clause ? section_clause : "",
+ limit_clause ? limit_clause : "");
+
+ free(machine_clause);
+ free(section_clause);
+ free(limit_clause);
+
+ if (query == NULL) {
+ *args->errmsg = estrdup("malloc failed");
+ return -1;
+ }
+ rc = sqlite3_prepare_v2(db, query, -1, &stmt, NULL);
+ if (rc == SQLITE_IOERR) {
+ warnx("Corrupt database. Please rerun makemandb");
+ sqlite3_free(query);
+ return -1;
+ } else if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ sqlite3_free(query);
+ return -1;
+ }
+
+ while (sqlite3_step(stmt) == SQLITE_ROW) {
+ section = (const char *) sqlite3_column_text(stmt, 0);
+ name_desc = (const char *) sqlite3_column_text(stmt, 2);
+ machine = (const char *) sqlite3_column_text(stmt, 3);
+ snippet = (const char *) sqlite3_column_text(stmt, 4);
+ if (machine && machine[0]) {
+ m = estrdup(machine);
+ easprintf(&name, "%s/%s", lower(m),
+ sqlite3_column_text(stmt, 1));
+ free(m);
+ } else {
+ name = estrdup((const char *) sqlite3_column_text(stmt,
1));
+ }
+
+ (args->callback)(args->callback_data, section, name, name_desc,
snippet,
+ strlen(snippet));
+
+ free(name);
+ }
+
+ sqlite3_finalize(stmt);
+ sqlite3_free(query);
+ return *(args->errmsg) == NULL ? 0 : -1;
+}
+
+/*
+ * callback_html --
+ * Callback function for run_query_html. It builds the html output and then
+ * calls the actual user supplied callback function.
+ */
+static int
+callback_html(void *data, const char *section, const char *name,
+ const char *name_desc, const char *snippet, size_t snippet_length)
+{
+ const char *temp = snippet;
+ int i = 0;
+ size_t sz = 0;
+ int count = 0;
+ struct orig_callback_data *orig_data = (struct orig_callback_data *)
data;
+ int (*callback) (void *, const char *, const char *, const char *,
+ const char *, size_t) = orig_data->callback;
+
+ /* First scan the snippet to find out the number of occurrences of
{'>', '<'
+ * '"', '&'}.
+ * Then allocate a new buffer with sufficient space to be able to store
the
+ * quoted versions of the special characters {>, <, ",
&}.
+ * Copy over the characters from the original snippet to this buffer
while
+ * replacing the special characters with their quoted versions.
+ */
+
+ while (*temp) {
+ sz = strcspn(temp, "<>\"&\002\003");
+ temp += sz + 1;
+ count++;
+ }
+ size_t qsnippet_length = snippet_length + count * 5;
+ char *qsnippet = emalloc(qsnippet_length + 1);
+ sz = 0;
+ while (*snippet) {
+ sz = strcspn(snippet, "<>\"&\002\003");
+ if (sz) {
+ memcpy(&qsnippet[i], snippet, sz);
+ snippet += sz;
+ i += sz;
+ }
+
+ switch (*snippet++) {
+ case '<':
+ memcpy(&qsnippet[i], "<", 4);
+ i += 4;
+ break;
+ case '>':
+ memcpy(&qsnippet[i], ">", 4);
+ i += 4;
+ break;
+ case '\"':
+ memcpy(&qsnippet[i], """, 6);
+ i += 6;
+ break;
+ case '&':
+ /* Don't perform the quoting if this & is part of an
mdoc escape
+ * sequence, e.g. \&
+ */
+ if (i && *(snippet - 2) != '\\') {
+ memcpy(&qsnippet[i], "&", 5);
+ i += 5;
+ } else {
+ qsnippet[i++] = '&';
+ }
+ break;
+ case '\002':
+ memcpy(&qsnippet[i], "<b>", 3);
+ i += 3;
+ break;
+ case '\003':
+ memcpy(&qsnippet[i], "</b>", 4);
+ i += 4;
+ break;
+ default:
+ break;
+ }
+ }
+ qsnippet[++i] = 0;
+ (*callback)(orig_data->data, section, name, name_desc,
+ (const char *)qsnippet, qsnippet_length);
+ free(qsnippet);
+ return 0;
+}
+
+/*
+ * run_query_html --
+ * Utility function to output query result in HTML format.
+ * It internally calls run_query only, but it first passes the output to it's
+ * own custom callback function, which preprocess the snippet for quoting
+ * inline HTML fragments.
+ * After that it delegates the call the actual user supplied callback
function.
+ */
+int
+run_query_html(sqlite3 *db, query_args *args)
+{
+ struct orig_callback_data orig_data;
+ orig_data.callback = args->callback;
+ orig_data.data = args->callback_data;
+ const char *snippet_args[] = {"\002", "\003", "..."};
+ args->callback = &callback_html;
+ args->callback_data = (void *) &orig_data;
+ return run_query(db, snippet_args, args);
+}
+
+/*
+ * callback_pager --
+ * A callback similar to callback_html. It overstrikes the matching text in
+ * the snippet so that it appears emboldened when viewed using a pager like
+ * more or less.
+ */
+static int
+callback_pager(void *data, const char *section, const char *name,
+ const char *name_desc, const char *snippet, size_t snippet_length)
+{
+ struct orig_callback_data *orig_data = (struct orig_callback_data *)
data;
+ char *psnippet;
+ const char *temp = snippet;
+ int count = 0;
+ int i = 0;
+ size_t sz = 0;
+ size_t psnippet_length;
+
+ /* Count the number of bytes of matching text. For each of these bytes
we
+ * will use 2 extra bytes to overstrike it so that it appears bold when
+ * viewed using a pager.
+ */
+ while (*temp) {
+ sz = strcspn(temp, "\002\003");
+ temp += sz;
+ if (*temp == '\003') {
+ count += 2 * (sz);
+ }
+ temp++;
+ }
+
+ psnippet_length = snippet_length + count;
+ psnippet = emalloc(psnippet_length + 1);
+
+ /* Copy the bytes from snippet to psnippet:
+ * 1. Copy the bytes before \002 as it is.
+ * 2. The bytes after \002 need to be overstriked till we encounter
\003.
+ * 3. To overstrike a byte 'A' we need to write 'A\bA'
+ */
+ while (*snippet) {
+ sz = strcspn(snippet, "\002");
+ memcpy(&psnippet[i], snippet, sz);
+ snippet += sz;
+ i += sz;
+
+ /* Don't change this. Advancing the pointer without reading the
byte
+ * is causing strange behavior.
+ */
+ if (*snippet == '\002')
+ snippet++;
+ while (*snippet && *snippet != '\003') {
+ psnippet[i++] = *snippet;
+ psnippet[i++] = '\b';
+ psnippet[i++] = *snippet++;
+ }
+ if (*snippet)
+ snippet++;
+ }
+
+ psnippet[i] = 0;
+ (orig_data->callback)(orig_data->data, section, name, name_desc,
psnippet,
+ psnippet_length);
+ free(psnippet);
+ return 0;
+}
+
+/*
+ * run_query_pager --
+ * Utility function similar to run_query_html. This function tries to
+ * pre-process the result assuming it will be piped to a pager.
+ * For this purpose it first calls it's own callback function callback_pager
+ * which then delegates the call to the user supplied callback.
+ */
+int run_query_pager(sqlite3 *db, query_args *args)
+{
+ struct orig_callback_data orig_data;
+ orig_data.callback = args->callback;
+ orig_data.data = args->callback_data;
+ const char *snippet_args[] = {"\002", "\003", "..."};
+ args->callback = &callback_pager;
+ args->callback_data = (void *) &orig_data;
+ return run_query(db, snippet_args, args);
+}
Index: usr.sbin/makemandb/apropos-utils.h
===================================================================
RCS file: usr.sbin/makemandb/apropos-utils.h
diff -N usr.sbin/makemandb/apropos-utils.h
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos-utils.h 4 Feb 2012 23:30:01 -0000
@@ -0,0 +1,92 @@
+/* $NetBSD */
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef APROPOS_UTILS_H
+#define APROPOS_UTILS_H
+
+#include "sqlite3.h"
+
+#define DBPATH "/var/db/man.db"
+#define SECMAX 9
+
+/* Flags for opening the database */
+#define MANDB_READONLY SQLITE_OPEN_READONLY
+#define MANDB_WRITE SQLITE_OPEN_READWRITE
+#define MANDB_CREATE SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE
+
+#define APROPOS_SCHEMA_VERSION 20120130
+
+/*
+ * Used to identify the section of a man(7) page.
+ * This is similar to the enum mdoc_sec defined in mdoc.h from mdocml project.
+ */
+enum man_sec {
+ MANSEC_NAME = 0,
+ MANSEC_SYNOPSIS,
+ MANSEC_LIBRARY,
+ MANSEC_ERRORS,
+ MANSEC_FILES,
+ MANSEC_RETURN_VALUES,
+ MANSEC_EXIT_STATUS,
+ MANSEC_DESCRIPTION,
+ MANSEC_ENVIRONMENT,
+ MANSEC_DIAGNOSTICS,
+ MANSEC_EXAMPLES,
+ MANSEC_STANDARDS,
+ MANSEC_HISTORY,
+ MANSEC_BUGS,
+ MANSEC_AUTHORS,
+ MANSEC_COPYRIGHT,
+ MANSEC_NONE
+};
+
+typedef struct query_args {
+ const char *search_str; // user query
+ int *sec_nums; // Section in which to do the search
+ int nrec; // number of records to fetch
+ int offset; //From which position to start processing the
records
+ const char *machine;
+ int (*callback) (void *, const char *, const char *, const char *,
+ const char *, size_t); // The callback function
+ void *callback_data; // data to pass to the callback function
+ char **errmsg; // buffer for storing the error msg
+} query_args;
+
+char *lower(char *);
+void concat(char **, const char *);
+void concat2(char **, const char *, size_t);
+sqlite3 *init_db(int);
+void close_db(sqlite3 *);
+int run_query(sqlite3 *, const char *[3], query_args *);
+int run_query_html(sqlite3 *, query_args *);
+int run_query_pager(sqlite3 *, query_args *);
+#endif
Index: usr.sbin/makemandb/apropos.1
===================================================================
RCS file: usr.sbin/makemandb/apropos.1
diff -N usr.sbin/makemandb/apropos.1
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos.1 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,106 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd January 22, 2012
+.Dt APROPOS 1
+.Os
+.Sh NAME
+.Nm apropos
+.Nd search the complete content of all man pages
+.Sh SYNOPSIS
+.Nm
+.Op Fl 123456789p
+.Op Fl n Ar Number of results
+.Op Fl S Ar machine
+.Ar query
+.Sh DESCRIPTION
+The
+.Nm
+utility performs a full text search over the complete content of all man pages.
+It uses the FTS engine of Sqlite to perform the search.
+The database is created with the help of
+.Xr makemandb 1
+utility.
+.Pp
+This implementation of the
+.Nm
+utility is more sophisticated than the classical version of
+.Nm .
+Like modern search applications, it uses advanced techniques like stemming
+and term weighting to rank the matching results in decreasing order of
+relevance.
+By default
+.Nm
+will only display the top 10 matches in the output.
+.Pp
+It supports the following options:
+.Bl -tag -width indent
+.It Fl 1
+Search only within section 1 manual pages.
+.It Fl 2
+Search only within section 2 manual pages.
+.It Fl 3
+Search only within section 3 manual pages.
+.It Fl 4
+Search only within section 4 manual pages.
+.It Fl 5
+Search only within section 5 manual pages.
+.It Fl 6
+Search only within section 6 manual pages.
+.It Fl 7
+Search only within section 7 manual pages.
+.It Fl 8
+Search only within section 8 manual pages.
+.It Fl 9
+Search only within section 9 manual pages.
+.It Fl n
+Output only the specified number of search results.
+This option can be used to override the default limit of maximum 10 results per
+query.
+.It Fl p
+Display all matching results (instead of the default of 10)
+and pipe them through a pager (defaulting to
+.Xr more 1 ) .
+.It Fl S
+Limit the search to the pages for the specified machine architecture.
+By default pages for all architectures are shown in the search results.
+.El
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr makemandb 1 ,
+.Xr man 1
+.Sh AUTHORS
+.An Abhinav Upadhyay
+
Index: usr.sbin/makemandb/apropos.c
===================================================================
RCS file: usr.sbin/makemandb/apropos.c
diff -N usr.sbin/makemandb/apropos.c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/apropos.c 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,259 @@
+/* $NetBSD */
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD$");
+
+#include <err.h>
+#include <search.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <util.h>
+
+#include "apropos-utils.h"
+#include "sqlite3.h"
+
+typedef struct apropos_flags {
+ int sec_nums[SECMAX];
+ int nresults;
+ int pager;
+ const char *machine;
+} apropos_flags;
+
+typedef struct callback_data {
+ int count;
+ FILE *out;
+} callback_data;
+
+static char *remove_stopwords(const char *);
+static int query_callback(void *, const char * , const char *, const char *,
+ const char *, size_t);
+__dead static void usage(void);
+
+int
+main(int argc, char *argv[])
+{
+ char *query = NULL; // the user query
+ char ch;
+ char *errmsg = NULL;
+ char *str;
+ int rc = 0;
+ callback_data cbdata;
+ const char *snippet_args[] = {"\033[1m", "\033[0m", "..."};
+ cbdata.out = stdout; // the default output stream
+ cbdata.count = 0;
+ apropos_flags aflags;
+ sqlite3 *db;
+ setprogname(argv[0]);
+ if (argc < 2)
+ usage();
+
+ memset(&aflags, 0, sizeof(aflags));
+
+ /*If the user specifies a section number as an option, the
corresponding
+ * index element in sec_nums is set to the string representing that
+ * section number.
+ */
+ while ((ch = getopt(argc, argv, "123456789n:pS:")) != -1) {
+ switch (ch) {
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ aflags.sec_nums[atoi(&ch) - 1] = 1;
+ break;
+ case 'n':
+ aflags.nresults = atoi(optarg);
+ break;
+ case 'p': //user wants to view more than 10 results and
page them
+ aflags.pager = 1;
+ aflags.nresults = -1; // Fetch all records
+ break;
+ case 'S':
+ aflags.machine = optarg;
+ break;
+ case '?':
+ default:
+ usage();
+ }
+ }
+
+ argc -= optind;
+ argv += optind;
+
+ if (!argc)
+ usage();
+
+ str = NULL;
+ while (argc--)
+ concat(&str, *argv++);
+ /* Eliminate any stopwords from the query */
+ query = remove_stopwords(lower(str));
+ free(str);
+
+ /* if any error occured in remove_stopwords, exit */
+ if (query == NULL)
+ errx(EXIT_FAILURE, "Try using more relevant keywords");
+
+ if ((db = init_db(MANDB_READONLY)) == NULL)
+ exit(EXIT_FAILURE);
+
+ /* If user wants to page the output, then set some settings */
+ if (aflags.pager) {
+ /* Open a pipe to the pager */
+ if ((cbdata.out = popen("more", "w")) == NULL) {
+ close_db(db);
+ err(EXIT_FAILURE, "pipe failed");
+ }
+ }
+
+ query_args args;
+ args.search_str = query;
+ args.sec_nums = aflags.sec_nums;
+ args.nrec = aflags.nresults ? aflags.nresults : 10;
+ args.offset = 0;
+ args.machine = aflags.machine;
+ args.callback = &query_callback;
+ args.callback_data = &cbdata;
+ args.errmsg = &errmsg;
+
+ if (aflags.pager) {
+ rc = run_query_pager(db, &args);
+ pclose(cbdata.out);
+ } else {
+ rc = run_query(db, snippet_args, &args);
+ }
+
+ free(query);
+ close_db(db);
+ if (errmsg) {
+ warnx("%s", errmsg);
+ free(errmsg);
+ exit(EXIT_FAILURE);
+ }
+
+ if (rc < 0) {
+ /* Something wrong with the database. Exit */
+ exit(EXIT_FAILURE);
+ }
+
+ if (cbdata.count == 0) {
+ warnx("No relevant results obtained.\n"
+ "Please make sure that you spelled all the terms
correctly\n"
+ "Or try using better keywords.");
+ }
+ return 0;
+}
+
+/*
+ * query_callback --
+ * Callback function for run_query.
+ * It simply outputs the results from do_query. If the user specified the -p
+ * option, then the output is sent to a pager, otherwise stdout is the default
+ * output stream.
+ */
+static int
+query_callback(void *data, const char *section, const char *name,
+ const char *name_desc, const char *snippet, size_t snippet_length)
+{
+ callback_data *cbdata = (callback_data *) data;
+ FILE *out = cbdata->out;
+ cbdata->count++;
+ fprintf(out, "%s(%s)\t%s\n%s\n\n", name, section, name_desc,
+ snippet);
+ return 0;
+}
+
+#include "stopwords.c"
+
+/*
+ * remove_stopwords--
+ * Scans the query and removes any stop words from it.
+ * Returns the modified query or NULL, if it contained only stop words.
+ */
+
+static char *
+remove_stopwords(const char *query)
+{
+ size_t len, idx;
+ char *output, *buf;
+ const char *sep, *next;
+
+ output = buf = emalloc(strlen(query) + 1);
+
+ for (; query[0] != '\0'; query = next) {
+ sep = strchr(query, ' ');
+ if (sep == NULL) {
+ len = strlen(query);
+ next = query + len;
+ } else {
+ len = sep - query;
+ next = sep + 1;
+ }
+ if (len == 0)
+ continue;
+ idx = stopwords_hash(query, len);
+ if (memcmp(stopwords[idx], query, len) == 0 &&
+ stopwords[idx][len] == '\0')
+ continue;
+ memcpy(buf, query, len);
+ buf += len;
+ *buf++ = ' ';
+ }
+
+ if (output == buf) {
+ free(output);
+ return NULL;
+ }
+ buf[-1] = '\0';
+ return output;
+}
+
+/*
+ * usage --
+ * print usage message and die
+ */
+static void
+usage(void)
+{
+ fprintf(stderr,
+ "Usage: %s [-n Number of records] [-p] [-123456789] [-S
machine] query\n",
+ getprogname());
+ exit(1);
+}
Index: usr.sbin/makemandb/close_db.3
===================================================================
RCS file: usr.sbin/makemandb/close_db.3
diff -N usr.sbin/makemandb/close_db.3
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/close_db.3 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,62 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt CLOSE_DB 3
+.Os
+.Sh NAME
+.Nm close_db
+.Nd close apropos database connection
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft void
+.Fn close_db "sqlite3 *db"
+.Sh DESCRIPTION
+The
+.Fn close_db
+function will close the connection to
+.Pa /var/db/man.db
+referenced by
+.Fa db
+and release any resources being used by it safely.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/init_db.3
===================================================================
RCS file: usr.sbin/makemandb/init_db.3
diff -N usr.sbin/makemandb/init_db.3
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/init_db.3 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,92 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt INIT_DB 3
+.Os
+.Sh NAME
+.Nm init_db
+.Nd open apropos database connection
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft sqlite3 *
+.Fn init_db "int db_flag"
+.Sh DESCRIPTION
+The
+.Fn init_db
+function will try to initialize a connection to
+.Pa man.db
+which is an
+Sqlite database and return a pointer to a sqlite3 structure.
+The connection should subsequentially be closed by calling
+.Fn close_db
+which will also release any resources being used by it.
+.Pp
+The argument
+.Fa db_flag
+can be one of the following:
+.Bl -hang -width -compact
+.It Dv MANDB_READONLY
+This will open the database in read-only mode.
+Use this when you only want to query the database.
+.It Dv MANDB_WRITE
+This will open the database in read/write mode.
+.It Dv MANDB_CREATE
+This will open the database in read/write mode, and will also create
+the database schema if it does not exist already.
+.El
+.Sh RETURN VALUES
+On successful execution the
+.Fn init_db
+function will return a pointer to a sqlite3 structure which represents
+a connection to the database.
+.Pp
+In case the file
+.Pa /var/db/man.db
+does not exist and
+.Dv DB_CREATE
+is not used as a value of
+.Fa db_flag ,
+.Dv NULL
+will be returned.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/makemandb.8
===================================================================
RCS file: usr.sbin/makemandb/makemandb.8
diff -N usr.sbin/makemandb/makemandb.8
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/makemandb.8 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,109 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd February 4, 2012
+.Dt MAKEMANDB 8
+.Os
+.Sh NAME
+.Nm makemandb
+.Nd parse the manual pages and build a search index over them
+.Sh SYNOPSIS
+.Nm
+.Op Fl C Ar path
+.Op Fl floqv
+.Sh DESCRIPTION
+The
+.Nm
+utility traverses the directories containing man pages, parses the manual
+pages with the help of libmandoc and builds an Sqlite database
+.Pa man.db
+to support full text searches.
+It obtains the list of directories to traverse using the
+.Fl Ar path
+and
+.Fl p
+options of
+.Xr man 1 .
+.Pp
+It supports the following options:
+.Bl -tag -width indent
+.It Fl C Ar path
+Use different
+.Xr man 1
+configuration file than the default,
+.Pa /etc/man.conf .
+.It Fl f
+Force rebuilding the index from scratch, pruning the existing one.
+.It Fl l
+Limit the parsing to only the NAME section of the pages.
+This option can be used to mimic the behavior of the classic
+.Xr apropos 1
+and also to substantially save disk space.
+.It Fl o
+Use this option to optimize the index for speed and also
+to significantly reduce disk space usage.
+This is a somewhat expensive operation.
+.It Fl q
+Print only error messages and no status updates.
+.It Fl v
+Enable verbose output.
+This prints the name of every file being parsed.
+.El
+.Pp
+As the database file is stored under
+.Pa /var/db ,
+root privileges are required to run
+.Nm .
+.Ss DATABASE SCHEMA
+The name of the FTS table is mandb and its schema is as follows:
+.Bl -column -offset indent "Column Name" "Column Description"
+.It Li section Ta The section number of the page
+.It Li name Ta The name of the page from the NAME section.
+.It Li name_desc Ta The one line description from the NAME section.
+.It Li desc Ta The DESCRIPTION section.
+.It Li lib Ta The LIBRARY section.
+.It Li return_vals Ta The RETURN VALUES section.
+.It Li env Ta The ENVIRONMENT section.
+.It Li files Ta The FILES section.
+.It Li exit_status Ta The EXIT STATUS section.
+.It Li errors Ta The ERRORS section.
+.El
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos 1 ,
+.Xr man 1 ,
+.Xr man.conf 5
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/makemandb.c
===================================================================
RCS file: usr.sbin/makemandb/makemandb.c
diff -N usr.sbin/makemandb/makemandb.c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/makemandb.c 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,1934 @@
+/* $NetBSD */
+/*
+ * Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+ * Copyright (c) 2011 Kristaps Dzonsons <kristaps%bsd.lv@localhost>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD$");
+
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <err.h>
+#include <md5.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <util.h>
+
+#include "apropos-utils.h"
+#include "man.h"
+#include "mandoc.h"
+#include "mdoc.h"
+#include "sqlite3.h"
+
+#define BUFLEN 1024
+#define MDOC 0 //If the page is of mdoc(7) type
+#define MAN 1 //If the page is of man(7) type
+
+/*
+ * A data structure for holding section specific data.
+ */
+typedef struct secbuff {
+ char *data;
+ size_t buflen; //Total length of buffer allocated initially
+ size_t offset; // Current offset in the buffer.
+} secbuff;
+
+typedef struct makemandb_flags {
+ int optimize;
+ int limit; // limit the indexing to only NAME section
+ int recreate; // Database was created from scratch
+ int verbosity; // 0: quiet, 1: default, 2: verbose
+} makemandb_flags;
+
+typedef struct mandb_rec {
+ /* Fields for mandb table */
+ char *name; // for storing the name of the man page
+ char *name_desc; // for storing the one line description (.Nd)
+ secbuff desc; // for storing the DESCRIPTION section
+ secbuff lib; // for the LIBRARY section
+ secbuff return_vals; // RETURN VALUES
+ secbuff env; // ENVIRONMENT
+ secbuff files; // FILES
+ secbuff exit_status; // EXIT STATUS
+ secbuff diagnostics; // DIAGNOSTICS
+ secbuff errors; // ERRORS
+ char section[2];
+
+ int xr_found;
+
+ /* Fields for mandb_meta table */
+ char *md5_hash;
+ dev_t device;
+ ino_t inode;
+ time_t mtime;
+
+ /* Fields for mandb_links table */
+ char *machine;
+ char *links; //all the links to a page in a space separated form
+ char *file_path;
+
+ /* Non-db fields */
+ int page_type; //Indicates the type of page: mdoc or man
+} mandb_rec;
+
+static void append(secbuff *sbuff, const char *src);
+static void init_secbuffs(mandb_rec *);
+static void free_secbuffs(mandb_rec *);
+static int check_md5(const char *, sqlite3 *, const char *, char **);
+static void cleanup(mandb_rec *);
+static void set_section(const struct mdoc *, const struct man *, mandb_rec *);
+static void set_machine(const struct mdoc *, mandb_rec *);
+static int insert_into_db(sqlite3 *, mandb_rec *);
+static void begin_parse(const char *, struct mparse *, mandb_rec *);
+static void pmdoc_node(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Nm(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Nd(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Sh(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Xr(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Pp(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_macro_handler(const struct mdoc_node *, mandb_rec *,
+ enum mdoct);
+static void pman_node(const struct man_node *n, mandb_rec *);
+static void pman_parse_node(const struct man_node *, secbuff *);
+static void pman_parse_name(const struct man_node *, mandb_rec *);
+static void pman_sh(const struct man_node *, mandb_rec *);
+static void pman_block(const struct man_node *, mandb_rec *);
+static void traversedir(const char *, sqlite3 *, struct mparse *);
+static void mdoc_parse_section(enum mdoc_sec, const char *, mandb_rec *);
+static void man_parse_section(enum man_sec, const struct man_node *, mandb_rec
*);
+static void build_file_cache(sqlite3 *, const char *, struct stat *);
+static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
+__dead static void usage(void);
+static void optimize(sqlite3 *);
+static char *parse_escape(const char *);
+static makemandb_flags mflags = { .verbosity = 1 };
+
+typedef void (*pman_nf)(const struct man_node *n, mandb_rec *);
+typedef void (*pmdoc_nf)(const struct mdoc_node *n, mandb_rec *);
+static const pmdoc_nf mdocs[MDOC_MAX] = {
+ NULL, /* Ap */
+ NULL, /* Dd */
+ NULL, /* Dt */
+ NULL, /* Os */
+ pmdoc_Sh, /* Sh */
+ NULL, /* Ss */
+ pmdoc_Pp, /* Pp */
+ NULL, /* D1 */
+ NULL, /* Dl */
+ NULL, /* Bd */
+ NULL, /* Ed */
+ NULL, /* Bl */
+ NULL, /* El */
+ NULL, /* It */
+ NULL, /* Ad */
+ NULL, /* An */
+ NULL, /* Ar */
+ NULL, /* Cd */
+ NULL, /* Cm */
+ NULL, /* Dv */
+ NULL, /* Er */
+ NULL, /* Ev */
+ NULL, /* Ex */
+ NULL, /* Fa */
+ NULL, /* Fd */
+ NULL, /* Fl */
+ NULL, /* Fn */
+ NULL, /* Ft */
+ NULL, /* Ic */
+ NULL, /* In */
+ NULL, /* Li */
+ pmdoc_Nd, /* Nd */
+ pmdoc_Nm, /* Nm */
+ NULL, /* Op */
+ NULL, /* Ot */
+ NULL, /* Pa */
+ NULL, /* Rv */
+ NULL, /* St */
+ NULL, /* Va */
+ NULL, /* Vt */
+ pmdoc_Xr, /* Xr */
+ NULL, /* %A */
+ NULL, /* %B */
+ NULL, /* %D */
+ NULL, /* %I */
+ NULL, /* %J */
+ NULL, /* %N */
+ NULL, /* %O */
+ NULL, /* %P */
+ NULL, /* %R */
+ NULL, /* %T */
+ NULL, /* %V */
+ NULL, /* Ac */
+ NULL, /* Ao */
+ NULL, /* Aq */
+ NULL, /* At */
+ NULL, /* Bc */
+ NULL, /* Bf */
+ NULL, /* Bo */
+ NULL, /* Bq */
+ NULL, /* Bsx */
+ NULL, /* Bx */
+ NULL, /* Db */
+ NULL, /* Dc */
+ NULL, /* Do */
+ NULL, /* Dq */
+ NULL, /* Ec */
+ NULL, /* Ef */
+ NULL, /* Em */
+ NULL, /* Eo */
+ NULL, /* Fx */
+ NULL, /* Ms */
+ NULL, /* No */
+ NULL, /* Ns */
+ NULL, /* Nx */
+ NULL, /* Ox */
+ NULL, /* Pc */
+ NULL, /* Pf */
+ NULL, /* Po */
+ NULL, /* Pq */
+ NULL, /* Qc */
+ NULL, /* Ql */
+ NULL, /* Qo */
+ NULL, /* Qq */
+ NULL, /* Re */
+ NULL, /* Rs */
+ NULL, /* Sc */
+ NULL, /* So */
+ NULL, /* Sq */
+ NULL, /* Sm */
+ NULL, /* Sx */
+ NULL, /* Sy */
+ NULL, /* Tn */
+ NULL, /* Ux */
+ NULL, /* Xc */
+ NULL, /* Xo */
+ NULL, /* Fo */
+ NULL, /* Fc */
+ NULL, /* Oo */
+ NULL, /* Oc */
+ NULL, /* Bk */
+ NULL, /* Ek */
+ NULL, /* Bt */
+ NULL, /* Hf */
+ NULL, /* Fr */
+ NULL, /* Ud */
+ NULL, /* Lb */
+ NULL, /* Lp */
+ NULL, /* Lk */
+ NULL, /* Mt */
+ NULL, /* Brq */
+ NULL, /* Bro */
+ NULL, /* Brc */
+ NULL, /* %C */
+ NULL, /* Es */
+ NULL, /* En */
+ NULL, /* Dx */
+ NULL, /* %Q */
+ NULL, /* br */
+ NULL, /* sp */
+ NULL, /* %U */
+ NULL, /* Ta */
+};
+
+static const pman_nf mans[MAN_MAX] = {
+ NULL, //br
+ NULL, //TH
+ pman_sh, //SH
+ NULL, //SS
+ NULL, //TP
+ NULL, //LP
+ NULL, //PP
+ NULL, //P
+ NULL, //IP
+ NULL, //HP
+ NULL, //SM
+ NULL, //SB
+ NULL, //BI
+ NULL, //IB
+ NULL, //BR
+ NULL, //RB
+ NULL, //R
+ pman_block, //B
+ NULL, //I
+ NULL, //IR
+ NULL, //RI
+ NULL, //na
+ NULL, //sp
+ NULL, //nf
+ NULL, //fi
+ NULL, //RE
+ NULL, //RS
+ NULL, //DT
+ NULL, //UC
+ NULL, //PD
+ NULL, //AT
+ NULL, //in
+ NULL, //ft
+};
+
+
+int
+main(int argc, char *argv[])
+{
+ FILE *file;
+ const char *sqlstr, *manconf = NULL;
+ char *line, *command;
+ char *errmsg;
+ int ch;
+ struct mparse *mp;
+ sqlite3 *db;
+ ssize_t len;
+ size_t linesize;
+ struct mandb_rec rec;
+
+ while ((ch = getopt(argc, argv, "C:floqv")) != -1) {
+ switch (ch) {
+ case 'C':
+ manconf = optarg;
+ break;
+ case 'f':
+ remove(DBPATH);
+ mflags.recreate = 1;
+ break;
+ case 'l':
+ mflags.limit = 1;
+ break;
+ case 'o':
+ mflags.optimize = 1;
+ break;
+ case 'q':
+ mflags.verbosity = 0;
+ break;
+ case 'v':
+ mflags.verbosity = 2;
+ break;
+ default:
+ usage();
+ }
+ }
+
+ memset(&rec, 0, sizeof(rec));
+
+ init_secbuffs(&rec);
+ mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL);
+
+ if ((db = init_db(MANDB_CREATE)) == NULL)
+ exit(EXIT_FAILURE);
+
+ sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, &errmsg);
+ if (errmsg != NULL) {
+ warnx("%s", errmsg);
+ free(errmsg);
+ close_db(db);
+ exit(EXIT_FAILURE);
+ }
+
+ sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
+ &errmsg);
+ if (errmsg != NULL) {
+ warnx("%s", errmsg);
+ free(errmsg);
+ close_db(db);
+ exit(EXIT_FAILURE);
+ }
+
+ if (manconf) {
+ char *arg;
+ size_t command_len = shquote(manconf, NULL, 0) + 1;
+ arg = malloc(command_len );
+ shquote(manconf, arg, command_len);
+ easprintf(&command, "man -p -C %s", arg);
+ free(arg);
+ } else {
+ command = estrdup("man -p");
+ }
+
+ /* Call man -p to get the list of man page dirs */
+ if ((file = popen(command, "r")) == NULL) {
+ close_db(db);
+ err(EXIT_FAILURE, "fopen failed");
+ }
+ free(command);
+
+ /* Begin the transaction for indexing the pages */
+ sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
+ if (errmsg != NULL) {
+ warnx("%s", errmsg);
+ free(errmsg);
+ exit(EXIT_FAILURE);
+ }
+
+ sqlstr = "CREATE TABLE IF NOT EXISTS metadb.file_cache(device, inode,"
+ " mtime, file PRIMARY KEY);"
+ "CREATE UNIQUE INDEX IF NOT EXISTS metadb.index_file_cache_dev"
+ " ON file_cache (device, inode)";
+
+ sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+ if (errmsg != NULL) {
+ warnx("%s", errmsg);
+ free(errmsg);
+ close_db(db);
+ exit(EXIT_FAILURE);
+ }
+
+ if (mflags.verbosity)
+ printf("Building temporary file cache\n");
+ line = NULL;
+ linesize = 0;
+ while ((len = getline(&line, &linesize, file)) != -1) {
+ /* Replace the new line character at the end of string with
'\0' */
+ line[len - 1] = '\0';
+ /* Traverse the man page directories and parse the pages */
+ traversedir(line, db, mp);
+ }
+ free(line);
+
+ if (pclose(file) == -1) {
+ close_db(db);
+ cleanup(&rec);
+ free_secbuffs(&rec);
+ err(EXIT_FAILURE, "pclose error");
+ }
+
+ update_db(db, mp, &rec);
+ mparse_free(mp);
+ free_secbuffs(&rec);
+
+ /* Commit the transaction */
+ sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
+ if (errmsg != NULL) {
+ warnx("%s", errmsg);
+ free(errmsg);
+ exit(EXIT_FAILURE);
+ }
+
+ if (mflags.optimize)
+ optimize(db);
+
+ close_db(db);
+ return 0;
+}
+
+/*
+ * traversedir --
+ * Traverses the given directory recursively and passes all the man page files
+ * in the way to build_file_cache()
+ */
+static void
+traversedir(const char *file, sqlite3 *db, struct mparse *mp)
+{
+ struct stat sb;
+ struct dirent *dirp;
+ DIR *dp;
+ char *buf;
+
+ if (stat(file, &sb) < 0) {
+ warn("stat failed: %s", file);
+ return;
+ }
+
+ /* If it is a regular file or a symlink, pass it to build_cache() */
+ if (S_ISREG(sb.st_mode) || S_ISLNK(sb.st_mode)) {
+ build_file_cache(db, file, &sb);
+ return;
+ }
+
+ /* If it is a directory, traverse it recursively */
+ if (S_ISDIR(sb.st_mode)) {
+ if ((dp = opendir(file)) == NULL) {
+ warn("opendir error: %s", file);
+ return;
+ }
+
+ while ((dirp = readdir(dp)) != NULL) {
+ /* Avoid . and .. entries in a directory */
+ if (strncmp(dirp->d_name, ".", 1)) {
+ easprintf(&buf, "%s/%s", file, dirp->d_name);
+ traversedir(buf, db, mp);
+ free(buf);
+ }
+ }
+ closedir(dp);
+ }
+}
+
+/* build_file_cache --
+ * This function generates an md5 hash of the file passed as it's 2nd
parameter
+ * and stores it in a temporary table file_cache along with the full file
path.
+ * This is done to support incremental updation of the database.
+ * The temporary table file_cache is dropped thereafter in the function
+ * update_db(), once the database has been updated.
+ */
+static void
+build_file_cache(sqlite3 *db, const char *file, struct stat *sb)
+{
+ const char *sqlstr;
+ sqlite3_stmt *stmt = NULL;
+ int rc, idx;
+ assert(file != NULL);
+ dev_t device_cache = sb->st_dev;
+ ino_t inode_cache = sb->st_ino;
+ time_t mtime_cache = sb->st_mtime;
+
+ sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
+ " :mtime, :file)";
+ rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ return;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":device");
+ rc = sqlite3_bind_int64(stmt, idx, device_cache);
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ sqlite3_finalize(stmt);
+ return;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":inode");
+ rc = sqlite3_bind_int64(stmt, idx, inode_cache);
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ sqlite3_finalize(stmt);
+ return;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+ rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ sqlite3_finalize(stmt);
+ return;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":file");
+ rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ sqlite3_finalize(stmt);
+ return;
+ }
+
+ sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+}
+
+static void
+update_existing_entry(sqlite3 *db, const char *file, const char *hash,
+ mandb_rec *rec, int *new_count, int *link_count, int *err_count)
+{
+ int update_count, rc, idx;
+ const char *inner_sqlstr;
+ sqlite3_stmt *inner_stmt;
+
+ update_count = sqlite3_total_changes(db);
+ inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
+ " inode = :inode, mtime = :mtime WHERE"
+ " md5_hash = :md5 AND file = :file AND"
+ " (device <> :device2 OR inode <> "
+ " :inode2 OR mtime <> :mtime2)";
+ rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ return;
+ }
+ idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
+ sqlite3_bind_int64(inner_stmt, idx, rec->device);
+ idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
+ sqlite3_bind_int64(inner_stmt, idx, rec->inode);
+ idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
+ sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
+ idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
+ sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
+ idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
+ sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
+ idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
+ sqlite3_bind_int64(inner_stmt, idx, rec->device);
+ idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
+ sqlite3_bind_int64(inner_stmt, idx, rec->inode);
+ idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
+ sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
+
+ rc = sqlite3_step(inner_stmt);
+ if (rc == SQLITE_DONE) {
+ /* Check if an update has been performed. */
+ if (update_count != sqlite3_total_changes(db)) {
+ if (mflags.verbosity)
+ printf("Updated %s\n", file);
+ (*new_count)++;
+ } else {
+ /* Otherwise it was a hardlink. */
+ (*link_count)++;
+ }
+ } else {
+ warnx("Could not update the meta data for %s", file);
+ (*err_count)++;
+ }
+ sqlite3_finalize(inner_stmt);
+}
+
+/* update_db --
+ * Does an incremental updation of the database by checking the file_cache.
+ * It parses and adds the pages which are present in file_cache,
+ * but not in the database.
+ * It also removes the pages which are present in the databse,
+ * but not in the file_cache.
+ */
+static void
+update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
+{
+ const char *sqlstr;
+ sqlite3_stmt *stmt = NULL;
+ const char *file;
+ char *errmsg = NULL;
+ char *buf = NULL;
+ int new_count = 0; /* Counter for newly indexed/updated pages */
+ int total_count = 0; /* Counter for total number of pages */
+ int err_count = 0; /* Counter for number of failed pages */
+ int link_count = 0; /* Counter for number of hard/sym links */
+ int md5_status;
+ int rc;
+
+ sqlstr = "SELECT device, inode, mtime, file FROM metadb.file_cache"
+ " EXCEPT SELECT device, inode, mtime, file from mandb_meta";
+
+ rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ close_db(db);
+ errx(EXIT_FAILURE, "Could not query file cache");
+ }
+
+ while (sqlite3_step(stmt) == SQLITE_ROW) {
+ total_count++;
+ rec->device = sqlite3_column_int64(stmt, 0);
+ rec->inode = sqlite3_column_int64(stmt, 1);
+ rec->mtime = sqlite3_column_int64(stmt, 2);
+ file = (const char *) sqlite3_column_text(stmt, 3);
+ md5_status = check_md5(file, db, "mandb_meta", &buf);
+ assert(buf != NULL);
+ if (md5_status == -1) {
+ warnx("An error occurred in checking md5 value"
+ " for file %s", file);
+ err_count++;
+ continue;
+ }
+
+ if (md5_status == 0) {
+ /*
+ * The MD5 hash is already present in the database,
+ * so simply update the metadata, ignoring symlinks.
+ */
+ struct stat sb;
+ stat(file, &sb);
+ if (S_ISLNK(sb.st_mode)) {
+ free(buf);
+ link_count++;
+ continue;
+ }
+ update_existing_entry(db, file, buf, rec,
+ &new_count, &link_count, &err_count);
+ free(buf);
+ continue;
+ }
+
+ if (md5_status == 1) {
+ /*
+ * The MD5 hash was not present in the database.
+ * This means is either a new file or an updated file.
+ * We should go ahead with parsing.
+ */
+ if (mflags.verbosity > 1)
+ printf("Parsing: %s\n", file);
+ rec->md5_hash = buf;
+ rec->file_path = estrdup(file);
+ // file_path is freed by insert_into_db itself.
+ begin_parse(file, mp, rec);
+ if (insert_into_db(db, rec) < 0) {
+ warnx("Error in indexing %s", file);
+ err_count++;
+ } else {
+ new_count++;
+ }
+ }
+ }
+
+ sqlite3_finalize(stmt);
+
+ if (mflags.verbosity) {
+ printf("Total Number of new or updated pages enountered = %d\n"
+ "Total number of pages that were successfully"
+ " indexed/updated = %d\n"
+ "Total number of (hard or symbolic) links found = %d\n"
+ "Total number of pages that could not be indexed"
+ " due to errors = %d\n",
+ total_count, new_count, link_count, err_count);
+ }
+
+ if (mflags.recreate == 0)
+ return;
+
+ if (mflags.verbosity)
+ printf("Deleting stale index entries\n");
+
+ sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
+ " (SELECT file FROM metadb.file_cache);"
+ "DROP TABLE metadb.file_cache;"
+ "DELETE FROM mandb WHERE rowid NOT IN"
+ " (SELECT id FROM mandb_meta);";
+
+ sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+ if (errmsg != NULL) {
+ warnx("Removing old entries failed: %s", errmsg);
+ warnx("Please rebuild database from scratch with -f.");
+ free(errmsg);
+ return;
+ }
+}
+
+/*
+ * begin_parse --
+ * parses the man page using libmandoc
+ */
+static void
+begin_parse(const char *file, struct mparse *mp, mandb_rec *rec)
+{
+ struct mdoc *mdoc;
+ struct man *man;
+ mparse_reset(mp);
+
+ rec->xr_found = 0;
+
+ if (mparse_readfd(mp, -1, file) >= MANDOCLEVEL_FATAL) {
+ warnx("%s: Parse failure", file);
+ return;
+ }
+
+ mparse_result(mp, &mdoc, &man);
+ if (mdoc == NULL && man == NULL) {
+ warnx("Not a man(7) or mdoc(7) page");
+ return;
+ }
+
+ set_machine(mdoc, rec);
+ set_section(mdoc, man, rec);
+ if (mdoc) {
+ rec->page_type = MDOC;
+ pmdoc_node(mdoc_node(mdoc), rec);
+ } else {
+ rec->page_type = MAN;
+ pman_node(man_node(man), rec);
+ }
+}
+
+/*
+ * set_section --
+ * Extracts the section number and normalizes it to only the numeric part
+ * (Which should be the first character of the string).
+ */
+static void
+set_section(const struct mdoc *md, const struct man *m, mandb_rec *rec)
+{
+ if (md) {
+ const struct mdoc_meta *md_meta = mdoc_meta(md);
+ rec->section[0] = md_meta->msec[0];
+ } else if (m) {
+ const struct man_meta *m_meta = man_meta(m);
+ rec->section[0] = m_meta->msec[0];
+ }
+}
+
+/*
+ * get_machine --
+ * Extracts the machine architecture information if available.
+ */
+static void
+set_machine(const struct mdoc *md, mandb_rec *rec)
+{
+ if (md == NULL)
+ return;
+ const struct mdoc_meta *md_meta = mdoc_meta(md);
+ if (md_meta->arch)
+ rec->machine = estrdup(md_meta->arch);
+}
+
+static void
+pmdoc_node(const struct mdoc_node *n, mandb_rec *rec)
+{
+
+ if (n == NULL)
+ return;
+
+ switch (n->type) {
+ case (MDOC_BODY):
+ /* FALLTHROUGH */
+ case (MDOC_TAIL):
+ /* FALLTHROUGH */
+ case (MDOC_ELEM):
+ if (mdocs[n->tok] == NULL)
+ break;
+ (*mdocs[n->tok])(n, rec);
+ break;
+ default:
+ break;
+ }
+
+ pmdoc_node(n->child, rec);
+ pmdoc_node(n->next, rec);
+}
+
+/*
+ * pmdoc_Nm --
+ * Extracts the Name of the manual page from the .Nm macro
+ */
+static void
+pmdoc_Nm(const struct mdoc_node *n, mandb_rec *rec)
+{
+ if (n->sec != SEC_NAME)
+ return;
+
+ for (n = n->child; n; n = n->next) {
+ if (n->type == MDOC_TEXT) {
+ concat(&rec->name, n->string);
+ }
+ }
+}
+
+/*
+ * pmdoc_Nd --
+ * Extracts the one line description of the man page from the .Nd macro
+ */
+static void
+pmdoc_Nd(const struct mdoc_node *n, mandb_rec *rec)
+{
+ /*
+ * A static variable for keeping track of whether a Xr macro was seen
+ * previously.
+ */
+ char *buf = NULL;
+ char *temp;
+
+ if (n == NULL)
+ return;
+
+ if (n->type == MDOC_TEXT) {
+ if (rec->xr_found && n->next) {
+ /*
+ * An Xr macro was seen previously, so parse this
+ * and the next node.
+ */
+ temp = estrdup(n->string);
+ n = n->next;
+ easprintf(&buf, "%s(%s)", temp, n->string);
+ concat(&rec->name_desc, buf);
+ free(buf);
+ free(temp);
+ } else {
+ concat(&rec->name_desc, n->string);
+ }
+ rec->xr_found = 0;
+ } else if (mdocs[n->tok] == pmdoc_Xr) {
+ /* Remember that we have encountered an Xr macro */
+ rec->xr_found = 1;
+ }
+
+ if (n->child)
+ pmdoc_Nd(n->child, rec);
+
+ if(n->next)
+ pmdoc_Nd(n->next, rec);
+}
+
+/*
+ * pmdoc_macro_handler--
+ * This function is a single point of handling all the special macros that we
+ * want to handle especially. For example the .Xr macro for properly parsing
+ * the referenced page name along with the section number, or the .Pp macro
+ * for adding a new line whenever we encounter it.
+ */
+static void
+pmdoc_macro_handler(const struct mdoc_node *n, mandb_rec *rec, enum mdoct doct)
+{
+ const struct mdoc_node *sn;
+ assert(n);
+
+ switch (doct) {
+ /* Parse the man page references.
+ * Basically the .Xr macros are used like:
+ * .Xr ls 1
+ * and formatted like this:
+ * ls(1)
+ * Prepare a buffer to format the data like the above example and call
+ * pmdoc_parse_section to append it.
+ */
+ case MDOC_Xr:
+ n = n->child;
+ while (n->type != MDOC_TEXT && n->next)
+ n = n->next;
+
+ if (n && n->type != MDOC_TEXT)
+ return;
+ sn = n;
+ if (n->next)
+ n = n->next;
+
+ while (n->type != MDOC_TEXT && n->next)
+ n = n->next;
+
+ if (n && n->type == MDOC_TEXT) {
+ size_t len = strlen(sn->string);
+ char *buf = emalloc(len + 4);
+ memcpy(buf, sn->string, len);
+ buf[len] = '(';
+ buf[len + 1] = n->string[0];
+ buf[len + 2] = ')';
+ buf[len + 3] = 0;
+ mdoc_parse_section(n->sec, buf, rec);
+ free(buf);
+ }
+
+ break;
+
+ /* Parse the .Pp macro to add a new line */
+ case MDOC_Pp:
+ if (n->type == MDOC_TEXT)
+ mdoc_parse_section(n->sec, "\n", rec);
+ break;
+ default:
+ break;
+ }
+
+}
+
+/*
+ * pmdoc_Xr, pmdoc_Pp--
+ * Empty stubs.
+ * The parser calls these functions each time it encounters
+ * a .Xr or .Pp macro. We are parsing all the data from
+ * the pmdoc_Sh function, so don't do anything here.
+ * (See if else blocks in pmdoc_Sh.)
+ */
+static void
+pmdoc_Xr(const struct mdoc_node *n, mandb_rec *rec)
+{
+}
+
+static void
+pmdoc_Pp(const struct mdoc_node *n, mandb_rec *rec)
+{
+}
+
+/*
+ * pmdoc_Sh --
+ * Called when a .Sh macro is encountered and loops through its body, calling
+ * mdoc_parse_section to append the data to the section specific buffer.
+ * Two special macros which may occur inside the body of Sh are .Nm and .Xr
and
+ * they need special handling, thus the separate if branches for them.
+ */
+static void
+pmdoc_Sh(const struct mdoc_node *n, mandb_rec *rec)
+{
+ if (n == NULL)
+ return;
+ int xr_found = 0;
+
+ if (n->type == MDOC_TEXT) {
+ mdoc_parse_section(n->sec, n->string, rec);
+ } else if (mdocs[n->tok] == pmdoc_Nm && rec->name != NULL) {
+ /*
+ * When encountering a .Nm macro, substitute it
+ * with its previously cached value of the argument.
+ */
+ mdoc_parse_section(n->sec, rec->name, rec);
+ } else if (mdocs[n->tok] == pmdoc_Xr) {
+ /*
+ * When encountering other inline macros,
+ * call pmdoc_macro_handler.
+ */
+ pmdoc_macro_handler(n, rec, MDOC_Xr);
+ xr_found = 1;
+ } else if (mdocs[n->tok] == pmdoc_Pp) {
+ pmdoc_macro_handler(n, rec, MDOC_Pp);
+ }
+
+ /*
+ * If an Xr macro was encountered then the child node has
+ * already been explored by pmdoc_macro_handler.
+ */
+ if (xr_found == 0)
+ pmdoc_Sh(n->child, rec);
+ pmdoc_Sh(n->next, rec);
+}
+
+/*
+ * mdoc_parse_section--
+ * Utility function for parsing sections of the mdoc type pages.
+ * Takes two params:
+ * 1. sec is an enum which indicates the section in which we are present
+ * 2. string is the string which we need to append to the secbuff for this
+ * particular section.
+ * The function appends string to the global section buffer and returns.
+ */
+static void
+mdoc_parse_section(enum mdoc_sec sec, const char *string, mandb_rec *rec)
+{
+ /*
+ * If the user specified the 'l' flag, then parse and store only the
+ * NAME section. Ignore the rest.
+ */
+ if (mflags.limit)
+ return;
+
+ switch (sec) {
+ case SEC_LIBRARY:
+ append(&rec->lib, string);
+ break;
+ case SEC_RETURN_VALUES:
+ append(&rec->return_vals, string);
+ break;
+ case SEC_ENVIRONMENT:
+ append(&rec->env, string);
+ break;
+ case SEC_FILES:
+ append(&rec->files, string);
+ break;
+ case SEC_EXIT_STATUS:
+ append(&rec->exit_status, string);
+ break;
+ case SEC_DIAGNOSTICS:
+ append(&rec->diagnostics, string);
+ break;
+ case SEC_ERRORS:
+ append(&rec->errors, string);
+ break;
+ case SEC_NAME:
+ case SEC_SYNOPSIS:
+ case SEC_EXAMPLES:
+ case SEC_STANDARDS:
+ case SEC_HISTORY:
+ case SEC_AUTHORS:
+ case SEC_BUGS:
+ break;
+ default:
+ append(&rec->desc, string);
+ break;
+ }
+}
+
+static void
+pman_node(const struct man_node *n, mandb_rec *rec)
+{
+ if (n == NULL)
+ return;
+
+ switch (n->type) {
+ case (MAN_BODY):
+ /* FALLTHROUGH */
+ case (MAN_TAIL):
+ /* FALLTHROUGH */
+ case (MAN_BLOCK):
+ /* FALLTHROUGH */
+ case (MAN_ELEM):
+ if (mans[n->tok] != NULL)
+ (*mans[n->tok])(n, rec);
+ break;
+ default:
+ break;
+ }
+
+ pman_node(n->child, rec);
+ pman_node(n->next, rec);
+}
+
+/*
+ * pman_parse_name --
+ * Parses the NAME section and puts the complete content in the name_desc
+ * variable.
+ */
+static void
+pman_parse_name(const struct man_node *n, mandb_rec *rec)
+{
+ if (n == NULL)
+ return;
+
+ if (n->type == MAN_TEXT)
+ concat(&rec->name_desc, n->string);
+
+ if (n->child)
+ pman_parse_name(n->child, rec);
+
+ if(n->next)
+ pman_parse_name(n->next, rec);
+}
+
+/*
+ * A stub function to be able to parse the macros like .B embedded inside
+ * a section.
+ */
+static void
+pman_block(const struct man_node *n, mandb_rec *rec)
+{
+}
+
+/*
+ * pman_sh --
+ * This function does one of the two things:
+ * 1. If the present section is NAME, then it will:
+ * (a) Extract the name of the page (in case of multiple comma separated
+ * names, it will pick up the first one).
+ * (b) Build a space spearated list of all the symlinks/hardlinks to
+ * this page and store in the buffer 'links'. These are extracted from
+ * the comma separated list of names in the NAME section as well.
+ * (c) Move on to the one line description section, which is after the list
+ * of names in the NAME section.
+ * 2. Otherwise, it will check the section name and call the man_parse_section
+ * function, passing the enum corresponding that section.
+ */
+static void
+pman_sh(const struct man_node *n, mandb_rec *rec)
+{
+ static const struct {
+ enum man_sec section;
+ const char *header;
+ } mapping[] = {
+ { MANSEC_DESCRIPTION, "DESCRIPTION" },
+ { MANSEC_SYNOPSIS, "SYNOPSIS" },
+ { MANSEC_LIBRARY, "LIBRARY" },
+ { MANSEC_ERRORS, "ERRORS" },
+ { MANSEC_FILES, "FILES" },
+ { MANSEC_RETURN_VALUES, "RETURN VALUE" },
+ { MANSEC_RETURN_VALUES, "RETURN VALUES" },
+ { MANSEC_EXIT_STATUS, "EXIT STATUS" },
+ { MANSEC_EXAMPLES, "EXAMPLES" },
+ { MANSEC_EXAMPLES, "EXAMPLE" },
+ { MANSEC_STANDARDS, "STANDARDS" },
+ { MANSEC_HISTORY, "HISTORY" },
+ { MANSEC_BUGS, "BUGS" },
+ { MANSEC_AUTHORS, "AUTHORS" },
+ { MANSEC_COPYRIGHT, "COPYRIGHT" },
+ };
+ const struct man_node *head;
+ char *name_desc;
+ int sz;
+ size_t i;
+
+ if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
+ head->type != MAN_TEXT)
+ return;
+
+ /*
+ * Check if this section should be extracted and
+ * where it should be stored. Handled the trival cases first.
+ */
+ for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
+ if (strcmp(head->string, mapping[i].header) == 0) {
+ man_parse_section(mapping[i].section, n, rec);
+ return;
+ }
+ }
+
+ if (strcmp(head->string, "NAME") == 0) {
+ /*
+ * We are in the NAME section.
+ * pman_parse_name will put the complete content in name_desc.
+ */
+ pman_parse_name(n, rec);
+
+ name_desc = rec->name_desc;
+
+ /* Remove any leading spaces. */
+ while (name_desc[0] == ' ')
+ name_desc++;
+
+ /* If the line begins with a "\&", avoid those */
+ if (name_desc[0] == '\\' && name_desc[1] == '&')
+ name_desc += 2;
+
+ /* Now name_desc should be left with a comma-space
+ * separated list of names and the one line description
+ * of the page:
+ * "a, b, c \- sample description"
+ * Take out the first name, before the first comma
+ * (or space) and store it in rec->name.
+ * If the page has aliases then they should be
+ * in the form of a comma separated list.
+ * Keep looping while there is a comma in name_desc,
+ * extract the alias name and store in rec->links.
+ * When there are no more commas left, break out.
+ */
+ int has_alias = 0; // Any more aliases left?
+ while (*name_desc) {
+ /* Remove any leading spaces. */
+ if (name_desc[0] == ' ') {
+ name_desc++;
+ continue;
+ }
+ sz = strcspn(name_desc, ", ");
+
+ /* Extract the first term and store it in rec->name. */
+ if (rec->name == NULL) {
+ if (name_desc[sz] == ',')
+ has_alias = 1;
+ name_desc[sz] = 0;
+ rec->name = emalloc(sz + 1);
+ memcpy(rec->name, name_desc, sz + 1);
+ name_desc += sz + 1;
+ continue;
+ }
+
+ /*
+ * Once rec->name is set, rest of the names
+ * are to be treated as links or aliases.
+ */
+ if (rec->name && has_alias) {
+ if (name_desc[sz] != ',') {
+ /* No more commas left -->
+ * no more aliases to take out
+ */
+ has_alias = 0;
+ }
+ name_desc[sz] = 0;
+ concat2(&rec->links, name_desc, sz);
+ name_desc += sz + 1;
+ continue;
+ }
+ break;
+ }
+
+ /* Parse any escape sequences that might be there */
+ char *temp = parse_escape(name_desc);
+ free(rec->name_desc);
+ rec->name_desc = temp;
+ temp = parse_escape(rec->name);
+ free(rec->name);
+ rec->name = temp;
+ return;
+ }
+
+ /* The RETURN VALUE section might be specified in multiple ways */
+ if (strcmp(head->string, "RETURN") == 0 &&
+ head->next != NULL && head->next->type == MAN_TEXT &&
+ (strcmp(head->next->string, "VALUE") == 0 ||
+ strcmp(head->next->string, "VALUES") == 0)) {
+ man_parse_section(MANSEC_RETURN_VALUES, n, rec);
+ return;
+ }
+
+ /*
+ * EXIT STATUS section can also be specified all on one line or on two
+ * separate lines.
+ */
+ if (strcmp(head->string, "EXIT") == 0 &&
+ head->next != NULL && head->next->type == MAN_TEXT &&
+ strcmp(head->next->string, "STATUS") == 0) {
+ man_parse_section(MANSEC_EXIT_STATUS, n, rec);
+ return;
+ }
+
+ /* Store the rest of the content in desc. */
+ man_parse_section(MANSEC_NONE, n, rec);
+}
+
+/*
+ * pman_parse_node --
+ * Generic function to iterate through a node. Usually called from
+ * man_parse_section to parse a particular section of the man page.
+ */
+static void
+pman_parse_node(const struct man_node *n, secbuff *s)
+{
+ if (n == NULL)
+ return;
+
+ if (n->type == MAN_TEXT)
+ append(s, n->string);
+
+ pman_parse_node(n->child, s);
+ pman_parse_node(n->next, s);
+}
+
+/*
+ * man_parse_section --
+ * Takes two parameters:
+ * sec: Tells which section we are present in
+ * n: Is the present node of the AST.
+ * Depending on the section, we call pman_parse_node to parse that section and
+ * concatenate the content from that section into the buffer for that section.
+ */
+static void
+man_parse_section(enum man_sec sec, const struct man_node *n, mandb_rec *rec)
+{
+ /*
+ * If the user sepecified the 'l' flag then just parse
+ * the NAME section, ignore the rest.
+ */
+ if (mflags.limit)
+ return;
+
+ switch (sec) {
+ case MANSEC_LIBRARY:
+ pman_parse_node(n, &rec->lib);
+ break;
+ case MANSEC_RETURN_VALUES:
+ pman_parse_node(n, &rec->return_vals);
+ break;
+ case MANSEC_ENVIRONMENT:
+ pman_parse_node(n, &rec->env);
+ break;
+ case MANSEC_FILES:
+ pman_parse_node(n, &rec->files);
+ break;
+ case MANSEC_EXIT_STATUS:
+ pman_parse_node(n, &rec->exit_status);
+ break;
+ case MANSEC_DIAGNOSTICS:
+ pman_parse_node(n, &rec->diagnostics);
+ break;
+ case MANSEC_ERRORS:
+ pman_parse_node(n, &rec->errors);
+ break;
+ case MANSEC_NAME:
+ case MANSEC_SYNOPSIS:
+ case MANSEC_EXAMPLES:
+ case MANSEC_STANDARDS:
+ case MANSEC_HISTORY:
+ case MANSEC_BUGS:
+ case MANSEC_AUTHORS:
+ case MANSEC_COPYRIGHT:
+ break;
+ default:
+ pman_parse_node(n, &rec->desc);
+ break;
+ }
+
+}
+
+/*
+ * insert_into_db --
+ * Inserts the parsed data of the man page in the Sqlite databse.
+ * If any of the values is NULL, then we cleanup and return -1 indicating
+ * an error.
+ * Otherwise, store the data in the database and return 0.
+ */
+static int
+insert_into_db(sqlite3 *db, mandb_rec *rec)
+{
+ int rc = 0;
+ int idx = -1;
+ const char *sqlstr = NULL;
+ sqlite3_stmt *stmt = NULL;
+ char *ln = NULL;
+ char *errmsg = NULL;
+ long int mandb_rowid;
+
+ /*
+ * At the very minimum we want to make sure that we store
+ * the following data:
+ * Name, one line description, and the MD5 hash
+ */
+ if (rec->name == NULL || rec->name_desc == NULL ||
+ rec->md5_hash == NULL) {
+ cleanup(rec);
+ return -1;
+ }
+
+ /* Write null byte at the end of all the sec_buffs */
+ rec->desc.data[rec->desc.offset] = 0;
+ rec->lib.data[rec->lib.offset] = 0;
+ rec->env.data[rec->env.offset] = 0;
+ rec->return_vals.data[rec->return_vals.offset] = 0;
+ rec->exit_status.data[rec->exit_status.offset] = 0;
+ rec->files.data[rec->files.offset] = 0;
+ rec->diagnostics.data[rec->diagnostics.offset] = 0;
+ rec->errors.data[rec->errors.offset] = 0;
+
+ /*
+ * In case of a mdoc page: (sorry, no better place to put this code)
+ * parse the comma separated list of names of man pages,
+ * the first name will be stored in the mandb table, rest will be
+ * treated as links and put in the mandb_links table.
+ */
+ if (rec->page_type == MDOC) {
+ char *tmp;
+ rec->links = estrdup(rec->name);
+ free(rec->name);
+ int sz = strcspn(rec->links, " \0");
+ rec->name = emalloc(sz + 1);
+ memcpy(rec->name, rec->links, sz);
+ if(rec->name[sz - 1] == ',')
+ rec->name[sz - 1] = 0;
+ else
+ rec->name[sz] = 0;
+ while (rec->links[sz] == ' ')
+ ++sz;
+ tmp = estrdup(rec->links + sz);
+ free(rec->links);
+ rec->links = tmp;
+ }
+
+/*------------------------ Populate the mandb
table---------------------------*/
+ sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
+ " :lib, :return_vals, :env, :files, :exit_status,"
+ " :diagnostics, :errors, :md5_hash, :machine)";
+
+ rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+ if (rc != SQLITE_OK)
+ goto Out;
+
+ idx = sqlite3_bind_parameter_index(stmt, ":name");
+ rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":section");
+ rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
+ rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":desc");
+ rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
+ rec->desc.offset + 1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":lib");
+ rc = sqlite3_bind_text(stmt, idx, rec->lib.data, rec->lib.offset + 1,
NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
+ rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
+ rec->return_vals.offset + 1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":env");
+ rc = sqlite3_bind_text(stmt, idx, rec->env.data, rec->env.offset + 1,
NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":files");
+ rc = sqlite3_bind_text(stmt, idx, rec->files.data,
+ rec->files.offset + 1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
+ rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
+ rec->exit_status.offset + 1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
+ rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
+ rec->diagnostics.offset + 1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":errors");
+ rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
+ rec->errors.offset + 1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+ rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":machine");
+ if (rec->machine)
+ rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
+ else
+ rc = sqlite3_bind_null(stmt, idx);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_DONE) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ sqlite3_finalize(stmt);
+
+ /* Get the row id of the last inserted row */
+ mandb_rowid = sqlite3_last_insert_rowid(db);
+
+/*------------------------Populate the mandb_meta
table-----------------------*/
+ sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
+ " :file, :md5_hash, :id)";
+ rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+ if (rc != SQLITE_OK)
+ goto Out;
+
+ idx = sqlite3_bind_parameter_index(stmt, ":device");
+ rc = sqlite3_bind_int64(stmt, idx, rec->device);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":inode");
+ rc = sqlite3_bind_int64(stmt, idx, rec->inode);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+ rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":file");
+ rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+ rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":id");
+ rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
+ if (rc != SQLITE_OK) {
+ sqlite3_finalize(stmt);
+ goto Out;
+ }
+
+ rc = sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+ if (rc == SQLITE_CONSTRAINT) {
+ /* The *most* probable reason for reaching here is that
+ * the UNIQUE contraint on the file column of the mandb_meta
+ * table was violated.
+ * This can happen when a file was updated/modified.
+ * To fix this we need to do two things:
+ * 1. Delete the row for the older version of this file
+ * from mandb table.
+ * 2. Run an UPDATE query to update the row for this file
+ * in the mandb_meta table.
+ */
+ warnx("Trying to update index for %s", rec->file_path);
+ char *sql = sqlite3_mprintf("DELETE FROM mandb "
+ "WHERE rowid = (SELECT id"
+ " FROM mandb_meta"
+ " WHERE file = %Q)",
+ rec->file_path);
+ sqlite3_exec(db, sql, NULL, NULL, &errmsg);
+ sqlite3_free(sql);
+ if (errmsg != NULL) {
+ warnx("%s", errmsg);
+ free(errmsg);
+ }
+ sqlstr = "UPDATE mandb_meta SET device = :device,"
+ " inode = :inode, mtime = :mtime, id = :id,"
+ " md5_hash = :md5 WHERE file = :file";
+ rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+ if (rc != SQLITE_OK) {
+ warnx("Update failed with error: %s",
+ sqlite3_errmsg(db));
+ close_db(db);
+ cleanup(rec);
+ errx(EXIT_FAILURE,
+ "Consider running makemandb with -f option");
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":device");
+ sqlite3_bind_int64(stmt, idx, rec->device);
+ idx = sqlite3_bind_parameter_index(stmt, ":inode");
+ sqlite3_bind_int64(stmt, idx, rec->inode);
+ idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+ sqlite3_bind_int64(stmt, idx, rec->mtime);
+ idx = sqlite3_bind_parameter_index(stmt, ":id");
+ sqlite3_bind_int64(stmt, idx, mandb_rowid);
+ idx = sqlite3_bind_parameter_index(stmt, ":md5");
+ sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+ idx = sqlite3_bind_parameter_index(stmt, ":file");
+ sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
+ rc = sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+
+ if (rc != SQLITE_DONE) {
+ warnx("%s", sqlite3_errmsg(db));
+ close_db(db);
+ cleanup(rec);
+ errx(EXIT_FAILURE,
+ "Consider running makemandb with -f option");
+ }
+ } else if (rc != SQLITE_DONE) {
+ /* Otherwise make this error fatal */
+ warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
+ cleanup(rec);
+ close_db(db);
+ exit(EXIT_FAILURE);
+ }
+
+/*------------------------ Populate the mandb_links
table---------------------*/
+ char *str = NULL;
+ char *links;
+ if (rec->links && strlen(rec->links)) {
+ links = rec->links;
+ for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
+ if (ln[0] == ',')
+ ln++;
+ if(ln[strlen(ln) - 1] == ',')
+ ln[strlen(ln) - 1] = 0;
+
+ str = sqlite3_mprintf("INSERT INTO mandb_links"
+ " VALUES (%Q, %Q, %Q, %Q)",
+ ln, rec->name, rec->section,
+ rec->machine);
+ sqlite3_exec(db, str, NULL, NULL, &errmsg);
+ sqlite3_free(str);
+ if (errmsg != NULL) {
+ warnx("%s", errmsg);
+ cleanup(rec);
+ free(errmsg);
+ return -1;
+ }
+ }
+ }
+
+ cleanup(rec);
+ return 0;
+
+ Out:
+ warnx("%s", sqlite3_errmsg(db));
+ cleanup(rec);
+ return -1;
+}
+
+/*
+ * check_md5--
+ * Generates the md5 hash of the file and checks if it already doesn't exist
+ * in the table (passed as the 3rd parameter).
+ * This function is being used to avoid hardlinks.
+ * On successful completion it will also set the value of the fourth parameter
+ * to the md5 hash of the file (computed previously). It is the responsibility
+ * of the caller to free this buffer.
+ * Return values:
+ * -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
+ * 0: If the md5 hash does not exist in the table.
+ * 1: If the hash exists in the database.
+ */
+static int
+check_md5(const char *file, sqlite3 *db, const char *table, char **buf)
+{
+ int rc = 0;
+ int idx = -1;
+ char *sqlstr = NULL;
+ sqlite3_stmt *stmt = NULL;
+
+ assert(file != NULL);
+ *buf = MD5File(file, NULL);
+ if (*buf == NULL) {
+ warn("md5 failed: %s", file);
+ return -1;
+ }
+
+ easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash",
+ table);
+ rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+ if (rc != SQLITE_OK) {
+ free(sqlstr);
+ free(*buf);
+ *buf = NULL;
+ return -1;
+ }
+
+ idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+ rc = sqlite3_bind_text(stmt, idx, *buf, -1, NULL);
+ if (rc != SQLITE_OK) {
+ warnx("%s", sqlite3_errmsg(db));
+ sqlite3_finalize(stmt);
+ free(sqlstr);
+ free(*buf);
+ *buf = NULL;
+ return -1;
+ }
+
+ if (sqlite3_step(stmt) == SQLITE_ROW) {
+ sqlite3_finalize(stmt);
+ free(sqlstr);
+ return 0;
+ }
+
+ sqlite3_finalize(stmt);
+ free(sqlstr);
+ return 1;
+}
+
+/* Optimize the index for faster search */
+static void
+optimize(sqlite3 *db)
+{
+ const char *sqlstr;
+ char *errmsg = NULL;
+
+ if (mflags.verbosity)
+ printf("Optimizing the database index\n");
+ sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
+ "VACUUM";
+ sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+ if (errmsg != NULL) {
+ warnx("%s", errmsg);
+ free(errmsg);
+ return;
+ }
+}
+
+/*
+ * cleanup --
+ * cleans up the global buffers
+ */
+static void
+cleanup(mandb_rec *rec)
+{
+ rec->desc.offset = 0;
+ rec->lib.offset = 0;
+ rec->return_vals.offset = 0;
+ rec->env.offset = 0;
+ rec->exit_status.offset = 0;
+ rec->diagnostics.offset = 0;
+ rec->errors.offset = 0;
+ rec->files.offset = 0;
+
+ free(rec->machine);
+ rec->machine = NULL;
+
+ free(rec->links);
+ rec->links = NULL;
+
+ free(rec->file_path);
+ rec->file_path = NULL;
+
+ free(rec->name);
+ rec->name = NULL;
+
+ free(rec->name_desc);
+ rec->name_desc = NULL;
+
+ free(rec->md5_hash);
+ rec->md5_hash = NULL;
+}
+
+/*
+ * init_secbuffs--
+ * Sets the value of buflen for all the sec_buff field of rec. And then
+ * allocate memory to each sec_buff member of rec.
+ */
+static void
+init_secbuffs(mandb_rec *rec)
+{
+ /*
+ * Some sec_buff might need more memory, for example desc,
+ * which stores the data of the DESCRIPTION section,
+ * while some might need very small amount of memory.
+ * Therefore explicitly setting the value of buflen field for
+ * each sec_buff.
+ */
+ rec->desc.buflen = 10 * BUFLEN;
+ rec->desc.data = emalloc(rec->desc.buflen);
+ rec->desc.offset = 0;
+
+ rec->lib.buflen = BUFLEN / 2;
+ rec->lib.data = emalloc(rec->lib.buflen);
+ rec->lib.offset = 0;
+
+ rec->return_vals.buflen = BUFLEN;
+ rec->return_vals.data = emalloc(rec->return_vals.buflen);
+ rec->return_vals.offset = 0;
+
+ rec->exit_status.buflen = BUFLEN;
+ rec->exit_status.data = emalloc(rec->exit_status.buflen);
+ rec->exit_status.offset = 0;
+
+ rec->env.buflen = BUFLEN;
+ rec->env.data = emalloc(rec->env.buflen);
+ rec->env.offset = 0;
+
+ rec->files.buflen = BUFLEN;
+ rec->files.data = emalloc(rec->files.buflen);
+ rec->files.offset = 0;
+
+ rec->diagnostics.buflen = BUFLEN;
+ rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
+ rec->diagnostics.offset = 0;
+
+ rec->errors.buflen = BUFLEN;
+ rec->errors.data = emalloc(rec->errors.buflen);
+ rec->errors.offset = 0;
+}
+
+/*
+ * free_secbuffs--
+ * This function should be called at the end, when all the pages have been
+ * parsed.
+ * It frees the memory allocated to sec_buffs by init_secbuffs in the
starting.
+ */
+static void
+free_secbuffs(mandb_rec *rec)
+{
+ free(rec->desc.data);
+ free(rec->lib.data);
+ free(rec->return_vals.data);
+ free(rec->exit_status.data);
+ free(rec->env.data);
+ free(rec->files.data);
+ free(rec->diagnostics.data);
+ free(rec->errors.data);
+}
+
+static char *
+parse_escape(const char *str)
+{
+ const char *backslash, *last_backslash;
+ char *result, *iter;
+ size_t len;
+
+ assert(str);
+
+ last_backslash = str;
+ backslash = strchr(str, '\\');
+ if (backslash == NULL)
+ return estrdup(str);
+
+ result = emalloc(strlen(str) + 1);
+ iter = result;
+
+ do {
+ len = backslash - last_backslash;
+ memcpy(iter, last_backslash, len);
+ iter += len;
+ if (backslash[1] == '-' || backslash[1] == ' ') {
+ *iter++ = backslash[1];
+ last_backslash = backslash + 2;
+ backslash = strchr(backslash + 2, '\\');
+ } else {
+ ++backslash;
+ mandoc_escape(&backslash, NULL, NULL);
+ last_backslash = backslash;
+ if (backslash == NULL)
+ break;
+ backslash = strchr(last_backslash, '\\');
+ }
+ } while (backslash != NULL);
+ if (last_backslash != NULL)
+ strcpy(iter, last_backslash);
+ return result;
+}
+
+/*
+ * append--
+ * Concatenates a space and src at the end of sbuff->data (much like concat in
+ * apropos-utils.c).
+ * Rather than reallocating space for writing data, it uses the value of the
+ * offset field of sec_buff to write new data at the free space left in the
+ * buffer.
+ * In case the size of the data to be appended exceeds the number of bytes
left
+ * in the buffer, it reallocates buflen number of bytes and then continues.
+ * Value of offset field should be adjusted as new data is written.
+ *
+ * NOTE: This function does not write the null byte at the end of the buffers,
+ * write a null byte at the position pointed to by offset before inserting
data
+ * in the db.
+ */
+static void
+append(secbuff *sbuff, const char *src)
+{
+ short flag = 0;
+ size_t srclen, newlen;
+ char *temp;
+
+ assert(src != NULL);
+ temp = parse_escape(src);
+ srclen = strlen(temp);
+
+ if (sbuff->data == NULL) {
+ sbuff->data = emalloc(sbuff->buflen);
+ sbuff->offset = 0;
+ }
+
+ newlen = sbuff->offset + srclen + 2;
+ if (newlen >= sbuff->buflen) {
+ while (sbuff->buflen < newlen)
+ sbuff->buflen += sbuff->buflen;
+ sbuff->data = erealloc(sbuff->data, sbuff->buflen);
+ flag = 1;
+ }
+
+ /* Append a space at the end of the buffer. */
+ if (sbuff->offset || flag)
+ sbuff->data[sbuff->offset++] = ' ';
+ /* Now, copy src at the end of the buffer. */
+ memcpy(sbuff->data + sbuff->offset, temp, srclen);
+ sbuff->offset += srclen;
+ free(temp);
+}
+
+static void
+usage(void)
+{
+ fprintf(stderr, "Usage: %s [-flo]\n", getprogname());
+ exit(1);
+}
Index: usr.sbin/makemandb/run_query.3
===================================================================
RCS file: usr.sbin/makemandb/run_query.3
diff -N usr.sbin/makemandb/run_query.3
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/run_query.3 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,223 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY 3
+.Os
+.Sh NAME
+.Nm run_query
+.Nd run a query against
+.Pa /var/db/man.db
+and process the results in a callback function.
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query "sqlite3 *db" "const char **snippet_args" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query
+function prepares the user supplied query in a form suitable to be run
+against
+.Pa /var/db/man.db
+and executes the query.
+For each row obtained in the result set,
+.Fn run_query
+will call the user supplied callback function, which should contain the
+logic for processing the data thus obtained.
+.Pp
+The
+.Fn run_query
+function takes following arguments:
+.Bl -tag -width 8n
+.It Fa sqlite3 *db
+Handle to the database connection which can be obtained by calling
+.Fn init_db .
+.It Fa const char *snippet_args
+An array of strings which specify the
+delimiters to the matching text in snippet.
+It is an optional argument and caller can supply a
+.Dv NULL
+value for it, in which case, a default value of
+.Brq \&"\&", \&"\&", \&"...\&"
+will be used.
+The 3 members of the array specify the following values:
+.Bl -enum -offset indent
+.It
+The first element marks the beginning of each matching token in the snippet.
+.It
+The second element the end of each matching token in the snippet.
+.It
+The third element is used to delimit the beginning and end of the snippet.
+.El
+For example for highlighting matching tokens in HTML style mark-up, use this
+value:
+.Bd -literal -offset indent
+ const char **snippet_args = {
+ "<b>",
+ "</b>",
+ "..."
+ };
+.Ed
+.It Fa query_args *args
+.Ft query_args
+is a struct which is defined in
+.Pa apropos-utils.h .
+It has the following fields:
+.Bl -tag -width 8n -offset indent
+.It Li const char *search_str
+This is the query as entered by the user.
+You may want to pre-process it to do sanitization etc.
+.It Li int *sec_nums
+This is an array of
+.Ft int
+wherein each index element represents the
+section number in which search should be performed.
+The sections whose corresponding index element in this array is set to
+.Dv NULL
+are not searched.
+If all the elements in the array are
+.Dv 0
+then all the sections are searched.
+Alternatively pass
+.Dv NULL
+as to search in all the sections.
+.It Li int nrec
+This specifies the number of matching rows to fetch from the database.
+Specifiy a negative value to fetch all the matching rows.
+.It Li int offset
+This specifies the offset within the result-set. Use it to specify the position
+from where to start processing the result-set.
+For example if the value of nrec is m and value of offset is n, then the first
+n records from the result-set will be omitted and rest m-n (or less) records
will
+be available for processing inside the callback.
+Use a negative value if you don't wish to offset any records.
+.It Li const char *machine
+The machine architecture to which the searches should be restricted.
+Specify NULL if the search should not be restricted a particular machine
architecture.
+.It Li int (*callback) (void *, const char *, const char *, const char *,\
+const char *, size_t)
+This is the callback function which will
+be called for each row retrieved from the database.
+The function should return a value of 0 on successful execution.
+A non-zero return value will indicate a failure and
+.Fn run_query
+will return.
+The interface of the callback function is described later in this section.
+.It Li void *callback_data
+Use this argument to pass any data to the callback function.
+It can be retrieved inside the callback function from its first argument.
+.It Li char **errmsg
+If an error occurs while fetching the data from the database,
+a human readable error message will be stored in
+.Fa errmsg .
+If no error occurs then
+.Fa errmsg
+will be set to
+.Dv NULL .
+In case
+.Fa errmsg
+is not
+.Dv NULL ,
+then the caller should make sure to free it.
+.El
+.El
+.Pp
+The interface of the callback function is
+.Ft int
+.Fn (*callback) "void *callback_data" "const char *section" "const char *name"\
+"const char *name_desc" "const char *snippet" "size_t snippet_length"
+.Pp
+Its arguments are:
+.Bl -column -offset indent "Function" "Argument Description"
+.It Li void *callback_data Ta This is the caller supplied data.
+.It Li const char *section Ta Ta \&It is the section number to which this man
+page belongs.
+.It Li const char *name Ta This is the name of the man page
+.It Li const char *name_desc Ta This is the one line description of this man
+page obtained from it's NAME section.
+.It Li const char *snippet Ta This is a snippet of the matching text from this
+man page.
+.It Li size_t snippet_length Ta This is the length of the snippet.
+.El
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Pa apropos.c
+uses
+.Fn run_query .
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = 10;
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = NULL;
+args.errmsg = &errmsg;
+if (run_query(db, NULL, &args) < 0)
+ errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+ const char *name_desc, const char *snippet, size_t snippet_length )
+{
+ /* The user supplied data could be obtained as follows */
+ // my_data *buf = (my_data *) data;
+
+ fprintf(stdout, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+ snippet);
+ return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/run_query_html.3
===================================================================
RCS file: usr.sbin/makemandb/run_query_html.3
diff -N usr.sbin/makemandb/run_query_html.3
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/run_query_html.3 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,116 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY_HTML 3
+.Os
+.Sh NAME
+.Nm run_query_html
+.Nd run a query against
+.Pa /var/db/man.db
+and returns the results in HTML format
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query_html "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query_html
+is very much similar to
+.Fn run_query
+but with the following three differences.
+.Bl -enum -offset indent
+.It
+.Nm
+takes only two arguments.
+.It
+The snippet returned by
+.Nm
+uses <b> and </b> tags to highlight the matching pieces of text so that it
+appears bold in a web browser.
+.It
+.Nm
+will do limited quoting of the inline HTML fragments in the snippet.
+Concretely, it will replace <, >, ", & with < > " and &
+respectively.
+.El
+For more details, refer to the manual page of
+.Xr run_query 3 .
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query_html
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Nm
+can be used.
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = 10;
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = NULL;
+args.errmsg = &errmsg;
+if (run_query(db, &args) < 0)
+ errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+ const char *name_desc, const char *snippet, size_t snippet_length)
+{
+ fprintf(stdout, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+ snippet);
+ return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/run_query_pager.3
===================================================================
RCS file: usr.sbin/makemandb/run_query_pager.3
diff -N usr.sbin/makemandb/run_query_pager.3
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/run_query_pager.3 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,106 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay
<er.abhinav.upadhyay%gmail.com@localhost>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY_PAGER 3
+.Os
+.Sh NAME
+.Nm run_query_pager
+.Nd run a query against
+.Pa /var/db/man.db
+and returns the results in a format which can be piped to a pager
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query_pager "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query_pager
+is very similar to
+.Xr run_query_html 3
+with the only difference that the snippet returned is formatted in a manner so
+that the matching text in it appears highlighted when viewed using a pager.
+.Pp
+For more details, refer to the manual page of
+.Xr run_query 3 .
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query_pager
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Nm
+can be used.
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+FILE *pager = popen("more", "w");
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = -1; //A negative value means fetch all matching records
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = (void *)pager;
+args.errmsg = &errmsg;
+if (run_query(db, &args) < 0)
+ errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+pclose(pager);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+ const char *name_desc, const char *snippet, size_t snippet_length)
+{
+ FILE *out = (FILE *) data;
+ fprintf(out, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+ snippet);
+ return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
Index: usr.sbin/makemandb/stopwords.txt
===================================================================
RCS file: usr.sbin/makemandb/stopwords.txt
diff -N usr.sbin/makemandb/stopwords.txt
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/stopwords.txt 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,251 @@
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+about
+again
+all
+also
+always
+an
+and
+another
+any
+are
+around
+as
+ask
+at
+b
+back
+be
+because
+been
+before
+below
+between
+but
+by
+bye
+can
+case
+consist
+could
+d
+did
+does
+down
+e
+each
+early
+either
+end
+enough
+even
+every
+f
+fact
+far
+few
+follow
+four
+from
+full
+further
+g
+general
+get
+give
+given
+good
+got
+great
+h
+had
+has
+have
+having
+here
+high
+him
+his
+how
+however
+i
+if
+important
+in
+interest
+into
+is
+it
+j
+just
+k
+keep
+keeps
+kind
+knew
+know
+l
+large
+larger
+last
+later
+latest
+latter
+least
+let
+like
+likely
+long
+longer
+m
+made
+many
+may
+me
+might
+most
+mostly
+much
+must
+my
+n
+names
+necessary
+need
+needs
+never
+new
+next
+no
+non
+noone
+not
+nothing
+o
+of
+off
+often
+old
+older
+on
+once
+only
+or
+order
+our
+out
+over
+p
+part
+per
+perhaps
+possible
+present
+problem
+q
+quite
+r
+rather
+really
+right
+room
+s
+said
+same
+saw
+say
+says
+second
+see
+seem
+seemed
+seems
+sees
+several
+shall
+should
+side
+sides
+small
+smaller
+so
+some
+something
+state
+states
+still
+such
+sure
+t
+take
+taken
+that
+the
+their
+them
+then
+there
+therefore
+these
+thing
+think
+thinks
+this
+those
+though
+three
+thus
+to
+together
+too
+took
+toward
+turn
+two
+u
+until
+up
+upon
+us
+use
+used
+uses
+v
+very
+w
+want
+wanted
+wants
+was
+way
+ways
+we
+well
+went
+were
+what
+when
+whether
+why
+will
+willing
+with
+within
+without
+work
+would
+x
+y
+year
+yet
+you
+z
Index: usr.sbin/makemandb/whatis.1
===================================================================
RCS file: usr.sbin/makemandb/whatis.1
diff -N usr.sbin/makemandb/whatis.1
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/whatis.1 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,58 @@
+.\" $NetBSD$
+.\"
+.\" Copyright (c) 2012 Joerg Sonnenberger <joerg%NetBSD.org@localhost>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in
+.\" the documentation and/or other materials provided with the
+.\" distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd February 1, 2012
+.Dt WHATIS 1
+.Os
+.Sh NAME
+.Nm whatis
+.Nd describe what a command is
+.Sh SYNOPSIS
+.Nm
+.Ar command Ar ...
+.Sh DESCRIPTION
+The
+.Nm
+program queries the database
+.Pa /var/db/man.db
+for manual pages with name
+.Ar command
+and prints the name of the manual page, the section and the description from
+the section NAME.
+.Sh SEE ALSO
+.Xr apropos 1 ,
+.Xr makemandb 8
+.Sh HISTORY
+The
+.Nm
+command appeared in 3.0BSD.
+It was rewritten in
+.Nx 6.0
+to use the SQLite index created by
+.Xr makemandb 8 .
Index: usr.sbin/makemandb/whatis.c
===================================================================
RCS file: usr.sbin/makemandb/whatis.c
diff -N usr.sbin/makemandb/whatis.c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ usr.sbin/makemandb/whatis.c 4 Feb 2012 23:30:02 -0000
@@ -0,0 +1,101 @@
+/* $NetBSD */
+/*-
+ * Copyright (c) 2012 Joerg Sonnenberger <joerg%NetBSD.org@localhost>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD$");
+
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "apropos-utils.h"
+
+__dead static void
+usage(void)
+{
+ fprintf(stderr, "%s ...\n", "whatis");
+ exit(EXIT_FAILURE);
+}
+
+static int
+whatis(sqlite3 *db, const char *cmd)
+{
+ static const char sqlstr[] = "SELECT name, section, name_desc"
+ " FROM mandb WHERE name=?"
+ " ORDER BY section, name";
+ sqlite3_stmt *stmt = NULL;
+ int retval;
+
+ if (sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL) != SQLITE_OK)
+ errx(EXIT_FAILURE, "Unable to query database");
+ if (sqlite3_bind_text(stmt, 1, cmd, -1, NULL) != SQLITE_OK)
+ errx(EXIT_FAILURE, "Unable to query database");
+ retval = 1;
+ while (sqlite3_step(stmt) == SQLITE_ROW) {
+ printf("%s(%s) - %s\n", sqlite3_column_text(stmt, 0),
+ sqlite3_column_text(stmt, 1),
+ sqlite3_column_text(stmt, 2));
+ retval = 0;
+ }
+ sqlite3_finalize(stmt);
+ if (retval)
+ fprintf(stderr, "%s: not found\n", cmd);
+ return retval;
+}
+
+int
+main(int argc, char *argv[])
+{
+ sqlite3 *db;
+ int ch, retval;
+
+ while ((ch = getopt(argc, argv, "")) != -1) {
+ switch (ch) {
+ default:
+ usage();
+ }
+ }
+ argc -= optind;
+ argv += optind;
+
+ if (argc == 0)
+ usage();
+
+ if ((db = init_db(MANDB_READONLY)) == NULL)
+ exit(EXIT_FAILURE);
+
+ retval = 0;
+ while (argc--)
+ retval |= whatis(db, *argv++);
+
+ close_db(db);
+ return retval;
+}
Home |
Main Index |
Thread Index |
Old Index