Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/lib/libedit Tokenization function enhancements:
details: https://anonhg.NetBSD.org/src/rev/b606b887a419
branches: trunk
changeset: 555986:b606b887a419
user: lukem <lukem%NetBSD.org@localhost>
date: Fri Dec 05 13:37:48 2003 +0000
description:
Tokenization function enhancements:
* Make tok_init(), tok_end(), tok_reset(), tok_line() and tok_str()
publically available in <histedit.h>
* Documented the public functions in editline(3)
* Renamed tok_line() -> tok_str()
* Added new tok_line() which takes a "const LineInfo *" instead of
"const char *" (the former has "cursor" information), and optionally
return the argv index ("int *cursorc") and offset within that index
("int *cursorv"). This means that completion routines can use the
tokenization code to crack the line and easily find which word the
cursor is at. (mmm, context sensitive completion :)
* Fixed TEST/test.c when using "continuation" lines (unmatched quote
or \ at EOL), and added some more DEBUG messages including highlighting
where the cursor is (with a `_').
diffstat:
lib/libedit/Makefile | 7 ++-
lib/libedit/TEST/Makefile | 6 ++-
lib/libedit/TEST/test.c | 47 ++++++++++++++++---
lib/libedit/editline.3 | 107 ++++++++++++++++++++++++++++++++++++++++++---
lib/libedit/histedit.h | 29 +++++++++--
lib/libedit/parse.c | 7 +-
lib/libedit/readline.c | 7 +-
lib/libedit/shlib_version | 4 +-
lib/libedit/tokenizer.c | 82 ++++++++++++++++++++++++++---------
lib/libedit/tokenizer.h | 50 ---------------------
10 files changed, 239 insertions(+), 107 deletions(-)
diffs (truncated from 647 to 300 lines):
diff -r ecb7b3361ecc -r b606b887a419 lib/libedit/Makefile
--- a/lib/libedit/Makefile Fri Dec 05 12:15:49 2003 +0000
+++ b/lib/libedit/Makefile Fri Dec 05 13:37:48 2003 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.28 2003/08/01 17:03:58 lukem Exp $
+# $NetBSD: Makefile,v 1.29 2003/12/05 13:37:48 lukem Exp $
# @(#)Makefile 8.1 (Berkeley) 6/4/93
USE_SHLIBDIR= yes
@@ -16,7 +16,10 @@
editline.3 el_parse.3 editline.3 el_set.3 editline.3 el_get.3 \
editline.3 el_source.3 editline.3 el_resize.3 editline.3 el_line.3 \
editline.3 el_insertstr.3 editline.3 el_deletestr.3 \
- editline.3 history_init.3 editline.3 history_end.3 editline.3 history.3
+ editline.3 history_init.3 editline.3 history_end.3 \
+ editline.3 history.3 \
+ editline.3 tok_init.3 editline.3 tok_end.3 editline.3 tok_reset.3 \
+ editline.3 tok_line.3 editline.3 tok_str.3
# For speed and debugging
#SRCS= ${OSRCS} tokenizer.c history.c readline.c
diff -r ecb7b3361ecc -r b606b887a419 lib/libedit/TEST/Makefile
--- a/lib/libedit/TEST/Makefile Fri Dec 05 12:15:49 2003 +0000
+++ b/lib/libedit/TEST/Makefile Fri Dec 05 13:37:48 2003 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.1 2003/10/16 21:41:46 christos Exp $
+# $NetBSD: Makefile,v 1.2 2003/12/05 13:37:48 lukem Exp $
NOMAN=1
PROG=test
@@ -6,4 +6,8 @@
LDADD+=-ledit -ltermcap
DPADD+=${LIBEDIT} ${LIBTERMCAP}
+.ifdef DEBUG
+CPPFLAGS+=-DDEBUG
+.endif
+
.include <bsd.prog.mk>
diff -r ecb7b3361ecc -r b606b887a419 lib/libedit/TEST/test.c
--- a/lib/libedit/TEST/test.c Fri Dec 05 12:15:49 2003 +0000
+++ b/lib/libedit/TEST/test.c Fri Dec 05 13:37:48 2003 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: test.c,v 1.13 2003/08/07 16:44:35 agc Exp $ */
+/* $NetBSD: test.c,v 1.14 2003/12/05 13:37:48 lukem Exp $ */
/*-
* Copyright (c) 1992, 1993
@@ -42,7 +42,7 @@
#if 0
static char sccsid[] = "@(#)test.c 8.1 (Berkeley) 6/4/93";
#else
-__RCSID("$NetBSD: test.c,v 1.13 2003/08/07 16:44:35 agc Exp $");
+__RCSID("$NetBSD: test.c,v 1.14 2003/12/05 13:37:48 lukem Exp $");
#endif
#endif /* not lint && not SCCSID */
@@ -59,7 +59,6 @@
#include <dirent.h>
#include "histedit.h"
-#include "tokenizer.h"
static int continuation = 0;
static EditLine *el = NULL;
@@ -72,8 +71,8 @@
static char *
prompt(EditLine *el)
{
- static char a[] = "Edit$";
- static char b[] = "Edit>";
+ static char a[] = "Edit$ ";
+ static char b[] = "Edit> ";
return (continuation ? b : a);
}
@@ -171,15 +170,35 @@
el_source(el, NULL);
while ((buf = el_gets(el, &num)) != NULL && num != 0) {
- int ac;
+ int ac, cc, co;
+#ifdef DEBUG
+ int i;
+#endif
const char **av;
+ const LineInfo *li;
+ li = el_line(el);
#ifdef DEBUG
- (void) fprintf(stderr, "got %d %s", num, buf);
+ (void) fprintf(stderr, "==> got %d %s", num, buf);
+ (void) fprintf(stderr, " > li `%.*s_%.*s'\n",
+ (li->cursor - li->buffer), li->buffer,
+ (li->lastchar - 1 - li->cursor),
+ (li->cursor >= li->lastchar) ? "" : li->cursor);
+
#endif
if (!continuation && num == 1)
continue;
- ncontinuation = tok_line(tok, buf, &ac, &av) > 0;
+ ac = cc = co = 0;
+ ncontinuation = tok_LineInfo(tok, li, &ac, &av, &cc, &co);
+ if (ncontinuation < 0) {
+ (void) fprintf(stderr, "Internal error\n");
+ continuation = 0;
+ continue;
+ }
+#ifdef DEBUG
+ (void) fprintf(stderr, " > nc %d ac %d cc %d co %d\n",
+ ncontinuation, ac, cc, co);
+#endif
#if 0
if (continuation) {
/*
@@ -200,6 +219,18 @@
continuation = ncontinuation;
ncontinuation = 0;
+ if (continuation)
+ continue;
+#ifdef DEBUG
+ for (i = 0; i < ac; i++) {
+ (void) fprintf(stderr, " > arg# %2d ", i);
+ if (i != cc)
+ (void) fprintf(stderr, "`%s'\n", av[i]);
+ else
+ (void) fprintf(stderr, "`%.*s_%s'\n",
+ co, av[i], av[i] + co);
+ }
+#endif
if (strcmp(av[0], "history") == 0) {
int rv;
diff -r ecb7b3361ecc -r b606b887a419 lib/libedit/editline.3
--- a/lib/libedit/editline.3 Fri Dec 05 12:15:49 2003 +0000
+++ b/lib/libedit/editline.3 Fri Dec 05 13:37:48 2003 +0000
@@ -1,6 +1,6 @@
-.\" $NetBSD: editline.3,v 1.42 2003/11/04 13:22:19 christos Exp $
+.\" $NetBSD: editline.3,v 1.43 2003/12/05 13:37:48 lukem Exp $
.\"
-.\" Copyright (c) 1997-1999 The NetBSD Foundation, Inc.
+.\" Copyright (c) 1997-2003 The NetBSD Foundation, Inc.
.\" All rights reserved.
.\"
.\" This file was contributed to The NetBSD Foundation by Luke Mewburn.
@@ -33,7 +33,7 @@
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
-.Dd October 17, 2003
+.Dd December 5, 2003
.Os
.Dt EDITLINE 3
.Sh NAME
@@ -53,8 +53,13 @@
.Nm el_deletestr ,
.Nm history_init ,
.Nm history_end ,
-.Nm history
-.Nd line editor and history functions
+.Nm history ,
+.Nm tok_init ,
+.Nm tok_end ,
+.Nm tok_reset ,
+.Nm tok_line ,
+.Nm tok_str
+.Nd line editor, history and tokenization functions
.Sh LIBRARY
.Lb libedit
.Sh SYNOPSIS
@@ -93,10 +98,20 @@
.Fn history_end "History *h"
.Ft int
.Fn history "History *h" "HistEvent *ev" "int op" "..."
+.Ft Tokenizer *
+.Fn tok_init "const char *IFS"
+.Ft void
+.Fn tok_end "Tokenizer *t"
+.Ft void
+.Fn tok_reset "Tokenizer *t"
+.Ft int
+.Fn tok_line "Tokenizer *t" "const LineInfo *li" "int *argc" "const char *argv[]" "int *cursorc" "int *cursoro"
+.Ft int
+.Fn tok_str "Tokenizer *t" "const char *str" "int *argc" "const char *argv[]"
.Sh DESCRIPTION
The
.Nm
-library provides generic line editing and history functions,
+library provides generic line editing, history and tokenization functions,
similar to those found in
.Xr sh 1 .
.Pp
@@ -476,6 +491,16 @@
const char *lastchar; /* address of last character */
} LineInfo;
.Ed
+.Pp
+.Fa buffer
+is not NUL terminated.
+This function may be called after
+.Fn el_gets
+to obtain the
+.Fa LineInfo
+structure pertaining to line returned by that function,
+and from within user defined functions added with
+.Dv EL_ADDFN .
.It Fn el_insertstr
Insert
.Fa str
@@ -619,6 +644,73 @@
.Fa ev
is updated to contain more details about the error.
.El
+.Sh TOKENIZATION FUNCTIONS
+The tokenization functions use a common data structure,
+.Fa Tokenizer ,
+which is created by
+.Fn tok_init
+and freed by
+.Fn tok_end .
+.Pp
+The following functions are available:
+.Bl -tag -width 4n
+.It Fn tok_init
+Initialise the tokenizer, and return a data structure
+to be used by all other tokenizer functions.
+.Fa IFS
+contains the Input Field Separators, which defaults to
+<space>, <tab>, and <newline> if
+.Dv NULL .
+.It Fn tok_end
+Clean up and finish with
+.Fa t ,
+assumed to have been created with
+.Fn tok_init .
+.It Fn tok_reset
+Reset the tokenizer state.
+Use after a line has been successfully tokenized
+by
+.Fn tok_line
+or
+.Fn tok_str
+and before a new line is to be tokenized.
+.It Fn tok_line
+Tokenize
+.Fa li ,
+If successful, modify:
+.Fa argv
+to contain the words,
+.Fa argc
+to contain the number of words,
+.Fa cursorc
+(if not
+.Dv NULL )
+to contain the index of the word containing the cursor,
+and
+.Fa cursoro
+(if not
+.Dv NULL )
+to contain the offset within
+.Fa argv[cursorc]
+of the cursor.
+.Pp
+Returns
+0 if successful,
+-1 for an internal error,
+1 for an unmatched single quote,
+2 for an unmatched double quote,
+and
+3 for a backslash quoted <newline>.
+A positive exit code indicates that another line should be read
+and tokenization attempted again.
+.
+.It Fn tok_str
+A simpler form of
+.Fn tok_line ;
+.Fa str
+is a NUL terminated string to tokenize.
+.El
+.
.\"XXX.Sh EXAMPLES
.\"XXX: provide some examples
.Sh SEE ALSO
@@ -653,9 +745,6 @@
.Dv EL_RPROMPT .
Jaromir Dolecek implemented the readline emulation.
.Sh BUGS
-The tokenization functions are not publicly defined in
-.Aq Pa histedit.h .
-.Pp
At this time, it is the responsibility of the caller to
check the result of the
.Dv EL_EDITMODE
diff -r ecb7b3361ecc -r b606b887a419 lib/libedit/histedit.h
--- a/lib/libedit/histedit.h Fri Dec 05 12:15:49 2003 +0000
+++ b/lib/libedit/histedit.h Fri Dec 05 13:37:48 2003 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: histedit.h,v 1.24 2003/10/16 22:26:32 christos Exp $ */
+/* $NetBSD: histedit.h,v 1.25 2003/12/05 13:37:48 lukem Exp $ */
Home |
Main Index |
Thread Index |
Old Index