lib/53983: libedit: Fix types for readline compatibility

[jperkins+netbsd%google.com@localhost]

>Number:         53983
>Category:       lib
>Synopsis:       libedit: Fix types for readline compatibility
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    lib-bug-people
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Fri Feb 15 19:35:00 +0000 2019
>Originator:     Jonathan Perkins
>Release:        Sources as of 2019/02/15
>Organization:
Google
>Environment:
>Description:
As part of working on readline compatibility, I ran into several type differences.  These were mainly in two categories:

1) readline returns int, libedit returns void, causing issues where the return is checked.  (rl_stuff_char, rl_reset_terminal)

2) readline uses "const char*", libedit uses "char*", causing issues where a string literal is assigned to the value.

Because I'm focused on type issues here, I'm also including a type fix in terminal_alloc_buffer, which returns wchar_t**, but should be wint_t**: both the return value b and the storage el_display are wint_t**.
>How-To-Repeat:
The readline type differences can be validated against readline documentation.

Compiling with clang/llvm using -Wincompatible-pointer-types should catch the terminal.c issue.
>Fix:
--- old/lib/libedit/readline/readline.h
+++ new/lib/libedit/readline/readline.h
@@ -99,14 +99,14 @@
 #endif
 extern const char	*rl_library_version;
 extern int		rl_readline_version;
-extern char		*rl_readline_name;
+extern const char	*rl_readline_name;
 extern FILE		*rl_instream;
 extern FILE		*rl_outstream;
 extern char		*rl_line_buffer;
 extern int		 rl_point, rl_end;
 extern int		 history_base, history_length;
 extern int		 max_input_history;
-extern char		*rl_basic_word_break_characters;
+extern const char	*rl_basic_word_break_characters;
 extern char		*rl_completer_word_break_characters;
 extern char		*rl_completer_quote_characters;
 extern rl_compentry_func_t *rl_completion_entry_function;
@@ -115,7 +115,7 @@
 extern int		 rl_attempted_completion_over;
 extern int		rl_completion_type;
 extern int		rl_completion_query_items;
-extern char		*rl_special_prefixes;
+extern const char	*rl_special_prefixes;
 extern int		rl_completion_append_character;
 extern int		rl_inhibit_completion;
 extern Function		*rl_pre_input_hook;
@@ -185,7 +185,7 @@
 
 int		 rl_insert(int, int);
 int		 rl_insert_text(const char *);
-void		 rl_reset_terminal(const char *);
+int		 rl_reset_terminal(const char *);
 void		 rl_resize_terminal(void);
 int		 rl_bind_key(int, rl_command_func_t *);
 int		 rl_newline(int, int);
@@ -199,7 +199,7 @@
 int		 rl_read_init_file(const char *);
 int		 rl_parse_and_bind(const char *);
 int		 rl_variable_bind(const char *, const char *);
-void		 rl_stuff_char(int);
+int		 rl_stuff_char(int);
 int		 rl_add_defun(const char *, rl_command_func_t *, int);
 HISTORY_STATE	*history_get_history_state(void);
 void		 rl_get_screen_size(int *, int *);
diff -r -u old/readline.c new/readline.c
--- old/lib/libedit/readline.c
+++ new/lib/libedit/readline.c
@@ -72,7 +72,7 @@
 static char expand_chars[] = { ' ', '\t', '\n', '=', '(', '\0' };
 static char break_chars[] = { ' ', '\t', '\n', '"', '\\', '\'', '`', '@', '$',
     '>', '<', '=', ';', '|', '&', '{', '(', '\0' };
-char *rl_readline_name = empty;
+const char *rl_readline_name = empty;
 FILE *rl_instream = NULL;
 FILE *rl_outstream = NULL;
 int rl_point = 0;
@@ -105,7 +105,7 @@
 
 int rl_inhibit_completion = 0;
 int rl_attempted_completion_over = 0;
-char *rl_basic_word_break_characters = break_chars;
+const char *rl_basic_word_break_characters = break_chars;
 char *rl_completer_word_break_characters = NULL;
 char *rl_completer_quote_characters = NULL;
 rl_compentry_func_t *rl_completion_entry_function = NULL;
@@ -150,7 +150,7 @@
  * in the parsed text when it is passed to the completion function.
  * Shell uses this to help determine what kind of completing to do.
  */
-char *rl_special_prefixes = NULL;
+const char *rl_special_prefixes = NULL;
 
 /*
  * This is the character appended to the completed words if at the end of
@@ -1885,7 +1885,7 @@
 rl_complete(int ignore __attribute__((__unused__)), int invoking_key)
 {
 	static ct_buffer_t wbreak_conv, sprefix_conv;
-	char *breakchars;
+	const char *breakchars;
 
 	if (h == NULL || e == NULL)
 		rl_initialize();
@@ -1971,13 +1971,14 @@
  * reset the terminal
  */
 /* ARGSUSED */
-void
+int
 rl_reset_terminal(const char *p __attribute__((__unused__)))
 {
 
 	if (h == NULL || e == NULL)
 		rl_initialize();
 	el_reset(e);
+	return 0;
 }
 
 
@@ -2166,7 +2167,7 @@
 	return el_set(e, EL_BIND, "", var, value, NULL) == -1 ? 1 : 0;
 }
 
-void
+int
 rl_stuff_char(int c)
 {
 	char buf[2];
@@ -2174,6 +2175,7 @@
 	buf[0] = (char)c;
 	buf[1] = '\0';
 	el_insertstr(e, buf);
+	return 1;
 }
 
 static int
--- old/lib/libedit/terminal.c
+++ new/lib/libedit/terminal.c
@@ -419,7 +419,7 @@
 	return 0;
 }
 
-static wchar_t **
+static wint_t **
 terminal_alloc_buffer(EditLine *el)
 {
 	wint_t **b;