Subject: HTTPS support patch for /usr/bin/ftp
To: None <current-users@netbsd.org>
From: YAMAMOTO Shigeru <shigeru@iij.ad.jp>
List: current-users
Date: 07/29/2003 14:30:02
----Next_Part(Tue_Jul_29_14:30:02_2003_728)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit


Hi, all,

I make a patch to support HTTPS for /usr/bin/ftp.
It is quick hack.

Please try and test it.

Thanks,
-------
YAMAMOTO Shigeru	<shigeru@iij.ad.jp>

----Next_Part(Tue_Jul_29_14:30:02_2003_728)--
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="lukem.ftp.ssl.patch"

--- Makefile.org	Tue Jul 29 14:14:50 2003
+++ Makefile	Tue Jul 29 14:19:23 2003
@@ -9,8 +9,8 @@
 #
 #CPPFLAGS+=-DGATE_SERVER=\"ftp-gw.host\" # -DGATE_PORT=21
 
-LDADD+=	-ledit -ltermcap -lutil
-DPADD+=	${LIBEDIT} ${LIBTERMCAP} ${LIBUTIL}
+LDADD+=	-ledit -ltermcap -lutil -lssl -lcrypto
+DPADD+=	${LIBEDIT} ${LIBTERMCAP} ${LIBUTIL} ${LIBSSL} ${LIBCRYPTO}
 
 CPPFLAGS+= -DINET6
 
--- fetch.c.org	Tue Jul 29 14:14:57 2003
+++ fetch.c	Tue Jul 29 12:21:29 2003
@@ -71,12 +71,18 @@
 #include <time.h>
 #include <util.h>
 
+#include <openssl/bio.h>
+#include <openssl/ssl.h>
+
 #include "ftp_var.h"
 #include "version.h"
 
+extern	SSL_CTX*	ssl_ctx;	/* SSL CTX */
+
 typedef enum {
 	UNKNOWN_URL_T=-1,
 	HTTP_URL_T,
+	HTTPS_URL_T,
 	FTP_URL_T,
 	FILE_URL_T,
 	CLASSIC_URL_T
@@ -99,6 +105,7 @@
 #define	FILE_URL	"file://"	/* file URL prefix */
 #define	FTP_URL		"ftp://"	/* ftp URL prefix */
 #define	HTTP_URL	"http://"	/* http URL prefix */
+#define	HTTPS_URL	"https://"	/* https URL prefix */
 
 
 /*
@@ -300,6 +307,11 @@
 		*type = HTTP_URL_T;
 		*portnum = HTTP_PORT;
 		tport = httpport;
+	} else if (strncasecmp(url, HTTPS_URL, sizeof(HTTPS_URL) - 1) == 0) {
+		url += sizeof(HTTPS_URL) - 1;
+		*type = HTTPS_URL_T;
+		*portnum = HTTPS_PORT;
+		tport = httpsport;
 	} else if (strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) {
 		url += sizeof(FTP_URL) - 1;
 		*type = FTP_URL_T;
@@ -412,6 +424,170 @@
 	return (0);
 }
 
+static
+int
+isescaped(const char* sp, char* p, int esc) {
+	int	result = 0;
+
+	if (esc == '\0') {
+		result = 1;
+	}
+	else {
+		size_t	ne;
+		char*	cp;
+
+		for (ne = 0, cp = p; --cp >= sp && *cp == esc; ne ++) {
+			continue;
+		}
+
+		result = ((ne & 1) != 0);
+	}
+
+	return(result);
+}
+
+static
+char*
+BIO_parseln(BIO* bio, size_t* len, size_t* lineno, const char delim[3], int flags) {
+	static const char	dstr[3] = { '\\', '\\', '#' };
+	int		cnt = 1;
+	char	esc, con, com;
+	char	nl;
+	char*	buffer = NULL;
+	size_t	length = 0;
+
+	if (delim == NULL) {
+		delim = dstr;
+	}
+
+	esc = delim[0];
+	con = delim[1];
+	com = delim[2];
+
+	nl = '\n';
+
+	cnt = 1;
+	while (cnt) {
+		char	read_buffer[BUFSIZ];
+		size_t	read_length = 0;
+
+		cnt = 0;
+
+		read_length = BIO_gets(bio, read_buffer, sizeof(read_buffer));
+		if (read_length > 0) {
+			if (read_length < sizeof(read_buffer) || read_buffer[read_length - 1] == nl) {
+				if (lineno) {
+					(*lineno) ++;
+				}
+			}
+
+			if (read_length && com) {
+				char*	cp;
+
+				for (cp = read_buffer; cp < read_buffer + read_length; cp ++) {
+					if (*cp == com && !isescaped(read_buffer, cp, esc)) {
+						read_length = cp - read_buffer;
+						cnt = (read_length == 0 && buffer == NULL);
+						break;
+					}
+				}
+			}
+
+			if (read_length && nl) {
+				char*	cp;
+
+				cp = &(read_buffer[read_length - 1]);
+				if (*cp == nl) {
+					read_length --;
+				}
+			}
+
+			if (read_length && con) {
+				char*	cp;
+
+				cp = &(read_buffer[read_length - 1]);
+				if (*cp == con && !isescaped(read_buffer, cp, esc)) {
+					read_length --;
+					cnt = 1;
+				}
+			}
+
+			if (read_length == 0 && buffer != NULL) {
+				continue;
+			}
+			else {
+				char*	new;
+
+				if ((new = realloc(buffer, length + read_length + 1)) == NULL) {
+					free(buffer);
+				}
+
+				buffer = new;
+
+				memcpy(buffer + length, read_buffer, read_length);
+				length += read_length;
+				buffer[length] = '\0';
+			}
+		}
+	}
+
+	if ((flags & FPARSELN_UNESCALL) != 0 && esc && buffer != NULL && strchr(buffer, esc) != NULL) {
+		char*	ptr;
+		char*	cp;
+
+		ptr = cp = buffer;
+		while (cp[0] != '\0') {
+			int	skipesc;
+
+			while (cp[0] != '\0' && cp[0] != esc) {
+				*ptr = *cp;
+				ptr ++;
+				cp ++;
+			}
+
+			if (cp[0] == '\0' || cp[1] == '\0') {
+				break;
+			}
+
+			skipesc = 0;
+			if (cp[1] == com) {
+				skipesc += (flags & FPARSELN_UNESCCOMM);
+			}
+			if (cp[1] == con) {
+				skipesc += (flags & FPARSELN_UNESCCONT);
+			}
+			if (cp[1] == esc) {
+				skipesc += (flags & FPARSELN_UNESCESC);
+			}
+			if (cp[1] == com && cp[1] != con && cp[1] != esc) {
+				skipesc += (flags & FPARSELN_UNESCREST);
+			}
+
+			if (skipesc) {
+				cp ++;
+			}
+			else {
+				*ptr = *cp;
+				ptr ++;
+				cp ++;
+			}
+
+			*ptr = *cp;
+			ptr ++;
+			cp ++;
+		}
+
+		*ptr = '\0';
+		length = strlen(buffer);
+	}
+
+	if (len) {
+		(*len) = length;
+	}
+
+	return(buffer);
+}
+
 sigjmp_buf	httpabort;
 
 /*
@@ -441,14 +617,16 @@
 	char			*puser, *ppass;
 	off_t			hashbytes, rangestart, rangeend, entitylen;
 	int			 (*closefunc)(FILE *);
-	FILE			*fin, *fout;
+	BIO*			bio;
+	FILE			*fout;
 	time_t			mtime;
 	url_t			urltype;
 	in_port_t		portnum;
 
 	oldintr = oldintp = NULL;
 	closefunc = NULL;
-	fin = fout = NULL;
+	bio = (BIO*)0;
+	fout = NULL;
 	s = -1;
 	buf = savefile = NULL;
 	auth = location = message = NULL;
@@ -458,7 +636,7 @@
 
 #ifdef __GNUC__			/* shut up gcc warnings */
 	(void)&closefunc;
-	(void)&fin;
+	(void)&bio;
 	(void)&fout;
 	(void)&buf;
 	(void)&savefile;
@@ -487,7 +665,7 @@
 			rval = fetch_ftp(url);
 			goto cleanup_fetch_url;
 		}
-		if (urltype != HTTP_URL_T || outfile == NULL)  {
+		if (urltype != HTTP_URL_T || urltype != HTTPS_URL_T || outfile == NULL)  {
 			warnx("Invalid URL (no file after host) `%s'", url);
 			goto cleanup_fetch_url;
 		}
@@ -527,18 +705,21 @@
 			restart_point = sb.st_size;
 	}
 	if (urltype == FILE_URL_T) {		/* file:// URLs */
+		int	fd;
+
 		direction = "copied";
-		fin = fopen(decodedpath, "r");
-		if (fin == NULL) {
+		bio = BIO_new_file(decodedpath, "r");
+		if (!bio) {
 			warn("Cannot open file `%s'", decodedpath);
 			goto cleanup_fetch_url;
 		}
-		if (fstat(fileno(fin), &sb) == 0) {
+		BIO_get_fd(bio, &fd);
+		if (fstat(fd, &sb) == 0) {
 			mtime = sb.st_mtime;
 			filesize = sb.st_size;
 		}
 		if (restart_point) {
-			if (lseek(fileno(fin), restart_point, SEEK_SET) < 0) {
+			if (BIO_seek(bio, restart_point) < 0) {
 				warn("Can't lseek to restart `%s'",
 				    decodedpath);
 				goto cleanup_fetch_url;
@@ -556,7 +737,7 @@
 		int hasleading;
 
 		if (proxyenv == NULL) {
-			if (urltype == HTTP_URL_T)
+			if (urltype == HTTP_URL_T || urltype == HTTPS_URL_T)
 				proxyenv = getoptionvalue("http_proxy");
 			else if (urltype == FTP_URL_T)
 				proxyenv = getoptionvalue("ftp_proxy");
@@ -613,6 +794,7 @@
 					goto cleanup_fetch_url;
 
 				if ((purltype != HTTP_URL_T
+				     && purltype != HTTPS_URL_T
 				     && purltype != FTP_URL_T) ||
 				    EMPTYSTRING(phost) ||
 				    (! EMPTYSTRING(ppath)
@@ -698,7 +880,54 @@
 			goto cleanup_fetch_url;
 		}
 
-		fin = fdopen(s, "r+");
+		bio = BIO_new(BIO_s_socket());
+		if (!bio) {
+			warn("Can't create BIO for socket");
+			goto cleanup_fetch_url;
+		}
+		else {
+			BIO_set_fd(bio, s, BIO_CLOSE);
+		}
+
+		if (urltype == HTTPS_URL_T) {
+			BIO*	new;
+			SSL*	ssl;
+
+			ssl = SSL_new(ssl_ctx);
+			if (!ssl) {
+				warn("Can't create SSL for HTTPS");
+				goto cleanup_fetch_url;
+			}
+			else {
+				SSL_set_connect_state(ssl);
+			}
+
+			new = BIO_new(BIO_f_ssl());
+			if (new) {
+				warn("Can't create BIO for SSL");
+				goto cleanup_fetch_url;
+			}
+			else {
+				BIO_set_ssl(new, ssl, BIO_CLOSE);
+				BIO_push(new, bio);
+				bio = new;
+			}
+		}
+
+		{
+			BIO*	new;
+
+			new = BIO_new(BIO_f_buffer());
+			if (new) {
+				warn("Can't create buffered BIO");
+				goto cleanup_fetch_url;
+			}
+			else {
+				BIO_push(new, bio);
+				bio = new;
+			}
+		}
+
 		/*
 		 * Construct and send the request.
 		 */
@@ -713,11 +942,11 @@
 				leading = ", ";
 				hasleading++;
 			}
-			fprintf(fin, "GET %s HTTP/1.0\r\n", path);
+			BIO_printf(bio, "GET %s HTTP/1.0\r\n", path);
 			if (flushcache)
-				fprintf(fin, "Pragma: no-cache\r\n");
+				BIO_printf(bio, "Pragma: no-cache\r\n");
 		} else {
-			fprintf(fin, "GET %s HTTP/1.1\r\n", path);
+			BIO_printf(bio, "GET %s HTTP/1.1\r\n", path);
 			if (strchr(host, ':')) {
 				char *h, *p;
 
@@ -730,18 +959,18 @@
 				    (p = strchr(h, '%')) != NULL) {
 					*p = '\0';
 				}
-				fprintf(fin, "Host: [%s]", h);
+				BIO_printf(bio, "Host: [%s]", h);
 				free(h);
 			} else
-				fprintf(fin, "Host: %s", host);
+				BIO_printf(bio, "Host: %s", host);
 			if (portnum != HTTP_PORT)
-				fprintf(fin, ":%u", portnum);
-			fprintf(fin, "\r\n");
-			fprintf(fin, "Accept: */*\r\n");
-			fprintf(fin, "Connection: close\r\n");
+				BIO_printf(bio, ":%u", portnum);
+			BIO_printf(bio, "\r\n");
+			BIO_printf(bio, "Accept: */*\r\n");
+			BIO_printf(bio, "Connection: close\r\n");
 			if (restart_point) {
 				fputs(leading, ttyout);
-				fprintf(fin, "Range: bytes=" LLF "-\r\n",
+				BIO_printf(bio, "Range: bytes=" LLF "-\r\n",
 				    (LLT)restart_point);
 				fprintf(ttyout, "restarting at " LLF,
 				    (LLT)restart_point);
@@ -749,9 +978,9 @@
 				hasleading++;
 			}
 			if (flushcache)
-				fprintf(fin, "Cache-Control: no-cache\r\n");
+				BIO_printf(bio, "Cache-Control: no-cache\r\n");
 		}
-		fprintf(fin, "User-Agent: %s/%s\r\n", FTP_PRODUCT, FTP_VERSION);
+		BIO_printf(bio, "User-Agent: %s/%s\r\n", FTP_PRODUCT, FTP_VERSION);
 		if (wwwauth) {
 			if (verbose) {
 				fprintf(ttyout, "%swith authorization",
@@ -759,7 +988,7 @@
 				leading = ", ";
 				hasleading++;
 			}
-			fprintf(fin, "Authorization: %s\r\n", wwwauth);
+			BIO_printf(bio, "Authorization: %s\r\n", wwwauth);
 		}
 		if (proxyauth) {
 			if (verbose) {
@@ -768,18 +997,18 @@
 				leading = ", ";
 				hasleading++;
 			}
-			fprintf(fin, "Proxy-Authorization: %s\r\n", proxyauth);
+			BIO_printf(bio, "Proxy-Authorization: %s\r\n", proxyauth);
 		}
 		if (verbose && hasleading)
 			fputs(")\n", ttyout);
-		fprintf(fin, "\r\n");
-		if (fflush(fin) == EOF) {
+		BIO_printf(bio, "\r\n");
+		if (BIO_flush(bio) == EOF) {
 			warn("Writing HTTP request");
 			goto cleanup_fetch_url;
 		}
 
 				/* Read the response */
-		if ((buf = fparseln(fin, &len, NULL, "\0\0\0", 0)) == NULL) {
+		if ((buf = BIO_parseln(bio, &len, NULL, "\0\0\0", 0)) == NULL) {
 			warn("Receiving HTTP reply");
 			goto cleanup_fetch_url;
 		}
@@ -802,7 +1031,7 @@
 				/* Read the rest of the header. */
 		FREEPTR(buf);
 		while (1) {
-			if ((buf = fparseln(fin, &len, NULL, "\0\0\0", 0))
+			if ((buf = BIO_parseln(bio, &len, NULL, "\0\0\0", 0))
 			    == NULL) {
 				warn("Receiving HTTP reply");
 				goto cleanup_fetch_url;
@@ -1114,7 +1343,7 @@
 		chunksize = 0;
 					/* read chunksize */
 		if (ischunked) {
-			if (fgets(xferbuf, bufsize, fin) == NULL) {
+			if (BIO_gets(bio, xferbuf, bufsize) == NULL) {
 				warnx("Unexpected EOF reading chunksize");
 				goto cleanup_fetch_url;
 			}
@@ -1149,8 +1378,7 @@
 			if (ischunked)
 				bufrem = MIN(chunksize, bufrem);
 			while (bufrem > 0) {
-				len = fread(xferbuf, sizeof(char),
-				    MIN(bufsize, bufrem), fin);
+				len = BIO_read(bio, xferbuf, (sizeof(char) * MIN(bufsize, bufrem)));
 				if (len <= 0)
 					goto chunkdone;
 				bytes += len;
@@ -1188,7 +1416,7 @@
 					/* read CRLF after chunk*/
  chunkdone:
 		if (ischunked) {
-			if (fgets(xferbuf, bufsize, fin) == NULL)
+			if (BIO_gets(bio, xferbuf, bufsize) == NULL)
 				break;
 			if (strcmp(xferbuf, "\r\n") != 0) {
 				warnx("Unexpected data following chunk");
@@ -1201,10 +1429,12 @@
 			(void)putc('#', ttyout);
 		(void)putc('\n', ttyout);
 	}
+#if	0	/* XXX: how to re-write? */
 	if (ferror(fin)) {
 		warn("Reading file");
 		goto cleanup_fetch_url;
 	}
+#endif
 	progressmeter(1);
 	bytes = 0;
 	(void)fflush(fout);
@@ -1237,8 +1467,8 @@
 		(void)xsignal(SIGINT, oldintr);
 	if (oldintp)
 		(void)xsignal(SIGPIPE, oldintp);
-	if (fin != NULL)
-		fclose(fin);
+	if (!bio)
+		BIO_free_all(bio);
 	else if (s != -1)
 		close(s);
 	if (closefunc != NULL && fout != NULL)
@@ -1651,6 +1881,7 @@
 	 * Check for file:// and http:// URLs.
 	 */
 	if (strncasecmp(url, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 ||
+	    strncasecmp(url, HTTPS_URL, sizeof(HTTPS_URL) - 1) == 0 ||
 	    strncasecmp(url, FILE_URL, sizeof(FILE_URL) - 1) == 0)
 		return (fetch_url(url, NULL, NULL, NULL));
 
--- ftp_var.h.org	Tue Jul 29 14:15:05 2003
+++ ftp_var.h	Tue Jul 29 11:40:22 2003
@@ -172,6 +172,14 @@
 	FEAT_max
 };
 
+struct	CA_Info {
+	char*	path;
+	char*	rootca;
+	char*	privatekey;
+	char*	certificate;
+};
+typedef	struct CA_Info	CA_Info_t;
+
 
 /*
  * Global defines
@@ -184,6 +192,7 @@
 
 #define	FTP_PORT	21	/* default if ! getservbyname("ftp/tcp") */
 #define	HTTP_PORT	80	/* default if ! getservbyname("http/tcp") */
+#define	HTTPS_PORT	443	/* default if ! getservbyname("https/tcp") */
 #ifndef	GATE_PORT
 #define	GATE_PORT	21	/* default if ! getservbyname("ftpgate/tcp") */
 #endif
@@ -279,7 +288,10 @@
 GLOBAL	sa_family_t family;	/* address family to use for connections */
 GLOBAL	char	*ftpport;	/* port number to use for FTP connections */
 GLOBAL	char	*httpport;	/* port number to use for HTTP connections */
+GLOBAL	char	*httpsport;	/* port number to use for HTTPS connections */
 GLOBAL	char	*gateport;	/* port number to use for gateftp connections */
+
+GLOBAL	CA_Info_t	CAinfo;	/* CA path, certificate and private key files */
 
 GLOBAL	char   *outfile;	/* filename to output URLs to */
 GLOBAL	int	restartautofetch; /* restart auto-fetch */
--- main.c.org	Tue Jul 29 14:18:18 2003
+++ main.c	Tue Jul 29 11:42:04 2003
@@ -129,6 +129,8 @@
 #include <unistd.h>
 #include <locale.h>
 
+#include <openssl/ssl.h>
+
 #define	GLOBAL		/* force GLOBAL decls in ftp_var.h to be declared */
 #include "ftp_var.h"
 
@@ -137,9 +139,20 @@
 #define	NO_PROXY	"no_proxy"	/* env var with list of non-proxied
 					 * hosts, comma or space separated */
 
+GLOBAL	SSL_CTX*	ssl_ctx = (SSL_CTX*)0;
+
 static void	setupoption(char *, char *, char *);
 int		main(int, char *[]);
 
+static
+void
+free_ssl_ctx(void) {
+	if (ssl_ctx) {
+		SSL_CTX_free(ssl_ctx);
+		ssl_ctx = (SSL_CTX*)0;
+	}
+}
+
 int
 main(int argc, char *argv[])
 {
@@ -152,6 +165,7 @@
 
 	ftpport = "ftp";
 	httpport = "http";
+	httpsport = "https";
 	gateport = NULL;
 	cp = getenv("FTPSERVERPORT");
 	if (cp != NULL)
@@ -280,6 +294,12 @@
 		}
 	}
 
+	/* set default key and CAs for SSL */
+	CAinfo.path = getenv("SSL_CAPATH");
+	CAinfo.rootca = getenv("SSL_ROOTCA");
+	CAinfo.privatekey = getenv("SSL_PRIVATEKEY");
+	CAinfo.certificate = getenv("SSL_CERTIFICATE");
+
 	while ((ch = getopt(argc, argv, "46AadefginN:o:pP:r:RtT:u:vV")) != -1) {
 		switch (ch) {
 		case '4':
@@ -490,6 +510,24 @@
 	(void)&argc;
 	(void)&argv;
 #endif
+
+	/* initialize SSL CTX */
+	SSL_library_init();
+	SSL_load_error_strings();
+
+	OpenSSL_add_ssl_algorithms();
+	ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+
+	if (CAinfo.certificate) {
+		SSL_CTX_use_certificate_chain_file(ssl_ctx, CAinfo.certificate);
+	}
+	if (CAinfo.privatekey) {
+		SSL_CTX_use_PrivateKey_file(ssl_ctx, CAinfo.privatekey, SSL_FILETYPE_PEM);
+	}
+	if (CAinfo.path || CAinfo.rootca) {
+		SSL_CTX_load_verify_locations(ssl_ctx, CAinfo.rootca, CAinfo.path);
+	}
+	atexit(free_ssl_ctx);
 
 	if (argc > 0) {
 		if (isupload) {

----Next_Part(Tue_Jul_29_14:30:02_2003_728)----