25d1500 Bundle libfetch — HardenedBSD-pkg

HardenedBSD-pkg

rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9

HardenedBSD Package Manager

Bundle libfetch

Baptiste Daroussin committed 11 years ago

commit 25d1500aed366366aafa90c349d725d8912bc2c2
parent 7e466e7

16 files changed +6708 -5

modified external/Makefile.am

@@ -64,18 +64,24 @@ noinst_HEADERS= expat/amiga/expat_68k.h \

 		uthash/utlist.h \
 		include/tree.h \
 		include/siphash.h \
 		config.h
 		config.h \
 		libfetch/common.h \
 		libfetch/fetch.h \
 		libfetch/httperr.h \
 		libfetch/ftperr.h
 DYNLIBS=		libucl.la \
 			libsqlite.la \
 			libexpat.la \
 			libsbuf.la \
 			libpicosat.la
 			libpicosat.la \
 			libfetch.la
 noinst_LTLIBRARIES=	libucl_static.la \
 			libsqlite_static.la \
 			libexpat_static.la \
 			libsbuf_static.la \
 			libpicosat_static.la
 			libpicosat_static.la \
 			libfetch_static.la
 if LIBELF_BUNDLED
 DYNLIBS+=		libelf.la
 noinst_LTLIBRARIES+=	libelf_static.la

@@ -237,6 +243,16 @@ libsqlite_static_la_SOURCES= $(libsqlite_la_SOURCES)

 libsqlite_static_la_CFLAGS=	$(sqlite_common_cflags) -static
 libsqlite_static_la_LDFLAGS=	-all-static
 libfetch_common_cflags=	-DWITH_SSL -Wno-pointer-sign
 libfetch_la_CFLAGS=	$(libfetch_common_cflags) -shared
 libfetch_la_SOURCES=	libfetch/common.c \
 			libfetch/fetch.c \
 			libfetch/file.c \
 			libfetch/ftp.c \
 			libfetch/http.c
 libfetch_static_la_CFLAGS=	$(libfetch_common_cflags) -static
 libfetch_static_la_SOURCES=	${libfetch_la_SOURCES}
 if HAVE_ELF_ABI
 CLEANFILES=	libelf/libelf_fsize.c \
 			libelf/libelf_msize.c \

@@ -260,4 +276,5 @@ libelf/libelf_msize.c:

 	@m4 -D SRCDIR=$(srcdir)/libelf $(srcdir)/libelf/elf_types.m4 $(srcdir)/libelf/libelf_msize.m4 > $@
 libelf/native-elf-format.h:
 	$(srcdir)/libelf/native-elf-format > libelf/native-elf-format.h
 endif

added external/libfetch/Makefile

@@ -0,0 +1,81 @@

 # $FreeBSD: head/lib/libfetch/Makefile 275024 2014-11-25 11:07:26Z bapt $
 .include <src.opts.mk>
 LIB=		fetch
 CFLAGS+=	-I.
 SRCS=		fetch.c common.c ftp.c http.c file.c \
 		ftperr.h httperr.h
 INCS=		fetch.h
 MAN=		fetch.3
 CLEANFILES=	ftperr.h httperr.h
 .if ${MK_INET6_SUPPORT} != "no"
 CFLAGS+=	-DINET6
 .endif
 .if ${MK_OPENSSL} != "no"
 CFLAGS+=	-DWITH_SSL
 LIBADD+=	ssl crypto
 .else
 LIBADD+=	md
 .endif
 CFLAGS+=	-DFTP_COMBINE_CWDS
 CSTD?=		c99
 SHLIB_MAJOR=    6
 ftperr.h: ftp.errors ${.CURDIR}/Makefile
 	@echo "static struct fetcherr ftp_errlist[] = {" > ${.TARGET}
 	@cat ${.CURDIR}/ftp.errors \
 	  | grep -v ^# \
 	  | sort \
 	  | while read NUM CAT STRING; do \
 	    echo "    { $${NUM}, FETCH_$${CAT}, \"$${STRING}\" },"; \
 	  done >> ${.TARGET}
 	@echo "    { -1, FETCH_UNKNOWN, \"Unknown FTP error\" }" >> ${.TARGET}
 	@echo "};" >> ${.TARGET}
 httperr.h: http.errors ${.CURDIR}/Makefile
 	@echo "static struct fetcherr http_errlist[] = {" > ${.TARGET}
 	@cat ${.CURDIR}/http.errors \
 	  | grep -v ^# \
 	  | sort \
 	  | while read NUM CAT STRING; do \
 	    echo "    { $${NUM}, FETCH_$${CAT}, \"$${STRING}\" },"; \
 	  done >> ${.TARGET}
 	@echo "    { -1, FETCH_UNKNOWN, \"Unknown HTTP error\" }" >> ${.TARGET}
 	@echo "};" >> ${.TARGET}
 MLINKS+= fetch.3 fetchFreeURL.3
 MLINKS+= fetch.3 fetchGet.3
 MLINKS+= fetch.3 fetchGetFTP.3
 MLINKS+= fetch.3 fetchGetFile.3
 MLINKS+= fetch.3 fetchGetHTTP.3
 MLINKS+= fetch.3 fetchGetURL.3
 MLINKS+= fetch.3 fetchList.3
 MLINKS+= fetch.3 fetchListFTP.3
 MLINKS+= fetch.3 fetchListFile.3
 MLINKS+= fetch.3 fetchListHTTP.3
 MLINKS+= fetch.3 fetchListURL.3
 MLINKS+= fetch.3 fetchMakeURL.3
 MLINKS+= fetch.3 fetchParseURL.3
 MLINKS+= fetch.3 fetchPut.3
 MLINKS+= fetch.3 fetchPutFTP.3
 MLINKS+= fetch.3 fetchPutFile.3
 MLINKS+= fetch.3 fetchPutHTTP.3
 MLINKS+= fetch.3 fetchPutURL.3
 MLINKS+= fetch.3 fetchStat.3
 MLINKS+= fetch.3 fetchStatFTP.3
 MLINKS+= fetch.3 fetchStatFile.3
 MLINKS+= fetch.3 fetchStatHTTP.3
 MLINKS+= fetch.3 fetchStatURL.3
 MLINKS+= fetch.3 fetchXGet.3
 MLINKS+= fetch.3 fetchXGetFTP.3
 MLINKS+= fetch.3 fetchXGetFile.3
 MLINKS+= fetch.3 fetchXGetHTTP.3
 MLINKS+= fetch.3 fetchXGetURL.3
 .include <bsd.lib.mk>

added external/libfetch/common.c

@@ -0,0 +1,1387 @@

 /*-
  * Copyright (c) 1998-2014 Dag-Erling Smørgrav
  * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD: head/lib/libfetch/common.c 273124 2014-10-15 07:35:50Z des $");
 #include <sys/param.h>
 #include <sys/socket.h>
 #include <sys/time.h>
 #include <sys/uio.h>
 #include <netinet/in.h>
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <netdb.h>
 #include <poll.h>
 #include <pwd.h>
 #include <stdarg.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 #ifdef WITH_SSL
 #include <openssl/x509v3.h>
 #endif
 #include "fetch.h"
 #include "common.h"
 /*** Local data **************************************************************/
 /*
  * Error messages for resolver errors
  */
 static struct fetcherr netdb_errlist[] = {
 #ifdef EAI_NODATA
 	{ EAI_NODATA,	FETCH_RESOLV,	"Host not found" },
 #endif
 	{ EAI_AGAIN,	FETCH_TEMP,	"Transient resolver failure" },
 	{ EAI_FAIL,	FETCH_RESOLV,	"Non-recoverable resolver failure" },
 	{ EAI_NONAME,	FETCH_RESOLV,	"No address record" },
 	{ -1,		FETCH_UNKNOWN,	"Unknown resolver error" }
 };
 /* End-of-Line */
 static const char ENDL[2] = "\r\n";
 /*** Error-reporting functions ***********************************************/
 /*
  * Map error code to string
  */
 static struct fetcherr *
 fetch_finderr(struct fetcherr *p, int e)
 {
 	while (p->num != -1 && p->num != e)
 		p++;
 	return (p);
 }
 /*
  * Set error code
  */
 void
 fetch_seterr(struct fetcherr *p, int e)
 {
 	p = fetch_finderr(p, e);
 	fetchLastErrCode = p->cat;
 	snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string);
 }
 /*
  * Set error code according to errno
  */
 void
 fetch_syserr(void)
 {
 	switch (errno) {
 	case 0:
 		fetchLastErrCode = FETCH_OK;
 		break;
 	case EPERM:
 	case EACCES:
 	case EROFS:
 	case EAUTH:
 	case ENEEDAUTH:
 		fetchLastErrCode = FETCH_AUTH;
 		break;
 	case ENOENT:
 	case EISDIR: /* XXX */
 		fetchLastErrCode = FETCH_UNAVAIL;
 		break;
 	case ENOMEM:
 		fetchLastErrCode = FETCH_MEMORY;
 		break;
 	case EBUSY:
 	case EAGAIN:
 		fetchLastErrCode = FETCH_TEMP;
 		break;
 	case EEXIST:
 		fetchLastErrCode = FETCH_EXISTS;
 		break;
 	case ENOSPC:
 		fetchLastErrCode = FETCH_FULL;
 		break;
 	case EADDRINUSE:
 	case EADDRNOTAVAIL:
 	case ENETDOWN:
 	case ENETUNREACH:
 	case ENETRESET:
 	case EHOSTUNREACH:
 		fetchLastErrCode = FETCH_NETWORK;
 		break;
 	case ECONNABORTED:
 	case ECONNRESET:
 		fetchLastErrCode = FETCH_ABORT;
 		break;
 	case ETIMEDOUT:
 		fetchLastErrCode = FETCH_TIMEOUT;
 		break;
 	case ECONNREFUSED:
 	case EHOSTDOWN:
 		fetchLastErrCode = FETCH_DOWN;
 		break;
 default:
 		fetchLastErrCode = FETCH_UNKNOWN;
 	}
 	snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno));
 }
 /*
  * Emit status message
  */
 void
 fetch_info(const char *fmt, ...)
 {
 	va_list ap;
 	va_start(ap, fmt);
 	vfprintf(stderr, fmt, ap);
 	va_end(ap);
 	fputc('\n', stderr);
 }
 /*** Network-related utility functions ***************************************/
 /*
  * Return the default port for a scheme
  */
 int
 fetch_default_port(const char *scheme)
 {
 	struct servent *se;
 	if ((se = getservbyname(scheme, "tcp")) != NULL)
 		return (ntohs(se->s_port));
 	if (strcasecmp(scheme, SCHEME_FTP) == 0)
 		return (FTP_DEFAULT_PORT);
 	if (strcasecmp(scheme, SCHEME_HTTP) == 0)
 		return (HTTP_DEFAULT_PORT);
 	return (0);
 }
 /*
  * Return the default proxy port for a scheme
  */
 int
 fetch_default_proxy_port(const char *scheme)
 {
 	if (strcasecmp(scheme, SCHEME_FTP) == 0)
 		return (FTP_DEFAULT_PROXY_PORT);
 	if (strcasecmp(scheme, SCHEME_HTTP) == 0)
 		return (HTTP_DEFAULT_PROXY_PORT);
 	return (0);
 }
 /*
  * Create a connection for an existing descriptor.
  */
 conn_t *
 fetch_reopen(int sd)
 {
 	conn_t *conn;
 	int opt = 1;
 	/* allocate and fill connection structure */
 	if ((conn = calloc(1, sizeof(*conn))) == NULL)
 		return (NULL);
 	fcntl(sd, F_SETFD, FD_CLOEXEC);
 	setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, &opt, sizeof opt);
 	conn->sd = sd;
 	++conn->ref;
 	return (conn);
 }
 /*
  * Bump a connection's reference count.
  */
 conn_t *
 fetch_ref(conn_t *conn)
 {
 	++conn->ref;
 	return (conn);
 }
 /*
  * Bind a socket to a specific local address
  */
 int
 fetch_bind(int sd, int af, const char *addr)
 {
 	struct addrinfo hints, *res, *res0;
 	int err;
 	memset(&hints, 0, sizeof(hints));
 	hints.ai_family = af;
 	hints.ai_socktype = SOCK_STREAM;
 	hints.ai_protocol = 0;
 	if ((err = getaddrinfo(addr, NULL, &hints, &res0)) != 0)
 		return (-1);
 	for (res = res0; res; res = res->ai_next)
 		if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)
 			return (0);
 	return (-1);
 }
 /*
  * Establish a TCP connection to the specified port on the specified host.
  */
 conn_t *
 fetch_connect(const char *host, int port, int af, int verbose)
 {
 	conn_t *conn;
 	char pbuf[10];
 	const char *bindaddr;
 	struct addrinfo hints, *res, *res0;
 	int sd, err;
 	DEBUG(fprintf(stderr, "---> %s:%d\n", host, port));
 	if (verbose)
 		fetch_info("looking up %s", host);
 	/* look up host name and set up socket address structure */
 	snprintf(pbuf, sizeof(pbuf), "%d", port);
 	memset(&hints, 0, sizeof(hints));
 	hints.ai_family = af;
 	hints.ai_socktype = SOCK_STREAM;
 	hints.ai_protocol = 0;
 	if ((err = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
 		netdb_seterr(err);
 		return (NULL);
 	}
 	bindaddr = getenv("FETCH_BIND_ADDRESS");
 	if (verbose)
 		fetch_info("connecting to %s:%d", host, port);
 	/* try to connect */
 	for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
 		if ((sd = socket(res->ai_family, res->ai_socktype,
 			 res->ai_protocol)) == -1)
 			continue;
 		if (bindaddr != NULL && *bindaddr != '\0' &&
 		    fetch_bind(sd, res->ai_family, bindaddr) != 0) {
 			fetch_info("failed to bind to '%s'", bindaddr);
 			close(sd);
 			continue;
 		}
 		if (connect(sd, res->ai_addr, res->ai_addrlen) == 0 &&
 		    fcntl(sd, F_SETFL, O_NONBLOCK) == 0)
 			break;
 		close(sd);
 	}
 	freeaddrinfo(res0);
 	if (sd == -1) {
 		fetch_syserr();
 		return (NULL);
 	}
 	if ((conn = fetch_reopen(sd)) == NULL) {
 		fetch_syserr();
 		close(sd);
 	}
 	return (conn);
 }
 #ifdef WITH_SSL
 /*
  * Convert characters A-Z to lowercase (intentionally avoid any locale
  * specific conversions).
  */
 static char
 fetch_ssl_tolower(char in)
 {
 	if (in >= 'A' && in <= 'Z')
 		return (in + 32);
 	else
 		return (in);
 }
 /*
  * isalpha implementation that intentionally avoids any locale specific
  * conversions.
  */
 static int
 fetch_ssl_isalpha(char in)
 {
 	return ((in >= 'A' && in <= 'Z') || (in >= 'a' && in <= 'z'));
 }
 /*
  * Check if passed hostnames a and b are equal.
  */
 static int
 fetch_ssl_hname_equal(const char *a, size_t alen, const char *b,
     size_t blen)
 {
 	size_t i;
 	if (alen != blen)
 		return (0);
 	for (i = 0; i < alen; ++i) {
 		if (fetch_ssl_tolower(a[i]) != fetch_ssl_tolower(b[i]))
 			return (0);
 	}
 	return (1);
 }
 /*
  * Check if domain label is traditional, meaning that only A-Z, a-z, 0-9
  * and '-' (hyphen) are allowed. Hyphens have to be surrounded by alpha-
  * numeric characters. Double hyphens (like they're found in IDN a-labels
  * 'xn--') are not allowed. Empty labels are invalid.
  */
 static int
 fetch_ssl_is_trad_domain_label(const char *l, size_t len, int wcok)
 {
 	size_t i;
 	if (!len || l[0] == '-' || l[len-1] == '-')
 		return (0);
 	for (i = 0; i < len; ++i) {
 		if (!isdigit(l[i]) &&
 		    !fetch_ssl_isalpha(l[i]) &&
 		    !(l[i] == '*' && wcok) &&
 		    !(l[i] == '-' && l[i - 1] != '-'))
 			return (0);
 	}
 	return (1);
 }
 /*
  * Check if host name consists only of numbers. This might indicate an IP
  * address, which is not a good idea for CN wildcard comparison.
  */
 static int
 fetch_ssl_hname_is_only_numbers(const char *hostname, size_t len)
 {
 	size_t i;
 	for (i = 0; i < len; ++i) {
 		if (!((hostname[i] >= '0' && hostname[i] <= '9') ||
 		    hostname[i] == '.'))
 			return (0);
 	}
 	return (1);
 }
 /*
  * Check if the host name h passed matches the pattern passed in m which
  * is usually part of subjectAltName or CN of a certificate presented to
  * the client. This includes wildcard matching. The algorithm is based on
  * RFC6125, sections 6.4.3 and 7.2, which clarifies RFC2818 and RFC3280.
  */
 static int
 fetch_ssl_hname_match(const char *h, size_t hlen, const char *m,
     size_t mlen)
 {
 	int delta, hdotidx, mdot1idx, wcidx;
 	const char *hdot, *mdot1, *mdot2;
 	const char *wc; /* wildcard */
 	if (!(h && *h && m && *m))
 		return (0);
 	if ((wc = strnstr(m, "*", mlen)) == NULL)
 		return (fetch_ssl_hname_equal(h, hlen, m, mlen));
 	wcidx = wc - m;
 	/* hostname should not be just dots and numbers */
 	if (fetch_ssl_hname_is_only_numbers(h, hlen))
 		return (0);
 	/* only one wildcard allowed in pattern */
 	if (strnstr(wc + 1, "*", mlen - wcidx - 1) != NULL)
 		return (0);
 	/*
 	 * there must be at least two more domain labels and
 	 * wildcard has to be in the leftmost label (RFC6125)
 	 */
 	mdot1 = strnstr(m, ".", mlen);
 	if (mdot1 == NULL || mdot1 < wc || (mlen - (mdot1 - m)) < 4)
 		return (0);
 	mdot1idx = mdot1 - m;
 	mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1);
 	if (mdot2 == NULL || (mlen - (mdot2 - m)) < 2)
 		return (0);
 	/* hostname must contain a dot and not be the 1st char */
 	hdot = strnstr(h, ".", hlen);
 	if (hdot == NULL || hdot == h)
 		return (0);
 	hdotidx = hdot - h;
 	/*
 	 * host part of hostname must be at least as long as
 	 * pattern it's supposed to match
 	 */
 	if (hdotidx < mdot1idx)
 		return (0);
 	/*
 	 * don't allow wildcards in non-traditional domain names
 	 * (IDN, A-label, U-label...)
 	 */
 	if (!fetch_ssl_is_trad_domain_label(h, hdotidx, 0) ||
 	    !fetch_ssl_is_trad_domain_label(m, mdot1idx, 1))
 		return (0);
 	/* match domain part (part after first dot) */
 	if (!fetch_ssl_hname_equal(hdot, hlen - hdotidx, mdot1,
 	    mlen - mdot1idx))
 		return (0);
 	/* match part left of wildcard */
 	if (!fetch_ssl_hname_equal(h, wcidx, m, wcidx))
 		return (0);
 	/* match part right of wildcard */
 	delta = mdot1idx - wcidx - 1;
 	if (!fetch_ssl_hname_equal(hdot - delta, delta,
 	    mdot1 - delta, delta))
 		return (0);
 	/* all tests succeded, it's a match */
 	return (1);
 }
 /*
  * Get numeric host address info - returns NULL if host was not an IP
  * address. The caller is responsible for deallocation using
  * freeaddrinfo(3).
  */
 static struct addrinfo *
 fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len)
 {
 	struct addrinfo hints, *res;
 	char *host;
 	host = (char *)malloc(len + 1);
 	memcpy(host, hostname, len);
 	host[len] = '\0';
 	memset(&hints, 0, sizeof(hints));
 	hints.ai_family = PF_UNSPEC;
 	hints.ai_socktype = SOCK_STREAM;
 	hints.ai_protocol = 0;
 	hints.ai_flags = AI_NUMERICHOST;
 	/* port is not relevant for this purpose */
 	getaddrinfo(host, "443", &hints, &res);
 	free(host);
 	return res;
 }
 /*
  * Compare ip address in addrinfo with address passes.
  */
 static int
 fetch_ssl_ipaddr_match_bin(const struct addrinfo *lhost, const char *rhost,
     size_t rhostlen)
 {
 	const void *left;
 	if (lhost->ai_family == AF_INET && rhostlen == 4) {
 		left = (void *)&((struct sockaddr_in*)(void *)
 		    lhost->ai_addr)->sin_addr.s_addr;
 #ifdef INET6
 	} else if (lhost->ai_family == AF_INET6 && rhostlen == 16) {
 		left = (void *)&((struct sockaddr_in6 *)(void *)
 		    lhost->ai_addr)->sin6_addr;
 #endif
 	} else
 		return (0);
 	return (!memcmp(left, (const void *)rhost, rhostlen) ? 1 : 0);
 }
 /*
  * Compare ip address in addrinfo with host passed. If host is not an IP
  * address, comparison will fail.
  */
 static int
 fetch_ssl_ipaddr_match(const struct addrinfo *laddr, const char *r,
     size_t rlen)
 {
 	struct addrinfo *raddr;
 	int ret;
 	char *rip;
 	ret = 0;
 	if ((raddr = fetch_ssl_get_numeric_addrinfo(r, rlen)) == NULL)
 		return 0; /* not a numeric host */
 	if (laddr->ai_family == raddr->ai_family) {
 		if (laddr->ai_family == AF_INET) {
 			rip = (char *)&((struct sockaddr_in *)(void *)
 			    raddr->ai_addr)->sin_addr.s_addr;
 			ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 4);
 #ifdef INET6
 		} else if (laddr->ai_family == AF_INET6) {
 			rip = (char *)&((struct sockaddr_in6 *)(void *)
 			    raddr->ai_addr)->sin6_addr;
 			ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 16);
 #endif
 		}
 	}
 	freeaddrinfo(raddr);
 	return (ret);
 }
 /*
  * Verify server certificate by subjectAltName.
  */
 static int
 fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames,
     const char *host, struct addrinfo *ip)
 {
 	const GENERAL_NAME *name;
 	size_t nslen;
 	int i;
 	const char *ns;
 	for (i = 0; i < sk_GENERAL_NAME_num(altnames); ++i) {
 #if OPENSSL_VERSION_NUMBER < 0x10000000L
 		/*
 		 * This is a workaround, since the following line causes
 		 * alignment issues in clang:
 		 * name = sk_GENERAL_NAME_value(altnames, i);
 		 * OpenSSL explicitly warns not to use those macros
 		 * directly, but there isn't much choice (and there
 		 * shouldn't be any ill side effects)
 		 */
 		name = (GENERAL_NAME *)SKM_sk_value(void, altnames, i);
 #else
 		name = sk_GENERAL_NAME_value(altnames, i);
 #endif
 		ns = (const char *)ASN1_STRING_data(name->d.ia5);
 		nslen = (size_t)ASN1_STRING_length(name->d.ia5);
 		if (name->type == GEN_DNS && ip == NULL &&
 		    fetch_ssl_hname_match(host, strlen(host), ns, nslen))
 			return (1);
 		else if (name->type == GEN_IPADD && ip != NULL &&
 		    fetch_ssl_ipaddr_match_bin(ip, ns, nslen))
 			return (1);
 	}
 	return (0);
 }
 /*
  * Verify server certificate by CN.
  */
 static int
 fetch_ssl_verify_cn(X509_NAME *subject, const char *host,
     struct addrinfo *ip)
 {
 	ASN1_STRING *namedata;
 	X509_NAME_ENTRY *nameentry;
 	int cnlen, lastpos, loc, ret;
 	unsigned char *cn;
 	ret = 0;
 	lastpos = -1;
 	loc = -1;
 	cn = NULL;
 	/* get most specific CN (last entry in list) and compare */
 	while ((lastpos = X509_NAME_get_index_by_NID(subject,
 	    NID_commonName, lastpos)) != -1)
 		loc = lastpos;
 	if (loc > -1) {
 		nameentry = X509_NAME_get_entry(subject, loc);
 		namedata = X509_NAME_ENTRY_get_data(nameentry);
 		cnlen = ASN1_STRING_to_UTF8(&cn, namedata);
 		if (ip == NULL &&
 		    fetch_ssl_hname_match(host, strlen(host), cn, cnlen))
 			ret = 1;
 		else if (ip != NULL && fetch_ssl_ipaddr_match(ip, cn, cnlen))
 			ret = 1;
 		OPENSSL_free(cn);
 	}
 	return (ret);
 }
 /*
  * Verify that server certificate subjectAltName/CN matches
  * hostname. First check, if there are alternative subject names. If yes,
  * those have to match. Only if those don't exist it falls back to
  * checking the subject's CN.
  */
 static int
 fetch_ssl_verify_hname(X509 *cert, const char *host)
 {
 	struct addrinfo *ip;
 	STACK_OF(GENERAL_NAME) *altnames;
 	X509_NAME *subject;
 	int ret;
 	ret = 0;
 	ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host));
 	altnames = X509_get_ext_d2i(cert, NID_subject_alt_name,
 	    NULL, NULL);
 	if (altnames != NULL) {
 		ret = fetch_ssl_verify_altname(altnames, host, ip);
 	} else {
 		subject = X509_get_subject_name(cert);
 		if (subject != NULL)
 			ret = fetch_ssl_verify_cn(subject, host, ip);
 	}
 	if (ip != NULL)
 		freeaddrinfo(ip);
 	if (altnames != NULL)
 		GENERAL_NAMES_free(altnames);
 	return (ret);
 }
 /*
  * Configure transport security layer based on environment.
  */
 static void
 fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose)
 {
 	long ssl_ctx_options;
 	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_TICKET;
 	if (getenv("SSL_ALLOW_SSL2") == NULL)
 		ssl_ctx_options |= SSL_OP_NO_SSLv2;
 	if (getenv("SSL_ALLOW_SSL3") == NULL)
 		ssl_ctx_options |= SSL_OP_NO_SSLv3;
 	if (getenv("SSL_NO_TLS1") != NULL)
 		ssl_ctx_options |= SSL_OP_NO_TLSv1;
 	if (getenv("SSL_NO_TLS1_1") != NULL)
 		ssl_ctx_options |= SSL_OP_NO_TLSv1_1;
 	if (getenv("SSL_NO_TLS1_2") != NULL)
 		ssl_ctx_options |= SSL_OP_NO_TLSv1_2;
 	if (verbose)
 		fetch_info("SSL options: %lx", ssl_ctx_options);
 	SSL_CTX_set_options(ctx, ssl_ctx_options);
 }
 /*
  * Configure peer verification based on environment.
  */
 #define LOCAL_CERT_FILE	"/usr/local/etc/ssl/cert.pem"
 #define BASE_CERT_FILE	"/etc/ssl/cert.pem"
 static int
 fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose)
 {
 	X509_LOOKUP *crl_lookup;
 	X509_STORE *crl_store;
 	const char *ca_cert_file, *ca_cert_path, *crl_file;
 	if (getenv("SSL_NO_VERIFY_PEER") == NULL) {
 		ca_cert_file = getenv("SSL_CA_CERT_FILE");
 		if (ca_cert_file == NULL &&
 		    access(LOCAL_CERT_FILE, R_OK) == 0)
 			ca_cert_file = LOCAL_CERT_FILE;
 		if (ca_cert_file == NULL)
 			ca_cert_file = BASE_CERT_FILE;
 		ca_cert_path = getenv("SSL_CA_CERT_PATH");
 		if (verbose) {
 			fetch_info("Peer verification enabled");
 			if (ca_cert_file != NULL)
 				fetch_info("Using CA cert file: %s",
 				    ca_cert_file);
 			if (ca_cert_path != NULL)
 				fetch_info("Using CA cert path: %s",
 				    ca_cert_path);
 		}
 		SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER,
 		    fetch_ssl_cb_verify_crt);
 		SSL_CTX_load_verify_locations(ctx, ca_cert_file,
 		    ca_cert_path);
 		if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) {
 			if (verbose)
 				fetch_info("Using CRL file: %s", crl_file);
 			crl_store = SSL_CTX_get_cert_store(ctx);
 			crl_lookup = X509_STORE_add_lookup(crl_store,
 			    X509_LOOKUP_file());
 			if (crl_lookup == NULL ||
 			    !X509_load_crl_file(crl_lookup, crl_file,
 				X509_FILETYPE_PEM)) {
 				fprintf(stderr,
 				    "Could not load CRL file %s\n",
 				    crl_file);
 				return (0);
 			}
 			X509_STORE_set_flags(crl_store,
 			    X509_V_FLAG_CRL_CHECK |
 			    X509_V_FLAG_CRL_CHECK_ALL);
 		}
 	}
 	return (1);
 }
 /*
  * Configure client certificate based on environment.
  */
 static int
 fetch_ssl_setup_client_certificate(SSL_CTX *ctx, int verbose)
 {
 	const char *client_cert_file, *client_key_file;
 	if ((client_cert_file = getenv("SSL_CLIENT_CERT_FILE")) != NULL) {
 		client_key_file = getenv("SSL_CLIENT_KEY_FILE") != NULL ?
 		    getenv("SSL_CLIENT_KEY_FILE") : client_cert_file;
 		if (verbose) {
 			fetch_info("Using client cert file: %s",
 			    client_cert_file);
 			fetch_info("Using client key file: %s",
 			    client_key_file);
 		}
 		if (SSL_CTX_use_certificate_chain_file(ctx,
 			client_cert_file) != 1) {
 			fprintf(stderr,
 			    "Could not load client certificate %s\n",
 			    client_cert_file);
 			return (0);
 		}
 		if (SSL_CTX_use_PrivateKey_file(ctx, client_key_file,
 			SSL_FILETYPE_PEM) != 1) {
 			fprintf(stderr,
 			    "Could not load client key %s\n",
 			    client_key_file);
 			return (0);
 		}
 	}
 	return (1);
 }
 /*
  * Callback for SSL certificate verification, this is called on server
  * cert verification. It takes no decision, but informs the user in case
  * verification failed.
  */
 int
 fetch_ssl_cb_verify_crt(int verified, X509_STORE_CTX *ctx)
 {
 	X509 *crt;
 	X509_NAME *name;
 	char *str;
 	str = NULL;
 	if (!verified) {
 		if ((crt = X509_STORE_CTX_get_current_cert(ctx)) != NULL &&
 		    (name = X509_get_subject_name(crt)) != NULL)
 			str = X509_NAME_oneline(name, 0, 0);
 		fprintf(stderr, "Certificate verification failed for %s\n",
 		    str != NULL ? str : "no relevant certificate");
 		OPENSSL_free(str);
 	}
 	return (verified);
 }
 #endif
 /*
  * Enable SSL on a connection.
  */
 int
 fetch_ssl(conn_t *conn, const struct url *URL, int verbose)
 {
 #ifdef WITH_SSL
 	int ret, ssl_err;
 	X509_NAME *name;
 	char *str;
 	/* Init the SSL library and context */
 	if (!SSL_library_init()){
 		fprintf(stderr, "SSL library init failed\n");
 		return (-1);
 	}
 	SSL_load_error_strings();
 	conn->ssl_meth = SSLv23_client_method();
 	conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);
 	SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
 	fetch_ssl_setup_transport_layer(conn->ssl_ctx, verbose);
 	if (!fetch_ssl_setup_peer_verification(conn->ssl_ctx, verbose))
 		return (-1);
 	if (!fetch_ssl_setup_client_certificate(conn->ssl_ctx, verbose))
 		return (-1);
 	conn->ssl = SSL_new(conn->ssl_ctx);
 	if (conn->ssl == NULL) {
 		fprintf(stderr, "SSL context creation failed\n");
 		return (-1);
 	}
 	SSL_set_fd(conn->ssl, conn->sd);
 #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
 	if (!SSL_set_tlsext_host_name(conn->ssl,
 	    __DECONST(struct url *, URL)->host)) {
 		fprintf(stderr,
 		    "TLS server name indication extension failed for host %s\n",
 		    URL->host);
 		return (-1);
 	}
 #endif
 	while ((ret = SSL_connect(conn->ssl)) == -1) {
 		ssl_err = SSL_get_error(conn->ssl, ret);
 		if (ssl_err != SSL_ERROR_WANT_READ &&
 		    ssl_err != SSL_ERROR_WANT_WRITE) {
 			ERR_print_errors_fp(stderr);
 			return (-1);
 		}
 	}
 	conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
 	if (conn->ssl_cert == NULL) {
 		fprintf(stderr, "No server SSL certificate\n");
 		return (-1);
 	}
 	if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) {
 		if (verbose)
 			fetch_info("Verify hostname");
 		if (!fetch_ssl_verify_hname(conn->ssl_cert, URL->host)) {
 			fprintf(stderr,
 			    "SSL certificate subject doesn't match host %s\n",
 			    URL->host);
 			return (-1);
 		}
 	}
 	if (verbose) {
 		fetch_info("%s connection established using %s",
 		    SSL_get_version(conn->ssl), SSL_get_cipher(conn->ssl));
 		name = X509_get_subject_name(conn->ssl_cert);
 		str = X509_NAME_oneline(name, 0, 0);
 		fetch_info("Certificate subject: %s", str);
 		OPENSSL_free(str);
 		name = X509_get_issuer_name(conn->ssl_cert);
 		str = X509_NAME_oneline(name, 0, 0);
 		fetch_info("Certificate issuer: %s", str);
 		OPENSSL_free(str);
 	}
 	return (0);
 #else
 	(void)conn;
 	(void)verbose;
 	fprintf(stderr, "SSL support disabled\n");
 	return (-1);
 #endif
 }
 #define FETCH_READ_WAIT		-2
 #define FETCH_READ_ERROR	-1
 #define FETCH_READ_DONE		 0
 #ifdef WITH_SSL
 static ssize_t
 fetch_ssl_read(SSL *ssl, char *buf, size_t len)
 {
 	ssize_t rlen;
 	int ssl_err;
 	rlen = SSL_read(ssl, buf, len);
 	if (rlen < 0) {
 		ssl_err = SSL_get_error(ssl, rlen);
 		if (ssl_err == SSL_ERROR_WANT_READ ||
 		    ssl_err == SSL_ERROR_WANT_WRITE) {
 			return (FETCH_READ_WAIT);
 		} else {
 			ERR_print_errors_fp(stderr);
 			return (FETCH_READ_ERROR);
 		}
 	}
 	return (rlen);
 }
 #endif
 static ssize_t
 fetch_socket_read(int sd, char *buf, size_t len)
 {
 	ssize_t rlen;
 	rlen = read(sd, buf, len);
 	if (rlen < 0) {
 		if (errno == EAGAIN || (errno == EINTR && fetchRestartCalls))
 			return (FETCH_READ_WAIT);
 		else
 			return (FETCH_READ_ERROR);
 	}
 	return (rlen);
 }
 /*
  * Read a character from a connection w/ timeout
  */
 ssize_t
 fetch_read(conn_t *conn, char *buf, size_t len)
 {
 	struct timeval now, timeout, delta;
 	struct pollfd pfd;
 	ssize_t rlen;
 	int deltams;
 	if (fetchTimeout > 0) {
 		gettimeofday(&timeout, NULL);
 		timeout.tv_sec += fetchTimeout;
 	}
 	deltams = INFTIM;
 	memset(&pfd, 0, sizeof pfd);
 	pfd.fd = conn->sd;
 	pfd.events = POLLIN | POLLERR;
 	for (;;) {
 		/*
 		 * The socket is non-blocking.  Instead of the canonical
 		 * poll() -> read(), we do the following:
 		 *
 		 * 1) call read() or SSL_read().
 		 * 2) if we received some data, return it.
 		 * 3) if an error occurred, return -1.
 		 * 4) if read() or SSL_read() signaled EOF, return.
 		 * 5) if we did not receive any data but we're not at EOF,
 		 *    call poll().
 		 *
 		 * In the SSL case, this is necessary because if we
 		 * receive a close notification, we have to call
 		 * SSL_read() one additional time after we've read
 		 * everything we received.
 		 *
 		 * In the non-SSL case, it may improve performance (very
 		 * slightly) when reading small amounts of data.
 		 */
 #ifdef WITH_SSL
 		if (conn->ssl != NULL)
 			rlen = fetch_ssl_read(conn->ssl, buf, len);
 		else
 #endif
 			rlen = fetch_socket_read(conn->sd, buf, len);
 		if (rlen >= 0) {
 			break;
 		} else if (rlen == FETCH_READ_ERROR) {
 			fetch_syserr();
 			return (-1);
 		}
 		// assert(rlen == FETCH_READ_WAIT);
 		if (fetchTimeout > 0) {
 			gettimeofday(&now, NULL);
 			if (!timercmp(&timeout, &now, >)) {
 				errno = ETIMEDOUT;
 				fetch_syserr();
 				return (-1);
 			}
 			timersub(&timeout, &now, &delta);
 			deltams = delta.tv_sec * 1000 +
 			    delta.tv_usec / 1000;;
 		}
 		errno = 0;
 		pfd.revents = 0;
 		if (poll(&pfd, 1, deltams) < 0) {
 			if (errno == EINTR && fetchRestartCalls)
 				continue;
 			fetch_syserr();
 			return (-1);
 		}
 	}
 	return (rlen);
 }
 /*
  * Read a line of text from a connection w/ timeout
  */
 #define MIN_BUF_SIZE 1024
 int
 fetch_getln(conn_t *conn)
 {
 	char *tmp;
 	size_t tmpsize;
 	ssize_t len;
 	char c;
 	if (conn->buf == NULL) {
 		if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
 			errno = ENOMEM;
 			return (-1);
 		}
 		conn->bufsize = MIN_BUF_SIZE;
 	}
 	conn->buf[0] = '\0';
 	conn->buflen = 0;
 	do {
 		len = fetch_read(conn, &c, 1);
 		if (len == -1)
 			return (-1);
 		if (len == 0)
 			break;
 		conn->buf[conn->buflen++] = c;
 		if (conn->buflen == conn->bufsize) {
 			tmp = conn->buf;
 			tmpsize = conn->bufsize * 2 + 1;
 			if ((tmp = realloc(tmp, tmpsize)) == NULL) {
 				errno = ENOMEM;
 				return (-1);
 			}
 			conn->buf = tmp;
 			conn->bufsize = tmpsize;
 		}
 	} while (c != '\n');
 	conn->buf[conn->buflen] = '\0';
 	DEBUG(fprintf(stderr, "<<< %s", conn->buf));
 	return (0);
 }
 /*
  * Write to a connection w/ timeout
  */
 ssize_t
 fetch_write(conn_t *conn, const char *buf, size_t len)
 {
 	struct iovec iov;
 	iov.iov_base = __DECONST(char *, buf);
 	iov.iov_len = len;
 	return fetch_writev(conn, &iov, 1);
 }
 /*
  * Write a vector to a connection w/ timeout
  * Note: can modify the iovec.
  */
 ssize_t
 fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
 {
 	struct timeval now, timeout, delta;
 	struct pollfd pfd;
 	ssize_t wlen, total;
 	int deltams;
 	memset(&pfd, 0, sizeof pfd);
 	if (fetchTimeout) {
 		pfd.fd = conn->sd;
 		pfd.events = POLLOUT | POLLERR;
 		gettimeofday(&timeout, NULL);
 		timeout.tv_sec += fetchTimeout;
 	}
 	total = 0;
 	while (iovcnt > 0) {
 		while (fetchTimeout && pfd.revents == 0) {
 			gettimeofday(&now, NULL);
 			if (!timercmp(&timeout, &now, >)) {
 				errno = ETIMEDOUT;
 				fetch_syserr();
 				return (-1);
 			}
 			timersub(&timeout, &now, &delta);
 			deltams = delta.tv_sec * 1000 +
 			    delta.tv_usec / 1000;
 			errno = 0;
 			pfd.revents = 0;
 			if (poll(&pfd, 1, deltams) < 0) {
 				/* POSIX compliance */
 				if (errno == EAGAIN)
 					continue;
 				if (errno == EINTR && fetchRestartCalls)
 					continue;
 				return (-1);
 			}
 		}
 		errno = 0;
 #ifdef WITH_SSL
 		if (conn->ssl != NULL)
 			wlen = SSL_write(conn->ssl,
 			    iov->iov_base, iov->iov_len);
 		else
 #endif
 			wlen = writev(conn->sd, iov, iovcnt);
 		if (wlen == 0) {
 			/* we consider a short write a failure */
 			/* XXX perhaps we shouldn't in the SSL case */
 			errno = EPIPE;
 			fetch_syserr();
 			return (-1);
 		}
 		if (wlen < 0) {
 			if (errno == EINTR && fetchRestartCalls)
 				continue;
 			return (-1);
 		}
 		total += wlen;
 		while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
 			wlen -= iov->iov_len;
 			iov++;
 			iovcnt--;
 		}
 		if (iovcnt > 0) {
 			iov->iov_len -= wlen;
 			iov->iov_base = __DECONST(char *, iov->iov_base) + wlen;
 		}
 	}
 	return (total);
 }
 /*
  * Write a line of text to a connection w/ timeout
  */
 int
 fetch_putln(conn_t *conn, const char *str, size_t len)
 {
 	struct iovec iov[2];
 	int ret;
 	DEBUG(fprintf(stderr, ">>> %s\n", str));
 	iov[0].iov_base = __DECONST(char *, str);
 	iov[0].iov_len = len;
 	iov[1].iov_base = __DECONST(char *, ENDL);
 	iov[1].iov_len = sizeof(ENDL);
 	if (len == 0)
 		ret = fetch_writev(conn, &iov[1], 1);
 	else
 		ret = fetch_writev(conn, iov, 2);
 	if (ret == -1)
 		return (-1);
 	return (0);
 }
 /*
  * Close connection
  */
 int
 fetch_close(conn_t *conn)
 {
 	int ret;
 	if (--conn->ref > 0)
 		return (0);
 #ifdef WITH_SSL
 	if (conn->ssl) {
 		SSL_shutdown(conn->ssl);
 		SSL_set_connect_state(conn->ssl);
 		SSL_free(conn->ssl);
 		conn->ssl = NULL;
 	}
 	if (conn->ssl_ctx) {
 		SSL_CTX_free(conn->ssl_ctx);
 		conn->ssl_ctx = NULL;
 	}
 	if (conn->ssl_cert) {
 		X509_free(conn->ssl_cert);
 		conn->ssl_cert = NULL;
 	}
 #endif
 	ret = close(conn->sd);
 	free(conn->buf);
 	free(conn);
 	return (ret);
 }
 /*** Directory-related utility functions *************************************/
 int
 fetch_add_entry(struct url_ent **p, int *size, int *len,
     const char *name, struct url_stat *us)
 {
 	struct url_ent *tmp;
 	if (*p == NULL) {
 		*size = 0;
 		*len = 0;
 	}
 	if (*len >= *size - 1) {
 		tmp = realloc(*p, (*size * 2 + 1) * sizeof(**p));
 		if (tmp == NULL) {
 			errno = ENOMEM;
 			fetch_syserr();
 			return (-1);
 		}
 		*size = (*size * 2 + 1);
 		*p = tmp;
 	}
 	tmp = *p + *len;
 	snprintf(tmp->name, PATH_MAX, "%s", name);
 	memcpy(&tmp->stat, us, sizeof(*us));
 	(*len)++;
 	(++tmp)->name[0] = 0;
 	return (0);
 }
 /*** Authentication-related utility functions ********************************/
 static const char *
 fetch_read_word(FILE *f)
 {
 	static char word[1024];
 	if (fscanf(f, " %1023s ", word) != 1)
 		return (NULL);
 	return (word);
 }
 /*
  * Get authentication data for a URL from .netrc
  */
 int
 fetch_netrc_auth(struct url *url)
 {
 	char fn[PATH_MAX];
 	const char *word;
 	char *p;
 	FILE *f;
 	if ((p = getenv("NETRC")) != NULL) {
 		if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
 			fetch_info("$NETRC specifies a file name "
 			    "longer than PATH_MAX");
 			return (-1);
 		}
 	} else {
 		if ((p = getenv("HOME")) != NULL) {
 			struct passwd *pwd;
 			if ((pwd = getpwuid(getuid())) == NULL ||
 			    (p = pwd->pw_dir) == NULL)
 				return (-1);
 		}
 		if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))
 			return (-1);
 	}
 	if ((f = fopen(fn, "r")) == NULL)
 		return (-1);
 	while ((word = fetch_read_word(f)) != NULL) {
 		if (strcmp(word, "default") == 0) {
 			DEBUG(fetch_info("Using default .netrc settings"));
 			break;
 		}
 		if (strcmp(word, "machine") == 0 &&
 		    (word = fetch_read_word(f)) != NULL &&
 		    strcasecmp(word, url->host) == 0) {
 			DEBUG(fetch_info("Using .netrc settings for %s", word));
 			break;
 		}
 	}
 	if (word == NULL)
 		goto ferr;
 	while ((word = fetch_read_word(f)) != NULL) {
 		if (strcmp(word, "login") == 0) {
 			if ((word = fetch_read_word(f)) == NULL)
 				goto ferr;
 			if (snprintf(url->user, sizeof(url->user),
 				"%s", word) > (int)sizeof(url->user)) {
 				fetch_info("login name in .netrc is too long");
 				url->user[0] = '\0';
 			}
 		} else if (strcmp(word, "password") == 0) {
 			if ((word = fetch_read_word(f)) == NULL)
 				goto ferr;
 			if (snprintf(url->pwd, sizeof(url->pwd),
 				"%s", word) > (int)sizeof(url->pwd)) {
 				fetch_info("password in .netrc is too long");
 				url->pwd[0] = '\0';
 			}
 		} else if (strcmp(word, "account") == 0) {
 			if ((word = fetch_read_word(f)) == NULL)
 				goto ferr;
 			/* XXX not supported! */
 		} else {
 			break;
 		}
 	}
 	fclose(f);
 	return (0);
  ferr:
 	fclose(f);
 	return (-1);
 }
 /*
  * The no_proxy environment variable specifies a set of domains for
  * which the proxy should not be consulted; the contents is a comma-,
  * or space-separated list of domain names.  A single asterisk will
  * override all proxy variables and no transactions will be proxied
  * (for compatability with lynx and curl, see the discussion at
  * <http://curl.haxx.se/mail/archive_pre_oct_99/0009.html>).
  */
 int
 fetch_no_proxy_match(const char *host)
 {
 	const char *no_proxy, *p, *q;
 	size_t h_len, d_len;
 	if ((no_proxy = getenv("NO_PROXY")) == NULL &&
 	    (no_proxy = getenv("no_proxy")) == NULL)
 		return (0);
 	/* asterisk matches any hostname */
 	if (strcmp(no_proxy, "*") == 0)
 		return (1);
 	h_len = strlen(host);
 	p = no_proxy;
 	do {
 		/* position p at the beginning of a domain suffix */
 		while (*p == ',' || isspace((unsigned char)*p))
 			p++;
 		/* position q at the first separator character */
 		for (q = p; *q; ++q)
 			if (*q == ',' || isspace((unsigned char)*q))
 				break;
 		d_len = q - p;
 		if (d_len > 0 && h_len >= d_len &&
 		    strncasecmp(host + h_len - d_len,
 			p, d_len) == 0) {
 			/* domain name matches */
 			return (1);
 		}
 		p = q + 1;
 	} while (*q);
 	return (0);
 }

added external/libfetch/common.h

@@ -0,0 +1,131 @@

 /*-
  * Copyright (c) 1998-2014 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD: head/lib/libfetch/common.h 267133 2014-06-05 22:16:26Z bapt $
  */
 #ifndef _COMMON_H_INCLUDED
 #define _COMMON_H_INCLUDED
 #define FTP_DEFAULT_PORT	21
 #define HTTP_DEFAULT_PORT	80
 #define FTP_DEFAULT_PROXY_PORT	21
 #define HTTP_DEFAULT_PROXY_PORT	3128
 #ifdef WITH_SSL
 #include <openssl/crypto.h>
 #include <openssl/x509.h>
 #include <openssl/pem.h>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 #endif
 /* Connection */
 typedef struct fetchconn conn_t;
 struct fetchconn {
 	int		 sd;		/* socket descriptor */
 	char		*buf;		/* buffer */
 	size_t		 bufsize;	/* buffer size */
 	size_t		 buflen;	/* length of buffer contents */
 	int		 err;		/* last protocol reply code */
 #ifdef WITH_SSL
 	SSL		*ssl;		/* SSL handle */
 	SSL_CTX		*ssl_ctx;	/* SSL context */
 	X509		*ssl_cert;	/* server certificate */
 	const SSL_METHOD *ssl_meth;	/* SSL method */
 #endif
 	int		 ref;		/* reference count */
 };
 /* Structure used for error message lists */
 struct fetcherr {
 	const int	 num;
 	const int	 cat;
 	const char	*string;
 };
 /* for fetch_writev */
 struct iovec;
 void		 fetch_seterr(struct fetcherr *, int);
 void		 fetch_syserr(void);
 void		 fetch_info(const char *, ...);
 int		 fetch_default_port(const char *);
 int		 fetch_default_proxy_port(const char *);
 int		 fetch_bind(int, int, const char *);
 conn_t		*fetch_connect(const char *, int, int, int);
 conn_t		*fetch_reopen(int);
 conn_t		*fetch_ref(conn_t *);
 #ifdef WITH_SSL
 int		 fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*);
 #endif
 int		 fetch_ssl(conn_t *, const struct url *, int);
 ssize_t		 fetch_read(conn_t *, char *, size_t);
 int		 fetch_getln(conn_t *);
 ssize_t		 fetch_write(conn_t *, const char *, size_t);
 ssize_t		 fetch_writev(conn_t *, struct iovec *, int);
 int		 fetch_putln(conn_t *, const char *, size_t);
 int		 fetch_close(conn_t *);
 int		 fetch_add_entry(struct url_ent **, int *, int *,
 		     const char *, struct url_stat *);
 int		 fetch_netrc_auth(struct url *url);
 int		 fetch_no_proxy_match(const char *);
 #define ftp_seterr(n)	 fetch_seterr(ftp_errlist, n)
 #define http_seterr(n)	 fetch_seterr(http_errlist, n)
 #define netdb_seterr(n)	 fetch_seterr(netdb_errlist, n)
 #define url_seterr(n)	 fetch_seterr(url_errlist, n)
 #ifndef NDEBUG
 #define DEBUG(x) do { if (fetchDebug) { x; } } while (0)
 #else
 #define DEBUG(x) do { } while (0)
 #endif
 /*
  * I don't really like exporting http_request() and ftp_request(),
  * but the HTTP and FTP code occasionally needs to cross-call
  * eachother, and this saves me from adding a lot of special-case code
  * to handle those cases.
  *
  * Note that _*_request() free purl, which is way ugly but saves us a
  * whole lot of trouble.
  */
 FILE		*http_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
 FILE		*http_request_body(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *,
 		     const char *, const char *);
 FILE		*ftp_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
 /*
  * Check whether a particular flag is set
  */
 #define CHECK_FLAG(x)	(flags && strchr(flags, (x)))
 #endif

added external/libfetch/fetch.3

@@ -0,0 +1,841 @@

 .\"-
 .\" Copyright (c) 1998-2013 Dag-Erling Smørgrav
 .\" Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
 .\" 1. Redistributions of source code must retain the above copyright
 .\"    notice, this list of conditions and the following disclaimer.
 .\" 2. Redistributions in binary form must reproduce the above copyright
 .\"    notice, this list of conditions and the following disclaimer in the
 .\"    documentation and/or other materials provided with the distribution.
 .\"
 .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 .\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD: head/lib/libfetch/fetch.3 273124 2014-10-15 07:35:50Z des $
 .\"
 .Dd October 15, 2014
 .Dt FETCH 3
 .Os
 .Sh NAME
 .Nm fetchMakeURL ,
 .Nm fetchParseURL ,
 .Nm fetchFreeURL ,
 .Nm fetchXGetURL ,
 .Nm fetchGetURL ,
 .Nm fetchPutURL ,
 .Nm fetchStatURL ,
 .Nm fetchListURL ,
 .Nm fetchXGet ,
 .Nm fetchGet ,
 .Nm fetchPut ,
 .Nm fetchStat ,
 .Nm fetchList ,
 .Nm fetchXGetFile ,
 .Nm fetchGetFile ,
 .Nm fetchPutFile ,
 .Nm fetchStatFile ,
 .Nm fetchListFile ,
 .Nm fetchXGetHTTP ,
 .Nm fetchGetHTTP ,
 .Nm fetchPutHTTP ,
 .Nm fetchStatHTTP ,
 .Nm fetchListHTTP ,
 .Nm fetchXGetFTP ,
 .Nm fetchGetFTP ,
 .Nm fetchPutFTP ,
 .Nm fetchStatFTP ,
 .Nm fetchListFTP
 .Nd file transfer functions
 .Sh LIBRARY
 .Lb libfetch
 .Sh SYNOPSIS
 .In sys/param.h
 .In stdio.h
 .In fetch.h
 .Ft struct url *
 .Fn fetchMakeURL "const char *scheme" "const char *host" "int port" "const char *doc" "const char *user" "const char *pwd"
 .Ft struct url *
 .Fn fetchParseURL "const char *URL"
 .Ft void
 .Fn fetchFreeURL "struct url *u"
 .Ft FILE *
 .Fn fetchXGetURL "const char *URL" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGetURL "const char *URL" "const char *flags"
 .Ft FILE *
 .Fn fetchPutURL "const char *URL" "const char *flags"
 .Ft int
 .Fn fetchStatURL "const char *URL" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchListURL "const char *URL" "const char *flags"
 .Ft FILE *
 .Fn fetchXGet "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGet "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchPut "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStat "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchList "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchXGetFile "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGetFile "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchPutFile "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatFile "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchListFile "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGetHTTP "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchPutHTTP "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatHTTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchListHTTP "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGetFTP "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchPutFTP "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatFTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchListFTP "struct url *u" "const char *flags"
 .Sh DESCRIPTION
 These functions implement a high-level library for retrieving and
 uploading files using Uniform Resource Locators (URLs).
 .Pp
 .Fn fetchParseURL
 takes a URL in the form of a null-terminated string and splits it into
 its components function according to the Common Internet Scheme Syntax
 detailed in RFC1738.
 A regular expression which produces this syntax is:
 .Bd -literal
     <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
 .Ed
 .Pp
 If the URL does not seem to begin with a scheme name, the following
 syntax is assumed:
 .Bd -literal
     ((<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
 .Ed
 .Pp
 Note that some components of the URL are not necessarily relevant to
 all URL schemes.
 For instance, the file scheme only needs the <scheme> and <document>
 components.
 .Pp
 .Fn fetchMakeURL
 and
 .Fn fetchParseURL
 return a pointer to a
 .Vt url
 structure, which is defined as follows in
 .In fetch.h :
 .Bd -literal
 #define URL_SCHEMELEN 16
 #define URL_USERLEN 256
 #define URL_PWDLEN 256
 struct url {
     char	 scheme[URL_SCHEMELEN+1];
     char	 user[URL_USERLEN+1];
     char	 pwd[URL_PWDLEN+1];
     char	 host[MAXHOSTNAMELEN+1];
     int		 port;
     char	*doc;
     off_t	 offset;
     size_t	 length;
     time_t	 ims_time;
 };
 .Ed
 .Pp
 The
 .Va ims_time
 field stores the time value for
 .Li If-Modified-Since
 HTTP requests.
 .Pp
 The pointer returned by
 .Fn fetchMakeURL
 or
 .Fn fetchParseURL
 should be freed using
 .Fn fetchFreeURL .
 .Pp
 .Fn fetchXGetURL ,
 .Fn fetchGetURL ,
 and
 .Fn fetchPutURL
 constitute the recommended interface to the
 .Nm fetch
 library.
 They examine the URL passed to them to determine the transfer
 method, and call the appropriate lower-level functions to perform the
 actual transfer.
 .Fn fetchXGetURL
 also returns the remote document's metadata in the
 .Vt url_stat
 structure pointed to by the
 .Fa us
 argument.
 .Pp
 The
 .Fa flags
 argument is a string of characters which specify transfer options.
 The
 meaning of the individual flags is scheme-dependent, and is detailed
 in the appropriate section below.
 .Pp
 .Fn fetchStatURL
 attempts to obtain the requested document's metadata and fill in the
 structure pointed to by its second argument.
 The
 .Vt url_stat
 structure is defined as follows in
 .In fetch.h :
 .Bd -literal
 struct url_stat {
     off_t	 size;
     time_t	 atime;
     time_t	 mtime;
 };
 .Ed
 .Pp
 If the size could not be obtained from the server, the
 .Fa size
 field is set to -1.
 If the modification time could not be obtained from the server, the
 .Fa mtime
 field is set to the epoch.
 If the access time could not be obtained from the server, the
 .Fa atime
 field is set to the modification time.
 .Pp
 .Fn fetchListURL
 attempts to list the contents of the directory pointed to by the URL
 provided.
 If successful, it returns a malloced array of
 .Vt url_ent
 structures.
 The
 .Vt url_ent
 structure is defined as follows in
 .In fetch.h :
 .Bd -literal
 struct url_ent {
     char         name[PATH_MAX];
     struct url_stat stat;
 };
 .Ed
 .Pp
 The list is terminated by an entry with an empty name.
 .Pp
 The pointer returned by
 .Fn fetchListURL
 should be freed using
 .Fn free .
 .Pp
 .Fn fetchXGet ,
 .Fn fetchGet ,
 .Fn fetchPut
 and
 .Fn fetchStat
 are similar to
 .Fn fetchXGetURL ,
 .Fn fetchGetURL ,
 .Fn fetchPutURL
 and
 .Fn fetchStatURL ,
 except that they expect a pre-parsed URL in the form of a pointer to
 a
 .Vt struct url
 rather than a string.
 .Pp
 All of the
 .Fn fetchXGetXXX ,
 .Fn fetchGetXXX
 and
 .Fn fetchPutXXX
 functions return a pointer to a stream which can be used to read or
 write data from or to the requested document, respectively.
 Note that
 although the implementation details of the individual access methods
 vary, it can generally be assumed that a stream returned by one of the
 .Fn fetchXGetXXX
 or
 .Fn fetchGetXXX
 functions is read-only, and that a stream returned by one of the
 .Fn fetchPutXXX
 functions is write-only.
 .Sh FILE SCHEME
 .Fn fetchXGetFile ,
 .Fn fetchGetFile
 and
 .Fn fetchPutFile
 provide access to documents which are files in a locally mounted file
 system.
 Only the <document> component of the URL is used.
 .Pp
 .Fn fetchXGetFile
 and
 .Fn fetchGetFile
 do not accept any flags.
 .Pp
 .Fn fetchPutFile
 accepts the
 .Ql a
 (append to file) flag.
 If that flag is specified, the data written to
 the stream returned by
 .Fn fetchPutFile
 will be appended to the previous contents of the file, instead of
 replacing them.
 .Sh FTP SCHEME
 .Fn fetchXGetFTP ,
 .Fn fetchGetFTP
 and
 .Fn fetchPutFTP
 implement the FTP protocol as described in RFC959.
 .Pp
 If the
 .Ql P
 (not passive) flag is specified, an active (rather than passive)
 connection will be attempted.
 .Pp
 The
 .Ql p
 flag is supported for compatibility with earlier versions where active
 connections were the default.
 It has precedence over the
 .Ql P
 flag, so if both are specified,
 .Nm
 will use a passive connection.
 .Pp
 If the
 .Ql l
 (low) flag is specified, data sockets will be allocated in the low (or
 default) port range instead of the high port range (see
 .Xr ip 4 ) .
 .Pp
 If the
 .Ql d
 (direct) flag is specified,
 .Fn fetchXGetFTP ,
 .Fn fetchGetFTP
 and
 .Fn fetchPutFTP
 will use a direct connection even if a proxy server is defined.
 .Pp
 If no user name or password is given, the
 .Nm fetch
 library will attempt an anonymous login, with user name "anonymous"
 and password "anonymous@<hostname>".
 .Sh HTTP SCHEME
 The
 .Fn fetchXGetHTTP ,
 .Fn fetchGetHTTP
 and
 .Fn fetchPutHTTP
 functions implement the HTTP/1.1 protocol.
 With a little luck, there is
 even a chance that they comply with RFC2616 and RFC2617.
 .Pp
 If the
 .Ql d
 (direct) flag is specified,
 .Fn fetchXGetHTTP ,
 .Fn fetchGetHTTP
 and
 .Fn fetchPutHTTP
 will use a direct connection even if a proxy server is defined.
 .Pp
 If the
 .Ql i
 (if-modified-since) flag is specified, and
 the
 .Va ims_time
 field is set in
 .Vt "struct url" ,
 then
 .Fn fetchXGetHTTP
 and
 .Fn fetchGetHTTP
 will send a conditional
 .Li If-Modified-Since
 HTTP header to only fetch the content if it is newer than
 .Va ims_time .
 .Pp
 Since there seems to be no good way of implementing the HTTP PUT
 method in a manner consistent with the rest of the
 .Nm fetch
 library,
 .Fn fetchPutHTTP
 is currently unimplemented.
 .Sh HTTPS SCHEME
 Based on HTTP SCHEME.
 By default the peer is verified using the CA bundle located in
 .Pa /etc/ssl/cert.pem .
 The file may contain multiple CA certificates.
 A common source of a current CA bundle is
 .Pa \%security/ca_root_nss .
 .Pp
 The CA bundle used for peer verification can be changed by setting the
 environment variables
 .Ev SSL_CA_CERT_FILE
 to point to a concatenated bundle of trusted certificates and
 .Ev SSL_CA_CERT_PATH
 to point to a directory containing hashes of trusted CAs (see
 .Xr verify 1 ) .
 .Pp
 A certificate revocation list (CRL) can be used by setting the
 environment variable
 .Ev SSL_CRL_FILE
 (see
 .Xr crl 1 ) .
 .Pp
 Peer verification can be disabled by setting the environment variable
 .Ev SSL_NO_VERIFY_PEER .
 Note that this also disables CRL checking.
 .Pp
 By default the service identity is verified according to the rules
 detailed in RFC6125 (also known as hostname verification).
 This feature can be disabled by setting the environment variable
 .Ev SSL_NO_VERIFY_HOSTNAME .
 .Pp
 Client certificate based authentication is supported.
 The environment variable
 .Ev SSL_CLIENT_CERT_FILE
 should be set to point to a file containing key and client certificate
 to be used in PEM format. In case the key is stored in a separate
 file, the environment variable
 .Ev SSL_CLIENT_KEY_FILE
 can be set to point to the key in PEM format.
 In case the key uses a password, the user will be prompted on standard
 input (see
 .Xr PEM 3 ) .
 .Pp
 By default
 .Nm libfetch
 allows TLSv1 and newer when negotiating the connecting with the remote
 peer.
 You can change this behavior by setting the
 .Ev SSL_ALLOW_SSL2
 and
 .Ev SSL_ALLOW_SSL3
 environment variables to allow SSLv2 and SSLv3, respectively, and
 .Ev SSL_NO_TLS1 ,
 .Ev SSL_NO_TLS1_1 and
 .Ev SSL_NO_TLS1_2
 to disable TLS 1.0, 1.1 and 1.2 respectively.
 .Sh AUTHENTICATION
 Apart from setting the appropriate environment variables and
 specifying the user name and password in the URL or the
 .Vt struct url ,
 the calling program has the option of defining an authentication
 function with the following prototype:
 .Pp
 .Ft int
 .Fn myAuthMethod "struct url *u"
 .Pp
 The callback function should fill in the
 .Fa user
 and
 .Fa pwd
 fields in the provided
 .Vt struct url
 and return 0 on success, or any other value to indicate failure.
 .Pp
 To register the authentication callback, simply set
 .Va fetchAuthMethod
 to point at it.
 The callback will be used whenever a site requires authentication and
 the appropriate environment variables are not set.
 .Pp
 This interface is experimental and may be subject to change.
 .Sh RETURN VALUES
 .Fn fetchParseURL
 returns a pointer to a
 .Vt struct url
 containing the individual components of the URL.
 If it is
 unable to allocate memory, or the URL is syntactically incorrect,
 .Fn fetchParseURL
 returns a NULL pointer.
 .Pp
 The
 .Fn fetchStat
 functions return 0 on success and -1 on failure.
 .Pp
 All other functions return a stream pointer which may be used to
 access the requested document, or NULL if an error occurred.
 .Pp
 The following error codes are defined in
 .In fetch.h :
 .Bl -tag -width 18n
 .It Bq Er FETCH_ABORT
 Operation aborted
 .It Bq Er FETCH_AUTH
 Authentication failed
 .It Bq Er FETCH_DOWN
 Service unavailable
 .It Bq Er FETCH_EXISTS
 File exists
 .It Bq Er FETCH_FULL
 File system full
 .It Bq Er FETCH_INFO
 Informational response
 .It Bq Er FETCH_MEMORY
 Insufficient memory
 .It Bq Er FETCH_MOVED
 File has moved
 .It Bq Er FETCH_NETWORK
 Network error
 .It Bq Er FETCH_OK
 No error
 .It Bq Er FETCH_PROTO
 Protocol error
 .It Bq Er FETCH_RESOLV
 Resolver error
 .It Bq Er FETCH_SERVER
 Server error
 .It Bq Er FETCH_TEMP
 Temporary error
 .It Bq Er FETCH_TIMEOUT
 Operation timed out
 .It Bq Er FETCH_UNAVAIL
 File is not available
 .It Bq Er FETCH_UNKNOWN
 Unknown error
 .It Bq Er FETCH_URL
 Invalid URL
 .El
 .Pp
 The accompanying error message includes a protocol-specific error code
 and message, e.g.\& "File is not available (404 Not Found)"
 .Sh ENVIRONMENT
 .Bl -tag -width ".Ev FETCH_BIND_ADDRESS"
 .It Ev FETCH_BIND_ADDRESS
 Specifies a hostname or IP address to which sockets used for outgoing
 connections will be bound.
 .It Ev FTP_LOGIN
 Default FTP login if none was provided in the URL.
 .It Ev FTP_PASSIVE_MODE
 If set to
 .Ql no ,
 forces the FTP code to use active mode.
 If set to any other value, forces passive mode even if the application
 requested active mode.
 .It Ev FTP_PASSWORD
 Default FTP password if the remote server requests one and none was
 provided in the URL.
 .It Ev FTP_PROXY
 URL of the proxy to use for FTP requests.
 The document part is ignored.
 FTP and HTTP proxies are supported; if no scheme is specified, FTP is
 assumed.
 If the proxy is an FTP proxy,
 .Nm libfetch
 will send
 .Ql user@host
 as user name to the proxy, where
 .Ql user
 is the real user name, and
 .Ql host
 is the name of the FTP server.
 .Pp
 If this variable is set to an empty string, no proxy will be used for
 FTP requests, even if the
 .Ev HTTP_PROXY
 variable is set.
 .It Ev ftp_proxy
 Same as
 .Ev FTP_PROXY ,
 for compatibility.
 .It Ev HTTP_ACCEPT
 Specifies the value of the
 .Va Accept
 header for HTTP requests.
 If empty, no
 .Va Accept
 header is sent.
 The default is
 .Dq */* .
 .It Ev HTTP_AUTH
 Specifies HTTP authorization parameters as a colon-separated list of
 items.
 The first and second item are the authorization scheme and realm
 respectively; further items are scheme-dependent.
 Currently, the
 .Dq basic
 and
 .Dq digest
 authorization methods are supported.
 .Pp
 Both methods require two parameters: the user name and
 password, in that order.
 .Pp
 This variable is only used if the server requires authorization and
 no user name or password was specified in the URL.
 .It Ev HTTP_PROXY
 URL of the proxy to use for HTTP requests.
 The document part is ignored.
 Only HTTP proxies are supported for HTTP requests.
 If no port number is specified, the default is 3128.
 .Pp
 Note that this proxy will also be used for FTP documents, unless the
 .Ev FTP_PROXY
 variable is set.
 .It Ev http_proxy
 Same as
 .Ev HTTP_PROXY ,
 for compatibility.
 .It Ev HTTP_PROXY_AUTH
 Specifies authorization parameters for the HTTP proxy in the same
 format as the
 .Ev HTTP_AUTH
 variable.
 .Pp
 This variable is used if and only if connected to an HTTP proxy, and
 is ignored if a user and/or a password were specified in the proxy
 URL.
 .It Ev HTTP_REFERER
 Specifies the referrer URL to use for HTTP requests.
 If set to
 .Dq auto ,
 the document URL will be used as referrer URL.
 .It Ev HTTP_USER_AGENT
 Specifies the User-Agent string to use for HTTP requests.
 This can be useful when working with HTTP origin or proxy servers that
 differentiate between user agents.
 If defined but empty, no User-Agent header is sent.
 .It Ev NETRC
 Specifies a file to use instead of
 .Pa ~/.netrc
 to look up login names and passwords for FTP sites.
 See
 .Xr ftp 1
 for a description of the file format.
 This feature is experimental.
 .It Ev NO_PROXY
 Either a single asterisk, which disables the use of proxies
 altogether, or a comma- or whitespace-separated list of hosts for
 which proxies should not be used.
 .It Ev no_proxy
 Same as
 .Ev NO_PROXY ,
 for compatibility.
 .It Ev SSL_ALLOW_SSL2
 Allow SSL version 2 when negotiating the connection (not recommended).
 .It Ev SSL_ALLOW_SSL3
 Allow SSL version 3 when negotiating the connection (not recommended).
 .It Ev SSL_CA_CERT_FILE
 CA certificate bundle containing trusted CA certificates.
 Default value:
 .Pa /etc/ssl/cert.pem .
 .It Ev SSL_CA_CERT_PATH
 Path containing trusted CA hashes.
 .It Ev SSL_CLIENT_CERT_FILE
 PEM encoded client certificate/key which will be used in
 client certificate authentication.
 .It Ev SSL_CLIENT_KEY_FILE
 PEM encoded client key in case key and client certificate
 are stored separately.
 .It Ev SSL_CRL_FILE
 File containing certificate revocation list.
 .It Ev SSL_NO_TLS1
 Do not allow TLS version 1.0 when negotiating the connection.
 .It Ev SSL_NO_TLS1_1
 Do not allow TLS version 1.1 when negotiating the connection.
 .It Ev SSL_NO_TLS1_2
 Do not allow TLS version 1.2 when negotiating the connection.
 .It Ev SSL_NO_VERIFY_HOSTNAME
 If set, do not verify that the hostname matches the subject of the
 certificate presented by the server.
 .It Ev SSL_NO_VERIFY_PEER
 If set, do not verify the peer certificate against trusted CAs.
 .El
 .Sh EXAMPLES
 To access a proxy server on
 .Pa proxy.example.com
 port 8080, set the
 .Ev HTTP_PROXY
 environment variable in a manner similar to this:
 .Pp
 .Dl HTTP_PROXY=http://proxy.example.com:8080
 .Pp
 If the proxy server requires authentication, there are
 two options available for passing the authentication data.
 The first method is by using the proxy URL:
 .Pp
 .Dl HTTP_PROXY=http://<user>:<pwd>@proxy.example.com:8080
 .Pp
 The second method is by using the
 .Ev HTTP_PROXY_AUTH
 environment variable:
 .Bd -literal -offset indent
 HTTP_PROXY=http://proxy.example.com:8080
 HTTP_PROXY_AUTH=basic:*:<user>:<pwd>
 .Ed
 .Pp
 To disable the use of a proxy for an HTTP server running on the local
 host, define
 .Ev NO_PROXY
 as follows:
 .Bd -literal -offset indent
 NO_PROXY=localhost,127.0.0.1
 .Ed
 .Pp
 Access HTTPS website without any certificate verification whatsoever:
 .Bd -literal -offset indent
 SSL_NO_VERIFY_PEER=1
 SSL_NO_VERIFY_HOSTNAME=1
 .Ed
 .Pp
 Access HTTPS website using client certificate based authentication
 and a private CA:
 .Bd -literal -offset indent
 SSL_CLIENT_CERT_FILE=/path/to/client.pem
 SSL_CA_CERT_FILE=/path/to/myca.pem
 .Ed
 .Sh SEE ALSO
 .Xr fetch 1 ,
 .Xr ftpio 3 ,
 .Xr ip 4
 .Rs
 .%A J. Postel
 .%A J. K. Reynolds
 .%D October 1985
 .%B File Transfer Protocol
 .%O RFC959
 .Re
 .Rs
 .%A P. Deutsch
 .%A A. Emtage
 .%A A. Marine.
 .%D May 1994
 .%T How to Use Anonymous FTP
 .%O RFC1635
 .Re
 .Rs
 .%A T. Berners-Lee
 .%A L. Masinter
 .%A M. McCahill
 .%D December 1994
 .%T Uniform Resource Locators (URL)
 .%O RFC1738
 .Re
 .Rs
 .%A R. Fielding
 .%A J. Gettys
 .%A J. Mogul
 .%A H. Frystyk
 .%A L. Masinter
 .%A P. Leach
 .%A T. Berners-Lee
 .%D January 1999
 .%B Hypertext Transfer Protocol -- HTTP/1.1
 .%O RFC2616
 .Re
 .Rs
 .%A J. Franks
 .%A P. Hallam-Baker
 .%A J. Hostetler
 .%A S. Lawrence
 .%A P. Leach
 .%A A. Luotonen
 .%A L. Stewart
 .%D June 1999
 .%B HTTP Authentication: Basic and Digest Access Authentication
 .%O RFC2617
 .Re
 .Sh HISTORY
 The
 .Nm fetch
 library first appeared in
 .Fx 3.0 .
 .Sh AUTHORS
 .An -nosplit
 The
 .Nm fetch
 library was mostly written by
 .An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org
 with numerous suggestions and contributions from
 .An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org ,
 .An Eugene Skepner Aq Mt eu@qub.com ,
 .An Hajimu Umemoto Aq Mt ume@FreeBSD.org ,
 .An Henry Whincup Aq Mt henry@techiebod.com ,
 .An Jukka A. Ukkonen Aq Mt jau@iki.fi ,
 .An Jean-Fran\(,cois Dockes Aq Mt jf@dockes.org ,
 .An Michael Gmelin Aq Mt freebsd@grem.de
 and others.
 It replaces the older
 .Nm ftpio
 library written by
 .An Poul-Henning Kamp Aq Mt phk@FreeBSD.org
 and
 .An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org .
 .Pp
 This manual page was written by
 .An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org
 and
 .An Michael Gmelin Aq Mt freebsd@grem.de .
 .Sh BUGS
 Some parts of the library are not yet implemented.
 The most notable
 examples of this are
 .Fn fetchPutHTTP ,
 .Fn fetchListHTTP ,
 .Fn fetchListFTP
 and FTP proxy support.
 .Pp
 There is no way to select a proxy at run-time other than setting the
 .Ev HTTP_PROXY
 or
 .Ev FTP_PROXY
 environment variables as appropriate.
 .Pp
 .Nm libfetch
 does not understand or obey 305 (Use Proxy) replies.
 .Pp
 Error numbers are unique only within a certain context; the error
 codes used for FTP and HTTP overlap, as do those used for resolver and
 system errors.
 For instance, error code 202 means "Command not
 implemented, superfluous at this site" in an FTP context and
 "Accepted" in an HTTP context.
 .Pp
 .Fn fetchStatFTP
 does not check that the result of an MDTM command is a valid date.
 .Pp
 In case password protected keys are used for client certificate based
 authentication the user is prompted for the password on each and every
 fetch operation.
 .Pp
 The man page is incomplete, poorly written and produces badly
 formatted text.
 .Pp
 The error reporting mechanism is unsatisfactory.
 .Pp
 Some parts of the code are not fully reentrant.

added external/libfetch/fetch.c

@@ -0,0 +1,469 @@

 /*-
  * Copyright (c) 1998-2004 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD: head/lib/libfetch/fetch.c 252375 2013-06-29 15:51:27Z kientzle $");
 #include <sys/param.h>
 #include <sys/errno.h>
 #include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "fetch.h"
 #include "common.h"
 auth_t	 fetchAuthMethod;
 int	 fetchLastErrCode;
 char	 fetchLastErrString[MAXERRSTRING];
 int	 fetchTimeout;
 int	 fetchRestartCalls = 1;
 int	 fetchDebug;
 /*** Local data **************************************************************/
 /*
  * Error messages for parser errors
  */
 #define URL_MALFORMED		1
 #define URL_BAD_SCHEME		2
 #define URL_BAD_PORT		3
 static struct fetcherr url_errlist[] = {
 	{ URL_MALFORMED,	FETCH_URL,	"Malformed URL" },
 	{ URL_BAD_SCHEME,	FETCH_URL,	"Invalid URL scheme" },
 	{ URL_BAD_PORT,		FETCH_URL,	"Invalid server port" },
 	{ -1,			FETCH_UNKNOWN,	"Unknown parser error" }
 };
 /*** Public API **************************************************************/
 /*
  * Select the appropriate protocol for the URL scheme, and return a
  * read-only stream connected to the document referenced by the URL.
  * Also fill out the struct url_stat.
  */
 FILE *
 fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
 {
 	if (us != NULL) {
 		us->size = -1;
 		us->atime = us->mtime = 0;
 	}
 	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchXGetFile(URL, us, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
 		return (fetchXGetFTP(URL, us, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchXGetHTTP(URL, us, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchXGetHTTP(URL, us, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
 }
 /*
  * Select the appropriate protocol for the URL scheme, and return a
  * read-only stream connected to the document referenced by the URL.
  */
 FILE *
 fetchGet(struct url *URL, const char *flags)
 {
 	return (fetchXGet(URL, NULL, flags));
 }
 /*
  * Select the appropriate protocol for the URL scheme, and return a
  * write-only stream connected to the document referenced by the URL.
  */
 FILE *
 fetchPut(struct url *URL, const char *flags)
 {
 	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchPutFile(URL, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
 		return (fetchPutFTP(URL, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchPutHTTP(URL, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchPutHTTP(URL, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
 }
 /*
  * Select the appropriate protocol for the URL scheme, and return the
  * size of the document referenced by the URL if it exists.
  */
 int
 fetchStat(struct url *URL, struct url_stat *us, const char *flags)
 {
 	if (us != NULL) {
 		us->size = -1;
 		us->atime = us->mtime = 0;
 	}
 	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchStatFile(URL, us, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
 		return (fetchStatFTP(URL, us, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchStatHTTP(URL, us, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchStatHTTP(URL, us, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (-1);
 }
 /*
  * Select the appropriate protocol for the URL scheme, and return a
  * list of files in the directory pointed to by the URL.
  */
 struct url_ent *
 fetchList(struct url *URL, const char *flags)
 {
 	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchListFile(URL, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
 		return (fetchListFTP(URL, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchListHTTP(URL, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchListHTTP(URL, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchXGet().
  */
 FILE *
 fetchXGetURL(const char *URL, struct url_stat *us, const char *flags)
 {
 	struct url *u;
 	FILE *f;
 	if ((u = fetchParseURL(URL)) == NULL)
 		return (NULL);
 	f = fetchXGet(u, us, flags);
 	fetchFreeURL(u);
 	return (f);
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchGet().
  */
 FILE *
 fetchGetURL(const char *URL, const char *flags)
 {
 	return (fetchXGetURL(URL, NULL, flags));
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchPut().
  */
 FILE *
 fetchPutURL(const char *URL, const char *flags)
 {
 	struct url *u;
 	FILE *f;
 	if ((u = fetchParseURL(URL)) == NULL)
 		return (NULL);
 	f = fetchPut(u, flags);
 	fetchFreeURL(u);
 	return (f);
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchStat().
  */
 int
 fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
 {
 	struct url *u;
 	int s;
 	if ((u = fetchParseURL(URL)) == NULL)
 		return (-1);
 	s = fetchStat(u, us, flags);
 	fetchFreeURL(u);
 	return (s);
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchList().
  */
 struct url_ent *
 fetchListURL(const char *URL, const char *flags)
 {
 	struct url *u;
 	struct url_ent *ue;
 	if ((u = fetchParseURL(URL)) == NULL)
 		return (NULL);
 	ue = fetchList(u, flags);
 	fetchFreeURL(u);
 	return (ue);
 }
 /*
  * Make a URL
  */
 struct url *
 fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
     const char *user, const char *pwd)
 {
 	struct url *u;
 	if (!scheme || (!host && !doc)) {
 		url_seterr(URL_MALFORMED);
 		return (NULL);
 	}
 	if (port < 0 || port > 65535) {
 		url_seterr(URL_BAD_PORT);
 		return (NULL);
 	}
 	/* allocate struct url */
 	if ((u = calloc(1, sizeof(*u))) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
 		fetch_syserr();
 		free(u);
 		return (NULL);
 	}
 #define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)
 	seturl(scheme);
 	seturl(host);
 	seturl(user);
 	seturl(pwd);
 #undef seturl
 	u->port = port;
 	return (u);
 }
 /*
  * Return value of the given hex digit.
  */
 static int
 fetch_hexval(char ch)
 {
 	if (ch >= '0' && ch <= '9')
 		return (ch - '0');
 	else if (ch >= 'a' && ch <= 'f')
 		return (ch - 'a' + 10);
 	else if (ch >= 'A' && ch <= 'F')
 		return (ch - 'A' + 10);
 	return (-1);
 }
 /*
  * Decode percent-encoded URL component from src into dst, stopping at end
  * of string, or at @ or : separators.  Returns a pointer to the unhandled
  * part of the input string (null terminator, @, or :).  No terminator is
  * written to dst (it is the caller's responsibility).
  */
 static const char *
 fetch_pctdecode(char *dst, const char *src, size_t dlen)
 {
 	int d1, d2;
 	char c;
 	const char *s;
 	for (s = src; *s != '\0' && *s != '@' && *s != ':'; s++) {
 		if (s[0] == '%' && (d1 = fetch_hexval(s[1])) >= 0 &&
 		    (d2 = fetch_hexval(s[2])) >= 0 && (d1 > 0 || d2 > 0)) {
 			c = d1 << 4 | d2;
 			s += 2;
 		} else {
 			c = *s;
 		}
 		if (dlen-- > 0)
 			*dst++ = c;
 	}
 	return (s);
 }
 /*
  * Split an URL into components. URL syntax is:
  * [method:/][/[user[:pwd]@]host[:port]/][document]
  * This almost, but not quite, RFC1738 URL syntax.
  */
 struct url *
 fetchParseURL(const char *URL)
 {
 	char *doc;
 	const char *p, *q;
 	struct url *u;
 	int i;
 	/* allocate struct url */
 	if ((u = calloc(1, sizeof(*u))) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	/* scheme name */
 	if ((p = strstr(URL, ":/"))) {
 		snprintf(u->scheme, URL_SCHEMELEN+1,
 		    "%.*s", (int)(p - URL), URL);
 		URL = ++p;
 		/*
 		 * Only one slash: no host, leave slash as part of document
 		 * Two slashes: host follows, strip slashes
 		 */
 		if (URL[1] == '/')
 			URL = (p += 2);
 	} else {
 		p = URL;
 	}
 	if (!*URL || *URL == '/' || *URL == '.' ||
 	    (u->scheme[0] == '\0' &&
 		strchr(URL, '/') == NULL && strchr(URL, ':') == NULL))
 		goto nohost;
 	p = strpbrk(URL, "/@");
 	if (p && *p == '@') {
 		/* username */
 		q = fetch_pctdecode(u->user, URL, URL_USERLEN);
 		/* password */
 		if (*q == ':')
 			q = fetch_pctdecode(u->pwd, q + 1, URL_PWDLEN);
 		p++;
 	} else {
 		p = URL;
 	}
 	/* hostname */
 #ifdef INET6
 	if (*p == '[' && (q = strchr(p + 1, ']')) != NULL &&
 	    (*++q == '\0' || *q == '/' || *q == ':')) {
 		if ((i = q - p - 2) > MAXHOSTNAMELEN)
 			i = MAXHOSTNAMELEN;
 		strncpy(u->host, ++p, i);
 		p = q;
 	} else
 #endif
 		for (i = 0; *p && (*p != '/') && (*p != ':'); p++)
 			if (i < MAXHOSTNAMELEN)
 				u->host[i++] = *p;
 	/* port */
 	if (*p == ':') {
 		for (q = ++p; *q && (*q != '/'); q++)
 			if (isdigit((unsigned char)*q))
 				u->port = u->port * 10 + (*q - '0');
 			else {
 				/* invalid port */
 				url_seterr(URL_BAD_PORT);
 				goto ouch;
 			}
 		p = q;
 	}
 nohost:
 	/* document */
 	if (!*p)
 		p = "/";
 	if (strcasecmp(u->scheme, SCHEME_HTTP) == 0 ||
 	    strcasecmp(u->scheme, SCHEME_HTTPS) == 0) {
 		const char hexnums[] = "0123456789abcdef";
 		/* percent-escape whitespace. */
 		if ((doc = malloc(strlen(p) * 3 + 1)) == NULL) {
 			fetch_syserr();
 			goto ouch;
 		}
 		u->doc = doc;
 		while (*p != '\0') {
 			if (!isspace((unsigned char)*p)) {
 				*doc++ = *p++;
 			} else {
 				*doc++ = '%';
 				*doc++ = hexnums[((unsigned int)*p) >> 4];
 				*doc++ = hexnums[((unsigned int)*p) & 0xf];
 				p++;
 			}
 		}
 		*doc = '\0';
 	} else if ((u->doc = strdup(p)) == NULL) {
 		fetch_syserr();
 		goto ouch;
 	}
 	DEBUG(fprintf(stderr,
 		  "scheme:   [%s]\n"
 		  "user:     [%s]\n"
 		  "password: [%s]\n"
 		  "host:     [%s]\n"
 		  "port:     [%d]\n"
 		  "document: [%s]\n",
 		  u->scheme, u->user, u->pwd,
 		  u->host, u->port, u->doc));
 	return (u);
 ouch:
 	free(u);
 	return (NULL);
 }
 /*
  * Free a URL
  */
 void
 fetchFreeURL(struct url *u)
 {
 	free(u->doc);
 	free(u);
 }

added external/libfetch/fetch.h

@@ -0,0 +1,153 @@

 /*-
  * Copyright (c) 1998-2004 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD: head/lib/libfetch/fetch.h 267133 2014-06-05 22:16:26Z bapt $
  */
 #ifndef _FETCH_H_INCLUDED
 #define _FETCH_H_INCLUDED
 #define _LIBFETCH_VER "libfetch/2.0"
 #define URL_SCHEMELEN 16
 #define URL_USERLEN 256
 #define URL_PWDLEN 256
 struct url {
 	char		 scheme[URL_SCHEMELEN+1];
 	char		 user[URL_USERLEN+1];
 	char		 pwd[URL_PWDLEN+1];
 	char		 host[MAXHOSTNAMELEN+1];
 	int		 port;
 	char		*doc;
 	off_t		 offset;
 	size_t		 length;
 	time_t		 ims_time;
 };
 struct url_stat {
 	off_t		 size;
 	time_t		 atime;
 	time_t		 mtime;
 };
 struct url_ent {
 	char		 name[PATH_MAX];
 	struct url_stat	 stat;
 };
 /* Recognized schemes */
 #define SCHEME_FTP	"ftp"
 #define SCHEME_HTTP	"http"
 #define SCHEME_HTTPS	"https"
 #define SCHEME_FILE	"file"
 /* Error codes */
 #define	FETCH_ABORT	 1
 #define	FETCH_AUTH	 2
 #define	FETCH_DOWN	 3
 #define	FETCH_EXISTS	 4
 #define	FETCH_FULL	 5
 #define	FETCH_INFO	 6
 #define	FETCH_MEMORY	 7
 #define	FETCH_MOVED	 8
 #define	FETCH_NETWORK	 9
 #define	FETCH_OK	10
 #define	FETCH_PROTO	11
 #define	FETCH_RESOLV	12
 #define	FETCH_SERVER	13
 #define	FETCH_TEMP	14
 #define	FETCH_TIMEOUT	15
 #define	FETCH_UNAVAIL	16
 #define	FETCH_UNKNOWN	17
 #define	FETCH_URL	18
 #define	FETCH_VERBOSE	19
 __BEGIN_DECLS
 /* FILE-specific functions */
 FILE		*fetchXGetFile(struct url *, struct url_stat *, const char *);
 FILE		*fetchGetFile(struct url *, const char *);
 FILE		*fetchPutFile(struct url *, const char *);
 int		 fetchStatFile(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListFile(struct url *, const char *);
 /* HTTP-specific functions */
 FILE		*fetchXGetHTTP(struct url *, struct url_stat *, const char *);
 FILE		*fetchGetHTTP(struct url *, const char *);
 FILE		*fetchPutHTTP(struct url *, const char *);
 int		 fetchStatHTTP(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListHTTP(struct url *, const char *);
 FILE		*fetchReqHTTP(struct url *, const char *, const char *,
 		    const char *, const char *);
 /* FTP-specific functions */
 FILE		*fetchXGetFTP(struct url *, struct url_stat *, const char *);
 FILE		*fetchGetFTP(struct url *, const char *);
 FILE		*fetchPutFTP(struct url *, const char *);
 int		 fetchStatFTP(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListFTP(struct url *, const char *);
 /* Generic functions */
 FILE		*fetchXGetURL(const char *, struct url_stat *, const char *);
 FILE		*fetchGetURL(const char *, const char *);
 FILE		*fetchPutURL(const char *, const char *);
 int		 fetchStatURL(const char *, struct url_stat *, const char *);
 struct url_ent	*fetchListURL(const char *, const char *);
 FILE		*fetchXGet(struct url *, struct url_stat *, const char *);
 FILE		*fetchGet(struct url *, const char *);
 FILE		*fetchPut(struct url *, const char *);
 int		 fetchStat(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchList(struct url *, const char *);
 /* URL parsing */
 struct url	*fetchMakeURL(const char *, const char *, int,
 		     const char *, const char *, const char *);
 struct url	*fetchParseURL(const char *);
 void		 fetchFreeURL(struct url *);
 __END_DECLS
 /* Authentication */
 typedef int (*auth_t)(struct url *);
 extern auth_t		 fetchAuthMethod;
 /* Last error code */
 extern int		 fetchLastErrCode;
 #define MAXERRSTRING 256
 extern char		 fetchLastErrString[MAXERRSTRING];
 /* I/O timeout */
 extern int		 fetchTimeout;
 /* Restart interrupted syscalls */
 extern int		 fetchRestartCalls;
 /* Extra verbosity */
 extern int		 fetchDebug;
 #endif

added external/libfetch/file.c

@@ -0,0 +1,155 @@

 /*-
  * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD: head/lib/libfetch/file.c 240495 2012-09-14 12:15:13Z eadler $");
 #include <sys/param.h>
 #include <sys/stat.h>
 #include <dirent.h>
 #include <fcntl.h>
 #include <stdio.h>
 #include <string.h>
 #include "fetch.h"
 #include "common.h"
 FILE *
 fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 {
 	FILE *f;
 	if (us && fetchStatFile(u, us, flags) == -1)
 		return (NULL);
 	f = fopen(u->doc, "r");
 	if (f == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	if (u->offset && fseeko(f, u->offset, SEEK_SET) == -1) {
 		fclose(f);
 		fetch_syserr();
 		return (NULL);
 	}
 	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 FILE *
 fetchGetFile(struct url *u, const char *flags)
 {
 	return (fetchXGetFile(u, NULL, flags));
 }
 FILE *
 fetchPutFile(struct url *u, const char *flags)
 {
 	FILE *f;
 	if (CHECK_FLAG('a'))
 		f = fopen(u->doc, "a");
 	else
 		f = fopen(u->doc, "w+");
 	if (f == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	if (u->offset && fseeko(f, u->offset, SEEK_SET) == -1) {
 		fclose(f);
 		fetch_syserr();
 		return (NULL);
 	}
 	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 static int
 fetch_stat_file(const char *fn, struct url_stat *us)
 {
 	struct stat sb;
 	us->size = -1;
 	us->atime = us->mtime = 0;
 	if (stat(fn, &sb) == -1) {
 		fetch_syserr();
 		return (-1);
 	}
 	us->size = sb.st_size;
 	us->atime = sb.st_atime;
 	us->mtime = sb.st_mtime;
 	return (0);
 }
 int
 fetchStatFile(struct url *u, struct url_stat *us, const char *flags __unused)
 {
 	return (fetch_stat_file(u->doc, us));
 }
 struct url_ent *
 fetchListFile(struct url *u, const char *flags __unused)
 {
 	struct dirent *de;
 	struct url_stat us;
 	struct url_ent *ue;
 	int size, len;
 	char fn[PATH_MAX], *p;
 	DIR *dir;
 	int l;
 	if ((dir = opendir(u->doc)) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	ue = NULL;
 	strncpy(fn, u->doc, sizeof(fn) - 2);
 	fn[sizeof(fn) - 2] = 0;
 	strcat(fn, "/");
 	p = strchr(fn, 0);
 	l = sizeof(fn) - strlen(fn) - 1;
 	while ((de = readdir(dir)) != NULL) {
 		strncpy(p, de->d_name, l - 1);
 		p[l - 1] = 0;
 		if (fetch_stat_file(fn, &us) == -1)
 			/* should I return a partial result, or abort? */
 			break;
 		fetch_add_entry(&ue, &size, &len, de->d_name, &us);
 	}
 	return (ue);
 }

added external/libfetch/ftp.c

@@ -0,0 +1,1206 @@

 /*-
  * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD: head/lib/libfetch/ftp.c 226537 2011-10-19 11:43:51Z des $");
 /*
  * Portions of this code were taken from or based on ftpio.c:
  *
  * ----------------------------------------------------------------------------
  * "THE BEER-WARE LICENSE" (Revision 42):
  * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
  * can do whatever you want with this stuff. If we meet some day, and you think
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
  * Major Changelog:
  *
  * Dag-Erling Smørgrav
  * 9 Jun 1998
  *
  * Incorporated into libfetch
  *
  * Jordan K. Hubbard
  * 17 Jan 1996
  *
  * Turned inside out. Now returns xfers as new file ids, not as a special
  * `state' of FTP_t
  *
  * $ftpioId: ftpio.c,v 1.30 1998/04/11 07:28:53 phk Exp $
  *
  */
 #include <sys/param.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <ctype.h>
 #include <err.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <netdb.h>
 #include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
 #include "fetch.h"
 #include "common.h"
 #include "ftperr.h"
 #define FTP_ANONYMOUS_USER	"anonymous"
 #define FTP_CONNECTION_ALREADY_OPEN	125
 #define FTP_OPEN_DATA_CONNECTION	150
 #define FTP_OK				200
 #define FTP_FILE_STATUS			213
 #define FTP_SERVICE_READY		220
 #define FTP_TRANSFER_COMPLETE		226
 #define FTP_PASSIVE_MODE		227
 #define FTP_LPASSIVE_MODE		228
 #define FTP_EPASSIVE_MODE		229
 #define FTP_LOGGED_IN			230
 #define FTP_FILE_ACTION_OK		250
 #define FTP_DIRECTORY_CREATED		257 /* multiple meanings */
 #define FTP_FILE_CREATED		257 /* multiple meanings */
 #define FTP_WORKING_DIRECTORY		257 /* multiple meanings */
 #define FTP_NEED_PASSWORD		331
 #define FTP_NEED_ACCOUNT		332
 #define FTP_FILE_OK			350
 #define FTP_SYNTAX_ERROR		500
 #define FTP_PROTOCOL_ERROR		999
 static struct url cached_host;
 static conn_t	*cached_connection;
 #define isftpreply(foo)				\
 	(isdigit((unsigned char)foo[0]) &&	\
 	    isdigit((unsigned char)foo[1]) &&	\
 	    isdigit((unsigned char)foo[2]) &&	\
 	    (foo[3] == ' ' || foo[3] == '\0'))
 #define isftpinfo(foo) \
 	(isdigit((unsigned char)foo[0]) &&	\
 	    isdigit((unsigned char)foo[1]) &&	\
 	    isdigit((unsigned char)foo[2]) &&	\
 	    foo[3] == '-')
 /*
  * Translate IPv4 mapped IPv6 address to IPv4 address
  */
 static void
 unmappedaddr(struct sockaddr_in6 *sin6)
 {
 	struct sockaddr_in *sin4;
 	u_int32_t addr;
 	int port;
 	if (sin6->sin6_family != AF_INET6 ||
 	    !IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr))
 		return;
 	sin4 = (struct sockaddr_in *)sin6;
 	addr = *(u_int32_t *)(uintptr_t)&sin6->sin6_addr.s6_addr[12];
 	port = sin6->sin6_port;
 	memset(sin4, 0, sizeof(struct sockaddr_in));
 	sin4->sin_addr.s_addr = addr;
 	sin4->sin_port = port;
 	sin4->sin_family = AF_INET;
 	sin4->sin_len = sizeof(struct sockaddr_in);
 }
 /*
  * Get server response
  */
 static int
 ftp_chkerr(conn_t *conn)
 {
 	if (fetch_getln(conn) == -1) {
 		fetch_syserr();
 		return (-1);
 	}
 	if (isftpinfo(conn->buf)) {
 		while (conn->buflen && !isftpreply(conn->buf)) {
 			if (fetch_getln(conn) == -1) {
 				fetch_syserr();
 				return (-1);
 			}
 		}
 	}
 	while (conn->buflen &&
 	    isspace((unsigned char)conn->buf[conn->buflen - 1]))
 		conn->buflen--;
 	conn->buf[conn->buflen] = '\0';
 	if (!isftpreply(conn->buf)) {
 		ftp_seterr(FTP_PROTOCOL_ERROR);
 		return (-1);
 	}
 	conn->err = (conn->buf[0] - '0') * 100
 	    + (conn->buf[1] - '0') * 10
 	    + (conn->buf[2] - '0');
 	return (conn->err);
 }
 /*
  * Send a command and check reply
  */
 static int
 ftp_cmd(conn_t *conn, const char *fmt, ...)
 {
 	va_list ap;
 	size_t len;
 	char *msg;
 	int r;
 	va_start(ap, fmt);
 	len = vasprintf(&msg, fmt, ap);
 	va_end(ap);
 	if (msg == NULL) {
 		errno = ENOMEM;
 		fetch_syserr();
 		return (-1);
 	}
 	r = fetch_putln(conn, msg, len);
 	free(msg);
 	if (r == -1) {
 		fetch_syserr();
 		return (-1);
 	}
 	return (ftp_chkerr(conn));
 }
 /*
  * Return a pointer to the filename part of a path
  */
 static const char *
 ftp_filename(const char *file, int *len, int *type)
 {
 	const char *s;
 	if ((s = strrchr(file, '/')) == NULL)
 		s = file;
 	else
 		s = s + 1;
 	*len = strlen(s);
 	if (*len > 7 && strncmp(s + *len - 7, ";type=", 6) == 0) {
 		*type = s[*len - 1];
 		*len -= 7;
 	} else {
 		*type = '\0';
 	}
 	return (s);
 }
 /*
  * Get current working directory from the reply to a CWD, PWD or CDUP
  * command.
  */
 static int
 ftp_pwd(conn_t *conn, char *pwd, size_t pwdlen)
 {
 	char *src, *dst, *end;
 	int q;
 	if (conn->err != FTP_WORKING_DIRECTORY &&
 	    conn->err != FTP_FILE_ACTION_OK)
 		return (FTP_PROTOCOL_ERROR);
 	end = conn->buf + conn->buflen;
 	src = conn->buf + 4;
 	if (src >= end || *src++ != '"')
 		return (FTP_PROTOCOL_ERROR);
 	for (q = 0, dst = pwd; src < end && pwdlen--; ++src) {
 		if (!q && *src == '"')
 			q = 1;
 		else if (q && *src != '"')
 			break;
 		else if (q)
 			*dst++ = '"', q = 0;
 		else
 			*dst++ = *src;
 	}
 	if (!pwdlen)
 		return (FTP_PROTOCOL_ERROR);
 	*dst = '\0';
 #if 0
 	DEBUG(fprintf(stderr, "pwd: [%s]\n", pwd));
 #endif
 	return (FTP_OK);
 }
 /*
  * Change working directory to the directory that contains the specified
  * file.
  */
 static int
 ftp_cwd(conn_t *conn, const char *file)
 {
 	const char *beg, *end;
 	char pwd[PATH_MAX];
 	int e, i, len;
 	/* If no slashes in name, no need to change dirs. */
 	if ((end = strrchr(file, '/')) == NULL)
 		return (0);
 	if ((e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY ||
 	    (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) {
 		ftp_seterr(e);
 		return (-1);
 	}
 	for (;;) {
 		len = strlen(pwd);
 		/* Look for a common prefix between PWD and dir to fetch. */
 		for (i = 0; i <= len && i <= end - file; ++i)
 			if (pwd[i] != file[i])
 				break;
 #if 0
 		DEBUG(fprintf(stderr, "have: [%.*s|%s]\n", i, pwd, pwd + i));
 		DEBUG(fprintf(stderr, "want: [%.*s|%s]\n", i, file, file + i));
 #endif
 		/* Keep going up a dir until we have a matching prefix. */
 		if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/'))
 			break;
 		if ((e = ftp_cmd(conn, "CDUP")) != FTP_FILE_ACTION_OK ||
 		    (e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY ||
 		    (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) {
 			ftp_seterr(e);
 			return (-1);
 		}
 	}
 #ifdef FTP_COMBINE_CWDS
 	/* Skip leading slashes, even "////". */
 	for (beg = file + i; beg < end && *beg == '/'; ++beg, ++i)
 		/* nothing */ ;
 	/* If there is no trailing dir, we're already there. */
 	if (beg >= end)
 		return (0);
 	/* Change to the directory all in one chunk (e.g., foo/bar/baz). */
 	e = ftp_cmd(conn, "CWD %.*s", (int)(end - beg), beg);
 	if (e == FTP_FILE_ACTION_OK)
 		return (0);
 #endif /* FTP_COMBINE_CWDS */
 	/* That didn't work so go back to legacy behavior (multiple CWDs). */
 	for (beg = file + i; beg < end; beg = file + i + 1) {
 		while (*beg == '/')
 			++beg, ++i;
 		for (++i; file + i < end && file[i] != '/'; ++i)
 			/* nothing */ ;
 		e = ftp_cmd(conn, "CWD %.*s", file + i - beg, beg);
 		if (e != FTP_FILE_ACTION_OK) {
 			ftp_seterr(e);
 			return (-1);
 		}
 	}
 	return (0);
 }
 /*
  * Set transfer mode and data type
  */
 static int
 ftp_mode_type(conn_t *conn, int mode, int type)
 {
 	int e;
 	switch (mode) {
 	case 0:
 	case 's':
 		mode = 'S';
 	case 'S':
 		break;
 	default:
 		return (FTP_PROTOCOL_ERROR);
 	}
 	if ((e = ftp_cmd(conn, "MODE %c", mode)) != FTP_OK) {
 		if (mode == 'S') {
 			/*
 			 * Stream mode is supposed to be the default - so
 			 * much so that some servers not only do not
 			 * support any other mode, but do not support the
 			 * MODE command at all.
 			 *
 			 * If "MODE S" fails, it is unlikely that we
 			 * previously succeeded in setting a different
 			 * mode.  Therefore, we simply hope that the
 			 * server is already in the correct mode, and
 			 * silently ignore the failure.
 			 */
 		} else {
 			return (e);
 		}
 	}
 	switch (type) {
 	case 0:
 	case 'i':
 		type = 'I';
 	case 'I':
 		break;
 	case 'a':
 		type = 'A';
 	case 'A':
 		break;
 	case 'd':
 		type = 'D';
 	case 'D':
 		/* can't handle yet */
 	default:
 		return (FTP_PROTOCOL_ERROR);
 	}
 	if ((e = ftp_cmd(conn, "TYPE %c", type)) != FTP_OK)
 		return (e);
 	return (FTP_OK);
 }
 /*
  * Request and parse file stats
  */
 static int
 ftp_stat(conn_t *conn, const char *file, struct url_stat *us)
 {
 	char *ln;
 	const char *filename;
 	int filenamelen, type;
 	struct tm tm;
 	time_t t;
 	int e;
 	us->size = -1;
 	us->atime = us->mtime = 0;
 	filename = ftp_filename(file, &filenamelen, &type);
 	if ((e = ftp_mode_type(conn, 0, type)) != FTP_OK) {
 		ftp_seterr(e);
 		return (-1);
 	}
 	e = ftp_cmd(conn, "SIZE %.*s", filenamelen, filename);
 	if (e != FTP_FILE_STATUS) {
 		ftp_seterr(e);
 		return (-1);
 	}
 	for (ln = conn->buf + 4; *ln && isspace((unsigned char)*ln); ln++)
 		/* nothing */ ;
 	for (us->size = 0; *ln && isdigit((unsigned char)*ln); ln++)
 		us->size = us->size * 10 + *ln - '0';
 	if (*ln && !isspace((unsigned char)*ln)) {
 		ftp_seterr(FTP_PROTOCOL_ERROR);
 		us->size = -1;
 		return (-1);
 	}
 	if (us->size == 0)
 		us->size = -1;
 	DEBUG(fprintf(stderr, "size: [%lld]\n", (long long)us->size));
 	e = ftp_cmd(conn, "MDTM %.*s", filenamelen, filename);
 	if (e != FTP_FILE_STATUS) {
 		ftp_seterr(e);
 		return (-1);
 	}
 	for (ln = conn->buf + 4; *ln && isspace((unsigned char)*ln); ln++)
 		/* nothing */ ;
 	switch (strspn(ln, "0123456789")) {
 	case 14:
 		break;
 	case 15:
 		ln++;
 		ln[0] = '2';
 		ln[1] = '0';
 		break;
 	default:
 		ftp_seterr(FTP_PROTOCOL_ERROR);
 		return (-1);
 	}
 	if (sscanf(ln, "%04d%02d%02d%02d%02d%02d",
 	    &tm.tm_year, &tm.tm_mon, &tm.tm_mday,
 	    &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) {
 		ftp_seterr(FTP_PROTOCOL_ERROR);
 		return (-1);
 	}
 	tm.tm_mon--;
 	tm.tm_year -= 1900;
 	tm.tm_isdst = -1;
 	t = timegm(&tm);
 	if (t == (time_t)-1)
 		t = time(NULL);
 	us->mtime = t;
 	us->atime = t;
 	DEBUG(fprintf(stderr,
 	    "last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n",
 	    tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
 	    tm.tm_hour, tm.tm_min, tm.tm_sec));
 	return (0);
 }
 /*
  * I/O functions for FTP
  */
 struct ftpio {
 	conn_t	*cconn;		/* Control connection */
 	conn_t	*dconn;		/* Data connection */
 	int	 dir;		/* Direction */
 	int	 eof;		/* EOF reached */
 	int	 err;		/* Error code */
 };
 static int	 ftp_readfn(void *, char *, int);
 static int	 ftp_writefn(void *, const char *, int);
 static fpos_t	 ftp_seekfn(void *, fpos_t, int);
 static int	 ftp_closefn(void *);
 static int
 ftp_readfn(void *v, char *buf, int len)
 {
 	struct ftpio *io;
 	int r;
 	io = (struct ftpio *)v;
 	if (io == NULL) {
 		errno = EBADF;
 		return (-1);
 	}
 	if (io->cconn == NULL || io->dconn == NULL || io->dir == O_WRONLY) {
 		errno = EBADF;
 		return (-1);
 	}
 	if (io->err) {
 		errno = io->err;
 		return (-1);
 	}
 	if (io->eof)
 		return (0);
 	r = fetch_read(io->dconn, buf, len);
 	if (r > 0)
 		return (r);
 	if (r == 0) {
 		io->eof = 1;
 		return (0);
 	}
 	if (errno != EINTR)
 		io->err = errno;
 	return (-1);
 }
 static int
 ftp_writefn(void *v, const char *buf, int len)
 {
 	struct ftpio *io;
 	int w;
 	io = (struct ftpio *)v;
 	if (io == NULL) {
 		errno = EBADF;
 		return (-1);
 	}
 	if (io->cconn == NULL || io->dconn == NULL || io->dir == O_RDONLY) {
 		errno = EBADF;
 		return (-1);
 	}
 	if (io->err) {
 		errno = io->err;
 		return (-1);
 	}
 	w = fetch_write(io->dconn, buf, len);
 	if (w >= 0)
 		return (w);
 	if (errno != EINTR)
 		io->err = errno;
 	return (-1);
 }
 static fpos_t
 ftp_seekfn(void *v, fpos_t pos __unused, int whence __unused)
 {
 	struct ftpio *io;
 	io = (struct ftpio *)v;
 	if (io == NULL) {
 		errno = EBADF;
 		return (-1);
 	}
 	errno = ESPIPE;
 	return (-1);
 }
 static int
 ftp_closefn(void *v)
 {
 	struct ftpio *io;
 	int r;
 	io = (struct ftpio *)v;
 	if (io == NULL) {
 		errno = EBADF;
 		return (-1);
 	}
 	if (io->dir == -1)
 		return (0);
 	if (io->cconn == NULL || io->dconn == NULL) {
 		errno = EBADF;
 		return (-1);
 	}
 	fetch_close(io->dconn);
 	io->dir = -1;
 	io->dconn = NULL;
 	DEBUG(fprintf(stderr, "Waiting for final status\n"));
 	r = ftp_chkerr(io->cconn);
 	if (io->cconn == cached_connection && io->cconn->ref == 1)
 		cached_connection = NULL;
 	fetch_close(io->cconn);
 	free(io);
 	return (r == FTP_TRANSFER_COMPLETE) ? 0 : -1;
 }
 static FILE *
 ftp_setup(conn_t *cconn, conn_t *dconn, int mode)
 {
 	struct ftpio *io;
 	FILE *f;
 	if (cconn == NULL || dconn == NULL)
 		return (NULL);
 	if ((io = malloc(sizeof(*io))) == NULL)
 		return (NULL);
 	io->cconn = cconn;
 	io->dconn = dconn;
 	io->dir = mode;
 	io->eof = io->err = 0;
 	f = funopen(io, ftp_readfn, ftp_writefn, ftp_seekfn, ftp_closefn);
 	if (f == NULL)
 		free(io);
 	return (f);
 }
 /*
  * Transfer file
  */
 static FILE *
 ftp_transfer(conn_t *conn, const char *oper, const char *file,
     int mode, off_t offset, const char *flags)
 {
 	struct sockaddr_storage sa;
 	struct sockaddr_in6 *sin6;
 	struct sockaddr_in *sin4;
 	const char *bindaddr;
 	const char *filename;
 	int filenamelen, type;
 	int low, pasv, verbose;
 	int e, sd = -1;
 	socklen_t l;
 	char *s;
 	FILE *df;
 	/* check flags */
 	low = CHECK_FLAG('l');
 	pasv = CHECK_FLAG('p') || !CHECK_FLAG('P');
 	verbose = CHECK_FLAG('v');
 	/* passive mode */
 	if ((s = getenv("FTP_PASSIVE_MODE")) != NULL)
 		pasv = (strncasecmp(s, "no", 2) != 0);
 	/* isolate filename */
 	filename = ftp_filename(file, &filenamelen, &type);
 	/* set transfer mode and data type */
 	if ((e = ftp_mode_type(conn, 0, type)) != FTP_OK)
 		goto ouch;
 	/* find our own address, bind, and listen */
 	l = sizeof(sa);
 	if (getsockname(conn->sd, (struct sockaddr *)&sa, &l) == -1)
 		goto sysouch;
 	if (sa.ss_family == AF_INET6)
 		unmappedaddr((struct sockaddr_in6 *)&sa);
 	/* open data socket */
 	if ((sd = socket(sa.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) {
 		fetch_syserr();
 		return (NULL);
 	}
 	if (pasv) {
 		u_char addr[64];
 		char *ln, *p;
 		unsigned int i;
 		int port;
 		/* send PASV command */
 		if (verbose)
 			fetch_info("setting passive mode");
 		switch (sa.ss_family) {
 		case AF_INET:
 			if ((e = ftp_cmd(conn, "PASV")) != FTP_PASSIVE_MODE)
 				goto ouch;
 			break;
 		case AF_INET6:
 			if ((e = ftp_cmd(conn, "EPSV")) != FTP_EPASSIVE_MODE) {
 				if (e == -1)
 					goto ouch;
 				if ((e = ftp_cmd(conn, "LPSV")) !=
 				    FTP_LPASSIVE_MODE)
 					goto ouch;
 			}
 			break;
 		default:
 			e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */
 			goto ouch;
 		}
 		/*
 		 * Find address and port number. The reply to the PASV command
 		 * is IMHO the one and only weak point in the FTP protocol.
 		 */
 		ln = conn->buf;
 		switch (e) {
 		case FTP_PASSIVE_MODE:
 		case FTP_LPASSIVE_MODE:
 			for (p = ln + 3; *p && !isdigit((unsigned char)*p); p++)
 				/* nothing */ ;
 			if (!*p) {
 				e = FTP_PROTOCOL_ERROR;
 				goto ouch;
 			}
 			l = (e == FTP_PASSIVE_MODE ? 6 : 21);
 			for (i = 0; *p && i < l; i++, p++)
 				addr[i] = strtol(p, &p, 10);
 			if (i < l) {
 				e = FTP_PROTOCOL_ERROR;
 				goto ouch;
 			}
 			break;
 		case FTP_EPASSIVE_MODE:
 			for (p = ln + 3; *p && *p != '('; p++)
 				/* nothing */ ;
 			if (!*p) {
 				e = FTP_PROTOCOL_ERROR;
 				goto ouch;
 			}
 			++p;
 			if (sscanf(p, "%c%c%c%d%c", &addr[0], &addr[1], &addr[2],
 				&port, &addr[3]) != 5 ||
 			    addr[0] != addr[1] ||
 			    addr[0] != addr[2] || addr[0] != addr[3]) {
 				e = FTP_PROTOCOL_ERROR;
 				goto ouch;
 			}
 			break;
 		}
 		/* seek to required offset */
 		if (offset)
 			if (ftp_cmd(conn, "REST %lu", (u_long)offset) != FTP_FILE_OK)
 				goto sysouch;
 		/* construct sockaddr for data socket */
 		l = sizeof(sa);
 		if (getpeername(conn->sd, (struct sockaddr *)&sa, &l) == -1)
 			goto sysouch;
 		if (sa.ss_family == AF_INET6)
 			unmappedaddr((struct sockaddr_in6 *)&sa);
 		switch (sa.ss_family) {
 		case AF_INET6:
 			sin6 = (struct sockaddr_in6 *)&sa;
 			if (e == FTP_EPASSIVE_MODE)
 				sin6->sin6_port = htons(port);
 			else {
 				memcpy(&sin6->sin6_addr, addr + 2, 16);
 				memcpy(&sin6->sin6_port, addr + 19, 2);
 			}
 			break;
 		case AF_INET:
 			sin4 = (struct sockaddr_in *)&sa;
 			if (e == FTP_EPASSIVE_MODE)
 				sin4->sin_port = htons(port);
 			else {
 				memcpy(&sin4->sin_addr, addr, 4);
 				memcpy(&sin4->sin_port, addr + 4, 2);
 			}
 			break;
 		default:
 			e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */
 			break;
 		}
 		/* connect to data port */
 		if (verbose)
 			fetch_info("opening data connection");
 		bindaddr = getenv("FETCH_BIND_ADDRESS");
 		if (bindaddr != NULL && *bindaddr != '\0' &&
 		    fetch_bind(sd, sa.ss_family, bindaddr) != 0)
 			goto sysouch;
 		if (connect(sd, (struct sockaddr *)&sa, sa.ss_len) == -1)
 			goto sysouch;
 		/* make the server initiate the transfer */
 		if (verbose)
 			fetch_info("initiating transfer");
 		e = ftp_cmd(conn, "%s %.*s", oper, filenamelen, filename);
 		if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION)
 			goto ouch;
 	} else {
 		u_int32_t a;
 		u_short p;
 		int arg, d;
 		char *ap;
 		char hname[INET6_ADDRSTRLEN];
 		switch (sa.ss_family) {
 		case AF_INET6:
 			((struct sockaddr_in6 *)&sa)->sin6_port = 0;
 #ifdef IPV6_PORTRANGE
 			arg = low ? IPV6_PORTRANGE_DEFAULT : IPV6_PORTRANGE_HIGH;
 			if (setsockopt(sd, IPPROTO_IPV6, IPV6_PORTRANGE,
 				(char *)&arg, sizeof(arg)) == -1)
 				goto sysouch;
 #endif
 			break;
 		case AF_INET:
 			((struct sockaddr_in *)&sa)->sin_port = 0;
 			arg = low ? IP_PORTRANGE_DEFAULT : IP_PORTRANGE_HIGH;
 			if (setsockopt(sd, IPPROTO_IP, IP_PORTRANGE,
 				(char *)&arg, sizeof(arg)) == -1)
 				goto sysouch;
 			break;
 		}
 		if (verbose)
 			fetch_info("binding data socket");
 		if (bind(sd, (struct sockaddr *)&sa, sa.ss_len) == -1)
 			goto sysouch;
 		if (listen(sd, 1) == -1)
 			goto sysouch;
 		/* find what port we're on and tell the server */
 		if (getsockname(sd, (struct sockaddr *)&sa, &l) == -1)
 			goto sysouch;
 		switch (sa.ss_family) {
 		case AF_INET:
 			sin4 = (struct sockaddr_in *)&sa;
 			a = ntohl(sin4->sin_addr.s_addr);
 			p = ntohs(sin4->sin_port);
 			e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d",
 			    (a >> 24) & 0xff, (a >> 16) & 0xff,
 			    (a >> 8) & 0xff, a & 0xff,
 			    (p >> 8) & 0xff, p & 0xff);
 			break;
 		case AF_INET6:
 #define UC(b)	(((int)b)&0xff)
 			e = -1;
 			sin6 = (struct sockaddr_in6 *)&sa;
 			sin6->sin6_scope_id = 0;
 			if (getnameinfo((struct sockaddr *)&sa, sa.ss_len,
 				hname, sizeof(hname),
 				NULL, 0, NI_NUMERICHOST) == 0) {
 				e = ftp_cmd(conn, "EPRT |%d|%s|%d|", 2, hname,
 				    htons(sin6->sin6_port));
 				if (e == -1)
 					goto ouch;
 			}
 			if (e != FTP_OK) {
 				ap = (char *)&sin6->sin6_addr;
 				e = ftp_cmd(conn,
 				    "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d",
 , 16,
 				    UC(ap[0]), UC(ap[1]), UC(ap[2]), UC(ap[3]),
 				    UC(ap[4]), UC(ap[5]), UC(ap[6]), UC(ap[7]),
 				    UC(ap[8]), UC(ap[9]), UC(ap[10]), UC(ap[11]),
 				    UC(ap[12]), UC(ap[13]), UC(ap[14]), UC(ap[15]),
 ,
 				    (ntohs(sin6->sin6_port) >> 8) & 0xff,
 				    ntohs(sin6->sin6_port)        & 0xff);
 			}
 			break;
 		default:
 			e = FTP_PROTOCOL_ERROR; /* XXX: error code should be prepared */
 			goto ouch;
 		}
 		if (e != FTP_OK)
 			goto ouch;
 		/* seek to required offset */
 		if (offset)
 			if (ftp_cmd(conn, "REST %ju", (uintmax_t)offset) != FTP_FILE_OK)
 				goto sysouch;
 		/* make the server initiate the transfer */
 		if (verbose)
 			fetch_info("initiating transfer");
 		e = ftp_cmd(conn, "%s %.*s", oper, filenamelen, filename);
 		if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION)
 			goto ouch;
 		/* accept the incoming connection and go to town */
 		if ((d = accept(sd, NULL, NULL)) == -1)
 			goto sysouch;
 		close(sd);
 		sd = d;
 	}
 	if ((df = ftp_setup(conn, fetch_reopen(sd), mode)) == NULL)
 		goto sysouch;
 	return (df);
 sysouch:
 	fetch_syserr();
 	if (sd >= 0)
 		close(sd);
 	return (NULL);
 ouch:
 	if (e != -1)
 		ftp_seterr(e);
 	if (sd >= 0)
 		close(sd);
 	return (NULL);
 }
 /*
  * Authenticate
  */
 static int
 ftp_authenticate(conn_t *conn, struct url *url, struct url *purl)
 {
 	const char *user, *pwd, *logname;
 	char pbuf[MAXHOSTNAMELEN + MAXLOGNAME + 1];
 	int e, len;
 	/* XXX FTP_AUTH, and maybe .netrc */
 	/* send user name and password */
 	if (url->user[0] == '\0')
 		fetch_netrc_auth(url);
 	user = url->user;
 	if (*user == '\0')
 		user = getenv("FTP_LOGIN");
 	if (user == NULL || *user == '\0')
 		user = FTP_ANONYMOUS_USER;
 	if (purl && url->port == fetch_default_port(url->scheme))
 		e = ftp_cmd(conn, "USER %s@%s", user, url->host);
 	else if (purl)
 		e = ftp_cmd(conn, "USER %s@%s@%d", user, url->host, url->port);
 	else
 		e = ftp_cmd(conn, "USER %s", user);
 	/* did the server request a password? */
 	if (e == FTP_NEED_PASSWORD) {
 		pwd = url->pwd;
 		if (*pwd == '\0')
 			pwd = getenv("FTP_PASSWORD");
 		if (pwd == NULL || *pwd == '\0') {
 			if ((logname = getlogin()) == 0)
 				logname = FTP_ANONYMOUS_USER;
 			if ((len = snprintf(pbuf, MAXLOGNAME + 1, "%s@", logname)) < 0)
 				len = 0;
 			else if (len > MAXLOGNAME)
 				len = MAXLOGNAME;
 			gethostname(pbuf + len, sizeof(pbuf) - len);
 			pwd = pbuf;
 		}
 		e = ftp_cmd(conn, "PASS %s", pwd);
 	}
 	return (e);
 }
 /*
  * Log on to FTP server
  */
 static conn_t *
 ftp_connect(struct url *url, struct url *purl, const char *flags)
 {
 	conn_t *conn;
 	int e, direct, verbose;
 #ifdef INET6
 	int af = AF_UNSPEC;
 #else
 	int af = AF_INET;
 #endif
 	direct = CHECK_FLAG('d');
 	verbose = CHECK_FLAG('v');
 	if (CHECK_FLAG('4'))
 		af = AF_INET;
 	else if (CHECK_FLAG('6'))
 		af = AF_INET6;
 	if (direct)
 		purl = NULL;
 	/* check for proxy */
 	if (purl) {
 		/* XXX proxy authentication! */
 		conn = fetch_connect(purl->host, purl->port, af, verbose);
 	} else {
 		/* no proxy, go straight to target */
 		conn = fetch_connect(url->host, url->port, af, verbose);
 		purl = NULL;
 	}
 	/* check connection */
 	if (conn == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
 	/* expect welcome message */
 	if ((e = ftp_chkerr(conn)) != FTP_SERVICE_READY)
 		goto fouch;
 	/* authenticate */
 	if ((e = ftp_authenticate(conn, url, purl)) != FTP_LOGGED_IN)
 		goto fouch;
 	/* TODO: Request extended features supported, if any (RFC 3659). */
 	/* done */
 	return (conn);
 fouch:
 	if (e != -1)
 		ftp_seterr(e);
 	fetch_close(conn);
 	return (NULL);
 }
 /*
  * Disconnect from server
  */
 static void
 ftp_disconnect(conn_t *conn)
 {
 	(void)ftp_cmd(conn, "QUIT");
 	if (conn == cached_connection && conn->ref == 1)
 		cached_connection = NULL;
 	fetch_close(conn);
 }
 /*
  * Check if we're already connected
  */
 static int
 ftp_isconnected(struct url *url)
 {
 	return (cached_connection
 	    && (strcmp(url->host, cached_host.host) == 0)
 	    && (strcmp(url->user, cached_host.user) == 0)
 	    && (strcmp(url->pwd, cached_host.pwd) == 0)
 	    && (url->port == cached_host.port));
 }
 /*
  * Check the cache, reconnect if no luck
  */
 static conn_t *
 ftp_cached_connect(struct url *url, struct url *purl, const char *flags)
 {
 	conn_t *conn;
 	int e;
 	/* set default port */
 	if (!url->port)
 		url->port = fetch_default_port(url->scheme);
 	/* try to use previously cached connection */
 	if (ftp_isconnected(url)) {
 		e = ftp_cmd(cached_connection, "NOOP");
 		if (e == FTP_OK || e == FTP_SYNTAX_ERROR)
 			return (fetch_ref(cached_connection));
 	}
 	/* connect to server */
 	if ((conn = ftp_connect(url, purl, flags)) == NULL)
 		return (NULL);
 	if (cached_connection)
 		ftp_disconnect(cached_connection);
 	cached_connection = fetch_ref(conn);
 	memcpy(&cached_host, url, sizeof(*url));
 	return (conn);
 }
 /*
  * Check the proxy settings
  */
 static struct url *
 ftp_get_proxy(struct url * url, const char *flags)
 {
 	struct url *purl;
 	char *p;
 	if (flags != NULL && strchr(flags, 'd') != NULL)
 		return (NULL);
 	if (fetch_no_proxy_match(url->host))
 		return (NULL);
 	if (((p = getenv("FTP_PROXY")) || (p = getenv("ftp_proxy")) ||
 		(p = getenv("HTTP_PROXY")) || (p = getenv("http_proxy"))) &&
 	    *p && (purl = fetchParseURL(p)) != NULL) {
 		if (!*purl->scheme) {
 			if (getenv("FTP_PROXY") || getenv("ftp_proxy"))
 				strcpy(purl->scheme, SCHEME_FTP);
 			else
 				strcpy(purl->scheme, SCHEME_HTTP);
 		}
 		if (!purl->port)
 			purl->port = fetch_default_proxy_port(purl->scheme);
 		if (strcasecmp(purl->scheme, SCHEME_FTP) == 0 ||
 		    strcasecmp(purl->scheme, SCHEME_HTTP) == 0)
 			return (purl);
 		fetchFreeURL(purl);
 	}
 	return (NULL);
 }
 /*
  * Process an FTP request
  */
 FILE *
 ftp_request(struct url *url, const char *op, struct url_stat *us,
     struct url *purl, const char *flags)
 {
 	conn_t *conn;
 	int oflag;
 	/* check if we should use HTTP instead */
 	if (purl && strcasecmp(purl->scheme, SCHEME_HTTP) == 0) {
 		if (strcmp(op, "STAT") == 0)
 			return (http_request(url, "HEAD", us, purl, flags));
 		else if (strcmp(op, "RETR") == 0)
 			return (http_request(url, "GET", us, purl, flags));
 		/*
 		 * Our HTTP code doesn't support PUT requests yet, so try
 		 * a direct connection.
 		 */
 	}
 	/* connect to server */
 	conn = ftp_cached_connect(url, purl, flags);
 	if (purl)
 		fetchFreeURL(purl);
 	if (conn == NULL)
 		return (NULL);
 	/* change directory */
 	if (ftp_cwd(conn, url->doc) == -1)
 		goto errsock;
 	/* stat file */
 	if (us && ftp_stat(conn, url->doc, us) == -1
 	    && fetchLastErrCode != FETCH_PROTO
 	    && fetchLastErrCode != FETCH_UNAVAIL)
 		goto errsock;
 	/* just a stat */
 	if (strcmp(op, "STAT") == 0) {
 		--conn->ref;
 		ftp_disconnect(conn);
 		return (FILE *)1; /* bogus return value */
 	}
 	if (strcmp(op, "STOR") == 0 || strcmp(op, "APPE") == 0)
 		oflag = O_WRONLY;
 	else
 		oflag = O_RDONLY;
 	/* initiate the transfer */
 	return (ftp_transfer(conn, op, url->doc, oflag, url->offset, flags));
 errsock:
 	ftp_disconnect(conn);
 	return (NULL);
 }
 /*
  * Get and stat file
  */
 FILE *
 fetchXGetFTP(struct url *url, struct url_stat *us, const char *flags)
 {
 	return (ftp_request(url, "RETR", us, ftp_get_proxy(url, flags), flags));
 }
 /*
  * Get file
  */
 FILE *
 fetchGetFTP(struct url *url, const char *flags)
 {
 	return (fetchXGetFTP(url, NULL, flags));
 }
 /*
  * Put file
  */
 FILE *
 fetchPutFTP(struct url *url, const char *flags)
 {
 	return (ftp_request(url, CHECK_FLAG('a') ? "APPE" : "STOR", NULL,
 	    ftp_get_proxy(url, flags), flags));
 }
 /*
  * Get file stats
  */
 int
 fetchStatFTP(struct url *url, struct url_stat *us, const char *flags)
 {
 	FILE *f;
 	f = ftp_request(url, "STAT", us, ftp_get_proxy(url, flags), flags);
 	if (f == NULL)
 		return (-1);
 	/*
 	 * When op is "STAT", ftp_request() will return either NULL or
 	 * (FILE *)1, never a valid FILE *, so we mustn't fclose(f) before
 	 * returning, as it would cause a segfault.
 	 */
 	return (0);
 }
 /*
  * List a directory
  */
 struct url_ent *
 fetchListFTP(struct url *url __unused, const char *flags __unused)
 {
 	warnx("fetchListFTP(): not implemented");
 	return (NULL);
 }

added external/libfetch/ftp.errors

@@ -0,0 +1,47 @@

 # $FreeBSD: head/lib/libfetch/ftp.errors 106190 2002-10-30 06:06:16Z des $
 #
 # This list is taken from RFC 959.
 # It probably needs a going over.
 #
 OK		Restart marker reply
 TEMP	Service ready in a few minutes
 OK		Data connection already open; transfer starting
 OK		File status okay; about to open data connection
 OK		Command okay
 PROTO	Command not implemented, superfluous at this site
 INFO	System status, or system help reply
 INFO	Directory status
 INFO	File status
 INFO	Help message
 INFO	Set system type
 OK		Service ready for new user
 OK		Service closing control connection
 OK		Data connection open; no transfer in progress
 OK		Requested file action successful
 OK		Entering Passive Mode
 OK		Entering Extended Passive Mode
 OK		User logged in, proceed
 OK		Requested file action okay, completed
 OK		File/directory created
 AUTH	User name okay, need password
 AUTH	Need account for login
 OK		Requested file action pending further information
 DOWN	Service not available, closing control connection
 NETWORK	Can't open data connection
 ABORT	Connection closed; transfer aborted
 UNAVAIL	File unavailable (e.g., file busy)
 SERVER	Requested action aborted: local error in processing
 FULL	Insufficient storage space in system
 PROTO	Syntax error, command unrecognized
 PROTO	Syntax error in parameters or arguments
 PROTO	Command not implemented
 PROTO	Bad sequence of commands
 PROTO	Command not implemented for that parameter
 AUTH	Not logged in
 AUTH	Need account for storing files
 PROTO	Bug in MediaHawk Video Kernel FTP server
 UNAVAIL	File unavailable (e.g., file not found, no access)
 PROTO	Requested action aborted. Page type unknown
 FULL	Exceeded storage allocation
 EXISTS	File name not allowed
 PROTO	Protocol error

added external/libfetch/ftperr.h

@@ -0,0 +1,45 @@

 static struct fetcherr ftp_errlist[] = {
     { 110, FETCH_OK, "Restart marker reply" },
     { 120, FETCH_TEMP, "Service ready in a few minutes" },
     { 125, FETCH_OK, "Data connection already open; transfer starting" },
     { 150, FETCH_OK, "File status okay; about to open data connection" },
     { 200, FETCH_OK, "Command okay" },
     { 202, FETCH_PROTO, "Command not implemented, superfluous at this site" },
     { 211, FETCH_INFO, "System status, or system help reply" },
     { 212, FETCH_INFO, "Directory status" },
     { 213, FETCH_INFO, "File status" },
     { 214, FETCH_INFO, "Help message" },
     { 215, FETCH_INFO, "Set system type" },
     { 220, FETCH_OK, "Service ready for new user" },
     { 221, FETCH_OK, "Service closing control connection" },
     { 225, FETCH_OK, "Data connection open; no transfer in progress" },
     { 226, FETCH_OK, "Requested file action successful" },
     { 227, FETCH_OK, "Entering Passive Mode" },
     { 229, FETCH_OK, "Entering Extended Passive Mode" },
     { 230, FETCH_OK, "User logged in, proceed" },
     { 250, FETCH_OK, "Requested file action okay, completed" },
     { 257, FETCH_OK, "File/directory created" },
     { 331, FETCH_AUTH, "User name okay, need password" },
     { 332, FETCH_AUTH, "Need account for login" },
     { 350, FETCH_OK, "Requested file action pending further information" },
     { 421, FETCH_DOWN, "Service not available, closing control connection" },
     { 425, FETCH_NETWORK, "Can't open data connection" },
     { 426, FETCH_ABORT, "Connection closed; transfer aborted" },
     { 450, FETCH_UNAVAIL, "File unavailable (e.g., file busy)" },
     { 451, FETCH_SERVER, "Requested action aborted: local error in processing" },
     { 452, FETCH_FULL, "Insufficient storage space in system" },
     { 500, FETCH_PROTO, "Syntax error, command unrecognized" },
     { 501, FETCH_PROTO, "Syntax error in parameters or arguments" },
     { 502, FETCH_PROTO, "Command not implemented" },
     { 503, FETCH_PROTO, "Bad sequence of commands" },
     { 504, FETCH_PROTO, "Command not implemented for that parameter" },
     { 530, FETCH_AUTH, "Not logged in" },
     { 532, FETCH_AUTH, "Need account for storing files" },
     { 535, FETCH_PROTO, "Bug in MediaHawk Video Kernel FTP server" },
     { 550, FETCH_UNAVAIL, "File unavailable (e.g., file not found, no access)" },
     { 551, FETCH_PROTO, "Requested action aborted. Page type unknown" },
     { 552, FETCH_FULL, "Exceeded storage allocation" },
     { 553, FETCH_EXISTS, "File name not allowed" },
     { 999, FETCH_PROTO, "Protocol error" },
     { -1, FETCH_UNKNOWN, "Unknown FTP error" }
 };

added external/libfetch/http.c

@@ -0,0 +1,2079 @@

 /*-
  * Copyright (c) 2000-2014 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD: head/lib/libfetch/http.c 267133 2014-06-05 22:16:26Z bapt $");
 /*
  * The following copyright applies to the base64 code:
  *
  *-
  * Copyright 1997 Massachusetts Institute of Technology
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
  * granted, provided that both the above copyright notice and this
  * permission notice appear in all copies, that both the above
  * copyright notice and this permission notice appear in all
  * supporting documentation, and that the name of M.I.T. not be used
  * in advertising or publicity pertaining to distribution of the
  * software without specific, written prior permission.  M.I.T. makes
  * no representations about the suitability of this software for any
  * purpose.  It is provided "as is" without express or implied
  * warranty.
  *
  * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
  * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
  * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 #include <sys/param.h>
 #include <sys/socket.h>
 #include <sys/time.h>
 #include <ctype.h>
 #include <err.h>
 #include <errno.h>
 #include <locale.h>
 #include <netdb.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
 #ifdef WITH_SSL
 #include <openssl/md5.h>
 #define MD5Init(c) MD5_Init(c)
 #define MD5Update(c, data, len) MD5_Update(c, data, len)
 #define MD5Final(md, c) MD5_Final(md, c)
 #else
 #include <md5.h>
 #endif
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include "fetch.h"
 #include "common.h"
 #include "httperr.h"
 /* Maximum number of redirects to follow */
 #define MAX_REDIRECT 20
 /* Symbolic names for reply codes we care about */
 #define HTTP_OK			200
 #define HTTP_PARTIAL		206
 #define HTTP_MOVED_PERM		301
 #define HTTP_MOVED_TEMP		302
 #define HTTP_SEE_OTHER		303
 #define HTTP_NOT_MODIFIED	304
 #define HTTP_USE_PROXY		305
 #define HTTP_TEMP_REDIRECT	307
 #define HTTP_PERM_REDIRECT	308
 #define HTTP_NEED_AUTH		401
 #define HTTP_NEED_PROXY_AUTH	407
 #define HTTP_BAD_RANGE		416
 #define HTTP_PROTOCOL_ERROR	999
 #define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \
 			    || (xyz) == HTTP_MOVED_TEMP \
 			    || (xyz) == HTTP_TEMP_REDIRECT \
 			    || (xyz) == HTTP_USE_PROXY \
 			    || (xyz) == HTTP_SEE_OTHER)
 #define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599)
 /*****************************************************************************
  * I/O functions for decoding chunked streams
  */
 struct httpio
 {
 	conn_t		*conn;		/* connection */
 	int		 chunked;	/* chunked mode */
 	char		*buf;		/* chunk buffer */
 	size_t		 bufsize;	/* size of chunk buffer */
 	ssize_t		 buflen;	/* amount of data currently in buffer */
 	int		 bufpos;	/* current read offset in buffer */
 	int		 eof;		/* end-of-file flag */
 	int		 error;		/* error flag */
 	size_t		 chunksize;	/* remaining size of current chunk */
 #ifndef NDEBUG
 	size_t		 total;
 #endif
 };
 /*
  * Get next chunk header
  */
 static int
 http_new_chunk(struct httpio *io)
 {
 	char *p;
 	if (fetch_getln(io->conn) == -1)
 		return (-1);
 	if (io->conn->buflen < 2 || !isxdigit((unsigned char)*io->conn->buf))
 		return (-1);
 	for (p = io->conn->buf; *p && !isspace((unsigned char)*p); ++p) {
 		if (*p == ';')
 			break;
 		if (!isxdigit((unsigned char)*p))
 			return (-1);
 		if (isdigit((unsigned char)*p)) {
 			io->chunksize = io->chunksize * 16 +
 			    *p - '0';
 		} else {
 			io->chunksize = io->chunksize * 16 +
 + tolower((unsigned char)*p) - 'a';
 		}
 	}
 #ifndef NDEBUG
 	if (fetchDebug) {
 		io->total += io->chunksize;
 		if (io->chunksize == 0)
 			fprintf(stderr, "%s(): end of last chunk\n", __func__);
 		else
 			fprintf(stderr, "%s(): new chunk: %lu (%lu)\n",
 			    __func__, (unsigned long)io->chunksize,
 			    (unsigned long)io->total);
 	}
 #endif
 	return (io->chunksize);
 }
 /*
  * Grow the input buffer to at least len bytes
  */
 static inline int
 http_growbuf(struct httpio *io, size_t len)
 {
 	char *tmp;
 	if (io->bufsize >= len)
 		return (0);
 	if ((tmp = realloc(io->buf, len)) == NULL)
 		return (-1);
 	io->buf = tmp;
 	io->bufsize = len;
 	return (0);
 }
 /*
  * Fill the input buffer, do chunk decoding on the fly
  */
 static ssize_t
 http_fillbuf(struct httpio *io, size_t len)
 {
 	ssize_t nbytes;
 	char ch;
 	if (io->error)
 		return (-1);
 	if (io->eof)
 		return (0);
 	if (io->chunked == 0) {
 		if (http_growbuf(io, len) == -1)
 			return (-1);
 		if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) {
 			io->error = errno;
 			return (-1);
 		}
 		io->buflen = nbytes;
 		io->bufpos = 0;
 		return (io->buflen);
 	}
 	if (io->chunksize == 0) {
 		switch (http_new_chunk(io)) {
 		case -1:
 			io->error = EPROTO;
 			return (-1);
 		case 0:
 			io->eof = 1;
 			return (0);
 		}
 	}
 	if (len > io->chunksize)
 		len = io->chunksize;
 	if (http_growbuf(io, len) == -1)
 		return (-1);
 	if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) {
 		io->error = errno;
 		return (-1);
 	}
 	io->buflen = nbytes;
 	io->chunksize -= io->buflen;
 	if (io->chunksize == 0) {
 		if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' ||
 		    fetch_read(io->conn, &ch, 1) != 1 || ch != '\n')
 			return (-1);
 	}
 	io->bufpos = 0;
 	return (io->buflen);
 }
 /*
  * Read function
  */
 static int
 http_readfn(void *v, char *buf, int len)
 {
 	struct httpio *io = (struct httpio *)v;
 	int rlen;
 	if (io->error)
 		return (-1);
 	if (io->eof)
 		return (0);
 	/* empty buffer */
 	if (!io->buf || io->bufpos == io->buflen) {
 		if ((rlen = http_fillbuf(io, len)) < 0) {
 			if ((errno = io->error) == EINTR)
 				io->error = 0;
 			return (-1);
 		} else if (rlen == 0) {
 			return (0);
 		}
 	}
 	rlen = io->buflen - io->bufpos;
 	if (len < rlen)
 		rlen = len;
 	memcpy(buf, io->buf + io->bufpos, rlen);
 	io->bufpos += rlen;
 	return (rlen);
 }
 /*
  * Write function
  */
 static int
 http_writefn(void *v, const char *buf, int len)
 {
 	struct httpio *io = (struct httpio *)v;
 	return (fetch_write(io->conn, buf, len));
 }
 /*
  * Close function
  */
 static int
 http_closefn(void *v)
 {
 	struct httpio *io = (struct httpio *)v;
 	int r;
 	r = fetch_close(io->conn);
 	if (io->buf)
 		free(io->buf);
 	free(io);
 	return (r);
 }
 /*
  * Wrap a file descriptor up
  */
 static FILE *
 http_funopen(conn_t *conn, int chunked)
 {
 	struct httpio *io;
 	FILE *f;
 	if ((io = calloc(1, sizeof(*io))) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	io->conn = conn;
 	io->chunked = chunked;
 	f = funopen(io, http_readfn, http_writefn, NULL, http_closefn);
 	if (f == NULL) {
 		fetch_syserr();
 		free(io);
 		return (NULL);
 	}
 	return (f);
 }
 /*****************************************************************************
  * Helper functions for talking to the server and parsing its replies
  */
 /* Header types */
 typedef enum {
 	hdr_syserror = -2,
 	hdr_error = -1,
 	hdr_end = 0,
 	hdr_unknown = 1,
 	hdr_content_length,
 	hdr_content_range,
 	hdr_last_modified,
 	hdr_location,
 	hdr_transfer_encoding,
 	hdr_www_authenticate,
 	hdr_proxy_authenticate,
 } hdr_t;
 /* Names of interesting headers */
 static struct {
 	hdr_t		 num;
 	const char	*name;
 } hdr_names[] = {
 	{ hdr_content_length,		"Content-Length" },
 	{ hdr_content_range,		"Content-Range" },
 	{ hdr_last_modified,		"Last-Modified" },
 	{ hdr_location,			"Location" },
 	{ hdr_transfer_encoding,	"Transfer-Encoding" },
 	{ hdr_www_authenticate,		"WWW-Authenticate" },
 	{ hdr_proxy_authenticate,	"Proxy-Authenticate" },
 	{ hdr_unknown,			NULL },
 };
 /*
  * Send a formatted line; optionally echo to terminal
  */
 static int
 http_cmd(conn_t *conn, const char *fmt, ...)
 {
 	va_list ap;
 	size_t len;
 	char *msg;
 	int r;
 	va_start(ap, fmt);
 	len = vasprintf(&msg, fmt, ap);
 	va_end(ap);
 	if (msg == NULL) {
 		errno = ENOMEM;
 		fetch_syserr();
 		return (-1);
 	}
 	r = fetch_putln(conn, msg, len);
 	free(msg);
 	if (r == -1) {
 		fetch_syserr();
 		return (-1);
 	}
 	return (0);
 }
 /*
  * Get and parse status line
  */
 static int
 http_get_reply(conn_t *conn)
 {
 	char *p;
 	if (fetch_getln(conn) == -1)
 		return (-1);
 	/*
 	 * A valid status line looks like "HTTP/m.n xyz reason" where m
 	 * and n are the major and minor protocol version numbers and xyz
 	 * is the reply code.
 	 * Unfortunately, there are servers out there (NCSA 1.5.1, to name
 	 * just one) that do not send a version number, so we can't rely
 	 * on finding one, but if we do, insist on it being 1.0 or 1.1.
 	 * We don't care about the reason phrase.
 	 */
 	if (strncmp(conn->buf, "HTTP", 4) != 0)
 		return (HTTP_PROTOCOL_ERROR);
 	p = conn->buf + 4;
 	if (*p == '/') {
 		if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1'))
 			return (HTTP_PROTOCOL_ERROR);
 		p += 4;
 	}
 	if (*p != ' ' ||
 	    !isdigit((unsigned char)p[1]) ||
 	    !isdigit((unsigned char)p[2]) ||
 	    !isdigit((unsigned char)p[3]))
 		return (HTTP_PROTOCOL_ERROR);
 	conn->err = (p[1] - '0') * 100 + (p[2] - '0') * 10 + (p[3] - '0');
 	return (conn->err);
 }
 /*
  * Check a header; if the type matches the given string, return a pointer
  * to the beginning of the value.
  */
 static const char *
 http_match(const char *str, const char *hdr)
 {
 	while (*str && *hdr &&
 	    tolower((unsigned char)*str++) == tolower((unsigned char)*hdr++))
 		/* nothing */;
 	if (*str || *hdr != ':')
 		return (NULL);
 	while (*hdr && isspace((unsigned char)*++hdr))
 		/* nothing */;
 	return (hdr);
 }
 /*
  * Get the next header and return the appropriate symbolic code.  We
  * need to read one line ahead for checking for a continuation line
  * belonging to the current header (continuation lines start with
  * white space).
  *
  * We get called with a fresh line already in the conn buffer, either
  * from the previous http_next_header() invocation, or, the first
  * time, from a fetch_getln() performed by our caller.
  *
  * This stops when we encounter an empty line (we dont read beyond the header
  * area).
  *
  * Note that the "headerbuf" is just a place to return the result. Its
  * contents are not used for the next call. This means that no cleanup
  * is needed when ie doing another connection, just call the cleanup when
  * fully done to deallocate memory.
  */
 /* Limit the max number of continuation lines to some reasonable value */
 #define HTTP_MAX_CONT_LINES 10
 /* Place into which to build a header from one or several lines */
 typedef struct {
 	char	*buf;		/* buffer */
 	size_t	 bufsize;	/* buffer size */
 	size_t	 buflen;	/* length of buffer contents */
 } http_headerbuf_t;
 static void
 init_http_headerbuf(http_headerbuf_t *buf)
 {
 	buf->buf = NULL;
 	buf->bufsize = 0;
 	buf->buflen = 0;
 }
 static void
 clean_http_headerbuf(http_headerbuf_t *buf)
 {
 	if (buf->buf)
 		free(buf->buf);
 	init_http_headerbuf(buf);
 }
 /* Remove whitespace at the end of the buffer */
 static void
 http_conn_trimright(conn_t *conn)
 {
 	while (conn->buflen &&
 	       isspace((unsigned char)conn->buf[conn->buflen - 1]))
 		conn->buflen--;
 	conn->buf[conn->buflen] = '\0';
 }
 static hdr_t
 http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p)
 {
 	unsigned int i, len;
 	/*
 	 * Have to do the stripping here because of the first line. So
 	 * it's done twice for the subsequent lines. No big deal
 	 */
 	http_conn_trimright(conn);
 	if (conn->buflen == 0)
 		return (hdr_end);
 	/* Copy the line to the headerbuf */
 	if (hbuf->bufsize < conn->buflen + 1) {
 		if ((hbuf->buf = realloc(hbuf->buf, conn->buflen + 1)) == NULL)
 			return (hdr_syserror);
 		hbuf->bufsize = conn->buflen + 1;
 	}
 	strcpy(hbuf->buf, conn->buf);
 	hbuf->buflen = conn->buflen;
 	/*
 	 * Fetch possible continuation lines. Stop at 1st non-continuation
 	 * and leave it in the conn buffer
 	 */
 	for (i = 0; i < HTTP_MAX_CONT_LINES; i++) {
 		if (fetch_getln(conn) == -1)
 			return (hdr_syserror);
 		/*
 		 * Note: we carry on the idea from the previous version
 		 * that a pure whitespace line is equivalent to an empty
 		 * one (so it's not continuation and will be handled when
 		 * we are called next)
 		 */
 		http_conn_trimright(conn);
 		if (conn->buf[0] != ' ' && conn->buf[0] != "\t"[0])
 			break;
 		/* Got a continuation line. Concatenate to previous */
 		len = hbuf->buflen + conn->buflen;
 		if (hbuf->bufsize < len + 1) {
 			len *= 2;
 			if ((hbuf->buf = realloc(hbuf->buf, len + 1)) == NULL)
 				return (hdr_syserror);
 			hbuf->bufsize = len + 1;
 		}
 		strcpy(hbuf->buf + hbuf->buflen, conn->buf);
 		hbuf->buflen += conn->buflen;
 	}
 	/*
 	 * We could check for malformed headers but we don't really care.
 	 * A valid header starts with a token immediately followed by a
 	 * colon; a token is any sequence of non-control, non-whitespace
 	 * characters except "()<>@,;:\\\"{}".
 	 */
 	for (i = 0; hdr_names[i].num != hdr_unknown; i++)
 		if ((*p = http_match(hdr_names[i].name, hbuf->buf)) != NULL)
 			return (hdr_names[i].num);
 	return (hdr_unknown);
 }
 /**************************
  * [Proxy-]Authenticate header parsing
  */
 /*
  * Read doublequote-delimited string into output buffer obuf (allocated
  * by caller, whose responsibility it is to ensure that it's big enough)
  * cp points to the first char after the initial '"'
  * Handles \ quoting
  * Returns pointer to the first char after the terminating double quote, or
  * NULL for error.
  */
 static const char *
 http_parse_headerstring(const char *cp, char *obuf)
 {
 	for (;;) {
 		switch (*cp) {
 		case 0: /* Unterminated string */
 			*obuf = 0;
 			return (NULL);
 		case '"': /* Ending quote */
 			*obuf = 0;
 			return (++cp);
 		case '\\':
 			if (*++cp == 0) {
 				*obuf = 0;
 				return (NULL);
 			}
 			/* FALLTHROUGH */
 		default:
 			*obuf++ = *cp++;
 		}
 	}
 }
 /* Http auth challenge schemes */
 typedef enum {HTTPAS_UNKNOWN, HTTPAS_BASIC,HTTPAS_DIGEST} http_auth_schemes_t;
 /* Data holder for a Basic or Digest challenge. */
 typedef struct {
 	http_auth_schemes_t scheme;
 	char	*realm;
 	char	*qop;
 	char	*nonce;
 	char	*opaque;
 	char	*algo;
 	int	 stale;
 	int	 nc; /* Nonce count */
 } http_auth_challenge_t;
 static void
 init_http_auth_challenge(http_auth_challenge_t *b)
 {
 	b->scheme = HTTPAS_UNKNOWN;
 	b->realm = b->qop = b->nonce = b->opaque = b->algo = NULL;
 	b->stale = b->nc = 0;
 }
 static void
 clean_http_auth_challenge(http_auth_challenge_t *b)
 {
 	if (b->realm)
 		free(b->realm);
 	if (b->qop)
 		free(b->qop);
 	if (b->nonce)
 		free(b->nonce);
 	if (b->opaque)
 		free(b->opaque);
 	if (b->algo)
 		free(b->algo);
 	init_http_auth_challenge(b);
 }
 /* Data holder for an array of challenges offered in an http response. */
 #define MAX_CHALLENGES 10
 typedef struct {
 	http_auth_challenge_t *challenges[MAX_CHALLENGES];
 	int	count; /* Number of parsed challenges in the array */
 	int	valid; /* We did parse an authenticate header */
 } http_auth_challenges_t;
 static void
 init_http_auth_challenges(http_auth_challenges_t *cs)
 {
 	int i;
 	for (i = 0; i < MAX_CHALLENGES; i++)
 		cs->challenges[i] = NULL;
 	cs->count = cs->valid = 0;
 }
 static void
 clean_http_auth_challenges(http_auth_challenges_t *cs)
 {
 	int i;
 	/* We rely on non-zero pointers being allocated, not on the count */
 	for (i = 0; i < MAX_CHALLENGES; i++) {
 		if (cs->challenges[i] != NULL) {
 			clean_http_auth_challenge(cs->challenges[i]);
 			free(cs->challenges[i]);
 		}
 	}
 	init_http_auth_challenges(cs);
 }
 /*
  * Enumeration for lexical elements. Separators will be returned as their own
  * ascii value
  */
 typedef enum {HTTPHL_WORD=256, HTTPHL_STRING=257, HTTPHL_END=258,
 	      HTTPHL_ERROR = 259} http_header_lex_t;
 /*
  * Determine what kind of token comes next and return possible value
  * in buf, which is supposed to have been allocated big enough by
  * caller. Advance input pointer and return element type.
  */
 static int
 http_header_lex(const char **cpp, char *buf)
 {
 	size_t l;
 	/* Eat initial whitespace */
 	*cpp += strspn(*cpp, " \t");
 	if (**cpp == 0)
 		return (HTTPHL_END);
 	/* Separator ? */
 	if (**cpp == ',' || **cpp == '=')
 		return (*((*cpp)++));
 	/* String ? */
 	if (**cpp == '"') {
 		*cpp = http_parse_headerstring(++*cpp, buf);
 		if (*cpp == NULL)
 			return (HTTPHL_ERROR);
 		return (HTTPHL_STRING);
 	}
 	/* Read other token, until separator or whitespace */
 	l = strcspn(*cpp, " \t,=");
 	memcpy(buf, *cpp, l);
 	buf[l] = 0;
 	*cpp += l;
 	return (HTTPHL_WORD);
 }
 /*
  * Read challenges from http xxx-authenticate header and accumulate them
  * in the challenges list structure.
  *
  * Headers with multiple challenges are specified by rfc2617, but
  * servers (ie: squid) often send them in separate headers instead,
  * which in turn is forbidden by the http spec (multiple headers with
  * the same name are only allowed for pure comma-separated lists, see
  * rfc2616 sec 4.2).
  *
  * We support both approaches anyway
  */
 static int
 http_parse_authenticate(const char *cp, http_auth_challenges_t *cs)
 {
 	int ret = -1;
 	http_header_lex_t lex;
 	char *key = malloc(strlen(cp) + 1);
 	char *value = malloc(strlen(cp) + 1);
 	char *buf = malloc(strlen(cp) + 1);
 	if (key == NULL || value == NULL || buf == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	/* In any case we've seen the header and we set the valid bit */
 	cs->valid = 1;
 	/* Need word first */
 	lex = http_header_lex(&cp, key);
 	if (lex != HTTPHL_WORD)
 		goto out;
 	/* Loop on challenges */
 	for (; cs->count < MAX_CHALLENGES; cs->count++) {
 		cs->challenges[cs->count] =
 			malloc(sizeof(http_auth_challenge_t));
 		if (cs->challenges[cs->count] == NULL) {
 			fetch_syserr();
 			goto out;
 		}
 		init_http_auth_challenge(cs->challenges[cs->count]);
 		if (!strcasecmp(key, "basic")) {
 			cs->challenges[cs->count]->scheme = HTTPAS_BASIC;
 		} else if (!strcasecmp(key, "digest")) {
 			cs->challenges[cs->count]->scheme = HTTPAS_DIGEST;
 		} else {
 			cs->challenges[cs->count]->scheme = HTTPAS_UNKNOWN;
 			/*
 			 * Continue parsing as basic or digest may
 			 * follow, and the syntax is the same for
 			 * all. We'll just ignore this one when
 			 * looking at the list
 			 */
 		}
 		/* Loop on attributes */
 		for (;;) {
 			/* Key */
 			lex = http_header_lex(&cp, key);
 			if (lex != HTTPHL_WORD)
 				goto out;
 			/* Equal sign */
 			lex = http_header_lex(&cp, buf);
 			if (lex != '=')
 				goto out;
 			/* Value */
 			lex = http_header_lex(&cp, value);
 			if (lex != HTTPHL_WORD && lex != HTTPHL_STRING)
 				goto out;
 			if (!strcasecmp(key, "realm"))
 				cs->challenges[cs->count]->realm =
 					strdup(value);
 			else if (!strcasecmp(key, "qop"))
 				cs->challenges[cs->count]->qop =
 					strdup(value);
 			else if (!strcasecmp(key, "nonce"))
 				cs->challenges[cs->count]->nonce =
 					strdup(value);
 			else if (!strcasecmp(key, "opaque"))
 				cs->challenges[cs->count]->opaque =
 					strdup(value);
 			else if (!strcasecmp(key, "algorithm"))
 				cs->challenges[cs->count]->algo =
 					strdup(value);
 			else if (!strcasecmp(key, "stale"))
 				cs->challenges[cs->count]->stale =
 					strcasecmp(value, "no");
 			/* Else ignore unknown attributes */
 			/* Comma or Next challenge or End */
 			lex = http_header_lex(&cp, key);
 			/*
 			 * If we get a word here, this is the beginning of the
 			 * next challenge. Break the attributes loop
 			 */
 			if (lex == HTTPHL_WORD)
 				break;
 			if (lex == HTTPHL_END) {
 				/* End while looking for ',' is normal exit */
 				cs->count++;
 				ret = 0;
 				goto out;
 			}
 			/* Anything else is an error */
 			if (lex != ',')
 				goto out;
 		} /* End attributes loop */
 	} /* End challenge loop */
 	/*
 	 * Challenges max count exceeded. This really can't happen
 	 * with normal data, something's fishy -> error
 	 */
 out:
 	if (key)
 		free(key);
 	if (value)
 		free(value);
 	if (buf)
 		free(buf);
 	return (ret);
 }
 /*
  * Parse a last-modified header
  */
 static int
 http_parse_mtime(const char *p, time_t *mtime)
 {
 	char locale[64], *r;
 	struct tm tm;
 	strncpy(locale, setlocale(LC_TIME, NULL), sizeof(locale));
 	setlocale(LC_TIME, "C");
 	r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
 	/*
 	 * Some proxies use UTC in response, but it should still be
 	 * parsed. RFC2616 states GMT and UTC are exactly equal for HTTP.
 	 */
 	if (r == NULL)
 		r = strptime(p, "%a, %d %b %Y %H:%M:%S UTC", &tm);
 	/* XXX should add support for date-2 and date-3 */
 	setlocale(LC_TIME, locale);
 	if (r == NULL)
 		return (-1);
 	DEBUG(fprintf(stderr, "last modified: [%04d-%02d-%02d "
 		  "%02d:%02d:%02d]\n",
 		  tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
 		  tm.tm_hour, tm.tm_min, tm.tm_sec));
 	*mtime = timegm(&tm);
 	return (0);
 }
 /*
  * Parse a content-length header
  */
 static int
 http_parse_length(const char *p, off_t *length)
 {
 	off_t len;
 	for (len = 0; *p && isdigit((unsigned char)*p); ++p)
 		len = len * 10 + (*p - '0');
 	if (*p)
 		return (-1);
 	DEBUG(fprintf(stderr, "content length: [%lld]\n",
 	    (long long)len));
 	*length = len;
 	return (0);
 }
 /*
  * Parse a content-range header
  */
 static int
 http_parse_range(const char *p, off_t *offset, off_t *length, off_t *size)
 {
 	off_t first, last, len;
 	if (strncasecmp(p, "bytes ", 6) != 0)
 		return (-1);
 	p += 6;
 	if (*p == '*') {
 		first = last = -1;
 		++p;
 	} else {
 		for (first = 0; *p && isdigit((unsigned char)*p); ++p)
 			first = first * 10 + *p - '0';
 		if (*p != '-')
 			return (-1);
 		for (last = 0, ++p; *p && isdigit((unsigned char)*p); ++p)
 			last = last * 10 + *p - '0';
 	}
 	if (first > last || *p != '/')
 		return (-1);
 	for (len = 0, ++p; *p && isdigit((unsigned char)*p); ++p)
 		len = len * 10 + *p - '0';
 	if (*p || len < last - first + 1)
 		return (-1);
 	if (first == -1) {
 		DEBUG(fprintf(stderr, "content range: [*/%lld]\n",
 		    (long long)len));
 		*length = 0;
 	} else {
 		DEBUG(fprintf(stderr, "content range: [%lld-%lld/%lld]\n",
 		    (long long)first, (long long)last, (long long)len));
 		*length = last - first + 1;
 	}
 	*offset = first;
 	*size = len;
 	return (0);
 }
 /*****************************************************************************
  * Helper functions for authorization
  */
 /*
  * Base64 encoding
  */
 static char *
 http_base64(const char *src)
 {
 	static const char base64[] =
 	    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 	    "abcdefghijklmnopqrstuvwxyz"
 	    "0123456789+/";
 	char *str, *dst;
 	size_t l;
 	int t, r;
 	l = strlen(src);
 	if ((str = malloc(((l + 2) / 3) * 4 + 1)) == NULL)
 		return (NULL);
 	dst = str;
 	r = 0;
 	while (l >= 3) {
 		t = (src[0] << 16) | (src[1] << 8) | src[2];
 		dst[0] = base64[(t >> 18) & 0x3f];
 		dst[1] = base64[(t >> 12) & 0x3f];
 		dst[2] = base64[(t >> 6) & 0x3f];
 		dst[3] = base64[(t >> 0) & 0x3f];
 		src += 3; l -= 3;
 		dst += 4; r += 4;
 	}
 	switch (l) {
 	case 2:
 		t = (src[0] << 16) | (src[1] << 8);
 		dst[0] = base64[(t >> 18) & 0x3f];
 		dst[1] = base64[(t >> 12) & 0x3f];
 		dst[2] = base64[(t >> 6) & 0x3f];
 		dst[3] = '=';
 		dst += 4;
 		r += 4;
 		break;
 	case 1:
 		t = src[0] << 16;
 		dst[0] = base64[(t >> 18) & 0x3f];
 		dst[1] = base64[(t >> 12) & 0x3f];
 		dst[2] = dst[3] = '=';
 		dst += 4;
 		r += 4;
 		break;
 	case 0:
 		break;
 	}
 	*dst = 0;
 	return (str);
 }
 /*
  * Extract authorization parameters from environment value.
  * The value is like scheme:realm:user:pass
  */
 typedef struct {
 	char	*scheme;
 	char	*realm;
 	char	*user;
 	char	*password;
 } http_auth_params_t;
 static void
 init_http_auth_params(http_auth_params_t *s)
 {
 	s->scheme = s->realm = s->user = s->password = NULL;
 }
 static void
 clean_http_auth_params(http_auth_params_t *s)
 {
 	if (s->scheme)
 		free(s->scheme);
 	if (s->realm)
 		free(s->realm);
 	if (s->user)
 		free(s->user);
 	if (s->password)
 		free(s->password);
 	init_http_auth_params(s);
 }
 static int
 http_authfromenv(const char *p, http_auth_params_t *parms)
 {
 	int ret = -1;
 	char *v, *ve;
 	char *str = strdup(p);
 	if (str == NULL) {
 		fetch_syserr();
 		return (-1);
 	}
 	v = str;
 	if ((ve = strchr(v, ':')) == NULL)
 		goto out;
 	*ve = 0;
 	if ((parms->scheme = strdup(v)) == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	v = ve + 1;
 	if ((ve = strchr(v, ':')) == NULL)
 		goto out;
 	*ve = 0;
 	if ((parms->realm = strdup(v)) == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	v = ve + 1;
 	if ((ve = strchr(v, ':')) == NULL)
 		goto out;
 	*ve = 0;
 	if ((parms->user = strdup(v)) == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	v = ve + 1;
 	if ((parms->password = strdup(v)) == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	ret = 0;
 out:
 	if (ret == -1)
 		clean_http_auth_params(parms);
 	if (str)
 		free(str);
 	return (ret);
 }
 /*
  * Digest response: the code to compute the digest is taken from the
  * sample implementation in RFC2616
  */
 #define IN const
 #define OUT
 #define HASHLEN 16
 typedef char HASH[HASHLEN];
 #define HASHHEXLEN 32
 typedef char HASHHEX[HASHHEXLEN+1];
 static const char *hexchars = "0123456789abcdef";
 static void
 CvtHex(IN HASH Bin, OUT HASHHEX Hex)
 {
 	unsigned short i;
 	unsigned char j;
 	for (i = 0; i < HASHLEN; i++) {
 		j = (Bin[i] >> 4) & 0xf;
 		Hex[i*2] = hexchars[j];
 		j = Bin[i] & 0xf;
 		Hex[i*2+1] = hexchars[j];
 	}
 	Hex[HASHHEXLEN] = '\0';
 };
 /* calculate H(A1) as per spec */
 static void
 DigestCalcHA1(
 	IN char * pszAlg,
 	IN char * pszUserName,
 	IN char * pszRealm,
 	IN char * pszPassword,
 	IN char * pszNonce,
 	IN char * pszCNonce,
 	OUT HASHHEX SessionKey
 	)
 {
 	MD5_CTX Md5Ctx;
 	HASH HA1;
 	MD5Init(&Md5Ctx);
 	MD5Update(&Md5Ctx, pszUserName, strlen(pszUserName));
 	MD5Update(&Md5Ctx, ":", 1);
 	MD5Update(&Md5Ctx, pszRealm, strlen(pszRealm));
 	MD5Update(&Md5Ctx, ":", 1);
 	MD5Update(&Md5Ctx, pszPassword, strlen(pszPassword));
 	MD5Final(HA1, &Md5Ctx);
 	if (strcasecmp(pszAlg, "md5-sess") == 0) {
 		MD5Init(&Md5Ctx);
 		MD5Update(&Md5Ctx, HA1, HASHLEN);
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce));
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
 		MD5Final(HA1, &Md5Ctx);
 	}
 	CvtHex(HA1, SessionKey);
 }
 /* calculate request-digest/response-digest as per HTTP Digest spec */
 static void
 DigestCalcResponse(
 	IN HASHHEX HA1,           /* H(A1) */
 	IN char * pszNonce,       /* nonce from server */
 	IN char * pszNonceCount,  /* 8 hex digits */
 	IN char * pszCNonce,      /* client nonce */
 	IN char * pszQop,         /* qop-value: "", "auth", "auth-int" */
 	IN char * pszMethod,      /* method from the request */
 	IN char * pszDigestUri,   /* requested URL */
 	IN HASHHEX HEntity,       /* H(entity body) if qop="auth-int" */
 	OUT HASHHEX Response      /* request-digest or response-digest */
 	)
 {
 /*	DEBUG(fprintf(stderr,
 		      "Calc: HA1[%s] Nonce[%s] qop[%s] method[%s] URI[%s]\n",
 		      HA1, pszNonce, pszQop, pszMethod, pszDigestUri));*/
 	MD5_CTX Md5Ctx;
 	HASH HA2;
 	HASH RespHash;
 	HASHHEX HA2Hex;
 	// calculate H(A2)
 	MD5Init(&Md5Ctx);
 	MD5Update(&Md5Ctx, pszMethod, strlen(pszMethod));
 	MD5Update(&Md5Ctx, ":", 1);
 	MD5Update(&Md5Ctx, pszDigestUri, strlen(pszDigestUri));
 	if (strcasecmp(pszQop, "auth-int") == 0) {
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, HEntity, HASHHEXLEN);
 	}
 	MD5Final(HA2, &Md5Ctx);
 	CvtHex(HA2, HA2Hex);
 	// calculate response
 	MD5Init(&Md5Ctx);
 	MD5Update(&Md5Ctx, HA1, HASHHEXLEN);
 	MD5Update(&Md5Ctx, ":", 1);
 	MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce));
 	MD5Update(&Md5Ctx, ":", 1);
 	if (*pszQop) {
 		MD5Update(&Md5Ctx, pszNonceCount, strlen(pszNonceCount));
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszQop, strlen(pszQop));
 		MD5Update(&Md5Ctx, ":", 1);
 	}
 	MD5Update(&Md5Ctx, HA2Hex, HASHHEXLEN);
 	MD5Final(RespHash, &Md5Ctx);
 	CvtHex(RespHash, Response);
 }
 /*
  * Generate/Send a Digest authorization header
  * This looks like: [Proxy-]Authorization: credentials
  *
  *  credentials      = "Digest" digest-response
  *  digest-response  = 1#( username | realm | nonce | digest-uri
  *                      | response | [ algorithm ] | [cnonce] |
  *                      [opaque] | [message-qop] |
  *                          [nonce-count]  | [auth-param] )
  *  username         = "username" "=" username-value
  *  username-value   = quoted-string
  *  digest-uri       = "uri" "=" digest-uri-value
  *  digest-uri-value = request-uri   ; As specified by HTTP/1.1
  *  message-qop      = "qop" "=" qop-value
  *  cnonce           = "cnonce" "=" cnonce-value
  *  cnonce-value     = nonce-value
  *  nonce-count      = "nc" "=" nc-value
  *  nc-value         = 8LHEX
  *  response         = "response" "=" request-digest
  *  request-digest = <"> 32LHEX <">
  */
 static int
 http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c,
 		 http_auth_params_t *parms, struct url *url)
 {
 	int r;
 	char noncecount[10];
 	char cnonce[40];
 	char *options = NULL;
 	if (!c->realm || !c->nonce) {
 		DEBUG(fprintf(stderr, "realm/nonce not set in challenge\n"));
 		return(-1);
 	}
 	if (!c->algo)
 		c->algo = strdup("");
 	if (asprintf(&options, "%s%s%s%s",
 		     *c->algo? ",algorithm=" : "", c->algo,
 		     c->opaque? ",opaque=" : "", c->opaque?c->opaque:"")== -1)
 		return (-1);
 	if (!c->qop) {
 		c->qop = strdup("");
 		*noncecount = 0;
 		*cnonce = 0;
 	} else {
 		c->nc++;
 		sprintf(noncecount, "%08x", c->nc);
 		/* We don't try very hard with the cnonce ... */
 		sprintf(cnonce, "%x%lx", getpid(), (unsigned long)time(0));
 	}
 	HASHHEX HA1;
 	DigestCalcHA1(c->algo, parms->user, c->realm,
 		      parms->password, c->nonce, cnonce, HA1);
 	DEBUG(fprintf(stderr, "HA1: [%s]\n", HA1));
 	HASHHEX digest;
 	DigestCalcResponse(HA1, c->nonce, noncecount, cnonce, c->qop,
 			   "GET", url->doc, "", digest);
 	if (c->qop[0]) {
 		r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\","
 			     "nonce=\"%s\",uri=\"%s\",response=\"%s\","
 			     "qop=\"auth\", cnonce=\"%s\", nc=%s%s",
 			     hdr, parms->user, c->realm,
 			     c->nonce, url->doc, digest,
 			     cnonce, noncecount, options);
 	} else {
 		r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\","
 			     "nonce=\"%s\",uri=\"%s\",response=\"%s\"%s",
 			     hdr, parms->user, c->realm,
 			     c->nonce, url->doc, digest, options);
 	}
 	if (options)
 		free(options);
 	return (r);
 }
 /*
  * Encode username and password
  */
 static int
 http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd)
 {
 	char *upw, *auth;
 	int r;
 	DEBUG(fprintf(stderr, "basic: usr: [%s]\n", usr));
 	DEBUG(fprintf(stderr, "basic: pwd: [%s]\n", pwd));
 	if (asprintf(&upw, "%s:%s", usr, pwd) == -1)
 		return (-1);
 	auth = http_base64(upw);
 	free(upw);
 	if (auth == NULL)
 		return (-1);
 	r = http_cmd(conn, "%s: Basic %s", hdr, auth);
 	free(auth);
 	return (r);
 }
 /*
  * Chose the challenge to answer and call the appropriate routine to
  * produce the header.
  */
 static int
 http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 	       http_auth_params_t *parms, struct url *url)
 {
 	http_auth_challenge_t *basic = NULL;
 	http_auth_challenge_t *digest = NULL;
 	int i;
 	/* If user or pass are null we're not happy */
 	if (!parms->user || !parms->password) {
 		DEBUG(fprintf(stderr, "NULL usr or pass\n"));
 		return (-1);
 	}
 	/* Look for a Digest and a Basic challenge */
 	for (i = 0; i < cs->count; i++) {
 		if (cs->challenges[i]->scheme == HTTPAS_BASIC)
 			basic = cs->challenges[i];
 		if (cs->challenges[i]->scheme == HTTPAS_DIGEST)
 			digest = cs->challenges[i];
 	}
 	/* Error if "Digest" was specified and there is no Digest challenge */
 	if (!digest && (parms->scheme &&
 			!strcasecmp(parms->scheme, "digest"))) {
 		DEBUG(fprintf(stderr,
 			      "Digest auth in env, not supported by peer\n"));
 		return (-1);
 	}
 	/*
 	 * If "basic" was specified in the environment, or there is no Digest
 	 * challenge, do the basic thing. Don't need a challenge for this,
 	 * so no need to check basic!=NULL
 	 */
 	if (!digest || (parms->scheme && !strcasecmp(parms->scheme,"basic")))
 		return (http_basic_auth(conn,hdr,parms->user,parms->password));
 	/* Else, prefer digest. We just checked that it's not NULL */
 	return (http_digest_auth(conn, hdr, digest, parms, url));
 }
 /*****************************************************************************
  * Helper functions for connecting to a server or proxy
  */
 /*
  * Connect to the correct HTTP server or proxy.
  */
 static conn_t *
 http_connect(struct url *URL, struct url *purl, const char *flags)
 {
 	struct url *curl;
 	conn_t *conn;
 	int verbose;
 	int af, val;
 #ifdef INET6
 	af = AF_UNSPEC;
 #else
 	af = AF_INET;
 #endif
 	verbose = CHECK_FLAG('v');
 	if (CHECK_FLAG('4'))
 		af = AF_INET;
 #ifdef INET6
 	else if (CHECK_FLAG('6'))
 		af = AF_INET6;
 #endif
 	curl = (purl != NULL) ? purl : URL;
 	if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
 		http_cmd(conn, "CONNECT %s:%d HTTP/1.1",
 		    URL->host, URL->port);
 		http_cmd(conn, "Host: %s:%d",
 		    URL->host, URL->port);
 		http_cmd(conn, "");
 		if (http_get_reply(conn) != HTTP_OK) {
 			fetch_close(conn);
 			return (NULL);
 		}
 		http_get_reply(conn);
 	}
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&
 	    fetch_ssl(conn, URL, verbose) == -1) {
 		fetch_close(conn);
 		/* grrr */
 		errno = EAUTH;
 		fetch_syserr();
 		return (NULL);
 	}
 	val = 1;
 	setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val));
 	return (conn);
 }
 static struct url *
 http_get_proxy(struct url * url, const char *flags)
 {
 	struct url *purl;
 	char *p;
 	if (flags != NULL && strchr(flags, 'd') != NULL)
 		return (NULL);
 	if (fetch_no_proxy_match(url->host))
 		return (NULL);
 	if (((p = getenv("HTTP_PROXY")) || (p = getenv("http_proxy"))) &&
 	    *p && (purl = fetchParseURL(p))) {
 		if (!*purl->scheme)
 			strcpy(purl->scheme, SCHEME_HTTP);
 		if (!purl->port)
 			purl->port = fetch_default_proxy_port(purl->scheme);
 		if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0)
 			return (purl);
 		fetchFreeURL(purl);
 	}
 	return (NULL);
 }
 static void
 http_print_html(FILE *out, FILE *in)
 {
 	size_t len;
 	char *line, *p, *q;
 	int comment, tag;
 	comment = tag = 0;
 	while ((line = fgetln(in, &len)) != NULL) {
 		while (len && isspace((unsigned char)line[len - 1]))
 			--len;
 		for (p = q = line; q < line + len; ++q) {
 			if (comment && *q == '-') {
 				if (q + 2 < line + len &&
 				    strcmp(q, "-->") == 0) {
 					tag = comment = 0;
 					q += 2;
 				}
 			} else if (tag && !comment && *q == '>') {
 				p = q + 1;
 				tag = 0;
 			} else if (!tag && *q == '<') {
 				if (q > p)
 					fwrite(p, q - p, 1, out);
 				tag = 1;
 				if (q + 3 < line + len &&
 				    strcmp(q, "<!--") == 0) {
 					comment = 1;
 					q += 3;
 				}
 			}
 		}
 		if (!tag && q > p)
 			fwrite(p, q - p, 1, out);
 		fputc('\n', out);
 	}
 }
 /*****************************************************************************
  * Core
  */
 FILE *
 http_request(struct url *URL, const char *op, struct url_stat *us,
 	struct url *purl, const char *flags)
 {
 	return (http_request_body(URL, op, us, purl, flags, NULL, NULL));
 }
 /*
  * Send a request and process the reply
  *
  * XXX This function is way too long, the do..while loop should be split
  * XXX off into a separate function.
  */
 FILE *
 http_request_body(struct url *URL, const char *op, struct url_stat *us,
 	struct url *purl, const char *flags, const char *content_type,
 	const char *body)
 {
 	char timebuf[80];
 	char hbuf[MAXHOSTNAMELEN + 7], *host;
 	conn_t *conn;
 	struct url *url, *new;
 	int chunked, direct, ims, noredirect, verbose;
 	int e, i, n, val;
 	off_t offset, clength, length, size;
 	time_t mtime;
 	const char *p;
 	FILE *f;
 	hdr_t h;
 	struct tm *timestruct;
 	http_headerbuf_t headerbuf;
 	http_auth_challenges_t server_challenges;
 	http_auth_challenges_t proxy_challenges;
 	size_t body_len;
 	/* The following calls don't allocate anything */
 	init_http_headerbuf(&headerbuf);
 	init_http_auth_challenges(&server_challenges);
 	init_http_auth_challenges(&proxy_challenges);
 	direct = CHECK_FLAG('d');
 	noredirect = CHECK_FLAG('A');
 	verbose = CHECK_FLAG('v');
 	ims = CHECK_FLAG('i');
 	if (direct && purl) {
 		fetchFreeURL(purl);
 		purl = NULL;
 	}
 	/* try the provided URL first */
 	url = URL;
 	n = MAX_REDIRECT;
 	i = 0;
 	e = HTTP_PROTOCOL_ERROR;
 	do {
 		new = NULL;
 		chunked = 0;
 		offset = 0;
 		clength = -1;
 		length = -1;
 		size = -1;
 		mtime = 0;
 		/* check port */
 		if (!url->port)
 			url->port = fetch_default_port(url->scheme);
 		/* were we redirected to an FTP URL? */
 		if (purl == NULL && strcmp(url->scheme, SCHEME_FTP) == 0) {
 			if (strcmp(op, "GET") == 0)
 				return (ftp_request(url, "RETR", us, purl, flags));
 			else if (strcmp(op, "HEAD") == 0)
 				return (ftp_request(url, "STAT", us, purl, flags));
 		}
 		/* connect to server or proxy */
 		if ((conn = http_connect(url, purl, flags)) == NULL)
 			goto ouch;
 		host = url->host;
 #ifdef INET6
 		if (strchr(url->host, ':')) {
 			snprintf(hbuf, sizeof(hbuf), "[%s]", url->host);
 			host = hbuf;
 		}
 #endif
 		if (url->port != fetch_default_port(url->scheme)) {
 			if (host != hbuf) {
 				strcpy(hbuf, host);
 				host = hbuf;
 			}
 			snprintf(hbuf + strlen(hbuf),
 			    sizeof(hbuf) - strlen(hbuf), ":%d", url->port);
 		}
 		/* send request */
 		if (verbose)
 			fetch_info("requesting %s://%s%s",
 			    url->scheme, host, url->doc);
 		if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) {
 			http_cmd(conn, "%s %s://%s%s HTTP/1.1",
 			    op, url->scheme, host, url->doc);
 		} else {
 			http_cmd(conn, "%s %s HTTP/1.1",
 			    op, url->doc);
 		}
 		if (ims && url->ims_time) {
 			timestruct = gmtime((time_t *)&url->ims_time);
 			(void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT",
 			    timestruct);
 			if (verbose)
 				fetch_info("If-Modified-Since: %s", timebuf);
 			http_cmd(conn, "If-Modified-Since: %s", timebuf);
 		}
 		/* virtual host */
 		http_cmd(conn, "Host: %s", host);
 		/*
 		 * Proxy authorization: we only send auth after we received
 		 * a 407 error. We do not first try basic anyway (changed
 		 * when support was added for digest-auth)
 		 */
 		if (purl && proxy_challenges.valid) {
 			http_auth_params_t aparams;
 			init_http_auth_params(&aparams);
 			if (*purl->user || *purl->pwd) {
 				aparams.user = purl->user ?
 					strdup(purl->user) : strdup("");
 				aparams.password = purl->pwd?
 					strdup(purl->pwd) : strdup("");
 			} else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL &&
 				   *p != '\0') {
 				if (http_authfromenv(p, &aparams) < 0) {
 					http_seterr(HTTP_NEED_PROXY_AUTH);
 					goto ouch;
 				}
 			}
 			http_authorize(conn, "Proxy-Authorization",
 				       &proxy_challenges, &aparams, url);
 			clean_http_auth_params(&aparams);
 		}
 		/*
 		 * Server authorization: we never send "a priori"
 		 * Basic auth, which used to be done if user/pass were
 		 * set in the url. This would be weird because we'd send the
 		 * password in the clear even if Digest is finally to be
 		 * used (it would have made more sense for the
 		 * pre-digest version to do this when Basic was specified
 		 * in the environment)
 		 */
 		if (server_challenges.valid) {
 			http_auth_params_t aparams;
 			init_http_auth_params(&aparams);
 			if (*url->user || *url->pwd) {
 				aparams.user = url->user ?
 					strdup(url->user) : strdup("");
 				aparams.password = url->pwd ?
 					strdup(url->pwd) : strdup("");
 			} else if ((p = getenv("HTTP_AUTH")) != NULL &&
 				   *p != '\0') {
 				if (http_authfromenv(p, &aparams) < 0) {
 					http_seterr(HTTP_NEED_AUTH);
 					goto ouch;
 				}
 			} else if (fetchAuthMethod &&
 				   fetchAuthMethod(url) == 0) {
 				aparams.user = url->user ?
 					strdup(url->user) : strdup("");
 				aparams.password = url->pwd ?
 					strdup(url->pwd) : strdup("");
 			} else {
 				http_seterr(HTTP_NEED_AUTH);
 				goto ouch;
 			}
 			http_authorize(conn, "Authorization",
 				       &server_challenges, &aparams, url);
 			clean_http_auth_params(&aparams);
 		}
 		/* other headers */
 		if ((p = getenv("HTTP_ACCEPT")) != NULL) {
 			if (*p != '\0')
 				http_cmd(conn, "Accept: %s", p);
 		} else {
 			http_cmd(conn, "Accept: */*");
 		}
 		if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') {
 			if (strcasecmp(p, "auto") == 0)
 				http_cmd(conn, "Referer: %s://%s%s",
 				    url->scheme, host, url->doc);
 			else
 				http_cmd(conn, "Referer: %s", p);
 		}
 		if ((p = getenv("HTTP_USER_AGENT")) != NULL) {
 			/* no User-Agent if defined but empty */
 			if  (*p != '\0')
 				http_cmd(conn, "User-Agent: %s", p);
 		} else {
 			/* default User-Agent */
 			http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER,
 			    getprogname());
 		}
 		if (url->offset > 0)
 			http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
 		http_cmd(conn, "Connection: close");
 		if (body) {
 			body_len = strlen(body);
 			http_cmd(conn, "Content-Length: %zu", body_len);
 			if (content_type != NULL)
 				http_cmd(conn, "Content-Type: %s", content_type);
 		}
 		http_cmd(conn, "");
 		if (body)
 			fetch_write(conn, body, body_len);
 		/*
 		 * Force the queued request to be dispatched.  Normally, one
 		 * would do this with shutdown(2) but squid proxies can be
 		 * configured to disallow such half-closed connections.  To
 		 * be compatible with such configurations, fiddle with socket
 		 * options to force the pending data to be written.
 		 */
 		val = 0;
 		setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val,
 			   sizeof(val));
 		val = 1;
 		setsockopt(conn->sd, IPPROTO_TCP, TCP_NODELAY, &val,
 			   sizeof(val));
 		/* get reply */
 		switch (http_get_reply(conn)) {
 		case HTTP_OK:
 		case HTTP_PARTIAL:
 		case HTTP_NOT_MODIFIED:
 			/* fine */
 			break;
 		case HTTP_MOVED_PERM:
 		case HTTP_MOVED_TEMP:
 		case HTTP_SEE_OTHER:
 		case HTTP_USE_PROXY:
 			/*
 			 * Not so fine, but we still have to read the
 			 * headers to get the new location.
 			 */
 			break;
 		case HTTP_NEED_AUTH:
 			if (server_challenges.valid) {
 				/*
 				 * We already sent out authorization code,
 				 * so there's nothing more we can do.
 				 */
 				http_seterr(conn->err);
 				goto ouch;
 			}
 			/* try again, but send the password this time */
 			if (verbose)
 				fetch_info("server requires authorization");
 			break;
 		case HTTP_NEED_PROXY_AUTH:
 			if (proxy_challenges.valid) {
 				/*
 				 * We already sent our proxy
 				 * authorization code, so there's
 				 * nothing more we can do. */
 				http_seterr(conn->err);
 				goto ouch;
 			}
 			/* try again, but send the password this time */
 			if (verbose)
 				fetch_info("proxy requires authorization");
 			break;
 		case HTTP_BAD_RANGE:
 			/*
 			 * This can happen if we ask for 0 bytes because
 			 * we already have the whole file.  Consider this
 			 * a success for now, and check sizes later.
 			 */
 			break;
 		case HTTP_PROTOCOL_ERROR:
 			/* fall through */
 		case -1:
 			fetch_syserr();
 			goto ouch;
 		default:
 			http_seterr(conn->err);
 			if (!verbose)
 				goto ouch;
 			/* fall through so we can get the full error message */
 		}
 		/* get headers. http_next_header expects one line readahead */
 		if (fetch_getln(conn) == -1) {
 			fetch_syserr();
 			goto ouch;
 		}
 		do {
 			switch ((h = http_next_header(conn, &headerbuf, &p))) {
 			case hdr_syserror:
 				fetch_syserr();
 				goto ouch;
 			case hdr_error:
 				http_seterr(HTTP_PROTOCOL_ERROR);
 				goto ouch;
 			case hdr_content_length:
 				http_parse_length(p, &clength);
 				break;
 			case hdr_content_range:
 				http_parse_range(p, &offset, &length, &size);
 				break;
 			case hdr_last_modified:
 				http_parse_mtime(p, &mtime);
 				break;
 			case hdr_location:
 				if (!HTTP_REDIRECT(conn->err))
 					break;
 				/*
 				 * if the A flag is set, we don't follow
 				 * temporary redirects.
 				 */
 				if (noredirect &&
 				    conn->err != HTTP_MOVED_PERM &&
 				    conn->err != HTTP_PERM_REDIRECT &&
 				    conn->err != HTTP_USE_PROXY) {
 					n = 1;
 					break;
 				}
 				if (new)
 					free(new);
 				if (verbose)
 					fetch_info("%d redirect to %s", conn->err, p);
 				if (*p == '/')
 					/* absolute path */
 					new = fetchMakeURL(url->scheme, url->host, url->port, p,
 					    url->user, url->pwd);
 				else
 					new = fetchParseURL(p);
 				if (new == NULL) {
 					/* XXX should set an error code */
 					DEBUG(fprintf(stderr, "failed to parse new URL\n"));
 					goto ouch;
 				}
 				/* Only copy credentials if the host matches */
 				if (!strcmp(new->host, url->host) && !*new->user && !*new->pwd) {
 					strcpy(new->user, url->user);
 					strcpy(new->pwd, url->pwd);
 				}
 				new->offset = url->offset;
 				new->length = url->length;
 				break;
 			case hdr_transfer_encoding:
 				/* XXX weak test*/
 				chunked = (strcasecmp(p, "chunked") == 0);
 				break;
 			case hdr_www_authenticate:
 				if (conn->err != HTTP_NEED_AUTH)
 					break;
 				if (http_parse_authenticate(p, &server_challenges) == 0)
 					++n;
 				break;
 			case hdr_proxy_authenticate:
 				if (conn->err != HTTP_NEED_PROXY_AUTH)
 					break;
 				if (http_parse_authenticate(p, &proxy_challenges) == 0)
 					++n;
 				break;
 			case hdr_end:
 				/* fall through */
 			case hdr_unknown:
 				/* ignore */
 				break;
 			}
 		} while (h > hdr_end);
 		/* we need to provide authentication */
 		if (conn->err == HTTP_NEED_AUTH ||
 		    conn->err == HTTP_NEED_PROXY_AUTH) {
 			e = conn->err;
 			if ((conn->err == HTTP_NEED_AUTH &&
 			     !server_challenges.valid) ||
 			    (conn->err == HTTP_NEED_PROXY_AUTH &&
 			     !proxy_challenges.valid)) {
 				/* 401/7 but no www/proxy-authenticate ?? */
 				DEBUG(fprintf(stderr, "401/7 and no auth header\n"));
 				goto ouch;
 			}
 			fetch_close(conn);
 			conn = NULL;
 			continue;
 		}
 		/* requested range not satisfiable */
 		if (conn->err == HTTP_BAD_RANGE) {
 			if (url->offset == size && url->length == 0) {
 				/* asked for 0 bytes; fake it */
 				offset = url->offset;
 				clength = -1;
 				conn->err = HTTP_OK;
 				break;
 			} else {
 				http_seterr(conn->err);
 				goto ouch;
 			}
 		}
 		/* we have a hit or an error */
 		if (conn->err == HTTP_OK
 		    || conn->err == HTTP_NOT_MODIFIED
 		    || conn->err == HTTP_PARTIAL
 		    || HTTP_ERROR(conn->err))
 			break;
 		/* all other cases: we got a redirect */
 		e = conn->err;
 		clean_http_auth_challenges(&server_challenges);
 		fetch_close(conn);
 		conn = NULL;
 		if (!new) {
 			DEBUG(fprintf(stderr, "redirect with no new location\n"));
 			break;
 		}
 		if (url != URL)
 			fetchFreeURL(url);
 		url = new;
 	} while (++i < n);
 	/* we failed, or ran out of retries */
 	if (conn == NULL) {
 		http_seterr(e);
 		goto ouch;
 	}
 	DEBUG(fprintf(stderr, "offset %lld, length %lld,"
 		  " size %lld, clength %lld\n",
 		  (long long)offset, (long long)length,
 		  (long long)size, (long long)clength));
 	if (conn->err == HTTP_NOT_MODIFIED) {
 		http_seterr(HTTP_NOT_MODIFIED);
 		return (NULL);
 	}
 	/* check for inconsistencies */
 	if (clength != -1 && length != -1 && clength != length) {
 		http_seterr(HTTP_PROTOCOL_ERROR);
 		goto ouch;
 	}
 	if (clength == -1)
 		clength = length;
 	if (clength != -1)
 		length = offset + clength;
 	if (length != -1 && size != -1 && length != size) {
 		http_seterr(HTTP_PROTOCOL_ERROR);
 		goto ouch;
 	}
 	if (size == -1)
 		size = length;
 	/* fill in stats */
 	if (us) {
 		us->size = size;
 		us->atime = us->mtime = mtime;
 	}
 	/* too far? */
 	if (URL->offset > 0 && offset > URL->offset) {
 		http_seterr(HTTP_PROTOCOL_ERROR);
 		goto ouch;
 	}
 	/* report back real offset and size */
 	URL->offset = offset;
 	URL->length = clength;
 	/* wrap it up in a FILE */
 	if ((f = http_funopen(conn, chunked)) == NULL) {
 		fetch_syserr();
 		goto ouch;
 	}
 	if (url != URL)
 		fetchFreeURL(url);
 	if (purl)
 		fetchFreeURL(purl);
 	if (HTTP_ERROR(conn->err)) {
 		http_print_html(stderr, f);
 		fclose(f);
 		f = NULL;
 	}
 	clean_http_headerbuf(&headerbuf);
 	clean_http_auth_challenges(&server_challenges);
 	clean_http_auth_challenges(&proxy_challenges);
 	return (f);
 ouch:
 	if (url != URL)
 		fetchFreeURL(url);
 	if (purl)
 		fetchFreeURL(purl);
 	if (conn != NULL)
 		fetch_close(conn);
 	clean_http_headerbuf(&headerbuf);
 	clean_http_auth_challenges(&server_challenges);
 	clean_http_auth_challenges(&proxy_challenges);
 	return (NULL);
 }
 /*****************************************************************************
  * Entry points
  */
 /*
  * Retrieve and stat a file by HTTP
  */
 FILE *
 fetchXGetHTTP(struct url *URL, struct url_stat *us, const char *flags)
 {
 	return (http_request(URL, "GET", us, http_get_proxy(URL, flags), flags));
 }
 /*
  * Retrieve a file by HTTP
  */
 FILE *
 fetchGetHTTP(struct url *URL, const char *flags)
 {
 	return (fetchXGetHTTP(URL, NULL, flags));
 }
 /*
  * Store a file by HTTP
  */
 FILE *
 fetchPutHTTP(struct url *URL __unused, const char *flags __unused)
 {
 	warnx("fetchPutHTTP(): not implemented");
 	return (NULL);
 }
 /*
  * Get an HTTP document's metadata
  */
 int
 fetchStatHTTP(struct url *URL, struct url_stat *us, const char *flags)
 {
 	FILE *f;
 	f = http_request(URL, "HEAD", us, http_get_proxy(URL, flags), flags);
 	if (f == NULL)
 		return (-1);
 	fclose(f);
 	return (0);
 }
 /*
  * List a directory
  */
 struct url_ent *
 fetchListHTTP(struct url *url __unused, const char *flags __unused)
 {
 	warnx("fetchListHTTP(): not implemented");
 	return (NULL);
 }
 FILE *
 fetchReqHTTP(struct url *URL, const char *method, const char *flags,
 	const char *content_type, const char *body)
 {
 	return (http_request_body(URL, method, NULL, http_get_proxy(URL, flags),
 	    flags, content_type, body));
 }

added external/libfetch/http.errors

@@ -0,0 +1,46 @@

 # $FreeBSD: head/lib/libfetch/http.errors 241840 2012-10-22 03:00:10Z eadler $
 #
 # This list is taken from RFC 2068.
 #
 OK		Continue
 OK		Switching Protocols
 OK		OK
 OK		Created
 OK		Accepted
 INFO	Non-Authoritative Information
 OK		No Content
 OK		Reset Content
 OK		Partial Content
 MOVED	Multiple Choices
 MOVED	Moved Permanently
 MOVED	Moved Temporarily
 MOVED	See Other
 OK		Not Modified
 INFO	Use Proxy
 MOVED	Temporary Redirect
 MOVED	Permanent Redirect
 PROTO	Bad Request
 AUTH	Unauthorized
 AUTH	Payment Required
 AUTH	Forbidden
 UNAVAIL	Not Found
 PROTO	Method Not Allowed
 PROTO	Not Acceptable
 AUTH	Proxy Authentication Required
 TIMEOUT	Request Time-out
 EXISTS	Conflict
 UNAVAIL	Gone
 PROTO	Length Required
 SERVER	Precondition Failed
 PROTO	Request Entity Too Large
 PROTO	Request-URI Too Large
 PROTO	Unsupported Media Type
 UNAVAIL	Requested Range Not Satisfiable
 SERVER	Expectation Failed
 SERVER	Internal Server Error
 PROTO	Not Implemented
 SERVER	Bad Gateway
 TEMP	Service Unavailable
 TIMEOUT	Gateway Time-out
 PROTO	HTTP Version not supported
 PROTO	Protocol error

added external/libfetch/httperr.h

@@ -0,0 +1,45 @@

 static struct fetcherr http_errlist[] = {
     { 100, FETCH_OK, "Continue" },
     { 101, FETCH_OK, "Switching Protocols" },
     { 200, FETCH_OK, "OK" },
     { 201, FETCH_OK, "Created" },
     { 202, FETCH_OK, "Accepted" },
     { 203, FETCH_INFO, "Non-Authoritative Information" },
     { 204, FETCH_OK, "No Content" },
     { 205, FETCH_OK, "Reset Content" },
     { 206, FETCH_OK, "Partial Content" },
     { 300, FETCH_MOVED, "Multiple Choices" },
     { 301, FETCH_MOVED, "Moved Permanently" },
     { 302, FETCH_MOVED, "Moved Temporarily" },
     { 303, FETCH_MOVED, "See Other" },
     { 304, FETCH_OK, "Not Modified" },
     { 305, FETCH_INFO, "Use Proxy" },
     { 307, FETCH_MOVED, "Temporary Redirect" },
     { 308, FETCH_MOVED, "Permanent Redirect" },
     { 400, FETCH_PROTO, "Bad Request" },
     { 401, FETCH_AUTH, "Unauthorized" },
     { 402, FETCH_AUTH, "Payment Required" },
     { 403, FETCH_AUTH, "Forbidden" },
     { 404, FETCH_UNAVAIL, "Not Found" },
     { 405, FETCH_PROTO, "Method Not Allowed" },
     { 406, FETCH_PROTO, "Not Acceptable" },
     { 407, FETCH_AUTH, "Proxy Authentication Required" },
     { 408, FETCH_TIMEOUT, "Request Time-out" },
     { 409, FETCH_EXISTS, "Conflict" },
     { 410, FETCH_UNAVAIL, "Gone" },
     { 411, FETCH_PROTO, "Length Required" },
     { 412, FETCH_SERVER, "Precondition Failed" },
     { 413, FETCH_PROTO, "Request Entity Too Large" },
     { 414, FETCH_PROTO, "Request-URI Too Large" },
     { 415, FETCH_PROTO, "Unsupported Media Type" },
     { 416, FETCH_UNAVAIL, "Requested Range Not Satisfiable" },
     { 417, FETCH_SERVER, "Expectation Failed" },
     { 500, FETCH_SERVER, "Internal Server Error" },
     { 501, FETCH_PROTO, "Not Implemented" },
     { 502, FETCH_SERVER, "Bad Gateway" },
     { 503, FETCH_TEMP, "Service Unavailable" },
     { 504, FETCH_TIMEOUT, "Gateway Time-out" },
     { 505, FETCH_PROTO, "HTTP Version not supported" },
     { 999, FETCH_PROTO, "Protocol error" },
     { -1, FETCH_UNKNOWN, "Unknown HTTP error" }
 };

modified libpkg/Makefile.am

@@ -2,6 +2,7 @@ EXTRA_DIST= libpkg.ver

 pkg_common_cflags=	-I$(top_srcdir)/libpkg -I$(top_builddir)/libpkg \
 			@LDNS_CFLAGS@ \
 			-I$(top_srcdir)/external/libfetch \
 			-I$(top_srcdir)/compat \
 			-I$(top_srcdir)/external/libsbuf \
 			-I$(top_srcdir)/external/expat/lib \

@@ -76,11 +77,11 @@ libpkg_la_LIBADD= $(top_builddir)/compat/libbsd_compat.la \

 			$(top_builddir)/external/libexpat.la \
 			$(top_builddir)/external/libsbuf.la \
 			$(top_builddir)/external/libpicosat.la \
 			$(top_builddir)/external/libfetch.la \
 			@REPOS_LDADD@ \
 			@LIBEXECINFO_LIB@ \
 			@LDNS_LIBS@ \
 			-larchive \
 			-lfetch \
 			-lutil \
 			-lssl \
 			-lcrypto \

@@ -120,6 +121,7 @@ libpkg_static_la_LIBADD= $(top_builddir)/external/libucl_static.la \

 			$(top_builddir)/external/libexpat_static.la \
 			$(top_builddir)/external/libpicosat_static.la \
 			$(top_builddir)/external/libsbuf_static.la \
 			$(top_builddir)/external/libfetch_static.la \
 			@REPOS_LDADD_STATIC@
 EXTRA_libpkg_static_la_DEPENDENCIES=	@REPOS_LDADD_STATIC@ libpkg.ver

modified src/Makefile.am

@@ -57,7 +57,6 @@ pkg_static_LDADD= $(top_builddir)/libpkg/libpkg_static.la \

 			@LIBJAIL_LIB@ \
 			@LIBEXECINFO_LIB@ \
 			@LDNS_LIBS@ \
 			-lfetch \
 			-larchive \
 			-lz \
 			-lutil \