Radish alpha
H
HardenedBSD-pkg
rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9
HardenedBSD Package Manager
Radicle
Git
Commits › 54cbe3f
fetch: readd libfetch++
Baptiste Daroussin committed 1 month ago
commit 54cbe3fb71964b95c48fc058c874c369110511a1
parent 995ba2b
14 files changed +6171 -2
modified auto.def
@@ -357,7 +357,7 @@ make-template Makefile.autosetup Makefile
define-append CONF_GEN_FILES Makefile.autosetup



foreach dir [list external/blake2 external/picosat \
- 
	external/linenoise external/sqlite \
+ 
	external/linenoise external/sqlite external/libfetch \

	external compat libpkg libpkg/repo libpkg/repo/binary src \

	external/libucl external/libelf tests docs \

	external/liblua external/yxml scripts external/libcurl external/libder \
modified external/Makefile.autosetup
@@ -1,5 +1,5 @@
include @builddir@/mk/defs.mk
- 
DIRS=	blake2 picosat linenoise sqlite libucl liblua yxml libder libecc
+ 
DIRS=	blake2 picosat linenoise libfetch sqlite libucl liblua yxml libder libecc

@if libelf-internal

DIRS+=	libelf

@endif
added external/libfetch/Makefile.autosetup
@@ -0,0 +1,17 @@
+ 
include @builddir@/mk/defs.mk
+ 
LIB=	fetch
+ 
SRCS=	common.c \
+ 
	fetch.c \
+ 
	http.c
+
+ 
LOCAL_CFLAGS=	-I$(top_srcdir)/compat \
+ 
		-I$(top_srcdir) \
+ 
		-I$(top_builddir) \
+ 
		-Wno-unused-parameter \
+ 
		-Wno-pointer-sign \
+ 
		-DWITH_SSL \
+ 
		-DINET6
+
+ 
VPATH=	$(top_srcdir)/external/libfetch
+
+ 
include $(MK)/static-lib.mk
added external/libfetch/common.c
@@ -0,0 +1,1782 @@
+ 
/*-
+ 
 * SPDX-License-Identifier: BSD-3-Clause
+ 
 *
+ 
 * Copyright (c) 1998-2016 Dag-Erling Smørgrav
+ 
 * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
+ 
 * All rights reserved.
+ 
 *
+ 
 * Redistribution and use in source and binary forms, with or without
+ 
 * modification, are permitted provided that the following conditions
+ 
 * are met:
+ 
 * 1. Redistributions of source code must retain the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer
+ 
 *    in this position and unchanged.
+ 
 * 2. Redistributions in binary form must reproduce the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer in the
+ 
 *    documentation and/or other materials provided with the distribution.
+ 
 * 3. The name of the author may not be used to endorse or promote products
+ 
 *    derived from this software without specific prior written permission
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ 
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ 
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ 
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ 
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ 
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ 
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ 
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
 */
+
+ 
#include "bsd_compat.h"
+
+ 
#include <sys/param.h>
+ 
#include <sys/socket.h>
+ 
#include <sys/time.h>
+ 
#include <sys/uio.h>
+
+ 
#include <netinet/in.h>
+
+ 
#include <ctype.h>
+ 
#include <errno.h>
+ 
#include <fcntl.h>
+ 
#include <inttypes.h>
+ 
#include <netdb.h>
+ 
#include <paths.h>
+ 
#include <poll.h>
+ 
#include <pwd.h>
+ 
#include <stdarg.h>
+ 
#include <stdlib.h>
+ 
#include <stdio.h>
+ 
#include <string.h>
+ 
#include <unistd.h>
+
+ 
#ifdef WITH_SSL
+ 
#include <openssl/x509v3.h>
+ 
#endif
+
+ 
#include "fetch.h"
+ 
#include "common.h"
+
+ 
#ifndef INFTIM
+ 
#define INFTIM (-1)
+ 
#endif
+
+ 
/*** Local data **************************************************************/
+
+ 
/*
+ 
 * Error messages for resolver errors
+ 
 */
+ 
static struct fetcherr netdb_errlist[] = {
+ 
#ifdef EAI_ADDRFAMILY
+ 
	{ EAI_ADDRFAMILY, FETCH_RESOLV, "Address family for host not supported" },
+ 
#endif
+ 
#ifdef EAI_NODATA
+ 
	{ EAI_NODATA,	FETCH_RESOLV,	"No address for host" },
+ 
#endif
+ 
	{ EAI_AGAIN,	FETCH_TEMP,	"Transient resolver failure" },
+ 
	{ EAI_FAIL,	FETCH_RESOLV,	"Non-recoverable resolver failure" },
+ 
	{ EAI_NONAME,	FETCH_RESOLV,	"Host does not resolve" },
+ 
	{ -1,		FETCH_UNKNOWN,	"Unknown resolver error" }
+ 
};
+
+ 
/*
+ 
 * SOCKS5 error enumerations
+ 
 */
+ 
enum SOCKS5_ERR {
+ 
/* Protocol errors */
+ 
	SOCKS5_ERR_SELECTION,
+ 
	SOCKS5_ERR_READ_METHOD,
+ 
	SOCKS5_ERR_VER5_ONLY,
+ 
	SOCKS5_ERR_NOMETHODS,
+ 
	SOCKS5_ERR_NOTIMPLEMENTED,
+ 
	SOCKS5_ERR_HOSTNAME_SIZE,
+ 
	SOCKS5_ERR_REQUEST,
+ 
	SOCKS5_ERR_REPLY,
+ 
	SOCKS5_ERR_NON_VER5_RESP,
+ 
	SOCKS5_ERR_GENERAL,
+ 
	SOCKS5_ERR_NOT_ALLOWED,
+ 
	SOCKS5_ERR_NET_UNREACHABLE,
+ 
	SOCKS5_ERR_HOST_UNREACHABLE,
+ 
	SOCKS5_ERR_CONN_REFUSED,
+ 
	SOCKS5_ERR_TTL_EXPIRED,
+ 
	SOCKS5_ERR_COM_UNSUPPORTED,
+ 
	SOCKS5_ERR_ADDR_UNSUPPORTED,
+ 
	SOCKS5_ERR_UNSPECIFIED,
+ 
/* Configuration errors */
+ 
	SOCKS5_ERR_BAD_HOST,
+ 
	SOCKS5_ERR_BAD_PROXY_FORMAT,
+ 
	SOCKS5_ERR_BAD_PORT
+ 
};
+
+ 
/*
+ 
 * Error messages for SOCKS5 errors
+ 
 */
+ 
static struct fetcherr socks5_errlist[] = {
+ 
/* SOCKS5 protocol errors */
+ 
	{ SOCKS5_ERR_SELECTION,		FETCH_ABORT,	"SOCKS5: Failed to send selection method" },
+ 
	{ SOCKS5_ERR_READ_METHOD,	FETCH_ABORT,	"SOCKS5: Failed to read method" },
+ 
	{ SOCKS5_ERR_VER5_ONLY,		FETCH_PROTO,	"SOCKS5: Only version 5 is implemented" },
+ 
	{ SOCKS5_ERR_NOMETHODS,		FETCH_PROTO,	"SOCKS5: No acceptable methods" },
+ 
	{ SOCKS5_ERR_NOTIMPLEMENTED,	FETCH_PROTO,	"SOCKS5: Method currently not implemented" },
+ 
	{ SOCKS5_ERR_HOSTNAME_SIZE,	FETCH_PROTO,	"SOCKS5: Hostname size is above 256 bytes" },
+ 
	{ SOCKS5_ERR_REQUEST,		FETCH_PROTO,	"SOCKS5: Failed to request" },
+ 
	{ SOCKS5_ERR_REPLY,		FETCH_PROTO,	"SOCKS5: Failed to receive reply" },
+ 
	{ SOCKS5_ERR_NON_VER5_RESP,	FETCH_PROTO,	"SOCKS5: Server responded with a non-version 5 response" },
+ 
	{ SOCKS5_ERR_GENERAL,		FETCH_ABORT,	"SOCKS5: General server failure" },
+ 
	{ SOCKS5_ERR_NOT_ALLOWED,	FETCH_AUTH,	"SOCKS5: Connection not allowed by ruleset" },
+ 
	{ SOCKS5_ERR_NET_UNREACHABLE,	FETCH_NETWORK,	"SOCKS5: Network unreachable" },
+ 
	{ SOCKS5_ERR_HOST_UNREACHABLE,	FETCH_ABORT,	"SOCKS5: Host unreachable" },
+ 
	{ SOCKS5_ERR_CONN_REFUSED,	FETCH_ABORT,	"SOCKS5: Connection refused" },
+ 
	{ SOCKS5_ERR_TTL_EXPIRED,	FETCH_TIMEOUT,	"SOCKS5: TTL expired" },
+ 
	{ SOCKS5_ERR_COM_UNSUPPORTED,	FETCH_PROTO,	"SOCKS5: Command not supported" },
+ 
	{ SOCKS5_ERR_ADDR_UNSUPPORTED,	FETCH_ABORT,	"SOCKS5: Address type not supported" },
+ 
	{ SOCKS5_ERR_UNSPECIFIED,	FETCH_UNKNOWN,	"SOCKS5: Unspecified error" },
+ 
/* Configuration error */
+ 
	{ SOCKS5_ERR_BAD_HOST,		FETCH_ABORT,	"SOCKS5: Bad proxy host" },
+ 
	{ SOCKS5_ERR_BAD_PROXY_FORMAT,	FETCH_ABORT,	"SOCKS5: Bad proxy format" },
+ 
	{ SOCKS5_ERR_BAD_PORT,		FETCH_ABORT,	"SOCKS5: Bad port" }
+ 
};
+
+ 
/* End-of-Line */
+ 
static const char ENDL[2] = { '\r', '\n' };
+
+
+ 
/*** Error-reporting functions ***********************************************/
+
+ 
/*
+ 
 * Map error code to string
+ 
 */
+ 
static struct fetcherr *
+ 
fetch_finderr(struct fetcherr *p, int e)
+ 
{
+ 
	while (p->num != -1 && p->num != e)
+ 
		p++;
+ 
	return (p);
+ 
}
+
+ 
/*
+ 
 * Set error code
+ 
 */
+ 
void
+ 
fetch_seterr(struct fetcherr *p, int e)
+ 
{
+ 
	p = fetch_finderr(p, e);
+ 
	fetchLastErrCode = p->cat;
+ 
	snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string);
+ 
}
+
+ 
/*
+ 
 * Set error code according to errno
+ 
 */
+ 
void
+ 
fetch_syserr(void)
+ 
{
+ 
	switch (errno) {
+ 
	case 0:
+ 
		fetchLastErrCode = FETCH_OK;
+ 
		break;
+ 
	case EPERM:
+ 
	case EACCES:
+ 
	case EROFS:
+ 
	case EAUTH:
+ 
	case ENEEDAUTH:
+ 
		fetchLastErrCode = FETCH_AUTH;
+ 
		break;
+ 
	case ENOENT:
+ 
	case EISDIR: /* XXX */
+ 
		fetchLastErrCode = FETCH_UNAVAIL;
+ 
		break;
+ 
	case ENOMEM:
+ 
		fetchLastErrCode = FETCH_MEMORY;
+ 
		break;
+ 
	case EBUSY:
+ 
	case EAGAIN:
+ 
		fetchLastErrCode = FETCH_TEMP;
+ 
		break;
+ 
	case EEXIST:
+ 
		fetchLastErrCode = FETCH_EXISTS;
+ 
		break;
+ 
	case ENOSPC:
+ 
		fetchLastErrCode = FETCH_FULL;
+ 
		break;
+ 
	case EADDRINUSE:
+ 
	case EADDRNOTAVAIL:
+ 
	case ENETDOWN:
+ 
	case ENETUNREACH:
+ 
	case ENETRESET:
+ 
	case EHOSTUNREACH:
+ 
		fetchLastErrCode = FETCH_NETWORK;
+ 
		break;
+ 
	case ECONNABORTED:
+ 
	case ECONNRESET:
+ 
		fetchLastErrCode = FETCH_ABORT;
+ 
		break;
+ 
	case ETIMEDOUT:
+ 
		fetchLastErrCode = FETCH_TIMEOUT;
+ 
		break;
+ 
	case ECONNREFUSED:
+ 
	case EHOSTDOWN:
+ 
		fetchLastErrCode = FETCH_DOWN;
+ 
		break;
+ 
	default:
+ 
		fetchLastErrCode = FETCH_UNKNOWN;
+ 
	}
+ 
	snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno));
+ 
}
+
+
+ 
/*
+ 
 * Emit status message
+ 
 */
+ 
void
+ 
fetch_info(const char *fmt, ...)
+ 
{
+ 
	va_list ap;
+ 
	int serrno = errno;
+
+ 
	va_start(ap, fmt);
+ 
	vfprintf(stderr, fmt, ap);
+ 
	va_end(ap);
+ 
	fputc('\n', stderr);
+ 
	errno = serrno;
+ 
}
+ 
#define fetch_verbose(...)						\
+ 
	do { if (verbose) fetch_info(__VA_ARGS__); } while (0)
+
+
+ 
/*** Network-related utility functions ***************************************/
+
+ 
/*
+ 
 * Return the default port for a scheme
+ 
 */
+ 
int
+ 
fetch_default_port(const char *scheme)
+ 
{
+ 
	struct servent *se;
+
+ 
	if ((se = getservbyname(scheme, "tcp")) != NULL)
+ 
		return (ntohs(se->s_port));
+ 
	if (strcmp(scheme, SCHEME_FTP) == 0)
+ 
		return (FTP_DEFAULT_PORT);
+ 
	if (strcmp(scheme, SCHEME_HTTP) == 0)
+ 
		return (HTTP_DEFAULT_PORT);
+ 
	return (0);
+ 
}
+
+ 
/*
+ 
 * Return the default proxy port for a scheme
+ 
 */
+ 
int
+ 
fetch_default_proxy_port(const char *scheme)
+ 
{
+ 
	if (strcmp(scheme, SCHEME_FTP) == 0)
+ 
		return (FTP_DEFAULT_PROXY_PORT);
+ 
	if (strcmp(scheme, SCHEME_HTTP) == 0)
+ 
		return (HTTP_DEFAULT_PROXY_PORT);
+ 
	return (0);
+ 
}
+
+
+ 
/*
+ 
 * Create a connection for an existing descriptor.
+ 
 */
+ 
conn_t *
+ 
fetch_reopen(int sd)
+ 
{
+ 
	conn_t *conn;
+ 
	int flags;
+ 
#ifdef SO_NOSIGPIPE
+ 
	int opt = 1;
+ 
#endif
+
+ 
	/* allocate and fill connection structure */
+ 
	if ((conn = calloc(1, sizeof(*conn))) == NULL)
+ 
		return (NULL);
+ 
	flags = fcntl(sd, F_GETFD);
+ 
	if (flags != -1 && (flags & FD_CLOEXEC) == 0)
+ 
		(void)fcntl(sd, F_SETFD, flags | FD_CLOEXEC);
+ 
	flags = fcntl(sd, F_GETFL);
+ 
	if (flags != -1 && (flags & O_NONBLOCK) == 0)
+ 
		(void)fcntl(sd, F_SETFL, flags | O_NONBLOCK);
+ 
#ifdef SO_NOSIGPIPE
+ 
	(void)setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, &opt, sizeof(opt));
+ 
#endif
+ 
	conn->sd = sd;
+ 
	++conn->ref;
+ 
	return (conn);
+ 
}
+
+
+ 
/*
+ 
 * Bump a connection's reference count.
+ 
 */
+ 
conn_t *
+ 
fetch_ref(conn_t *conn)
+ 
{
+ 
	++conn->ref;
+ 
	return (conn);
+ 
}
+
+
+ 
/*
+ 
 * Resolve an address
+ 
 */
+ 
struct addrinfo *
+ 
fetch_resolve(const char *addr, int port, int af)
+ 
{
+ 
	char hbuf[256], sbuf[8];
+ 
	struct addrinfo hints, *res;
+ 
	const char *hb, *he, *sep;
+ 
	const char *host, *service;
+ 
	int err, len;
+
+ 
	/* first, check for a bracketed IPv6 address */
+ 
	if (*addr == '[') {
+ 
		hb = addr + 1;
+ 
		if ((sep = strchr(hb, ']')) == NULL) {
+ 
			errno = EINVAL;
+ 
			goto syserr;
+ 
		}
+ 
		he = sep++;
+ 
	} else {
+ 
		hb = addr;
+ 
		sep = strchrnul(hb, ':');
+ 
		he = sep;
+ 
	}
+
+ 
	/* see if we need to copy the host name */
+ 
	if (*he != '\0') {
+ 
		len = snprintf(hbuf, sizeof(hbuf),
+ 
		    "%.*s", (int)(he - hb), hb);
+ 
		if (len < 0)
+ 
			goto syserr;
+ 
		if (len >= (int)sizeof(hbuf)) {
+ 
			errno = ENAMETOOLONG;
+ 
			goto syserr;
+ 
		}
+ 
		host = hbuf;
+ 
	} else {
+ 
		host = hb;
+ 
	}
+
+ 
	/* was it followed by a service name? */
+ 
	if (*sep == '\0' && port != 0) {
+ 
		if (port < 1 || port > 65535) {
+ 
			errno = EINVAL;
+ 
			goto syserr;
+ 
		}
+ 
		if (snprintf(sbuf, sizeof(sbuf), "%d", port) < 0)
+ 
			goto syserr;
+ 
		service = sbuf;
+ 
	} else if (*sep != '\0') {
+ 
		service = sep + 1;
+ 
	} else {
+ 
		service = NULL;
+ 
	}
+
+ 
	/* resolve */
+ 
	memset(&hints, 0, sizeof(hints));
+ 
	hints.ai_family = af;
+ 
	hints.ai_socktype = SOCK_STREAM;
+ 
	hints.ai_flags = AI_ADDRCONFIG;
+ 
	if ((err = getaddrinfo(host, service, &hints, &res)) != 0) {
+ 
		netdb_seterr(err);
+ 
		return (NULL);
+ 
	}
+ 
	return (res);
+ 
syserr:
+ 
	fetch_syserr();
+ 
	return (NULL);
+ 
}
+
+
+ 
/*
+ 
 * Bind a socket to a specific local address
+ 
 */
+ 
int
+ 
fetch_bind(int sd, int af, const char *addr)
+ 
{
+ 
	struct addrinfo *cliai, *ai;
+ 
	int err;
+
+ 
	if ((cliai = fetch_resolve(addr, 0, af)) == NULL)
+ 
		return (-1);
+ 
	for (ai = cliai; ai != NULL; ai = ai->ai_next)
+ 
		if ((err = bind(sd, ai->ai_addr, ai->ai_addrlen)) == 0)
+ 
			break;
+ 
	if (err != 0)
+ 
		fetch_syserr();
+ 
	freeaddrinfo(cliai);
+ 
	return (err == 0 ? 0 : -1);
+ 
}
+
+
+ 
/*
+ 
 * SOCKS5 connection initiation, based on RFC 1928
+ 
 * Default DNS resolution over SOCKS5
+ 
 */
+ 
int
+ 
fetch_socks5_init(conn_t *conn, const char *host, int port, int verbose)
+ 
{
+ 
	/*
+ 
	 * Size is based on largest packet prefix (4 bytes) +
+ 
	 * Largest FQDN (256) + one byte size (1) +
+ 
	 * Port (2)
+ 
	 */
+ 
	unsigned char buf[BUFF_SIZE];
+ 
	unsigned char *ptr;
+ 
	int ret = 1;
+
+ 
	fetch_verbose("Initializing SOCKS5 connection: %s:%d", host, port);
+
+ 
	/* Connection initialization */
+ 
	ptr = buf;
+ 
	*ptr++ = SOCKS_VERSION_5;
+ 
	*ptr++ = SOCKS_CONNECTION;
+ 
	*ptr++ = SOCKS_RSV;
+
+ 
	if (fetch_write(conn, buf, 3) != 3) {
+ 
		ret = SOCKS5_ERR_SELECTION;
+ 
		goto fail;
+ 
	}
+
+ 
	/* Verify response from SOCKS5 server */
+ 
	if (fetch_read(conn, buf, 2) != 2) {
+ 
		ret = SOCKS5_ERR_READ_METHOD;
+ 
		goto fail;
+ 
	}
+
+ 
	ptr = buf;
+ 
	if (ptr[0] != SOCKS_VERSION_5) {
+ 
		ret = SOCKS5_ERR_VER5_ONLY;
+ 
		goto fail;
+ 
	}
+ 
	if (ptr[1] == SOCKS_NOMETHODS) {
+ 
		ret = SOCKS5_ERR_NOMETHODS;
+ 
		goto fail;
+ 
	}
+ 
	else if (ptr[1] != SOCKS5_NOTIMPLEMENTED) {
+ 
		ret = SOCKS5_ERR_NOTIMPLEMENTED;
+ 
		goto fail;
+ 
	}
+
+ 
	/* Send Request */
+ 
	*ptr++ = SOCKS_VERSION_5;
+ 
	*ptr++ = SOCKS_CONNECTION;
+ 
	*ptr++ = SOCKS_RSV;
+ 
	/* Encode all targets as a hostname to avoid DNS leaks */
+ 
	*ptr++ = SOCKS_ATYP_DOMAINNAME;
+ 
	if (strlen(host) > FQDN_SIZE) {
+ 
		ret = SOCKS5_ERR_HOSTNAME_SIZE;
+ 
		goto fail;
+ 
	}
+ 
	*ptr++ = strlen(host);
+ 
	memcpy(ptr, host, strlen(host));
+ 
	ptr = ptr + strlen(host);
+
+ 
	port = htons(port);
+ 
	*ptr++ = port & 0x00ff;
+ 
	*ptr++ = (port & 0xff00) >> 8;
+
+ 
	if (fetch_write(conn, buf, ptr - buf) != ptr - buf) {
+ 
		ret = SOCKS5_ERR_REQUEST;
+ 
		goto fail;
+ 
	}
+
+ 
	/* BND.ADDR is variable length, read the largest on non-blocking socket */
+ 
	if (!fetch_read(conn, buf, BUFF_SIZE)) {
+ 
		ret = SOCKS5_ERR_REPLY;
+ 
		goto fail;
+ 
	}
+
+ 
	ptr = buf;
+ 
	if (*ptr++ != SOCKS_VERSION_5) {
+ 
		ret = SOCKS5_ERR_NON_VER5_RESP;
+ 
		goto fail;
+ 
	}
+
+ 
	switch (*ptr++) {
+ 
	case SOCKS_SUCCESS:
+ 
		break;
+ 
	case SOCKS_GENERAL_FAILURE:
+ 
		ret = SOCKS5_ERR_GENERAL;
+ 
		goto fail;
+ 
	case SOCKS_CONNECTION_NOT_ALLOWED:
+ 
		ret = SOCKS5_ERR_NOT_ALLOWED;
+ 
		goto fail;
+ 
	case SOCKS_NETWORK_UNREACHABLE:
+ 
		ret = SOCKS5_ERR_NET_UNREACHABLE;
+ 
		goto fail;
+ 
	case SOCKS_HOST_UNREACHABLE:
+ 
		ret = SOCKS5_ERR_HOST_UNREACHABLE;
+ 
		goto fail;
+ 
	case SOCKS_CONNECTION_REFUSED:
+ 
		ret = SOCKS5_ERR_CONN_REFUSED;
+ 
		goto fail;
+ 
	case SOCKS_TTL_EXPIRED:
+ 
		ret = SOCKS5_ERR_TTL_EXPIRED;
+ 
		goto fail;
+ 
	case SOCKS_COMMAND_NOT_SUPPORTED:
+ 
		ret = SOCKS5_ERR_COM_UNSUPPORTED;
+ 
		goto fail;
+ 
	case SOCKS_ADDRESS_NOT_SUPPORTED:
+ 
		ret = SOCKS5_ERR_ADDR_UNSUPPORTED;
+ 
		goto fail;
+ 
	default:
+ 
		ret = SOCKS5_ERR_UNSPECIFIED;
+ 
		goto fail;
+ 
	}
+
+ 
	return (ret);
+
+ 
fail:
+ 
	socks5_seterr(ret);
+ 
	return (0);
+ 
}
+
+ 
/*
+ 
 * Perform SOCKS5 initialization
+ 
 */
+ 
int
+ 
fetch_socks5_getenv(char **host, int *port)
+ 
{
+ 
	char *socks5env, *endptr, *ext;
+ 
	const char *portDelim;
+ 
	size_t slen;
+
+ 
	portDelim = ":";
+ 
	if ((socks5env = getenv("SOCKS5_PROXY")) == NULL || *socks5env == '\0') {
+ 
		*host = NULL;
+ 
		*port = -1;
+ 
		return (-1);
+ 
	}
+
+ 
	/*
+ 
	 * IPv6 addresses begin and end in brackets.  Set the port delimiter
+ 
	 * accordingly and search for it so we can do appropriate validation.
+ 
	 */
+ 
	if (socks5env[0] == '[')
+ 
		portDelim = "]:";
+
+ 
	slen = strlen(socks5env);
+ 
	ext = strstr(socks5env, portDelim);
+ 
	if (socks5env[0] == '[') {
+ 
		if (socks5env[slen - 1] == ']') {
+ 
			*host = strndup(socks5env, slen);
+ 
		} else if (ext != NULL) {
+ 
			*host = strndup(socks5env, ext - socks5env + 1);
+ 
		} else {
+ 
			socks5_seterr(SOCKS5_ERR_BAD_PROXY_FORMAT);
+ 
			return (0);
+ 
		}
+ 
	} else {
+ 
		*host = strndup(socks5env, ext - socks5env);
+ 
	}
+
+ 
	if (*host == NULL)
+ 
		return (-1);
+ 
	if (ext == NULL) {
+ 
		*port = 1080; /* Default port as defined in RFC1928 */
+ 
	} else {
+ 
		ext += strlen(portDelim);
+ 
		errno = 0;
+ 
		*port = strtoimax(ext, (char **)&endptr, 10);
+ 
		if (*endptr != '\0' || errno != 0 || *port < 0 ||
+ 
		    *port > 65535) {
+ 
			free(*host);
+ 
			*host = NULL;
+ 
			socks5_seterr(SOCKS5_ERR_BAD_PORT);
+ 
			return (0);
+ 
		}
+ 
	}
+
+ 
	return (2);
+ 
}
+
+
+ 
/*
+ 
 * Establish a TCP connection to the specified port on the specified host.
+ 
 */
+ 
conn_t *
+ 
fetch_connect(const char *host, int port, int af, int verbose)
+ 
{
+ 
	struct addrinfo *cais = NULL, *sais = NULL, *cai, *sai;
+ 
	const char *bindaddr;
+ 
	conn_t *conn = NULL;
+ 
	int err = 0, sd = -1;
+ 
	char *sockshost;
+ 
	int socksport;
+
+ 
	DEBUGF("---> %s:%d\n", host, port);
+
+ 
	/*
+ 
	 * Check if SOCKS5_PROXY env variable is set.  fetch_socks5_getenv
+ 
	 * will either set sockshost = NULL or allocate memory in all cases.
+ 
	 */
+ 
	sockshost = NULL;
+ 
	if (!fetch_socks5_getenv(&sockshost, &socksport))
+ 
		goto fail;
+
+ 
	/* Not using SOCKS5 proxy */
+ 
	if (sockshost == NULL) {
+ 
		/* resolve server address */
+ 
		fetch_verbose("resolving server address: %s:%d", host, port);
+ 
		if ((sais = fetch_resolve(host, port, af)) == NULL)
+ 
			goto fail;
+
+ 
		/* resolve client address */
+ 
		bindaddr = getenv("FETCH_BIND_ADDRESS");
+ 
		if (bindaddr != NULL && *bindaddr != '\0') {
+ 
			fetch_verbose("resolving client address: %s", bindaddr);
+ 
			if ((cais = fetch_resolve(bindaddr, 0, af)) == NULL)
+ 
				goto fail;
+ 
		}
+ 
	} else {
+ 
		/* resolve socks5 proxy address */
+ 
		fetch_verbose("resolving SOCKS5 server address: %s:%d",
+ 
		    sockshost, socksport);
+ 
		if ((sais = fetch_resolve(sockshost, socksport, af)) == NULL) {
+ 
			socks5_seterr(SOCKS5_ERR_BAD_HOST);
+ 
			goto fail;
+ 
		}
+ 
	}
+
+ 
	/* try each server address in turn */
+ 
	for (err = 0, sai = sais; sai != NULL; sai = sai->ai_next) {
+ 
		/* open socket */
+ 
		if ((sd = socket(sai->ai_family, SOCK_STREAM, 0)) < 0) {
+ 
			err = -1;
+ 
			if (errno == EAFNOSUPPORT || errno == EPROTONOSUPPORT)
+ 
				continue;
+ 
			goto syserr;
+ 
		}
+ 
		/* attempt to bind to client address */
+ 
		for (err = 0, cai = cais; cai != NULL; cai = cai->ai_next) {
+ 
			if (cai->ai_family != sai->ai_family)
+ 
				continue;
+ 
			if ((err = bind(sd, cai->ai_addr, cai->ai_addrlen)) == 0)
+ 
				break;
+ 
		}
+ 
		if (err != 0) {
+ 
			fetch_verbose("failed to bind to %s", bindaddr);
+ 
			goto syserr;
+ 
		}
+ 
		/* attempt to connect to server address */
+ 
		while ((err = connect(sd, sai->ai_addr, sai->ai_addrlen)) < 0) {
+ 
			if (errno == EINTR && fetchRestartCalls)
+ 
				continue;
+ 
			break;
+ 
		}
+ 
		/* success? */
+ 
		if (err == 0)
+ 
			break;
+ 
		/* clean up before next attempt */
+ 
		close(sd);
+ 
		sd = -1;
+ 
	}
+ 
	if (err != 0) {
+ 
		if (verbose && sockshost == NULL) {
+ 
			fetch_info("failed to connect to %s:%d", host, port);
+ 
			goto syserr;
+ 
		} else if (sockshost != NULL) {
+ 
			fetch_verbose("failed to connect to SOCKS5 server %s:%d",
+ 
			    sockshost, socksport);
+ 
			socks5_seterr(SOCKS5_ERR_CONN_REFUSED);
+ 
			goto fail;
+ 
		}
+ 
		goto syserr;
+ 
	}
+
+ 
	if ((conn = fetch_reopen(sd)) == NULL)
+ 
		goto syserr;
+
+ 
	if (sockshost)
+ 
		if (!fetch_socks5_init(conn, host, port, verbose))
+ 
			goto fail;
+ 
	free(sockshost);
+ 
	if (cais != NULL)
+ 
		freeaddrinfo(cais);
+ 
	if (sais != NULL)
+ 
		freeaddrinfo(sais);
+ 
	return (conn);
+ 
syserr:
+ 
	fetch_syserr();
+ 
fail:
+ 
	free(sockshost);
+ 
	/* Fully close if it was opened; otherwise just don't leak the fd. */
+ 
	if (conn != NULL)
+ 
		fetch_close(conn);
+ 
	else if (sd >= 0)
+ 
		close(sd);
+ 
	if (cais != NULL)
+ 
		freeaddrinfo(cais);
+ 
	if (sais != NULL)
+ 
		freeaddrinfo(sais);
+ 
	return (NULL);
+ 
}
+
+ 
#ifdef WITH_SSL
+ 
/*
+ 
 * Convert characters A-Z to lowercase (intentionally avoid any locale
+ 
 * specific conversions).
+ 
 */
+ 
static char
+ 
fetch_ssl_tolower(char in)
+ 
{
+ 
	if (in >= 'A' && in <= 'Z')
+ 
		return (in + 32);
+ 
	else
+ 
		return (in);
+ 
}
+
+ 
/*
+ 
 * isalpha implementation that intentionally avoids any locale specific
+ 
 * conversions.
+ 
 */
+ 
static int
+ 
fetch_ssl_isalpha(char in)
+ 
{
+ 
	return ((in >= 'A' && in <= 'Z') || (in >= 'a' && in <= 'z'));
+ 
}
+
+ 
/*
+ 
 * Check if passed hostnames a and b are equal.
+ 
 */
+ 
static int
+ 
fetch_ssl_hname_equal(const char *a, size_t alen, const char *b,
+ 
    size_t blen)
+ 
{
+ 
	size_t i;
+
+ 
	if (alen != blen)
+ 
		return (0);
+ 
	for (i = 0; i < alen; ++i) {
+ 
		if (fetch_ssl_tolower(a[i]) != fetch_ssl_tolower(b[i]))
+ 
			return (0);
+ 
	}
+ 
	return (1);
+ 
}
+
+ 
/*
+ 
 * Check if domain label is traditional, meaning that only A-Z, a-z, 0-9
+ 
 * and '-' (hyphen) are allowed. Hyphens have to be surrounded by alpha-
+ 
 * numeric characters. Double hyphens (like they're found in IDN a-labels
+ 
 * 'xn--') are not allowed. Empty labels are invalid.
+ 
 */
+ 
static int
+ 
fetch_ssl_is_trad_domain_label(const char *l, size_t len, int wcok)
+ 
{
+ 
	size_t i;
+
+ 
	if (!len || l[0] == '-' || l[len-1] == '-')
+ 
		return (0);
+ 
	for (i = 0; i < len; ++i) {
+ 
		if (!isdigit(l[i]) &&
+ 
		    !fetch_ssl_isalpha(l[i]) &&
+ 
		    !(l[i] == '*' && wcok) &&
+ 
		    !(l[i] == '-' && l[i - 1] != '-'))
+ 
			return (0);
+ 
	}
+ 
	return (1);
+ 
}
+
+ 
/*
+ 
 * Check if host name consists only of numbers. This might indicate an IP
+ 
 * address, which is not a good idea for CN wildcard comparison.
+ 
 */
+ 
static int
+ 
fetch_ssl_hname_is_only_numbers(const char *hostname, size_t len)
+ 
{
+ 
	size_t i;
+
+ 
	for (i = 0; i < len; ++i) {
+ 
		if (!((hostname[i] >= '0' && hostname[i] <= '9') ||
+ 
		    hostname[i] == '.'))
+ 
			return (0);
+ 
	}
+ 
	return (1);
+ 
}
+
+ 
/*
+ 
 * Check if the host name h passed matches the pattern passed in m which
+ 
 * is usually part of subjectAltName or CN of a certificate presented to
+ 
 * the client. This includes wildcard matching. The algorithm is based on
+ 
 * RFC6125, sections 6.4.3 and 7.2, which clarifies RFC2818 and RFC3280.
+ 
 */
+ 
static int
+ 
fetch_ssl_hname_match(const char *h, size_t hlen, const char *m,
+ 
    size_t mlen)
+ 
{
+ 
	int delta, hdotidx, mdot1idx, wcidx;
+ 
	const char *hdot, *mdot1, *mdot2;
+ 
	const char *wc; /* wildcard */
+
+ 
	if (!(h && *h && m && *m))
+ 
		return (0);
+ 
	if ((wc = strnstr(m, "*", mlen)) == NULL)
+ 
		return (fetch_ssl_hname_equal(h, hlen, m, mlen));
+ 
	wcidx = wc - m;
+ 
	/* hostname should not be just dots and numbers */
+ 
	if (fetch_ssl_hname_is_only_numbers(h, hlen))
+ 
		return (0);
+ 
	/* only one wildcard allowed in pattern */
+ 
	if (strnstr(wc + 1, "*", mlen - wcidx - 1) != NULL)
+ 
		return (0);
+ 
	/*
+ 
	 * there must be at least two more domain labels and
+ 
	 * wildcard has to be in the leftmost label (RFC6125)
+ 
	 */
+ 
	mdot1 = strnstr(m, ".", mlen);
+ 
	if (mdot1 == NULL || mdot1 < wc || (mlen - (mdot1 - m)) < 4)
+ 
		return (0);
+ 
	mdot1idx = mdot1 - m;
+ 
	mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1);
+ 
	if (mdot2 == NULL || (mlen - (mdot2 - m)) < 2)
+ 
		return (0);
+ 
	/* hostname must contain a dot and not be the 1st char */
+ 
	hdot = strnstr(h, ".", hlen);
+ 
	if (hdot == NULL || hdot == h)
+ 
		return (0);
+ 
	hdotidx = hdot - h;
+ 
	/*
+ 
	 * host part of hostname must be at least as long as
+ 
	 * pattern it's supposed to match
+ 
	 */
+ 
	if (hdotidx < mdot1idx)
+ 
		return (0);
+ 
	/*
+ 
	 * don't allow wildcards in non-traditional domain names
+ 
	 * (IDN, A-label, U-label...)
+ 
	 */
+ 
	if (!fetch_ssl_is_trad_domain_label(h, hdotidx, 0) ||
+ 
	    !fetch_ssl_is_trad_domain_label(m, mdot1idx, 1))
+ 
		return (0);
+ 
	/* match domain part (part after first dot) */
+ 
	if (!fetch_ssl_hname_equal(hdot, hlen - hdotidx, mdot1,
+ 
	    mlen - mdot1idx))
+ 
		return (0);
+ 
	/* match part left of wildcard */
+ 
	if (!fetch_ssl_hname_equal(h, wcidx, m, wcidx))
+ 
		return (0);
+ 
	/* match part right of wildcard */
+ 
	delta = mdot1idx - wcidx - 1;
+ 
	if (!fetch_ssl_hname_equal(hdot - delta, delta,
+ 
	    mdot1 - delta, delta))
+ 
		return (0);
+ 
	/* all tests succeeded, it's a match */
+ 
	return (1);
+ 
}
+
+ 
/*
+ 
 * Get numeric host address info - returns NULL if host was not an IP
+ 
 * address. The caller is responsible for deallocation using
+ 
 * freeaddrinfo(3).
+ 
 */
+ 
static struct addrinfo *
+ 
fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len)
+ 
{
+ 
	struct addrinfo hints, *res;
+ 
	char *host;
+
+ 
	host = (char *)malloc(len + 1);
+ 
	memcpy(host, hostname, len);
+ 
	host[len] = '\0';
+ 
	memset(&hints, 0, sizeof(hints));
+ 
	hints.ai_family = PF_UNSPEC;
+ 
	hints.ai_socktype = SOCK_STREAM;
+ 
	hints.ai_protocol = 0;
+ 
	hints.ai_flags = AI_NUMERICHOST;
+ 
	/* port is not relevant for this purpose */
+ 
	if (getaddrinfo(host, "443", &hints, &res) != 0)
+ 
		res = NULL;
+ 
	free(host);
+ 
	return res;
+ 
}
+
+ 
/*
+ 
 * Compare ip address in addrinfo with address passes.
+ 
 */
+ 
static int
+ 
fetch_ssl_ipaddr_match_bin(const struct addrinfo *lhost, const char *rhost,
+ 
    size_t rhostlen)
+ 
{
+ 
	const void *left;
+
+ 
	if (lhost->ai_family == AF_INET && rhostlen == 4) {
+ 
		left = (void *)&((struct sockaddr_in*)(void *)
+ 
		    lhost->ai_addr)->sin_addr.s_addr;
+ 
#ifdef INET6
+ 
	} else if (lhost->ai_family == AF_INET6 && rhostlen == 16) {
+ 
		left = (void *)&((struct sockaddr_in6 *)(void *)
+ 
		    lhost->ai_addr)->sin6_addr;
+ 
#endif
+ 
	} else
+ 
		return (0);
+ 
	return (!memcmp(left, (const void *)rhost, rhostlen) ? 1 : 0);
+ 
}
+
+ 
/*
+ 
 * Compare ip address in addrinfo with host passed. If host is not an IP
+ 
 * address, comparison will fail.
+ 
 */
+ 
static int
+ 
fetch_ssl_ipaddr_match(const struct addrinfo *laddr, const char *r,
+ 
    size_t rlen)
+ 
{
+ 
	struct addrinfo *raddr;
+ 
	int ret;
+ 
	char *rip;
+
+ 
	ret = 0;
+ 
	if ((raddr = fetch_ssl_get_numeric_addrinfo(r, rlen)) == NULL)
+ 
		return 0; /* not a numeric host */
+
+ 
	if (laddr->ai_family == raddr->ai_family) {
+ 
		if (laddr->ai_family == AF_INET) {
+ 
			rip = (char *)&((struct sockaddr_in *)(void *)
+ 
			    raddr->ai_addr)->sin_addr.s_addr;
+ 
			ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 4);
+ 
#ifdef INET6
+ 
		} else if (laddr->ai_family == AF_INET6) {
+ 
			rip = (char *)&((struct sockaddr_in6 *)(void *)
+ 
			    raddr->ai_addr)->sin6_addr;
+ 
			ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 16);
+ 
#endif
+ 
		}
+
+ 
	}
+ 
	freeaddrinfo(raddr);
+ 
	return (ret);
+ 
}
+
+ 
/*
+ 
 * Verify server certificate by subjectAltName.
+ 
 */
+ 
static int
+ 
fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames,
+ 
    const char *host, struct addrinfo *ip)
+ 
{
+ 
	const GENERAL_NAME *name;
+ 
	size_t nslen;
+ 
	int i;
+ 
	const char *ns;
+
+ 
	for (i = 0; i < sk_GENERAL_NAME_num(altnames); ++i) {
+ 
		name = sk_GENERAL_NAME_value(altnames, i);
+ 
		ns = (const char *)ASN1_STRING_get0_data(name->d.ia5);
+ 
		nslen = (size_t)ASN1_STRING_length(name->d.ia5);
+
+ 
		if (name->type == GEN_DNS && ip == NULL &&
+ 
		    fetch_ssl_hname_match(host, strlen(host), ns, nslen))
+ 
			return (1);
+ 
		else if (name->type == GEN_IPADD && ip != NULL &&
+ 
		    fetch_ssl_ipaddr_match_bin(ip, ns, nslen))
+ 
			return (1);
+ 
	}
+ 
	return (0);
+ 
}
+
+ 
/*
+ 
 * Verify server certificate by CN.
+ 
 */
+ 
static int
+ 
fetch_ssl_verify_cn(X509_NAME *subject, const char *host,
+ 
    struct addrinfo *ip)
+ 
{
+ 
	ASN1_STRING *namedata;
+ 
	X509_NAME_ENTRY *nameentry;
+ 
	int cnlen, lastpos, loc, ret;
+ 
	unsigned char *cn;
+
+ 
	ret = 0;
+ 
	lastpos = -1;
+ 
	loc = -1;
+ 
	cn = NULL;
+ 
	/* get most specific CN (last entry in list) and compare */
+ 
	while ((lastpos = X509_NAME_get_index_by_NID(subject,
+ 
	    NID_commonName, lastpos)) != -1)
+ 
		loc = lastpos;
+
+ 
	if (loc > -1) {
+ 
		nameentry = X509_NAME_get_entry(subject, loc);
+ 
		namedata = X509_NAME_ENTRY_get_data(nameentry);
+ 
		cnlen = ASN1_STRING_to_UTF8(&cn, namedata);
+ 
		if (ip == NULL &&
+ 
		    fetch_ssl_hname_match(host, strlen(host), cn, cnlen))
+ 
			ret = 1;
+ 
		else if (ip != NULL && fetch_ssl_ipaddr_match(ip, cn, cnlen))
+ 
			ret = 1;
+ 
		OPENSSL_free(cn);
+ 
	}
+ 
	return (ret);
+ 
}
+
+ 
/*
+ 
 * Verify that server certificate subjectAltName/CN matches
+ 
 * hostname. First check, if there are alternative subject names. If yes,
+ 
 * those have to match. Only if those don't exist it falls back to
+ 
 * checking the subject's CN.
+ 
 */
+ 
static int
+ 
fetch_ssl_verify_hname(X509 *cert, const char *host)
+ 
{
+ 
	struct addrinfo *ip;
+ 
	STACK_OF(GENERAL_NAME) *altnames;
+ 
	X509_NAME *subject;
+ 
	int ret;
+
+ 
	ret = 0;
+ 
	ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host));
+ 
	altnames = X509_get_ext_d2i(cert, NID_subject_alt_name,
+ 
	    NULL, NULL);
+
+ 
	if (altnames != NULL) {
+ 
		ret = fetch_ssl_verify_altname(altnames, host, ip);
+ 
	} else {
+ 
		subject = X509_get_subject_name(cert);
+ 
		if (subject != NULL)
+ 
			ret = fetch_ssl_verify_cn(subject, host, ip);
+ 
	}
+
+ 
	if (ip != NULL)
+ 
		freeaddrinfo(ip);
+ 
	if (altnames != NULL)
+ 
		GENERAL_NAMES_free(altnames);
+ 
	return (ret);
+ 
}
+
+ 
/*
+ 
 * Configure transport security layer based on environment.
+ 
 */
+ 
static void
+ 
fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose)
+ 
{
+ 
	long ssl_ctx_options;
+
+ 
	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_SSLv3 | SSL_OP_NO_TICKET;
+ 
	if (getenv("SSL_NO_TLS1") != NULL)
+ 
		ssl_ctx_options |= SSL_OP_NO_TLSv1;
+ 
	if (getenv("SSL_NO_TLS1_1") != NULL)
+ 
		ssl_ctx_options |= SSL_OP_NO_TLSv1_1;
+ 
	if (getenv("SSL_NO_TLS1_2") != NULL)
+ 
		ssl_ctx_options |= SSL_OP_NO_TLSv1_2;
+ 
	if (getenv("SSL_NO_TLS1_3") != NULL)
+ 
		ssl_ctx_options |= SSL_OP_NO_TLSv1_3;
+ 
	fetch_verbose("SSL options: %lx", ssl_ctx_options);
+ 
	SSL_CTX_set_options(ctx, ssl_ctx_options);
+ 
}
+
+
+ 
/*
+ 
 * Configure peer verification based on environment.
+ 
 */
+ 
static int
+ 
fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose)
+ 
{
+ 
	X509_LOOKUP *crl_lookup;
+ 
	X509_STORE *crl_store;
+ 
	const char *ca_cert_file, *ca_cert_path, *crl_file;
+
+ 
	if (getenv("SSL_NO_VERIFY_PEER") == NULL) {
+ 
		ca_cert_file = getenv("SSL_CA_CERT_FILE");
+ 
		ca_cert_path = getenv("SSL_CA_CERT_PATH");
+ 
		if (verbose) {
+ 
			fetch_info("Peer verification enabled");
+ 
			if (ca_cert_file != NULL)
+ 
				fetch_info("Using CA cert file: %s",
+ 
				    ca_cert_file);
+ 
			if (ca_cert_path != NULL)
+ 
				fetch_info("Using CA cert path: %s",
+ 
				    ca_cert_path);
+ 
			if (ca_cert_file == NULL && ca_cert_path == NULL)
+ 
				fetch_info("Using OpenSSL default "
+ 
				    "CA cert file and path");
+ 
		}
+ 
		SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER,
+ 
		    fetch_ssl_cb_verify_crt);
+ 
		if (ca_cert_file != NULL || ca_cert_path != NULL)
+ 
			SSL_CTX_load_verify_locations(ctx, ca_cert_file,
+ 
			    ca_cert_path);
+ 
		else
+ 
			SSL_CTX_set_default_verify_paths(ctx);
+ 
		if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) {
+ 
			fetch_verbose("Using CRL file: %s", crl_file);
+ 
			crl_store = SSL_CTX_get_cert_store(ctx);
+ 
			crl_lookup = X509_STORE_add_lookup(crl_store,
+ 
			    X509_LOOKUP_file());
+ 
			if (crl_lookup == NULL ||
+ 
			    !X509_load_crl_file(crl_lookup, crl_file,
+ 
				X509_FILETYPE_PEM)) {
+ 
				fetch_info("Could not load CRL file %s",
+ 
				    crl_file);
+ 
				return (0);
+ 
			}
+ 
			X509_STORE_set_flags(crl_store,
+ 
			    X509_V_FLAG_CRL_CHECK |
+ 
			    X509_V_FLAG_CRL_CHECK_ALL);
+ 
		}
+ 
	}
+ 
	return (1);
+ 
}
+
+ 
/*
+ 
 * Configure client certificate based on environment.
+ 
 */
+ 
static int
+ 
fetch_ssl_setup_client_certificate(SSL_CTX *ctx, int verbose)
+ 
{
+ 
	const char *client_cert_file, *client_key_file;
+
+ 
	if ((client_cert_file = getenv("SSL_CLIENT_CERT_FILE")) != NULL) {
+ 
		client_key_file = getenv("SSL_CLIENT_KEY_FILE") != NULL ?
+ 
		    getenv("SSL_CLIENT_KEY_FILE") : client_cert_file;
+ 
		fetch_verbose("Using client cert file: %s", client_cert_file);
+ 
		fetch_verbose("Using client key file: %s", client_key_file);
+ 
		if (SSL_CTX_use_certificate_chain_file(ctx,
+ 
			client_cert_file) != 1) {
+ 
			fetch_info("Could not load client certificate %s",
+ 
			    client_cert_file);
+ 
			return (0);
+ 
		}
+ 
		if (SSL_CTX_use_PrivateKey_file(ctx, client_key_file,
+ 
			SSL_FILETYPE_PEM) != 1) {
+ 
			fetch_info("Could not load client key %s",
+ 
			    client_key_file);
+ 
			return (0);
+ 
		}
+ 
	}
+ 
	return (1);
+ 
}
+
+ 
/*
+ 
 * Callback for SSL certificate verification, this is called on server
+ 
 * cert verification. It takes no decision, but informs the user in case
+ 
 * verification failed.
+ 
 */
+ 
int
+ 
fetch_ssl_cb_verify_crt(int verified, X509_STORE_CTX *ctx)
+ 
{
+ 
	X509 *crt;
+ 
	X509_NAME *name;
+ 
	char *str;
+
+ 
	str = NULL;
+ 
	if (!verified) {
+ 
		if ((crt = X509_STORE_CTX_get_current_cert(ctx)) != NULL &&
+ 
		    (name = X509_get_subject_name(crt)) != NULL)
+ 
			str = X509_NAME_oneline(name, 0, 0);
+ 
		fetch_info("Certificate verification failed for %s",
+ 
		    str != NULL ? str : "no relevant certificate");
+ 
		OPENSSL_free(str);
+ 
	}
+ 
	return (verified);
+ 
}
+
+ 
#endif
+
+ 
/*
+ 
 * Enable SSL on a connection.
+ 
 */
+ 
int
+ 
fetch_ssl(conn_t *conn, const struct url *URL, int verbose)
+ 
{
+ 
#ifdef WITH_SSL
+ 
	int ret, ssl_err;
+ 
	X509_NAME *name;
+ 
	char *str;
+
+ 
	if ((conn->ssl_ctx = SSL_CTX_new(TLS_client_method())) == NULL) {
+ 
		fetch_info("SSL context creation failed");
+ 
		ERR_print_errors_fp(stderr);
+ 
		return (-1);
+ 
	}
+ 
	SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
+
+ 
	fetch_ssl_setup_transport_layer(conn->ssl_ctx, verbose);
+ 
	if (!fetch_ssl_setup_peer_verification(conn->ssl_ctx, verbose))
+ 
		return (-1);
+ 
	if (!fetch_ssl_setup_client_certificate(conn->ssl_ctx, verbose))
+ 
		return (-1);
+
+ 
	conn->ssl = SSL_new(conn->ssl_ctx);
+ 
	if (conn->ssl == NULL) {
+ 
		fetch_info("SSL connection creation failed");
+ 
		ERR_print_errors_fp(stderr);
+ 
		return (-1);
+ 
	}
+ 
	SSL_set_fd(conn->ssl, conn->sd);
+
+ 
#if !defined(OPENSSL_NO_TLSEXT)
+ 
	if (!SSL_set_tlsext_host_name(conn->ssl, __DECONST(char *, URL->host))) {
+ 
		fetch_info("Failed to set TLS server name indication for host %s",
+ 
		    URL->host);
+ 
		return (-1);
+ 
	}
+ 
#endif
+ 
	while ((ret = SSL_connect(conn->ssl)) == -1) {
+ 
		ssl_err = SSL_get_error(conn->ssl, ret);
+ 
		if (ssl_err != SSL_ERROR_WANT_READ &&
+ 
		    ssl_err != SSL_ERROR_WANT_WRITE) {
+ 
			ERR_print_errors_fp(stderr);
+ 
			return (-1);
+ 
		}
+ 
	}
+ 
	conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
+
+ 
	if (conn->ssl_cert == NULL) {
+ 
		fetch_info("No server SSL certificate");
+ 
		return (-1);
+ 
	}
+
+ 
	if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) {
+ 
		fetch_verbose("Verify hostname");
+ 
		if (!fetch_ssl_verify_hname(conn->ssl_cert, URL->host)) {
+ 
			fetch_info("SSL certificate subject does not match host %s",
+ 
			    URL->host);
+ 
			return (-1);
+ 
		}
+ 
	}
+
+ 
	if (verbose) {
+ 
		fetch_info("%s connection established using %s",
+ 
		    SSL_get_version(conn->ssl), SSL_get_cipher(conn->ssl));
+ 
		name = X509_get_subject_name(conn->ssl_cert);
+ 
		str = X509_NAME_oneline(name, 0, 0);
+ 
		fetch_info("Certificate subject: %s", str);
+ 
		OPENSSL_free(str);
+ 
		name = X509_get_issuer_name(conn->ssl_cert);
+ 
		str = X509_NAME_oneline(name, 0, 0);
+ 
		fetch_info("Certificate issuer: %s", str);
+ 
		OPENSSL_free(str);
+ 
	}
+
+ 
	return (0);
+ 
#else
+ 
	(void)conn;
+ 
	(void)verbose;
+ 
	(void)URL;
+ 
	fetch_info("SSL support disabled");
+ 
	return (-1);
+ 
#endif
+ 
}
+
+ 
#define FETCH_READ_WAIT		-2
+ 
#define FETCH_READ_ERROR	-1
+ 
#define FETCH_READ_DONE		 0
+
+ 
#ifdef WITH_SSL
+ 
static ssize_t
+ 
fetch_ssl_read(SSL *ssl, char *buf, size_t len)
+ 
{
+ 
	ssize_t rlen;
+ 
	int ssl_err;
+
+ 
	rlen = SSL_read(ssl, buf, len);
+ 
	if (rlen < 0) {
+ 
		ssl_err = SSL_get_error(ssl, rlen);
+ 
		if (ssl_err == SSL_ERROR_WANT_READ ||
+ 
		    ssl_err == SSL_ERROR_WANT_WRITE) {
+ 
			return (FETCH_READ_WAIT);
+ 
		} else {
+ 
			ERR_print_errors_fp(stderr);
+ 
			return (FETCH_READ_ERROR);
+ 
		}
+ 
	}
+ 
	return (rlen);
+ 
}
+ 
#endif
+
+ 
static ssize_t
+ 
fetch_socket_read(int sd, char *buf, size_t len)
+ 
{
+ 
	ssize_t rlen;
+
+ 
	rlen = read(sd, buf, len);
+ 
	if (rlen < 0) {
+ 
		if (errno == EAGAIN || (errno == EINTR && fetchRestartCalls)) {
+ 
			return (FETCH_READ_WAIT);
+ 
		} else {
+ 
			return (FETCH_READ_ERROR);
+ 
		}
+ 
	}
+ 
	return (rlen);
+ 
}
+
+ 
/*
+ 
 * Read a character from a connection w/ timeout
+ 
 */
+ 
ssize_t
+ 
fetch_read(conn_t *conn, char *buf, size_t len)
+ 
{
+ 
	struct timeval now, timeout, delta;
+ 
	struct pollfd pfd;
+ 
	ssize_t rlen;
+ 
	int deltams;
+
+ 
	if (fetchTimeout > 0) {
+ 
		gettimeofday(&timeout, NULL);
+ 
		timeout.tv_sec += fetchTimeout;
+ 
	}
+
+ 
	deltams = INFTIM;
+ 
	memset(&pfd, 0, sizeof pfd);
+ 
	pfd.fd = conn->sd;
+ 
	pfd.events = POLLIN | POLLERR;
+
+ 
	for (;;) {
+ 
		/*
+ 
		 * The socket is non-blocking.  Instead of the canonical
+ 
		 * poll() -> read(), we do the following:
+ 
		 *
+ 
		 * 1) call read() or SSL_read().
+ 
		 * 2) if we received some data, return it.
+ 
		 * 3) if an error occurred, return -1.
+ 
		 * 4) if read() or SSL_read() signaled EOF, return.
+ 
		 * 5) if we did not receive any data but we're not at EOF,
+ 
		 *    call poll().
+ 
		 *
+ 
		 * In the SSL case, this is necessary because if we
+ 
		 * receive a close notification, we have to call
+ 
		 * SSL_read() one additional time after we've read
+ 
		 * everything we received.
+ 
		 *
+ 
		 * In the non-SSL case, it may improve performance (very
+ 
		 * slightly) when reading small amounts of data.
+ 
		 */
+ 
#ifdef WITH_SSL
+ 
		if (conn->ssl != NULL)
+ 
			rlen = fetch_ssl_read(conn->ssl, buf, len);
+ 
		else
+ 
#endif
+ 
			rlen = fetch_socket_read(conn->sd, buf, len);
+ 
		if (rlen >= 0) {
+ 
			break;
+ 
		} else if (rlen == FETCH_READ_ERROR) {
+ 
			fetch_syserr();
+ 
			return (-1);
+ 
		}
+ 
		// assert(rlen == FETCH_READ_WAIT);
+ 
		if (fetchTimeout > 0) {
+ 
			gettimeofday(&now, NULL);
+ 
			if (!timercmp(&timeout, &now, >)) {
+ 
				errno = ETIMEDOUT;
+ 
				fetch_syserr();
+ 
				return (-1);
+ 
			}
+ 
			timersub(&timeout, &now, &delta);
+ 
			deltams = delta.tv_sec * 1000 +
+ 
			    delta.tv_usec / 1000;
+ 
		}
+ 
		errno = 0;
+ 
		pfd.revents = 0;
+ 
		if (poll(&pfd, 1, deltams) < 0) {
+ 
			if (errno == EINTR && fetchRestartCalls)
+ 
				continue;
+ 
			fetch_syserr();
+ 
			return (-1);
+ 
		}
+ 
	}
+ 
	return (rlen);
+ 
}
+
+
+ 
/*
+ 
 * Read a line of text from a connection w/ timeout
+ 
 */
+ 
#define MIN_BUF_SIZE 1024
+
+ 
int
+ 
fetch_getln(conn_t *conn)
+ 
{
+ 
	char *tmp;
+ 
	size_t tmpsize;
+ 
	ssize_t len;
+ 
	char c;
+
+ 
	if (conn->buf == NULL) {
+ 
		if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
+ 
			errno = ENOMEM;
+ 
			return (-1);
+ 
		}
+ 
		conn->bufsize = MIN_BUF_SIZE;
+ 
	}
+
+ 
	conn->buf[0] = '\0';
+ 
	conn->buflen = 0;
+
+ 
	do {
+ 
		len = fetch_read(conn, &c, 1);
+ 
		if (len == -1)
+ 
			return (-1);
+ 
		if (len == 0)
+ 
			break;
+ 
		conn->buf[conn->buflen++] = c;
+ 
		if (conn->buflen == conn->bufsize) {
+ 
			tmp = conn->buf;
+ 
			tmpsize = conn->bufsize * 2 + 1;
+ 
			if ((tmp = realloc(tmp, tmpsize)) == NULL) {
+ 
				errno = ENOMEM;
+ 
				return (-1);
+ 
			}
+ 
			conn->buf = tmp;
+ 
			conn->bufsize = tmpsize;
+ 
		}
+ 
	} while (c != '\n');
+
+ 
	conn->buf[conn->buflen] = '\0';
+ 
	DEBUGF("<<< %s", conn->buf);
+ 
	return (0);
+ 
}
+
+
+ 
/*
+ 
 * Write to a connection w/ timeout
+ 
 */
+ 
ssize_t
+ 
fetch_write(conn_t *conn, const char *buf, size_t len)
+ 
{
+ 
	struct iovec iov;
+
+ 
	iov.iov_base = __DECONST(char *, buf);
+ 
	iov.iov_len = len;
+ 
	return (fetch_writev(conn, &iov, 1));
+ 
}
+
+ 
/*
+ 
 * Write a vector to a connection w/ timeout
+ 
 * Note: can modify the iovec.
+ 
 */
+ 
ssize_t
+ 
fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
+ 
{
+ 
	struct timeval now, timeout, delta;
+ 
	struct pollfd pfd;
+ 
	ssize_t wlen, total;
+ 
	int deltams;
+
+ 
	memset(&pfd, 0, sizeof pfd);
+ 
	if (fetchTimeout) {
+ 
		pfd.fd = conn->sd;
+ 
		pfd.events = POLLOUT | POLLERR;
+ 
		gettimeofday(&timeout, NULL);
+ 
		timeout.tv_sec += fetchTimeout;
+ 
	}
+
+ 
	total = 0;
+ 
	while (iovcnt > 0) {
+ 
		while (fetchTimeout && pfd.revents == 0) {
+ 
			gettimeofday(&now, NULL);
+ 
			if (!timercmp(&timeout, &now, >)) {
+ 
				errno = ETIMEDOUT;
+ 
				fetch_syserr();
+ 
				return (-1);
+ 
			}
+ 
			timersub(&timeout, &now, &delta);
+ 
			deltams = delta.tv_sec * 1000 +
+ 
			    delta.tv_usec / 1000;
+ 
			errno = 0;
+ 
			pfd.revents = 0;
+ 
			if (poll(&pfd, 1, deltams) < 0) {
+ 
				/* POSIX compliance */
+ 
				if (errno == EAGAIN)
+ 
					continue;
+ 
				if (errno == EINTR && fetchRestartCalls)
+ 
					continue;
+ 
				return (-1);
+ 
			}
+ 
		}
+ 
		errno = 0;
+ 
#ifdef WITH_SSL
+ 
		if (conn->ssl != NULL)
+ 
			wlen = SSL_write(conn->ssl,
+ 
			    iov->iov_base, iov->iov_len);
+ 
		else
+ 
#endif
+ 
			wlen = writev(conn->sd, iov, iovcnt);
+ 
		if (wlen == 0) {
+ 
			/* we consider a short write a failure */
+ 
			/* XXX perhaps we shouldn't in the SSL case */
+ 
			errno = EPIPE;
+ 
			fetch_syserr();
+ 
			return (-1);
+ 
		}
+ 
		if (wlen < 0) {
+ 
			if (errno == EINTR && fetchRestartCalls)
+ 
				continue;
+ 
			return (-1);
+ 
		}
+ 
		total += wlen;
+ 
		while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
+ 
			wlen -= iov->iov_len;
+ 
			iov++;
+ 
			iovcnt--;
+ 
		}
+ 
		if (iovcnt > 0) {
+ 
			iov->iov_len -= wlen;
+ 
			iov->iov_base = __DECONST(char *, iov->iov_base) + wlen;
+ 
		}
+ 
	}
+ 
	return (total);
+ 
}
+
+
+ 
/*
+ 
 * Write a line of text to a connection w/ timeout
+ 
 */
+ 
int
+ 
fetch_putln(conn_t *conn, const char *str, size_t len)
+ 
{
+ 
	struct iovec iov[2];
+ 
	int ret;
+
+ 
	DEBUGF(">>> %s\n", str);
+ 
	iov[0].iov_base = __DECONST(char *, str);
+ 
	iov[0].iov_len = len;
+ 
	iov[1].iov_base = __DECONST(char *, ENDL);
+ 
	iov[1].iov_len = sizeof(ENDL);
+ 
	if (len == 0)
+ 
		ret = fetch_writev(conn, &iov[1], 1);
+ 
	else
+ 
		ret = fetch_writev(conn, iov, 2);
+ 
	if (ret == -1)
+ 
		return (-1);
+ 
	return (0);
+ 
}
+
+
+ 
/*
+ 
 * Close connection
+ 
 */
+ 
int
+ 
fetch_close(conn_t *conn)
+ 
{
+ 
	int ret;
+
+ 
	if (--conn->ref > 0)
+ 
		return (0);
+ 
#ifdef WITH_SSL
+ 
	if (conn->ssl) {
+ 
		SSL_shutdown(conn->ssl);
+ 
		SSL_set_connect_state(conn->ssl);
+ 
		SSL_free(conn->ssl);
+ 
		conn->ssl = NULL;
+ 
	}
+ 
	if (conn->ssl_ctx) {
+ 
		SSL_CTX_free(conn->ssl_ctx);
+ 
		conn->ssl_ctx = NULL;
+ 
	}
+ 
	if (conn->ssl_cert) {
+ 
		X509_free(conn->ssl_cert);
+ 
		conn->ssl_cert = NULL;
+ 
	}
+ 
#endif
+ 
	ret = close(conn->sd);
+ 
	free(conn->buf);
+ 
	free(conn);
+ 
	return (ret);
+ 
}
+
+
+ 
/*** Directory-related utility functions *************************************/
+
+ 
int
+ 
fetch_add_entry(struct url_ent **p, int *size, int *len,
+ 
    const char *name, struct url_stat *us)
+ 
{
+ 
	struct url_ent *tmp;
+
+ 
	if (*p == NULL) {
+ 
		*size = 0;
+ 
		*len = 0;
+ 
	}
+
+ 
	if (*len >= *size - 1) {
+ 
		tmp = reallocarray(*p, *size * 2 + 1, sizeof(**p));
+ 
		if (tmp == NULL) {
+ 
			errno = ENOMEM;
+ 
			fetch_syserr();
+ 
			return (-1);
+ 
		}
+ 
		*size = (*size * 2 + 1);
+ 
		*p = tmp;
+ 
	}
+
+ 
	tmp = *p + *len;
+ 
	snprintf(tmp->name, PATH_MAX, "%s", name);
+ 
	memcpy(&tmp->stat, us, sizeof(*us));
+
+ 
	(*len)++;
+ 
	(++tmp)->name[0] = 0;
+
+ 
	return (0);
+ 
}
+
+
+ 
/*** Authentication-related utility functions ********************************/
+
+ 
static const char *
+ 
fetch_read_word(FILE *f)
+ 
{
+ 
	static char word[1024];
+
+ 
	if (fscanf(f, " %1023s ", word) != 1)
+ 
		return (NULL);
+ 
	return (word);
+ 
}
+
+ 
static int
+ 
fetch_netrc_open(void)
+ 
{
+ 
	struct passwd *pwd;
+ 
	char fn[PATH_MAX];
+ 
	const char *p;
+ 
	int fd, serrno;
+
+ 
	if ((p = getenv("NETRC")) != NULL) {
+ 
		DEBUGF("NETRC=%s\n", p);
+ 
		if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
+ 
			fetch_info("$NETRC specifies a file name "
+ 
			    "longer than PATH_MAX");
+ 
			return (-1);
+ 
		}
+ 
	} else {
+ 
		if ((p = getenv("HOME")) == NULL) {
+ 
			if ((pwd = getpwuid(getuid())) == NULL ||
+ 
			    (p = pwd->pw_dir) == NULL)
+ 
				return (-1);
+ 
		}
+ 
		if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))
+ 
			return (-1);
+ 
	}
+
+ 
	if ((fd = open(fn, O_RDONLY)) < 0) {
+ 
		serrno = errno;
+ 
		DEBUGF("%s: %s\n", fn, strerror(serrno));
+ 
		errno = serrno;
+ 
	}
+ 
	return (fd);
+ 
}
+
+ 
/*
+ 
 * Get authentication data for a URL from .netrc
+ 
 */
+ 
int
+ 
fetch_netrc_auth(struct url *url)
+ 
{
+ 
	const char *word;
+ 
	int serrno;
+ 
	FILE *f;
+
+ 
	if (url->netrcfd < 0)
+ 
		url->netrcfd = fetch_netrc_open();
+ 
	if (url->netrcfd < 0)
+ 
		return (-1);
+ 
	if ((f = fdopen(url->netrcfd, "r")) == NULL) {
+ 
		serrno = errno;
+ 
		DEBUGF("fdopen(netrcfd): %s", strerror(errno));
+ 
		close(url->netrcfd);
+ 
		url->netrcfd = -1;
+ 
		errno = serrno;
+ 
		return (-1);
+ 
	}
+ 
	rewind(f);
+ 
	DEBUGF("searching netrc for %s\n", url->host);
+ 
	while ((word = fetch_read_word(f)) != NULL) {
+ 
		if (strcmp(word, "default") == 0) {
+ 
			DEBUGF("using default netrc settings\n");
+ 
			break;
+ 
		}
+ 
		if (strcmp(word, "machine") == 0 &&
+ 
		    (word = fetch_read_word(f)) != NULL &&
+ 
		    strcasecmp(word, url->host) == 0) {
+ 
			DEBUGF("using netrc settings for %s\n", word);
+ 
			break;
+ 
		}
+ 
	}
+ 
	if (word == NULL)
+ 
		goto ferr;
+ 
	while ((word = fetch_read_word(f)) != NULL) {
+ 
		if (strcmp(word, "login") == 0) {
+ 
			if ((word = fetch_read_word(f)) == NULL)
+ 
				goto ferr;
+ 
			if (snprintf(url->user, sizeof(url->user),
+ 
				"%s", word) > (int)sizeof(url->user)) {
+ 
				fetch_info("login name in .netrc is too long");
+ 
				url->user[0] = '\0';
+ 
			}
+ 
		} else if (strcmp(word, "password") == 0) {
+ 
			if ((word = fetch_read_word(f)) == NULL)
+ 
				goto ferr;
+ 
			if (snprintf(url->pwd, sizeof(url->pwd),
+ 
				"%s", word) > (int)sizeof(url->pwd)) {
+ 
				fetch_info("password in .netrc is too long");
+ 
				url->pwd[0] = '\0';
+ 
			}
+ 
		} else if (strcmp(word, "account") == 0) {
+ 
			if ((word = fetch_read_word(f)) == NULL)
+ 
				goto ferr;
+ 
			/* XXX not supported! */
+ 
		} else {
+ 
			break;
+ 
		}
+ 
	}
+ 
	fclose(f);
+ 
	url->netrcfd = -1;
+ 
	return (0);
+ 
ferr:
+ 
	serrno = errno;
+ 
	fclose(f);
+ 
	url->netrcfd = -1;
+ 
	errno = serrno;
+ 
	return (-1);
+ 
}
+
+ 
/*
+ 
 * The no_proxy environment variable specifies a set of domains for
+ 
 * which the proxy should not be consulted; the contents is a comma-,
+ 
 * or space-separated list of domain names.  A single asterisk will
+ 
 * override all proxy variables and no transactions will be proxied
+ 
 * (for compatibility with lynx and curl, see the discussion at
+ 
 * <http://curl.haxx.se/mail/archive_pre_oct_99/0009.html>).
+ 
 */
+ 
int
+ 
fetch_no_proxy_match(const char *host)
+ 
{
+ 
	const char *no_proxy, *p, *q;
+ 
	size_t h_len, d_len;
+
+ 
	if ((no_proxy = getenv("NO_PROXY")) == NULL &&
+ 
	    (no_proxy = getenv("no_proxy")) == NULL)
+ 
		return (0);
+
+ 
	/* asterisk matches any hostname */
+ 
	if (strcmp(no_proxy, "*") == 0)
+ 
		return (1);
+
+ 
	h_len = strlen(host);
+ 
	p = no_proxy;
+ 
	do {
+ 
		/* position p at the beginning of a domain suffix */
+ 
		while (*p == ',' || isspace((unsigned char)*p))
+ 
			p++;
+
+ 
		/* position q at the first separator character */
+ 
		for (q = p; *q; ++q)
+ 
			if (*q == ',' || isspace((unsigned char)*q))
+ 
				break;
+
+ 
		d_len = q - p;
+ 
		if (d_len > 0 && h_len >= d_len &&
+ 
		    strncasecmp(host + h_len - d_len,
+ 
			p, d_len) == 0) {
+ 
			/* domain name matches */
+ 
			return (1);
+ 
		}
+
+ 
		p = q + 1;
+ 
	} while (*q);
+
+ 
	return (0);
+ 
}
added external/libfetch/common.h
@@ -0,0 +1,175 @@
+ 
/*-
+ 
 * SPDX-License-Identifier: BSD-3-Clause
+ 
 *
+ 
 * Copyright (c) 1998-2014 Dag-Erling Smørgrav
+ 
 * All rights reserved.
+ 
 *
+ 
 * Redistribution and use in source and binary forms, with or without
+ 
 * modification, are permitted provided that the following conditions
+ 
 * are met:
+ 
 * 1. Redistributions of source code must retain the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer
+ 
 *    in this position and unchanged.
+ 
 * 2. Redistributions in binary form must reproduce the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer in the
+ 
 *    documentation and/or other materials provided with the distribution.
+ 
 * 3. The name of the author may not be used to endorse or promote products
+ 
 *    derived from this software without specific prior written permission
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ 
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ 
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ 
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ 
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ 
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ 
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ 
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
 */
+
+ 
#ifndef _COMMON_H_INCLUDED
+ 
#define _COMMON_H_INCLUDED
+
+ 
#define FTP_DEFAULT_PORT	21
+ 
#define HTTP_DEFAULT_PORT	80
+ 
#define FTP_DEFAULT_PROXY_PORT	21
+ 
#define HTTP_DEFAULT_PROXY_PORT	3128
+
+ 
#ifdef WITH_SSL
+ 
#include <openssl/crypto.h>
+ 
#include <openssl/x509.h>
+ 
#include <openssl/pem.h>
+ 
#include <openssl/ssl.h>
+ 
#include <openssl/err.h>
+ 
#endif
+
+ 
/* Connection */
+ 
typedef struct fetchconn conn_t;
+ 
struct fetchconn {
+ 
	int		 sd;		/* socket descriptor */
+ 
	char		*buf;		/* buffer */
+ 
	size_t		 bufsize;	/* buffer size */
+ 
	size_t		 buflen;	/* length of buffer contents */
+ 
	int		 err;		/* last protocol reply code */
+ 
#ifdef WITH_SSL
+ 
	SSL		*ssl;		/* SSL handle */
+ 
	SSL_CTX		*ssl_ctx;	/* SSL context */
+ 
	X509		*ssl_cert;	/* server certificate */
+ 
#endif
+ 
	int		 ref;		/* reference count */
+ 
	char		 cache_host[MAXHOSTNAMELEN + 1];
+ 
	int		 cache_port;
+ 
	int		 cache_ssl;	/* 1 if HTTPS */
+ 
};
+
+ 
/* Structure used for error message lists */
+ 
struct fetcherr {
+ 
	const int	 num;
+ 
	const int	 cat;
+ 
	const char	*string;
+ 
};
+
+ 
/* For SOCKS header size */
+ 
#define HEAD_SIZE	4
+ 
#define FQDN_SIZE	256
+ 
#define PACK_SIZE	1
+ 
#define PORT_SIZE	2
+ 
#define BUFF_SIZE	HEAD_SIZE + FQDN_SIZE + PACK_SIZE + PORT_SIZE
+
+ 
/* SOCKS5 Request Header */
+ 
#define SOCKS_VERSION_5		0x05
+ 
/* SOCKS5 CMD */
+ 
#define SOCKS_CONNECTION	0x01
+ 
#define SOCKS_BIND		0x02
+ 
#define SOCKS_UDP		0x03
+ 
#define SOCKS_NOMETHODS		0xFF
+ 
#define SOCKS5_NOTIMPLEMENTED	0x00
+ 
/* SOCKS5 Reserved */
+ 
#define SOCKS_RSV		0x00
+ 
/* SOCKS5 Address Type */
+ 
#define SOCKS_ATYP_IPV4		0x01
+ 
#define SOCKS_ATYP_DOMAINNAME	0x03
+ 
#define SOCKS_ATYP_IPV6		0x04
+ 
/* SOCKS5 Reply Field */
+ 
#define SOCKS_SUCCESS			0x00
+ 
#define SOCKS_GENERAL_FAILURE		0x01
+ 
#define SOCKS_CONNECTION_NOT_ALLOWED	0x02
+ 
#define SOCKS_NETWORK_UNREACHABLE	0x03
+ 
#define SOCKS_HOST_UNREACHABLE		0x04
+ 
#define SOCKS_CONNECTION_REFUSED	0x05
+ 
#define SOCKS_TTL_EXPIRED		0x06
+ 
#define SOCKS_COMMAND_NOT_SUPPORTED	0x07
+ 
#define SOCKS_ADDRESS_NOT_SUPPORTED	0x08
+
+ 
/* for fetch_writev */
+ 
struct iovec;
+
+ 
void		 fetch_seterr(struct fetcherr *, int);
+ 
void		 fetch_syserr(void);
+ 
void		 fetch_info(const char *, ...) __printflike(1, 2);
+ 
int		 fetch_socks5_getenv(char **host, int *port);
+ 
int		 fetch_socks5_init(conn_t *conn, const char *host,
+ 
		     int port, int verbose);
+ 
int		 fetch_default_port(const char *);
+ 
int		 fetch_default_proxy_port(const char *);
+ 
struct addrinfo *fetch_resolve(const char *, int, int);
+ 
int		 fetch_bind(int, int, const char *);
+ 
conn_t		*fetch_connect(const char *, int, int, int);
+ 
conn_t		*fetch_reopen(int);
+ 
conn_t		*fetch_ref(conn_t *);
+ 
#ifdef WITH_SSL
+ 
int		 fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*);
+ 
#endif
+ 
int		 fetch_ssl(conn_t *, const struct url *, int);
+ 
ssize_t		 fetch_read(conn_t *, char *, size_t);
+ 
int		 fetch_getln(conn_t *);
+ 
ssize_t		 fetch_write(conn_t *, const char *, size_t);
+ 
ssize_t		 fetch_writev(conn_t *, struct iovec *, int);
+ 
int		 fetch_putln(conn_t *, const char *, size_t);
+ 
int		 fetch_close(conn_t *);
+ 
int		 fetch_add_entry(struct url_ent **, int *, int *,
+ 
		     const char *, struct url_stat *);
+ 
int		 fetch_netrc_auth(struct url *url);
+ 
int		 fetch_no_proxy_match(const char *);
+
+ 
#define ftp_seterr(n)	 fetch_seterr(ftp_errlist, n)
+ 
#define http_seterr(n)	 fetch_seterr(http_errlist, n)
+ 
#define netdb_seterr(n)	 fetch_seterr(netdb_errlist, n)
+ 
#define url_seterr(n)	 fetch_seterr(url_errlist, n)
+ 
#define socks5_seterr(n) fetch_seterr(socks5_errlist, n)
+
+ 
#ifndef NDEBUG
+ 
#define DEBUGF(...)							\
+ 
	do {								\
+ 
		if (fetchDebug)						\
+ 
			fprintf(stderr, __VA_ARGS__);			\
+ 
	} while (0)
+ 
#else
+ 
#define DEBUGF(...)							\
+ 
	do {								\
+ 
		/* nothing */						\
+ 
	} while (0)
+ 
#endif
+
+ 
/*
+ 
 * I don't really like exporting http_request(),
+ 
 * but the HTTP and FTP code occasionally needs to cross-call
+ 
 * eachother, and this saves me from adding a lot of special-case code
+ 
 * to handle those cases.
+ 
 *
+ 
 * Note that _*_request() free purl, which is way ugly but saves us a
+ 
 * whole lot of trouble.
+ 
 */
+ 
FILE		*http_request(struct url *, const char *,
+ 
		     struct url_stat *, struct url *, const char *);
+ 
FILE		*http_request_body(struct url *, const char *,
+ 
		     struct url_stat *, struct url *, const char *,
+ 
		     const char *, const char *);
+
+ 
/*
+ 
 * Check whether a particular flag is set
+ 
 */
+ 
#define CHECK_FLAG(x)	(flags && strchr(flags, (x)))
+
+ 
#endif
added external/libfetch/fetch.3
@@ -0,0 +1,841 @@
+ 
.\"-
+ 
.\" Copyright (c) 1998-2013 Dag-Erling Smørgrav
+ 
.\" Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
+ 
.\" All rights reserved.
+ 
.\"
+ 
.\" Redistribution and use in source and binary forms, with or without
+ 
.\" modification, are permitted provided that the following conditions
+ 
.\" are met:
+ 
.\" 1. Redistributions of source code must retain the above copyright
+ 
.\"    notice, this list of conditions and the following disclaimer.
+ 
.\" 2. Redistributions in binary form must reproduce the above copyright
+ 
.\"    notice, this list of conditions and the following disclaimer in the
+ 
.\"    documentation and/or other materials provided with the distribution.
+ 
.\"
+ 
.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ 
.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ 
.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ 
.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ 
.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ 
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ 
.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ 
.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ 
.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ 
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ 
.\" SUCH DAMAGE.
+ 
.\"
+ 
.\" $FreeBSD: head/lib/libfetch/fetch.3 273124 2014-10-15 07:35:50Z des $
+ 
.\"
+ 
.Dd October 15, 2014
+ 
.Dt FETCH 3
+ 
.Os
+ 
.Sh NAME
+ 
.Nm fetchMakeURL ,
+ 
.Nm fetchParseURL ,
+ 
.Nm fetchFreeURL ,
+ 
.Nm fetchXGetURL ,
+ 
.Nm fetchGetURL ,
+ 
.Nm fetchPutURL ,
+ 
.Nm fetchStatURL ,
+ 
.Nm fetchListURL ,
+ 
.Nm fetchXGet ,
+ 
.Nm fetchGet ,
+ 
.Nm fetchPut ,
+ 
.Nm fetchStat ,
+ 
.Nm fetchList ,
+ 
.Nm fetchXGetFile ,
+ 
.Nm fetchGetFile ,
+ 
.Nm fetchPutFile ,
+ 
.Nm fetchStatFile ,
+ 
.Nm fetchListFile ,
+ 
.Nm fetchXGetHTTP ,
+ 
.Nm fetchGetHTTP ,
+ 
.Nm fetchPutHTTP ,
+ 
.Nm fetchStatHTTP ,
+ 
.Nm fetchListHTTP ,
+ 
.Nm fetchXGetFTP ,
+ 
.Nm fetchGetFTP ,
+ 
.Nm fetchPutFTP ,
+ 
.Nm fetchStatFTP ,
+ 
.Nm fetchListFTP
+ 
.Nd file transfer functions
+ 
.Sh LIBRARY
+ 
.Lb libfetch
+ 
.Sh SYNOPSIS
+ 
.In sys/param.h
+ 
.In stdio.h
+ 
.In fetch.h
+ 
.Ft struct url *
+ 
.Fn fetchMakeURL "const char *scheme" "const char *host" "int port" "const char *doc" "const char *user" "const char *pwd"
+ 
.Ft struct url *
+ 
.Fn fetchParseURL "const char *URL"
+ 
.Ft void
+ 
.Fn fetchFreeURL "struct url *u"
+ 
.Ft FILE *
+ 
.Fn fetchXGetURL "const char *URL" "struct url_stat *us" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchGetURL "const char *URL" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchPutURL "const char *URL" "const char *flags"
+ 
.Ft int
+ 
.Fn fetchStatURL "const char *URL" "struct url_stat *us" "const char *flags"
+ 
.Ft struct url_ent *
+ 
.Fn fetchListURL "const char *URL" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchXGet "struct url *u" "struct url_stat *us" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchGet "struct url *u" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchPut "struct url *u" "const char *flags"
+ 
.Ft int
+ 
.Fn fetchStat "struct url *u" "struct url_stat *us" "const char *flags"
+ 
.Ft struct url_ent *
+ 
.Fn fetchList "struct url *u" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchXGetFile "struct url *u" "struct url_stat *us" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchGetFile "struct url *u" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchPutFile "struct url *u" "const char *flags"
+ 
.Ft int
+ 
.Fn fetchStatFile "struct url *u" "struct url_stat *us" "const char *flags"
+ 
.Ft struct url_ent *
+ 
.Fn fetchListFile "struct url *u" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchGetHTTP "struct url *u" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchPutHTTP "struct url *u" "const char *flags"
+ 
.Ft int
+ 
.Fn fetchStatHTTP "struct url *u" "struct url_stat *us" "const char *flags"
+ 
.Ft struct url_ent *
+ 
.Fn fetchListHTTP "struct url *u" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchGetFTP "struct url *u" "const char *flags"
+ 
.Ft FILE *
+ 
.Fn fetchPutFTP "struct url *u" "const char *flags"
+ 
.Ft int
+ 
.Fn fetchStatFTP "struct url *u" "struct url_stat *us" "const char *flags"
+ 
.Ft struct url_ent *
+ 
.Fn fetchListFTP "struct url *u" "const char *flags"
+ 
.Sh DESCRIPTION
+ 
These functions implement a high-level library for retrieving and
+ 
uploading files using Uniform Resource Locators (URLs).
+ 
.Pp
+ 
.Fn fetchParseURL
+ 
takes a URL in the form of a null-terminated string and splits it into
+ 
its components function according to the Common Internet Scheme Syntax
+ 
detailed in RFC1738.
+ 
A regular expression which produces this syntax is:
+ 
.Bd -literal
+ 
    <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
+ 
.Ed
+ 
.Pp
+ 
If the URL does not seem to begin with a scheme name, the following
+ 
syntax is assumed:
+ 
.Bd -literal
+ 
    ((<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
+ 
.Ed
+ 
.Pp
+ 
Note that some components of the URL are not necessarily relevant to
+ 
all URL schemes.
+ 
For instance, the file scheme only needs the <scheme> and <document>
+ 
components.
+ 
.Pp
+ 
.Fn fetchMakeURL
+ 
and
+ 
.Fn fetchParseURL
+ 
return a pointer to a
+ 
.Vt url
+ 
structure, which is defined as follows in
+ 
.In fetch.h :
+ 
.Bd -literal
+ 
#define URL_SCHEMELEN 16
+ 
#define URL_USERLEN 256
+ 
#define URL_PWDLEN 256
+
+ 
struct url {
+ 
    char	 scheme[URL_SCHEMELEN+1];
+ 
    char	 user[URL_USERLEN+1];
+ 
    char	 pwd[URL_PWDLEN+1];
+ 
    char	 host[MAXHOSTNAMELEN+1];
+ 
    int		 port;
+ 
    char	*doc;
+ 
    off_t	 offset;
+ 
    size_t	 length;
+ 
    time_t	 ims_time;
+ 
};
+ 
.Ed
+ 
.Pp
+ 
The
+ 
.Va ims_time
+ 
field stores the time value for
+ 
.Li If-Modified-Since
+ 
HTTP requests.
+ 
.Pp
+ 
The pointer returned by
+ 
.Fn fetchMakeURL
+ 
or
+ 
.Fn fetchParseURL
+ 
should be freed using
+ 
.Fn fetchFreeURL .
+ 
.Pp
+ 
.Fn fetchXGetURL ,
+ 
.Fn fetchGetURL ,
+ 
and
+ 
.Fn fetchPutURL
+ 
constitute the recommended interface to the
+ 
.Nm fetch
+ 
library.
+ 
They examine the URL passed to them to determine the transfer
+ 
method, and call the appropriate lower-level functions to perform the
+ 
actual transfer.
+ 
.Fn fetchXGetURL
+ 
also returns the remote document's metadata in the
+ 
.Vt url_stat
+ 
structure pointed to by the
+ 
.Fa us
+ 
argument.
+ 
.Pp
+ 
The
+ 
.Fa flags
+ 
argument is a string of characters which specify transfer options.
+ 
The
+ 
meaning of the individual flags is scheme-dependent, and is detailed
+ 
in the appropriate section below.
+ 
.Pp
+ 
.Fn fetchStatURL
+ 
attempts to obtain the requested document's metadata and fill in the
+ 
structure pointed to by its second argument.
+ 
The
+ 
.Vt url_stat
+ 
structure is defined as follows in
+ 
.In fetch.h :
+ 
.Bd -literal
+ 
struct url_stat {
+ 
    off_t	 size;
+ 
    time_t	 atime;
+ 
    time_t	 mtime;
+ 
};
+ 
.Ed
+ 
.Pp
+ 
If the size could not be obtained from the server, the
+ 
.Fa size
+ 
field is set to -1.
+ 
If the modification time could not be obtained from the server, the
+ 
.Fa mtime
+ 
field is set to the epoch.
+ 
If the access time could not be obtained from the server, the
+ 
.Fa atime
+ 
field is set to the modification time.
+ 
.Pp
+ 
.Fn fetchListURL
+ 
attempts to list the contents of the directory pointed to by the URL
+ 
provided.
+ 
If successful, it returns a malloced array of
+ 
.Vt url_ent
+ 
structures.
+ 
The
+ 
.Vt url_ent
+ 
structure is defined as follows in
+ 
.In fetch.h :
+ 
.Bd -literal
+ 
struct url_ent {
+ 
    char         name[PATH_MAX];
+ 
    struct url_stat stat;
+ 
};
+ 
.Ed
+ 
.Pp
+ 
The list is terminated by an entry with an empty name.
+ 
.Pp
+ 
The pointer returned by
+ 
.Fn fetchListURL
+ 
should be freed using
+ 
.Fn free .
+ 
.Pp
+ 
.Fn fetchXGet ,
+ 
.Fn fetchGet ,
+ 
.Fn fetchPut
+ 
and
+ 
.Fn fetchStat
+ 
are similar to
+ 
.Fn fetchXGetURL ,
+ 
.Fn fetchGetURL ,
+ 
.Fn fetchPutURL
+ 
and
+ 
.Fn fetchStatURL ,
+ 
except that they expect a pre-parsed URL in the form of a pointer to
+ 
a
+ 
.Vt struct url
+ 
rather than a string.
+ 
.Pp
+ 
All of the
+ 
.Fn fetchXGetXXX ,
+ 
.Fn fetchGetXXX
+ 
and
+ 
.Fn fetchPutXXX
+ 
functions return a pointer to a stream which can be used to read or
+ 
write data from or to the requested document, respectively.
+ 
Note that
+ 
although the implementation details of the individual access methods
+ 
vary, it can generally be assumed that a stream returned by one of the
+ 
.Fn fetchXGetXXX
+ 
or
+ 
.Fn fetchGetXXX
+ 
functions is read-only, and that a stream returned by one of the
+ 
.Fn fetchPutXXX
+ 
functions is write-only.
+ 
.Sh FILE SCHEME
+ 
.Fn fetchXGetFile ,
+ 
.Fn fetchGetFile
+ 
and
+ 
.Fn fetchPutFile
+ 
provide access to documents which are files in a locally mounted file
+ 
system.
+ 
Only the <document> component of the URL is used.
+ 
.Pp
+ 
.Fn fetchXGetFile
+ 
and
+ 
.Fn fetchGetFile
+ 
do not accept any flags.
+ 
.Pp
+ 
.Fn fetchPutFile
+ 
accepts the
+ 
.Ql a
+ 
(append to file) flag.
+ 
If that flag is specified, the data written to
+ 
the stream returned by
+ 
.Fn fetchPutFile
+ 
will be appended to the previous contents of the file, instead of
+ 
replacing them.
+ 
.Sh FTP SCHEME
+ 
.Fn fetchXGetFTP ,
+ 
.Fn fetchGetFTP
+ 
and
+ 
.Fn fetchPutFTP
+ 
implement the FTP protocol as described in RFC959.
+ 
.Pp
+ 
If the
+ 
.Ql P
+ 
(not passive) flag is specified, an active (rather than passive)
+ 
connection will be attempted.
+ 
.Pp
+ 
The
+ 
.Ql p
+ 
flag is supported for compatibility with earlier versions where active
+ 
connections were the default.
+ 
It has precedence over the
+ 
.Ql P
+ 
flag, so if both are specified,
+ 
.Nm
+ 
will use a passive connection.
+ 
.Pp
+ 
If the
+ 
.Ql l
+ 
(low) flag is specified, data sockets will be allocated in the low (or
+ 
default) port range instead of the high port range (see
+ 
.Xr ip 4 ) .
+ 
.Pp
+ 
If the
+ 
.Ql d
+ 
(direct) flag is specified,
+ 
.Fn fetchXGetFTP ,
+ 
.Fn fetchGetFTP
+ 
and
+ 
.Fn fetchPutFTP
+ 
will use a direct connection even if a proxy server is defined.
+ 
.Pp
+ 
If no user name or password is given, the
+ 
.Nm fetch
+ 
library will attempt an anonymous login, with user name "anonymous"
+ 
and password "anonymous@<hostname>".
+ 
.Sh HTTP SCHEME
+ 
The
+ 
.Fn fetchXGetHTTP ,
+ 
.Fn fetchGetHTTP
+ 
and
+ 
.Fn fetchPutHTTP
+ 
functions implement the HTTP/1.1 protocol.
+ 
With a little luck, there is
+ 
even a chance that they comply with RFC2616 and RFC2617.
+ 
.Pp
+ 
If the
+ 
.Ql d
+ 
(direct) flag is specified,
+ 
.Fn fetchXGetHTTP ,
+ 
.Fn fetchGetHTTP
+ 
and
+ 
.Fn fetchPutHTTP
+ 
will use a direct connection even if a proxy server is defined.
+ 
.Pp
+ 
If the
+ 
.Ql i
+ 
(if-modified-since) flag is specified, and
+ 
the
+ 
.Va ims_time
+ 
field is set in
+ 
.Vt "struct url" ,
+ 
then
+ 
.Fn fetchXGetHTTP
+ 
and
+ 
.Fn fetchGetHTTP
+ 
will send a conditional
+ 
.Li If-Modified-Since
+ 
HTTP header to only fetch the content if it is newer than
+ 
.Va ims_time .
+ 
.Pp
+ 
Since there seems to be no good way of implementing the HTTP PUT
+ 
method in a manner consistent with the rest of the
+ 
.Nm fetch
+ 
library,
+ 
.Fn fetchPutHTTP
+ 
is currently unimplemented.
+ 
.Sh HTTPS SCHEME
+ 
Based on HTTP SCHEME.
+ 
By default the peer is verified using the CA bundle located in
+ 
.Pa /etc/ssl/cert.pem .
+ 
The file may contain multiple CA certificates.
+ 
A common source of a current CA bundle is
+ 
.Pa \%security/ca_root_nss .
+ 
.Pp
+ 
The CA bundle used for peer verification can be changed by setting the
+ 
environment variables
+ 
.Ev SSL_CA_CERT_FILE
+ 
to point to a concatenated bundle of trusted certificates and
+ 
.Ev SSL_CA_CERT_PATH
+ 
to point to a directory containing hashes of trusted CAs (see
+ 
.Xr verify 1 ) .
+ 
.Pp
+ 
A certificate revocation list (CRL) can be used by setting the
+ 
environment variable
+ 
.Ev SSL_CRL_FILE
+ 
(see
+ 
.Xr crl 1 ) .
+ 
.Pp
+ 
Peer verification can be disabled by setting the environment variable
+ 
.Ev SSL_NO_VERIFY_PEER .
+ 
Note that this also disables CRL checking.
+ 
.Pp
+ 
By default the service identity is verified according to the rules
+ 
detailed in RFC6125 (also known as hostname verification).
+ 
This feature can be disabled by setting the environment variable
+ 
.Ev SSL_NO_VERIFY_HOSTNAME .
+ 
.Pp
+ 
Client certificate based authentication is supported.
+ 
The environment variable
+ 
.Ev SSL_CLIENT_CERT_FILE
+ 
should be set to point to a file containing key and client certificate
+ 
to be used in PEM format. In case the key is stored in a separate
+ 
file, the environment variable
+ 
.Ev SSL_CLIENT_KEY_FILE
+ 
can be set to point to the key in PEM format.
+ 
In case the key uses a password, the user will be prompted on standard
+ 
input (see
+ 
.Xr PEM 3 ) .
+ 
.Pp
+ 
By default
+ 
.Nm libfetch
+ 
allows TLSv1 and newer when negotiating the connecting with the remote
+ 
peer.
+ 
You can change this behavior by setting the
+ 
.Ev SSL_ALLOW_SSL2
+ 
and
+ 
.Ev SSL_ALLOW_SSL3
+ 
environment variables to allow SSLv2 and SSLv3, respectively, and
+ 
.Ev SSL_NO_TLS1 ,
+ 
.Ev SSL_NO_TLS1_1 and
+ 
.Ev SSL_NO_TLS1_2
+ 
to disable TLS 1.0, 1.1 and 1.2 respectively.
+ 
.Sh AUTHENTICATION
+ 
Apart from setting the appropriate environment variables and
+ 
specifying the user name and password in the URL or the
+ 
.Vt struct url ,
+ 
the calling program has the option of defining an authentication
+ 
function with the following prototype:
+ 
.Pp
+ 
.Ft int
+ 
.Fn myAuthMethod "struct url *u"
+ 
.Pp
+ 
The callback function should fill in the
+ 
.Fa user
+ 
and
+ 
.Fa pwd
+ 
fields in the provided
+ 
.Vt struct url
+ 
and return 0 on success, or any other value to indicate failure.
+ 
.Pp
+ 
To register the authentication callback, simply set
+ 
.Va fetchAuthMethod
+ 
to point at it.
+ 
The callback will be used whenever a site requires authentication and
+ 
the appropriate environment variables are not set.
+ 
.Pp
+ 
This interface is experimental and may be subject to change.
+ 
.Sh RETURN VALUES
+ 
.Fn fetchParseURL
+ 
returns a pointer to a
+ 
.Vt struct url
+ 
containing the individual components of the URL.
+ 
If it is
+ 
unable to allocate memory, or the URL is syntactically incorrect,
+ 
.Fn fetchParseURL
+ 
returns a NULL pointer.
+ 
.Pp
+ 
The
+ 
.Fn fetchStat
+ 
functions return 0 on success and -1 on failure.
+ 
.Pp
+ 
All other functions return a stream pointer which may be used to
+ 
access the requested document, or NULL if an error occurred.
+ 
.Pp
+ 
The following error codes are defined in
+ 
.In fetch.h :
+ 
.Bl -tag -width 18n
+ 
.It Bq Er FETCH_ABORT
+ 
Operation aborted
+ 
.It Bq Er FETCH_AUTH
+ 
Authentication failed
+ 
.It Bq Er FETCH_DOWN
+ 
Service unavailable
+ 
.It Bq Er FETCH_EXISTS
+ 
File exists
+ 
.It Bq Er FETCH_FULL
+ 
File system full
+ 
.It Bq Er FETCH_INFO
+ 
Informational response
+ 
.It Bq Er FETCH_MEMORY
+ 
Insufficient memory
+ 
.It Bq Er FETCH_MOVED
+ 
File has moved
+ 
.It Bq Er FETCH_NETWORK
+ 
Network error
+ 
.It Bq Er FETCH_OK
+ 
No error
+ 
.It Bq Er FETCH_PROTO
+ 
Protocol error
+ 
.It Bq Er FETCH_RESOLV
+ 
Resolver error
+ 
.It Bq Er FETCH_SERVER
+ 
Server error
+ 
.It Bq Er FETCH_TEMP
+ 
Temporary error
+ 
.It Bq Er FETCH_TIMEOUT
+ 
Operation timed out
+ 
.It Bq Er FETCH_UNAVAIL
+ 
File is not available
+ 
.It Bq Er FETCH_UNKNOWN
+ 
Unknown error
+ 
.It Bq Er FETCH_URL
+ 
Invalid URL
+ 
.El
+ 
.Pp
+ 
The accompanying error message includes a protocol-specific error code
+ 
and message, e.g.\& "File is not available (404 Not Found)"
+ 
.Sh ENVIRONMENT
+ 
.Bl -tag -width ".Ev FETCH_BIND_ADDRESS"
+ 
.It Ev FETCH_BIND_ADDRESS
+ 
Specifies a hostname or IP address to which sockets used for outgoing
+ 
connections will be bound.
+ 
.It Ev FTP_LOGIN
+ 
Default FTP login if none was provided in the URL.
+ 
.It Ev FTP_PASSIVE_MODE
+ 
If set to
+ 
.Ql no ,
+ 
forces the FTP code to use active mode.
+ 
If set to any other value, forces passive mode even if the application
+ 
requested active mode.
+ 
.It Ev FTP_PASSWORD
+ 
Default FTP password if the remote server requests one and none was
+ 
provided in the URL.
+ 
.It Ev FTP_PROXY
+ 
URL of the proxy to use for FTP requests.
+ 
The document part is ignored.
+ 
FTP and HTTP proxies are supported; if no scheme is specified, FTP is
+ 
assumed.
+ 
If the proxy is an FTP proxy,
+ 
.Nm libfetch
+ 
will send
+ 
.Ql user@host
+ 
as user name to the proxy, where
+ 
.Ql user
+ 
is the real user name, and
+ 
.Ql host
+ 
is the name of the FTP server.
+ 
.Pp
+ 
If this variable is set to an empty string, no proxy will be used for
+ 
FTP requests, even if the
+ 
.Ev HTTP_PROXY
+ 
variable is set.
+ 
.It Ev ftp_proxy
+ 
Same as
+ 
.Ev FTP_PROXY ,
+ 
for compatibility.
+ 
.It Ev HTTP_ACCEPT
+ 
Specifies the value of the
+ 
.Va Accept
+ 
header for HTTP requests.
+ 
If empty, no
+ 
.Va Accept
+ 
header is sent.
+ 
The default is
+ 
.Dq */* .
+ 
.It Ev HTTP_AUTH
+ 
Specifies HTTP authorization parameters as a colon-separated list of
+ 
items.
+ 
The first and second item are the authorization scheme and realm
+ 
respectively; further items are scheme-dependent.
+ 
Currently, the
+ 
.Dq basic
+ 
and
+ 
.Dq digest
+ 
authorization methods are supported.
+ 
.Pp
+ 
Both methods require two parameters: the user name and
+ 
password, in that order.
+ 
.Pp
+ 
This variable is only used if the server requires authorization and
+ 
no user name or password was specified in the URL.
+ 
.It Ev HTTP_PROXY
+ 
URL of the proxy to use for HTTP requests.
+ 
The document part is ignored.
+ 
Only HTTP proxies are supported for HTTP requests.
+ 
If no port number is specified, the default is 3128.
+ 
.Pp
+ 
Note that this proxy will also be used for FTP documents, unless the
+ 
.Ev FTP_PROXY
+ 
variable is set.
+ 
.It Ev http_proxy
+ 
Same as
+ 
.Ev HTTP_PROXY ,
+ 
for compatibility.
+ 
.It Ev HTTP_PROXY_AUTH
+ 
Specifies authorization parameters for the HTTP proxy in the same
+ 
format as the
+ 
.Ev HTTP_AUTH
+ 
variable.
+ 
.Pp
+ 
This variable is used if and only if connected to an HTTP proxy, and
+ 
is ignored if a user and/or a password were specified in the proxy
+ 
URL.
+ 
.It Ev HTTP_REFERER
+ 
Specifies the referrer URL to use for HTTP requests.
+ 
If set to
+ 
.Dq auto ,
+ 
the document URL will be used as referrer URL.
+ 
.It Ev HTTP_USER_AGENT
+ 
Specifies the User-Agent string to use for HTTP requests.
+ 
This can be useful when working with HTTP origin or proxy servers that
+ 
differentiate between user agents.
+ 
If defined but empty, no User-Agent header is sent.
+ 
.It Ev NETRC
+ 
Specifies a file to use instead of
+ 
.Pa ~/.netrc
+ 
to look up login names and passwords for FTP sites.
+ 
See
+ 
.Xr ftp 1
+ 
for a description of the file format.
+ 
This feature is experimental.
+ 
.It Ev NO_PROXY
+ 
Either a single asterisk, which disables the use of proxies
+ 
altogether, or a comma- or whitespace-separated list of hosts for
+ 
which proxies should not be used.
+ 
.It Ev no_proxy
+ 
Same as
+ 
.Ev NO_PROXY ,
+ 
for compatibility.
+ 
.It Ev SSL_ALLOW_SSL2
+ 
Allow SSL version 2 when negotiating the connection (not recommended).
+ 
.It Ev SSL_ALLOW_SSL3
+ 
Allow SSL version 3 when negotiating the connection (not recommended).
+ 
.It Ev SSL_CA_CERT_FILE
+ 
CA certificate bundle containing trusted CA certificates.
+ 
Default value:
+ 
.Pa /etc/ssl/cert.pem .
+ 
.It Ev SSL_CA_CERT_PATH
+ 
Path containing trusted CA hashes.
+ 
.It Ev SSL_CLIENT_CERT_FILE
+ 
PEM encoded client certificate/key which will be used in
+ 
client certificate authentication.
+ 
.It Ev SSL_CLIENT_KEY_FILE
+ 
PEM encoded client key in case key and client certificate
+ 
are stored separately.
+ 
.It Ev SSL_CRL_FILE
+ 
File containing certificate revocation list.
+ 
.It Ev SSL_NO_TLS1
+ 
Do not allow TLS version 1.0 when negotiating the connection.
+ 
.It Ev SSL_NO_TLS1_1
+ 
Do not allow TLS version 1.1 when negotiating the connection.
+ 
.It Ev SSL_NO_TLS1_2
+ 
Do not allow TLS version 1.2 when negotiating the connection.
+ 
.It Ev SSL_NO_VERIFY_HOSTNAME
+ 
If set, do not verify that the hostname matches the subject of the
+ 
certificate presented by the server.
+ 
.It Ev SSL_NO_VERIFY_PEER
+ 
If set, do not verify the peer certificate against trusted CAs.
+ 
.El
+ 
.Sh EXAMPLES
+ 
To access a proxy server on
+ 
.Pa proxy.example.com
+ 
port 8080, set the
+ 
.Ev HTTP_PROXY
+ 
environment variable in a manner similar to this:
+ 
.Pp
+ 
.Dl HTTP_PROXY=http://proxy.example.com:8080
+ 
.Pp
+ 
If the proxy server requires authentication, there are
+ 
two options available for passing the authentication data.
+ 
The first method is by using the proxy URL:
+ 
.Pp
+ 
.Dl HTTP_PROXY=http://<user>:<pwd>@proxy.example.com:8080
+ 
.Pp
+ 
The second method is by using the
+ 
.Ev HTTP_PROXY_AUTH
+ 
environment variable:
+ 
.Bd -literal -offset indent
+ 
HTTP_PROXY=http://proxy.example.com:8080
+ 
HTTP_PROXY_AUTH=basic:*:<user>:<pwd>
+ 
.Ed
+ 
.Pp
+ 
To disable the use of a proxy for an HTTP server running on the local
+ 
host, define
+ 
.Ev NO_PROXY
+ 
as follows:
+ 
.Bd -literal -offset indent
+ 
NO_PROXY=localhost,127.0.0.1
+ 
.Ed
+ 
.Pp
+ 
Access HTTPS website without any certificate verification whatsoever:
+ 
.Bd -literal -offset indent
+ 
SSL_NO_VERIFY_PEER=1
+ 
SSL_NO_VERIFY_HOSTNAME=1
+ 
.Ed
+ 
.Pp
+ 
Access HTTPS website using client certificate based authentication
+ 
and a private CA:
+ 
.Bd -literal -offset indent
+ 
SSL_CLIENT_CERT_FILE=/path/to/client.pem
+ 
SSL_CA_CERT_FILE=/path/to/myca.pem
+ 
.Ed
+ 
.Sh SEE ALSO
+ 
.Xr fetch 1 ,
+ 
.Xr ftpio 3 ,
+ 
.Xr ip 4
+ 
.Rs
+ 
.%A J. Postel
+ 
.%A J. K. Reynolds
+ 
.%D October 1985
+ 
.%B File Transfer Protocol
+ 
.%O RFC959
+ 
.Re
+ 
.Rs
+ 
.%A P. Deutsch
+ 
.%A A. Emtage
+ 
.%A A. Marine.
+ 
.%D May 1994
+ 
.%T How to Use Anonymous FTP
+ 
.%O RFC1635
+ 
.Re
+ 
.Rs
+ 
.%A T. Berners-Lee
+ 
.%A L. Masinter
+ 
.%A M. McCahill
+ 
.%D December 1994
+ 
.%T Uniform Resource Locators (URL)
+ 
.%O RFC1738
+ 
.Re
+ 
.Rs
+ 
.%A R. Fielding
+ 
.%A J. Gettys
+ 
.%A J. Mogul
+ 
.%A H. Frystyk
+ 
.%A L. Masinter
+ 
.%A P. Leach
+ 
.%A T. Berners-Lee
+ 
.%D January 1999
+ 
.%B Hypertext Transfer Protocol -- HTTP/1.1
+ 
.%O RFC2616
+ 
.Re
+ 
.Rs
+ 
.%A J. Franks
+ 
.%A P. Hallam-Baker
+ 
.%A J. Hostetler
+ 
.%A S. Lawrence
+ 
.%A P. Leach
+ 
.%A A. Luotonen
+ 
.%A L. Stewart
+ 
.%D June 1999
+ 
.%B HTTP Authentication: Basic and Digest Access Authentication
+ 
.%O RFC2617
+ 
.Re
+ 
.Sh HISTORY
+ 
The
+ 
.Nm fetch
+ 
library first appeared in
+ 
.Fx 3.0 .
+ 
.Sh AUTHORS
+ 
.An -nosplit
+ 
The
+ 
.Nm fetch
+ 
library was mostly written by
+ 
.An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org
+ 
with numerous suggestions and contributions from
+ 
.An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org ,
+ 
.An Eugene Skepner Aq Mt eu@qub.com ,
+ 
.An Hajimu Umemoto Aq Mt ume@FreeBSD.org ,
+ 
.An Henry Whincup Aq Mt henry@techiebod.com ,
+ 
.An Jukka A. Ukkonen Aq Mt jau@iki.fi ,
+ 
.An Jean-Fran\(,cois Dockes Aq Mt jf@dockes.org ,
+ 
.An Michael Gmelin Aq Mt freebsd@grem.de
+ 
and others.
+ 
It replaces the older
+ 
.Nm ftpio
+ 
library written by
+ 
.An Poul-Henning Kamp Aq Mt phk@FreeBSD.org
+ 
and
+ 
.An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org .
+ 
.Pp
+ 
This manual page was written by
+ 
.An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org
+ 
and
+ 
.An Michael Gmelin Aq Mt freebsd@grem.de .
+ 
.Sh BUGS
+ 
Some parts of the library are not yet implemented.
+ 
The most notable
+ 
examples of this are
+ 
.Fn fetchPutHTTP ,
+ 
.Fn fetchListHTTP ,
+ 
.Fn fetchListFTP
+ 
and FTP proxy support.
+ 
.Pp
+ 
There is no way to select a proxy at run-time other than setting the
+ 
.Ev HTTP_PROXY
+ 
or
+ 
.Ev FTP_PROXY
+ 
environment variables as appropriate.
+ 
.Pp
+ 
.Nm libfetch
+ 
does not understand or obey 305 (Use Proxy) replies.
+ 
.Pp
+ 
Error numbers are unique only within a certain context; the error
+ 
codes used for FTP and HTTP overlap, as do those used for resolver and
+ 
system errors.
+ 
For instance, error code 202 means "Command not
+ 
implemented, superfluous at this site" in an FTP context and
+ 
"Accepted" in an HTTP context.
+ 
.Pp
+ 
.Fn fetchStatFTP
+ 
does not check that the result of an MDTM command is a valid date.
+ 
.Pp
+ 
In case password protected keys are used for client certificate based
+ 
authentication the user is prompted for the password on each and every
+ 
fetch operation.
+ 
.Pp
+ 
The man page is incomplete, poorly written and produces badly
+ 
formatted text.
+ 
.Pp
+ 
The error reporting mechanism is unsatisfactory.
+ 
.Pp
+ 
Some parts of the code are not fully reentrant.
added external/libfetch/fetch.c
@@ -0,0 +1,485 @@
+ 
/*-
+ 
 * SPDX-License-Identifier: BSD-3-Clause
+ 
 *
+ 
 * Copyright (c) 1998-2004 Dag-Erling Smørgrav
+ 
 * All rights reserved.
+ 
 *
+ 
 * Redistribution and use in source and binary forms, with or without
+ 
 * modification, are permitted provided that the following conditions
+ 
 * are met:
+ 
 * 1. Redistributions of source code must retain the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer
+ 
 *    in this position and unchanged.
+ 
 * 2. Redistributions in binary form must reproduce the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer in the
+ 
 *    documentation and/or other materials provided with the distribution.
+ 
 * 3. The name of the author may not be used to endorse or promote products
+ 
 *    derived from this software without specific prior written permission
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ 
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ 
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ 
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ 
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ 
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ 
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ 
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
 */
+
+ 
#include "bsd_compat.h"
+
+ 
#include <sys/param.h>
+
+ 
#include <netinet/in.h>
+
+ 
#include <errno.h>
+ 
#include <ctype.h>
+ 
#include <stdio.h>
+ 
#include <stdlib.h>
+ 
#include <string.h>
+
+ 
#include "fetch.h"
+ 
#include "common.h"
+
+ 
auth_t	 fetchAuthMethod;
+ 
int	 fetchLastErrCode;
+ 
char	 fetchLastErrString[MAXERRSTRING];
+ 
int	 fetchTimeout;
+ 
int	 fetchRestartCalls = 1;
+ 
int	 fetchDebug;
+ 
const char	*fetchCustomHTTPHeaders;
+ 
int	 fetchSpeedLimit;
+ 
int	 fetchSpeedTime;
+
+
+ 
/*** Local data **************************************************************/
+
+ 
/*
+ 
 * Error messages for parser errors
+ 
 */
+ 
#define URL_MALFORMED		1
+ 
#define URL_BAD_SCHEME		2
+ 
#define URL_BAD_PORT		3
+ 
static struct fetcherr url_errlist[] = {
+ 
	{ URL_MALFORMED,	FETCH_URL,	"Malformed URL" },
+ 
	{ URL_BAD_SCHEME,	FETCH_URL,	"Invalid URL scheme" },
+ 
	{ URL_BAD_PORT,		FETCH_URL,	"Invalid server port" },
+ 
	{ -1,			FETCH_UNKNOWN,	"Unknown parser error" }
+ 
};
+
+
+ 
/*** Public API **************************************************************/
+
+ 
/*
+ 
 * Select the appropriate protocol for the URL scheme, and return a
+ 
 * read-only stream connected to the document referenced by the URL.
+ 
 * Also fill out the struct url_stat.
+ 
 */
+ 
FILE *
+ 
fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
+ 
{
+
+ 
	if (us != NULL) {
+ 
		us->size = -1;
+ 
		us->atime = us->mtime = 0;
+ 
	}
+ 
	if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
+ 
		return (fetchXGetHTTP(URL, us, flags));
+ 
	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
+ 
		return (fetchXGetHTTP(URL, us, flags));
+ 
	url_seterr(URL_BAD_SCHEME);
+ 
	return (NULL);
+ 
}
+
+ 
/*
+ 
 * Select the appropriate protocol for the URL scheme, and return a
+ 
 * read-only stream connected to the document referenced by the URL.
+ 
 */
+ 
FILE *
+ 
fetchGet(struct url *URL, const char *flags)
+ 
{
+ 
	return (fetchXGet(URL, NULL, flags));
+ 
}
+
+ 
/*
+ 
 * Select the appropriate protocol for the URL scheme, and return a
+ 
 * write-only stream connected to the document referenced by the URL.
+ 
 */
+ 
FILE *
+ 
fetchPut(struct url *URL, const char *flags)
+ 
{
+
+ 
	if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
+ 
		return (fetchPutHTTP(URL, flags));
+ 
	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
+ 
		return (fetchPutHTTP(URL, flags));
+ 
	url_seterr(URL_BAD_SCHEME);
+ 
	return (NULL);
+ 
}
+
+ 
/*
+ 
 * Select the appropriate protocol for the URL scheme, and return the
+ 
 * size of the document referenced by the URL if it exists.
+ 
 */
+ 
int
+ 
fetchStat(struct url *URL, struct url_stat *us, const char *flags)
+ 
{
+
+ 
	if (us != NULL) {
+ 
		us->size = -1;
+ 
		us->atime = us->mtime = 0;
+ 
	}
+ 
	if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
+ 
		return (fetchStatHTTP(URL, us, flags));
+ 
	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
+ 
		return (fetchStatHTTP(URL, us, flags));
+ 
	url_seterr(URL_BAD_SCHEME);
+ 
	return (-1);
+ 
}
+
+ 
/*
+ 
 * Select the appropriate protocol for the URL scheme, and return a
+ 
 * list of files in the directory pointed to by the URL.
+ 
 */
+ 
struct url_ent *
+ 
fetchList(struct url *URL, const char *flags)
+ 
{
+
+ 
	if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
+ 
		return (fetchListHTTP(URL, flags));
+ 
	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
+ 
		return (fetchListHTTP(URL, flags));
+ 
	url_seterr(URL_BAD_SCHEME);
+ 
	return (NULL);
+ 
}
+
+ 
/*
+ 
 * Attempt to parse the given URL; if successful, call fetchXGet().
+ 
 */
+ 
FILE *
+ 
fetchXGetURL(const char *URL, struct url_stat *us, const char *flags)
+ 
{
+ 
	struct url *u;
+ 
	FILE *f;
+
+ 
	if ((u = fetchParseURL(URL)) == NULL)
+ 
		return (NULL);
+
+ 
	f = fetchXGet(u, us, flags);
+
+ 
	fetchFreeURL(u);
+ 
	return (f);
+ 
}
+
+ 
/*
+ 
 * Attempt to parse the given URL; if successful, call fetchGet().
+ 
 */
+ 
FILE *
+ 
fetchGetURL(const char *URL, const char *flags)
+ 
{
+ 
	return (fetchXGetURL(URL, NULL, flags));
+ 
}
+
+ 
/*
+ 
 * Attempt to parse the given URL; if successful, call fetchPut().
+ 
 */
+ 
FILE *
+ 
fetchPutURL(const char *URL, const char *flags)
+ 
{
+ 
	struct url *u;
+ 
	FILE *f;
+
+ 
	if ((u = fetchParseURL(URL)) == NULL)
+ 
		return (NULL);
+
+ 
	f = fetchPut(u, flags);
+
+ 
	fetchFreeURL(u);
+ 
	return (f);
+ 
}
+
+ 
/*
+ 
 * Attempt to parse the given URL; if successful, call fetchStat().
+ 
 */
+ 
int
+ 
fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
+ 
{
+ 
	struct url *u;
+ 
	int s;
+
+ 
	if ((u = fetchParseURL(URL)) == NULL)
+ 
		return (-1);
+
+ 
	s = fetchStat(u, us, flags);
+
+ 
	fetchFreeURL(u);
+ 
	return (s);
+ 
}
+
+ 
/*
+ 
 * Attempt to parse the given URL; if successful, call fetchList().
+ 
 */
+ 
struct url_ent *
+ 
fetchListURL(const char *URL, const char *flags)
+ 
{
+ 
	struct url *u;
+ 
	struct url_ent *ue;
+
+ 
	if ((u = fetchParseURL(URL)) == NULL)
+ 
		return (NULL);
+
+ 
	ue = fetchList(u, flags);
+
+ 
	fetchFreeURL(u);
+ 
	return (ue);
+ 
}
+
+ 
/*
+ 
 * Make a URL
+ 
 */
+ 
struct url *
+ 
fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
+ 
    const char *user, const char *pwd)
+ 
{
+ 
	struct url *u;
+
+ 
	if (!scheme || (!host && !doc)) {
+ 
		url_seterr(URL_MALFORMED);
+ 
		return (NULL);
+ 
	}
+
+ 
	if (port < 0 || port > 65535) {
+ 
		url_seterr(URL_BAD_PORT);
+ 
		return (NULL);
+ 
	}
+
+ 
	/* allocate struct url */
+ 
	if ((u = calloc(1, sizeof(*u))) == NULL) {
+ 
		fetch_syserr();
+ 
		return (NULL);
+ 
	}
+ 
	u->netrcfd = -1;
+
+ 
	if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
+ 
		fetch_syserr();
+ 
		free(u);
+ 
		return (NULL);
+ 
	}
+
+ 
#define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)
+ 
	seturl(scheme);
+ 
	seturl(host);
+ 
	seturl(user);
+ 
	seturl(pwd);
+ 
#undef seturl
+ 
	u->port = port;
+
+ 
	return (u);
+ 
}
+
+ 
/*
+ 
 * Return value of the given hex digit.
+ 
 */
+ 
static int
+ 
fetch_hexval(char ch)
+ 
{
+
+ 
	if (ch >= '0' && ch <= '9')
+ 
		return (ch - '0');
+ 
	else if (ch >= 'a' && ch <= 'f')
+ 
		return (ch - 'a' + 10);
+ 
	else if (ch >= 'A' && ch <= 'F')
+ 
		return (ch - 'A' + 10);
+ 
	return (-1);
+ 
}
+
+ 
/*
+ 
 * Decode percent-encoded URL component from src into dst, stopping at end
+ 
 * of string, or at @ or : separators.  Returns a pointer to the unhandled
+ 
 * part of the input string (null terminator, @, or :).  No terminator is
+ 
 * written to dst (it is the caller's responsibility).
+ 
 */
+ 
static const char *
+ 
fetch_pctdecode(char *dst, const char *src, size_t dlen)
+ 
{
+ 
	int d1, d2;
+ 
	char c;
+ 
	const char *s;
+
+ 
	for (s = src; *s != '\0' && *s != '@' && *s != ':'; s++) {
+ 
		if (s[0] == '%' && (d1 = fetch_hexval(s[1])) >= 0 &&
+ 
		    (d2 = fetch_hexval(s[2])) >= 0 && (d1 > 0 || d2 > 0)) {
+ 
			c = d1 << 4 | d2;
+ 
			s += 2;
+ 
		} else if (s[0] == '%') {
+ 
			/* Invalid escape sequence. */
+ 
			return (NULL);
+ 
		} else {
+ 
			c = *s;
+ 
		}
+ 
		if (dlen-- > 0)
+ 
			*dst++ = c;
+ 
		else
+ 
			return (NULL);
+ 
	}
+ 
	return (s);
+ 
}
+
+ 
/*
+ 
 * Split an URL into components. URL syntax is:
+ 
 * [method:/][/[user[:pwd]@]host[:port]/][document]
+ 
 * This almost, but not quite, RFC1738 URL syntax.
+ 
 */
+ 
struct url *
+ 
fetchParseURL(const char *URL)
+ 
{
+ 
	char *doc;
+ 
	const char *p, *q;
+ 
	struct url *u;
+ 
	int i, n;
+
+ 
	/* allocate struct url */
+ 
	if ((u = calloc(1, sizeof(*u))) == NULL) {
+ 
		fetch_syserr();
+ 
		return (NULL);
+ 
	}
+ 
	u->netrcfd = -1;
+
+ 
	/* scheme name */
+ 
	if ((p = strstr(URL, ":/"))) {
+ 
                if (p - URL > URL_SCHEMELEN)
+ 
                        goto ouch;
+ 
                for (i = 0; URL + i < p; i++)
+ 
                        u->scheme[i] = tolower((unsigned char)URL[i]);
+ 
		URL = ++p;
+ 
		/*
+ 
		 * Only one slash: no host, leave slash as part of document
+ 
		 * Two slashes: host follows, strip slashes
+ 
		 */
+ 
		if (URL[1] == '/')
+ 
			URL = (p += 2);
+ 
	} else {
+ 
		p = URL;
+ 
	}
+ 
	if (!*URL || *URL == '/' || *URL == '.' ||
+ 
	    (u->scheme[0] == '\0' &&
+ 
		strchr(URL, '/') == NULL && strchr(URL, ':') == NULL))
+ 
		goto nohost;
+
+ 
	p = strpbrk(URL, "/@");
+ 
	if (p && *p == '@') {
+ 
		/* username */
+ 
		q = fetch_pctdecode(u->user, URL, URL_USERLEN);
+ 
		if (q == NULL)
+ 
			goto ouch;
+
+ 
		/* password */
+ 
		if (*q == ':') {
+ 
			q = fetch_pctdecode(u->pwd, q + 1, URL_PWDLEN);
+ 
			if (q == NULL)
+ 
				goto ouch;
+ 
		}
+ 
		p++;
+ 
	} else {
+ 
		p = URL;
+ 
	}
+
+ 
	/* hostname */
+ 
	if (*p == '[') {
+ 
		q = p + 1 + strspn(p + 1, ":0123456789ABCDEFabcdef.");
+ 
		if (*q++ != ']')
+ 
			goto ouch;
+ 
	} else {
+ 
		/* valid characters in a DNS name */
+ 
		q = p + strspn(p, "-." "0123456789"
+ 
		    "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "_"
+ 
		    "abcdefghijklmnopqrstuvwxyz");
+ 
	}
+ 
	if ((*q != '\0' && *q != '/' && *q != ':') || q - p > MAXHOSTNAMELEN)
+ 
		goto ouch;
+ 
	for (i = 0; p + i < q; i++)
+ 
		u->host[i] = tolower((unsigned char)p[i]);
+ 
	u->host[i] = '\0';
+ 
	p = q;
+
+ 
	/* port */
+ 
	if (*p == ':') {
+ 
		for (n = 0, q = ++p; *q && (*q != '/'); q++) {
+ 
			if (*q >= '0' && *q <= '9' && n < INT_MAX / 10) {
+ 
				n = n * 10 + (*q - '0');
+ 
			} else {
+ 
				/* invalid port */
+ 
				url_seterr(URL_BAD_PORT);
+ 
				goto ouch;
+ 
			}
+ 
		}
+ 
#ifndef IPPORT_MAX
+ 
#define IPPORT_MAX 65535
+ 
#endif
+ 
		if (p != q && (n < 1 || n > IPPORT_MAX))
+ 
			goto ouch;
+ 
		u->port = n;
+ 
		p = q;
+ 
	}
+
+ 
nohost:
+ 
	/* document */
+ 
	if (!*p)
+ 
		p = "/";
+
+ 
	if (strcmp(u->scheme, SCHEME_HTTP) == 0 ||
+ 
	    strcmp(u->scheme, SCHEME_HTTPS) == 0) {
+ 
		const char hexnums[] = "0123456789abcdef";
+
+ 
		/* percent-escape whitespace. */
+ 
		if ((doc = malloc(strlen(p) * 3 + 1)) == NULL) {
+ 
			fetch_syserr();
+ 
			goto ouch;
+ 
		}
+ 
		u->doc = doc;
+ 
		/* fragments are reserved for client-side processing, see
+ 
		 * https://www.rfc-editor.org/rfc/rfc9110.html#section-7.1
+ 
		 */
+ 
		while (*p != '\0' && *p != '#') {
+ 
			if (!isspace((unsigned char)*p)) {
+ 
				*doc++ = *p++;
+ 
			} else {
+ 
				*doc++ = '%';
+ 
				*doc++ = hexnums[((unsigned int)*p) >> 4];
+ 
				*doc++ = hexnums[((unsigned int)*p) & 0xf];
+ 
				p++;
+ 
			}
+ 
		}
+ 
		*doc = '\0';
+ 
	} else if ((u->doc = strdup(p)) == NULL) {
+ 
		fetch_syserr();
+ 
		goto ouch;
+ 
	}
+
+ 
	DEBUGF("scheme:   \"%s\"\n"
+ 
	    "user:     \"%s\"\n"
+ 
	    "password: \"%s\"\n"
+ 
	    "host:     \"%s\"\n"
+ 
	    "port:     \"%d\"\n"
+ 
	    "document: \"%s\"\n",
+ 
	    u->scheme, u->user, u->pwd,
+ 
	    u->host, u->port, u->doc);
+
+ 
	return (u);
+
+ 
ouch:
+ 
	free(u);
+ 
	return (NULL);
+ 
}
+
+ 
/*
+ 
 * Free a URL
+ 
 */
+ 
void
+ 
fetchFreeURL(struct url *u)
+ 
{
+ 
	free(u->doc);
+ 
	free(u);
+ 
}
added external/libfetch/fetch.h
@@ -0,0 +1,159 @@
+ 
/*-
+ 
 * SPDX-License-Identifier: BSD-3-Clause
+ 
 *
+ 
 * Copyright (c) 1998-2004 Dag-Erling Smørgrav
+ 
 * All rights reserved.
+ 
 *
+ 
 * Redistribution and use in source and binary forms, with or without
+ 
 * modification, are permitted provided that the following conditions
+ 
 * are met:
+ 
 * 1. Redistributions of source code must retain the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer
+ 
 *    in this position and unchanged.
+ 
 * 2. Redistributions in binary form must reproduce the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer in the
+ 
 *    documentation and/or other materials provided with the distribution.
+ 
 * 3. The name of the author may not be used to endorse or promote products
+ 
 *    derived from this software without specific prior written permission
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ 
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ 
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ 
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ 
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ 
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ 
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ 
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
 */
+
+ 
#ifndef _FETCH_H_INCLUDED
+ 
#define _FETCH_H_INCLUDED
+
+ 
#define _LIBFETCH_VER "libfetch/2.0"
+
+ 
#define URL_SCHEMELEN 16
+ 
#define URL_USERLEN 256
+ 
#define URL_PWDLEN 256
+
+ 
struct url {
+ 
	char		 scheme[URL_SCHEMELEN+1];
+ 
	char		 user[URL_USERLEN+1];
+ 
	char		 pwd[URL_PWDLEN+1];
+ 
	char		 host[MAXHOSTNAMELEN+1];
+ 
	int		 port;
+ 
	char		*doc;
+ 
	off_t		 offset;
+ 
	size_t		 length;
+ 
	time_t		 ims_time;
+ 
	int		 netrcfd;
+ 
};
+
+ 
struct url_stat {
+ 
	off_t		 size;
+ 
	time_t		 atime;
+ 
	time_t		 mtime;
+ 
};
+
+ 
struct url_ent {
+ 
	char		 name[PATH_MAX];
+ 
	struct url_stat	 stat;
+ 
};
+
+ 
/* Recognized schemes */
+ 
#define SCHEME_FTP	"ftp"
+ 
#define SCHEME_HTTP	"http"
+ 
#define SCHEME_HTTPS	"https"
+ 
#define SCHEME_FILE	"file"
+
+ 
/* Error codes */
+ 
#define	FETCH_ABORT	 1
+ 
#define	FETCH_AUTH	 2
+ 
#define	FETCH_DOWN	 3
+ 
#define	FETCH_EXISTS	 4
+ 
#define	FETCH_FULL	 5
+ 
#define	FETCH_INFO	 6
+ 
#define	FETCH_MEMORY	 7
+ 
#define	FETCH_MOVED	 8
+ 
#define	FETCH_NETWORK	 9
+ 
#define	FETCH_OK	10
+ 
#define	FETCH_PROTO	11
+ 
#define	FETCH_RESOLV	12
+ 
#define	FETCH_SERVER	13
+ 
#define	FETCH_TEMP	14
+ 
#define	FETCH_TIMEOUT	15
+ 
#define	FETCH_UNAVAIL	16
+ 
#define	FETCH_UNKNOWN	17
+ 
#define	FETCH_URL	18
+ 
#define	FETCH_VERBOSE	19
+
+ 
#ifdef __cplusplus
+ 
extern "C" {
+ 
#endif
+
+ 
/* FILE-specific functions */
+ 
FILE		*fetchXGetFile(struct url *, struct url_stat *, const char *);
+ 
FILE		*fetchGetFile(struct url *, const char *);
+ 
FILE		*fetchPutFile(struct url *, const char *);
+ 
int		 fetchStatFile(struct url *, struct url_stat *, const char *);
+ 
struct url_ent	*fetchListFile(struct url *, const char *);
+
+ 
/* HTTP-specific functions */
+ 
FILE		*fetchXGetHTTP(struct url *, struct url_stat *, const char *);
+ 
FILE		*fetchGetHTTP(struct url *, const char *);
+ 
FILE		*fetchPutHTTP(struct url *, const char *);
+ 
int		 fetchStatHTTP(struct url *, struct url_stat *, const char *);
+ 
struct url_ent	*fetchListHTTP(struct url *, const char *);
+ 
FILE		*fetchReqHTTP(struct url *, const char *, const char *,
+ 
		    const char *, const char *);
+
+ 
/* Generic functions */
+ 
FILE		*fetchXGetURL(const char *, struct url_stat *, const char *);
+ 
FILE		*fetchGetURL(const char *, const char *);
+ 
FILE		*fetchPutURL(const char *, const char *);
+ 
int		 fetchStatURL(const char *, struct url_stat *, const char *);
+ 
struct url_ent	*fetchListURL(const char *, const char *);
+ 
FILE		*fetchXGet(struct url *, struct url_stat *, const char *);
+ 
FILE		*fetchGet(struct url *, const char *);
+ 
FILE		*fetchPut(struct url *, const char *);
+ 
int		 fetchStat(struct url *, struct url_stat *, const char *);
+ 
struct url_ent	*fetchList(struct url *, const char *);
+
+ 
/* URL parsing */
+ 
struct url	*fetchMakeURL(const char *, const char *, int,
+ 
		     const char *, const char *, const char *);
+ 
struct url	*fetchParseURL(const char *);
+ 
void		 fetchFreeURL(struct url *);
+
+ 
#ifdef __cplusplus
+ 
}
+ 
#endif
+
+ 
/* Authentication */
+ 
typedef int (*auth_t)(struct url *);
+ 
extern auth_t		 fetchAuthMethod;
+
+ 
/* Last error code */
+ 
extern int		 fetchLastErrCode;
+ 
#define MAXERRSTRING 256
+ 
extern char		 fetchLastErrString[MAXERRSTRING];
+
+ 
/* I/O timeout */
+ 
extern int		 fetchTimeout;
+
+ 
/* Restart interrupted syscalls */
+ 
extern int		 fetchRestartCalls;
+
+ 
/* Extra verbosity */
+ 
extern int		 fetchDebug;
+
+ 
/* Custom HTTP headers (newline-separated) */
+ 
extern const char	*fetchCustomHTTPHeaders;
+
+ 
/* Low speed limit: abort if speed drops below fetchSpeedLimit bytes/sec
+ 
 * for fetchSpeedTime consecutive seconds.  Both must be set to enable. */
+ 
extern int		 fetchSpeedLimit;
+ 
extern int		 fetchSpeedTime;
+
+ 
#endif
added external/libfetch/http.c
@@ -0,0 +1,2318 @@
+ 
/*-
+ 
 * SPDX-License-Identifier: BSD-3-Clause
+ 
 *
+ 
 * Copyright (c) 2000-2014 Dag-Erling Smørgrav
+ 
 * All rights reserved.
+ 
 *
+ 
 * Redistribution and use in source and binary forms, with or without
+ 
 * modification, are permitted provided that the following conditions
+ 
 * are met:
+ 
 * 1. Redistributions of source code must retain the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer
+ 
 *    in this position and unchanged.
+ 
 * 2. Redistributions in binary form must reproduce the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer in the
+ 
 *    documentation and/or other materials provided with the distribution.
+ 
 * 3. The name of the author may not be used to endorse or promote products
+ 
 *    derived from this software without specific prior written permission.
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ 
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ 
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ 
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ 
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ 
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ 
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ 
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
 */
+
+ 
#include "bsd_compat.h"
+
+ 
/*
+ 
 * The following copyright applies to the base64 code:
+ 
 *
+ 
 *-
+ 
 * Copyright 1997 Massachusetts Institute of Technology
+ 
 *
+ 
 * Permission to use, copy, modify, and distribute this software and
+ 
 * its documentation for any purpose and without fee is hereby
+ 
 * granted, provided that both the above copyright notice and this
+ 
 * permission notice appear in all copies, that both the above
+ 
 * copyright notice and this permission notice appear in all
+ 
 * supporting documentation, and that the name of M.I.T. not be used
+ 
 * in advertising or publicity pertaining to distribution of the
+ 
 * software without specific, written prior permission.  M.I.T. makes
+ 
 * no representations about the suitability of this software for any
+ 
 * purpose.  It is provided "as is" without express or implied
+ 
 * warranty.
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
+ 
 * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ 
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ 
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
+ 
 * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ 
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ 
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ 
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ 
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ 
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ 
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ 
 * SUCH DAMAGE.
+ 
 */
+
+ 
#include <sys/param.h>
+ 
#include <sys/socket.h>
+ 
#include <sys/time.h>
+
+ 
#include <ctype.h>
+ 
#include <err.h>
+ 
#include <errno.h>
+ 
#include <locale.h>
+ 
#include <netdb.h>
+ 
#include <stdarg.h>
+ 
#include <stdbool.h>
+ 
#include <stdio.h>
+ 
#include <stdlib.h>
+ 
#include <string.h>
+ 
#include <time.h>
+ 
#include <unistd.h>
+
+ 
#ifdef WITH_SSL
+ 
#include <openssl/md5.h>
+ 
#define MD5Init(c) MD5_Init(c)
+ 
#define MD5Update(c, data, len) MD5_Update(c, data, len)
+ 
#define MD5Final(md, c) MD5_Final(md, c)
+ 
#else
+ 
#include <md5.h>
+ 
#endif
+
+ 
#include <netinet/in.h>
+ 
#include <netinet/tcp.h>
+
+ 
#include "fetch.h"
+ 
#include "common.h"
+ 
#include "httperr.h"
+
+ 
/* Maximum number of redirects to follow */
+ 
#define MAX_REDIRECT 20
+
+ 
/* Persistent connection cache */
+ 
static conn_t *http_cached_conn;
+
+ 
static void
+ 
http_cache_flush(void)
+ 
{
+ 
	if (http_cached_conn != NULL) {
+ 
		fetch_close(http_cached_conn);
+ 
		http_cached_conn = NULL;
+ 
	}
+ 
}
+
+ 
/* Symbolic names for reply codes we care about */
+ 
#define HTTP_OK			200
+ 
#define HTTP_PARTIAL		206
+ 
#define HTTP_MOVED_PERM		301
+ 
#define HTTP_MOVED_TEMP		302
+ 
#define HTTP_SEE_OTHER		303
+ 
#define HTTP_NOT_MODIFIED	304
+ 
#define HTTP_USE_PROXY		305
+ 
#define HTTP_TEMP_REDIRECT	307
+ 
#define HTTP_PERM_REDIRECT	308
+ 
#define HTTP_NEED_AUTH		401
+ 
#define HTTP_NEED_PROXY_AUTH	407
+ 
#define HTTP_BAD_RANGE		416
+ 
#define HTTP_PROTOCOL_ERROR	999
+
+ 
#define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \
+ 
			    || (xyz) == HTTP_MOVED_TEMP \
+ 
			    || (xyz) == HTTP_TEMP_REDIRECT \
+ 
			    || (xyz) == HTTP_PERM_REDIRECT \
+ 
			    || (xyz) == HTTP_USE_PROXY \
+ 
			    || (xyz) == HTTP_SEE_OTHER)
+
+ 
#define HTTP_ERROR(xyz) ((xyz) >= 400 && (xyz) <= 599)
+
+
+ 
/*****************************************************************************
+ 
 * I/O functions for decoding chunked streams
+ 
 */
+
+ 
struct httpio
+ 
{
+ 
	conn_t		*conn;		/* connection */
+ 
	int		 chunked;	/* chunked mode */
+ 
	int		 keep_alive;	/* eligible for connection reuse */
+ 
	char		*buf;		/* chunk buffer */
+ 
	size_t		 bufsize;	/* size of chunk buffer */
+ 
	size_t		 buflen;	/* amount of data currently in buffer */
+ 
	size_t		 bufpos;	/* current read offset in buffer */
+ 
	int		 eof;		/* end-of-file flag */
+ 
	int		 error;		/* error flag */
+ 
	size_t		 chunksize;	/* remaining size of current chunk */
+ 
	off_t		 contentlength;	/* Content-Length or -1 if unknown */
+ 
	off_t		 bytes_read;	/* total bytes read from body */
+ 
#ifndef NDEBUG
+ 
	size_t		 total;
+ 
#endif
+ 
	/* low speed limit tracking */
+ 
	time_t		 lowspeed_start;  /* when slow period began (0=none) */
+ 
	size_t		 lowspeed_bytes;  /* bytes received since lowspeed_start */
+ 
};
+
+ 
/*
+ 
 * Get next chunk header
+ 
 */
+ 
static int
+ 
http_new_chunk(struct httpio *io)
+ 
{
+ 
	char *p;
+
+ 
	if (fetch_getln(io->conn) == -1)
+ 
		return (-1);
+
+ 
	if (io->conn->buflen < 2 || !isxdigit((unsigned char)*io->conn->buf))
+ 
		return (-1);
+
+ 
	for (p = io->conn->buf; *p && !isspace((unsigned char)*p); ++p) {
+ 
		if (*p == ';')
+ 
			break;
+ 
		if (!isxdigit((unsigned char)*p))
+ 
			return (-1);
+ 
		if (isdigit((unsigned char)*p)) {
+ 
			io->chunksize = io->chunksize * 16 +
+ 
			    *p - '0';
+ 
		} else {
+ 
			io->chunksize = io->chunksize * 16 +
+ 
			    10 + tolower((unsigned char)*p) - 'a';
+ 
		}
+ 
	}
+
+ 
#ifndef NDEBUG
+ 
	if (fetchDebug) {
+ 
		io->total += io->chunksize;
+ 
		if (io->chunksize == 0)
+ 
			fprintf(stderr, "%s(): end of last chunk\n", __func__);
+ 
		else
+ 
			fprintf(stderr, "%s(): new chunk: %lu (%lu)\n",
+ 
			    __func__, (unsigned long)io->chunksize,
+ 
			    (unsigned long)io->total);
+ 
	}
+ 
#endif
+
+ 
	return (io->chunksize);
+ 
}
+
+ 
/*
+ 
 * Grow the input buffer to at least len bytes
+ 
 */
+ 
static inline int
+ 
http_growbuf(struct httpio *io, size_t len)
+ 
{
+ 
	char *tmp;
+
+ 
	if (io->bufsize >= len)
+ 
		return (0);
+
+ 
	if ((tmp = realloc(io->buf, len)) == NULL)
+ 
		return (-1);
+ 
	io->buf = tmp;
+ 
	io->bufsize = len;
+ 
	return (0);
+ 
}
+
+ 
/*
+ 
 * Fill the input buffer, do chunk decoding on the fly
+ 
 */
+ 
static ssize_t
+ 
http_fillbuf(struct httpio *io, size_t len)
+ 
{
+ 
	ssize_t nbytes;
+ 
	char ch;
+
+ 
	if (io->error)
+ 
		return (-1);
+ 
	if (io->eof)
+ 
		return (0);
+
+ 
	/* not chunked: just fetch the requested amount */
+ 
	if (io->chunked == 0) {
+ 
		/* limit read to remaining content if Content-Length is known */
+ 
		if (io->contentlength >= 0) {
+ 
			off_t remaining = io->contentlength - io->bytes_read;
+ 
			if (remaining <= 0) {
+ 
				io->eof = 1;
+ 
				return (0);
+ 
			}
+ 
			if ((off_t)len > remaining)
+ 
				len = (size_t)remaining;
+ 
		}
+ 
		if (http_growbuf(io, len) == -1)
+ 
			return (-1);
+ 
		if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) {
+ 
			io->error = errno;
+ 
			return (-1);
+ 
		}
+ 
		if (nbytes == 0) {
+ 
			io->eof = 1;
+ 
			return (0);
+ 
		}
+ 
		io->bytes_read += nbytes;
+ 
		io->buflen = nbytes;
+ 
		io->bufpos = 0;
+ 
		return (io->buflen);
+ 
	}
+
+ 
	/* chunked, but we ran out: get the next chunk header */
+ 
	if (io->chunksize == 0) {
+ 
		switch (http_new_chunk(io)) {
+ 
		case -1:
+ 
			io->error = EPROTO;
+ 
			return (-1);
+ 
		case 0:
+ 
			io->eof = 1;
+ 
			return (0);
+ 
		}
+ 
	}
+
+ 
	/* fetch the requested amount, but no more than the current chunk */
+ 
	if (len > io->chunksize)
+ 
		len = io->chunksize;
+ 
	if (http_growbuf(io, len) == -1)
+ 
		return (-1);
+ 
	if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) {
+ 
		io->error = errno;
+ 
		return (-1);
+ 
	}
+ 
	io->bufpos = 0;
+ 
	io->buflen = nbytes;
+ 
	io->chunksize -= nbytes;
+
+ 
	if (io->chunksize == 0) {
+ 
		if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' ||
+ 
		    fetch_read(io->conn, &ch, 1) != 1 || ch != '\n')
+ 
			return (-1);
+ 
	}
+
+ 
	return (io->buflen);
+ 
}
+
+ 
/*
+ 
 * Read function
+ 
 */
+ 
static int
+ 
http_readfn(void *v, char *buf, int len)
+ 
{
+ 
	struct httpio *io = (struct httpio *)v;
+ 
	int rlen;
+
+ 
	if (io->error)
+ 
		return (-1);
+ 
	if (io->eof)
+ 
		return (0);
+
+ 
	/* empty buffer */
+ 
	if (!io->buf || io->bufpos == io->buflen) {
+ 
		if ((rlen = http_fillbuf(io, len)) < 0) {
+ 
			if ((errno = io->error) == EINTR)
+ 
				io->error = 0;
+ 
			return (-1);
+ 
		} else if (rlen == 0) {
+ 
			return (0);
+ 
		}
+ 
	}
+
+ 
	rlen = io->buflen - io->bufpos;
+ 
	if (len < rlen)
+ 
		rlen = len;
+ 
	memcpy(buf, io->buf + io->bufpos, rlen);
+ 
	io->bufpos += rlen;
+
+ 
	/* low speed limit check */
+ 
	if (fetchSpeedLimit > 0 && fetchSpeedTime > 0) {
+ 
		time_t now = time(NULL);
+ 
		if (io->lowspeed_start == 0) {
+ 
			io->lowspeed_start = now;
+ 
			io->lowspeed_bytes = rlen;
+ 
		} else {
+ 
			io->lowspeed_bytes += rlen;
+ 
			time_t elapsed = now - io->lowspeed_start;
+ 
			if (elapsed >= fetchSpeedTime) {
+ 
				if ((off_t)io->lowspeed_bytes / elapsed <
+ 
				    fetchSpeedLimit) {
+ 
					io->error = ETIMEDOUT;
+ 
					errno = ETIMEDOUT;
+ 
					fetch_syserr();
+ 
					return (-1);
+ 
				}
+ 
				/* reset window */
+ 
				io->lowspeed_start = now;
+ 
				io->lowspeed_bytes = 0;
+ 
			}
+ 
		}
+ 
	}
+
+ 
	return (rlen);
+ 
}
+
+ 
/*
+ 
 * Write function
+ 
 */
+ 
static int
+ 
http_writefn(void *v, const char *buf, int len)
+ 
{
+ 
	struct httpio *io = (struct httpio *)v;
+
+ 
	return (fetch_write(io->conn, buf, len));
+ 
}
+
+ 
/*
+ 
 * Drain any unread response body data
+ 
 */
+ 
static void
+ 
http_drain(struct httpio *io)
+ 
{
+ 
	char buf[4096];
+
+ 
	while (!io->eof && !io->error) {
+ 
		if (http_fillbuf(io, sizeof(buf)) <= 0)
+ 
			break;
+ 
		io->bufpos = io->buflen;
+ 
	}
+ 
}
+
+ 
/*
+ 
 * Close function — caches the connection for reuse when possible
+ 
 */
+ 
static int
+ 
http_closefn(void *v)
+ 
{
+ 
	struct httpio *io = (struct httpio *)v;
+ 
	int r;
+
+ 
	if (io->keep_alive && !io->error) {
+ 
		/* drain unread body so the connection is at a clean state */
+ 
		http_drain(io);
+ 
		if (!io->error) {
+ 
			/* connection is reusable — cache it */
+ 
			if (http_cached_conn != NULL)
+ 
				fetch_close(http_cached_conn);
+ 
			http_cached_conn = io->conn;
+ 
			/* clear leftover line buffer */
+ 
			io->conn->buflen = 0;
+ 
			DEBUGF("cached connection to %s:%d\n",
+ 
			    io->conn->cache_host, io->conn->cache_port);
+ 
			if (io->buf)
+ 
				free(io->buf);
+ 
			free(io);
+ 
			return (0);
+ 
		}
+ 
	}
+ 
	r = fetch_close(io->conn);
+ 
	if (io->buf)
+ 
		free(io->buf);
+ 
	free(io);
+ 
	return (r);
+ 
}
+
+ 
/*
+ 
 * Wrap a file descriptor up
+ 
 */
+ 
static FILE *
+ 
http_funopen(conn_t *conn, int chunked, int keep_alive, off_t clength)
+ 
{
+ 
	struct httpio *io;
+ 
	FILE *f;
+
+ 
	if ((io = calloc(1, sizeof(*io))) == NULL) {
+ 
		fetch_syserr();
+ 
		return (NULL);
+ 
	}
+ 
	io->conn = conn;
+ 
	io->chunked = chunked;
+ 
	io->keep_alive = keep_alive;
+ 
	io->contentlength = clength;
+ 
	io->bytes_read = 0;
+ 
	f = funopen(io, http_readfn, http_writefn, NULL, http_closefn);
+ 
	if (f == NULL) {
+ 
		fetch_syserr();
+ 
		free(io);
+ 
		return (NULL);
+ 
	}
+ 
	return (f);
+ 
}
+
+
+ 
/*****************************************************************************
+ 
 * Helper functions for talking to the server and parsing its replies
+ 
 */
+
+ 
/* Header types */
+ 
typedef enum {
+ 
	hdr_syserror = -2,
+ 
	hdr_error = -1,
+ 
	hdr_end = 0,
+ 
	hdr_unknown = 1,
+ 
	hdr_content_length,
+ 
	hdr_content_range,
+ 
	hdr_last_modified,
+ 
	hdr_location,
+ 
	hdr_transfer_encoding,
+ 
	hdr_www_authenticate,
+ 
	hdr_proxy_authenticate,
+ 
	hdr_connection,
+ 
} hdr_t;
+
+ 
/* Names of interesting headers */
+ 
static struct {
+ 
	hdr_t		 num;
+ 
	const char	*name;
+ 
} hdr_names[] = {
+ 
	{ hdr_content_length,		"Content-Length" },
+ 
	{ hdr_content_range,		"Content-Range" },
+ 
	{ hdr_last_modified,		"Last-Modified" },
+ 
	{ hdr_location,			"Location" },
+ 
	{ hdr_transfer_encoding,	"Transfer-Encoding" },
+ 
	{ hdr_www_authenticate,		"WWW-Authenticate" },
+ 
	{ hdr_proxy_authenticate,	"Proxy-Authenticate" },
+ 
	{ hdr_connection,		"Connection" },
+ 
	{ hdr_unknown,			NULL },
+ 
};
+
+ 
/*
+ 
 * Send a formatted line; optionally echo to terminal
+ 
 */
+ 
static int
+ 
http_cmd(conn_t *conn, const char *fmt, ...)
+ 
{
+ 
	va_list ap;
+ 
	size_t len;
+ 
	char *msg;
+ 
	int r;
+
+ 
	va_start(ap, fmt);
+ 
	len = vasprintf(&msg, fmt, ap);
+ 
	va_end(ap);
+
+ 
	if (msg == NULL) {
+ 
		errno = ENOMEM;
+ 
		fetch_syserr();
+ 
		return (-1);
+ 
	}
+
+ 
	r = fetch_putln(conn, msg, len);
+ 
	free(msg);
+
+ 
	if (r == -1) {
+ 
		fetch_syserr();
+ 
		return (-1);
+ 
	}
+
+ 
	return (0);
+ 
}
+
+ 
/*
+ 
 * Get and parse status line
+ 
 */
+ 
static int
+ 
http_get_reply(conn_t *conn)
+ 
{
+ 
	char *p;
+
+ 
	if (fetch_getln(conn) == -1)
+ 
		return (-1);
+ 
	/*
+ 
	 * A valid status line looks like "HTTP/m.n xyz reason" where m
+ 
	 * and n are the major and minor protocol version numbers and xyz
+ 
	 * is the reply code.
+ 
	 * Unfortunately, there are servers out there (NCSA 1.5.1, to name
+ 
	 * just one) that do not send a version number, so we can't rely
+ 
	 * on finding one, but if we do, insist on it being 1.0 or 1.1.
+ 
	 * We don't care about the reason phrase.
+ 
	 */
+ 
	if (strncmp(conn->buf, "HTTP", 4) != 0)
+ 
		return (HTTP_PROTOCOL_ERROR);
+ 
	p = conn->buf + 4;
+ 
	if (*p == '/') {
+ 
		if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1'))
+ 
			return (HTTP_PROTOCOL_ERROR);
+ 
		p += 4;
+ 
	}
+ 
	if (*p != ' ' ||
+ 
	    !isdigit((unsigned char)p[1]) ||
+ 
	    !isdigit((unsigned char)p[2]) ||
+ 
	    !isdigit((unsigned char)p[3]))
+ 
		return (HTTP_PROTOCOL_ERROR);
+
+ 
	conn->err = (p[1] - '0') * 100 + (p[2] - '0') * 10 + (p[3] - '0');
+ 
	return (conn->err);
+ 
}
+
+ 
/*
+ 
 * Check a header; if the type matches the given string, return a pointer
+ 
 * to the beginning of the value.
+ 
 */
+ 
static const char *
+ 
http_match(const char *str, const char *hdr)
+ 
{
+ 
	while (*str && *hdr &&
+ 
	    tolower((unsigned char)*str++) == tolower((unsigned char)*hdr++))
+ 
		/* nothing */;
+ 
	if (*str || *hdr != ':')
+ 
		return (NULL);
+ 
	while (*hdr && isspace((unsigned char)*++hdr))
+ 
		/* nothing */;
+ 
	return (hdr);
+ 
}
+
+
+ 
/*
+ 
 * Get the next header and return the appropriate symbolic code.  We
+ 
 * need to read one line ahead for checking for a continuation line
+ 
 * belonging to the current header (continuation lines start with
+ 
 * white space).
+ 
 *
+ 
 * We get called with a fresh line already in the conn buffer, either
+ 
 * from the previous http_next_header() invocation, or, the first
+ 
 * time, from a fetch_getln() performed by our caller.
+ 
 *
+ 
 * This stops when we encounter an empty line (we dont read beyond the header
+ 
 * area).
+ 
 *
+ 
 * Note that the "headerbuf" is just a place to return the result. Its
+ 
 * contents are not used for the next call. This means that no cleanup
+ 
 * is needed when ie doing another connection, just call the cleanup when
+ 
 * fully done to deallocate memory.
+ 
 */
+
+ 
/* Limit the max number of continuation lines to some reasonable value */
+ 
#define HTTP_MAX_CONT_LINES 10
+
+ 
/* Place into which to build a header from one or several lines */
+ 
typedef struct {
+ 
	char	*buf;		/* buffer */
+ 
	size_t	 bufsize;	/* buffer size */
+ 
	size_t	 buflen;	/* length of buffer contents */
+ 
} http_headerbuf_t;
+
+ 
static void
+ 
init_http_headerbuf(http_headerbuf_t *buf)
+ 
{
+ 
	buf->buf = NULL;
+ 
	buf->bufsize = 0;
+ 
	buf->buflen = 0;
+ 
}
+
+ 
static void
+ 
clean_http_headerbuf(http_headerbuf_t *buf)
+ 
{
+ 
	if (buf->buf)
+ 
		free(buf->buf);
+ 
	init_http_headerbuf(buf);
+ 
}
+
+ 
/* Remove whitespace at the end of the buffer */
+ 
static void
+ 
http_conn_trimright(conn_t *conn)
+ 
{
+ 
	while (conn->buflen &&
+ 
	       isspace((unsigned char)conn->buf[conn->buflen - 1]))
+ 
		conn->buflen--;
+ 
	conn->buf[conn->buflen] = '\0';
+ 
}
+
+ 
static hdr_t
+ 
http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p)
+ 
{
+ 
	unsigned int i, len;
+
+ 
	/*
+ 
	 * Have to do the stripping here because of the first line. So
+ 
	 * it's done twice for the subsequent lines. No big deal
+ 
	 */
+ 
	http_conn_trimright(conn);
+ 
	if (conn->buflen == 0)
+ 
		return (hdr_end);
+
+ 
	/* Copy the line to the headerbuf */
+ 
	if (hbuf->bufsize < conn->buflen + 1) {
+ 
		if ((hbuf->buf = realloc(hbuf->buf, conn->buflen + 1)) == NULL)
+ 
			return (hdr_syserror);
+ 
		hbuf->bufsize = conn->buflen + 1;
+ 
	}
+ 
	strcpy(hbuf->buf, conn->buf);
+ 
	hbuf->buflen = conn->buflen;
+
+ 
	/*
+ 
	 * Fetch possible continuation lines. Stop at 1st non-continuation
+ 
	 * and leave it in the conn buffer
+ 
	 */
+ 
	for (i = 0; i < HTTP_MAX_CONT_LINES; i++) {
+ 
		if (fetch_getln(conn) == -1)
+ 
			return (hdr_syserror);
+
+ 
		/*
+ 
		 * Note: we carry on the idea from the previous version
+ 
		 * that a pure whitespace line is equivalent to an empty
+ 
		 * one (so it's not continuation and will be handled when
+ 
		 * we are called next)
+ 
		 */
+ 
		http_conn_trimright(conn);
+ 
		if (conn->buf[0] != ' ' && conn->buf[0] != "\t"[0])
+ 
			break;
+
+ 
		/* Got a continuation line. Concatenate to previous */
+ 
		len = hbuf->buflen + conn->buflen;
+ 
		if (hbuf->bufsize < len + 1) {
+ 
			len *= 2;
+ 
			if ((hbuf->buf = realloc(hbuf->buf, len + 1)) == NULL)
+ 
				return (hdr_syserror);
+ 
			hbuf->bufsize = len + 1;
+ 
		}
+ 
		strcpy(hbuf->buf + hbuf->buflen, conn->buf);
+ 
		hbuf->buflen += conn->buflen;
+ 
	}
+
+ 
	/*
+ 
	 * We could check for malformed headers but we don't really care.
+ 
	 * A valid header starts with a token immediately followed by a
+ 
	 * colon; a token is any sequence of non-control, non-whitespace
+ 
	 * characters except "()<>@,;:\\\"{}".
+ 
	 */
+ 
	for (i = 0; hdr_names[i].num != hdr_unknown; i++)
+ 
		if ((*p = http_match(hdr_names[i].name, hbuf->buf)) != NULL)
+ 
			return (hdr_names[i].num);
+
+ 
	return (hdr_unknown);
+ 
}
+
+ 
/**************************
+ 
 * [Proxy-]Authenticate header parsing
+ 
 */
+
+ 
/*
+ 
 * Read doublequote-delimited string into output buffer obuf (allocated
+ 
 * by caller, whose responsibility it is to ensure that it's big enough)
+ 
 * cp points to the first char after the initial '"'
+ 
 * Handles \ quoting
+ 
 * Returns pointer to the first char after the terminating double quote, or
+ 
 * NULL for error.
+ 
 */
+ 
static const char *
+ 
http_parse_headerstring(const char *cp, char *obuf)
+ 
{
+ 
	for (;;) {
+ 
		switch (*cp) {
+ 
		case 0: /* Unterminated string */
+ 
			*obuf = 0;
+ 
			return (NULL);
+ 
		case '"': /* Ending quote */
+ 
			*obuf = 0;
+ 
			return (++cp);
+ 
		case '\\':
+ 
			if (*++cp == 0) {
+ 
				*obuf = 0;
+ 
				return (NULL);
+ 
			}
+ 
			/* FALLTHROUGH */
+ 
		default:
+ 
			*obuf++ = *cp++;
+ 
		}
+ 
	}
+ 
}
+
+ 
/* Http auth challenge schemes */
+ 
typedef enum {HTTPAS_UNKNOWN, HTTPAS_BASIC,HTTPAS_DIGEST} http_auth_schemes_t;
+
+ 
/* Data holder for a Basic or Digest challenge. */
+ 
typedef struct {
+ 
	http_auth_schemes_t scheme;
+ 
	char	*realm;
+ 
	char	*qop;
+ 
	char	*nonce;
+ 
	char	*opaque;
+ 
	char	*algo;
+ 
	int	 stale;
+ 
	int	 nc; /* Nonce count */
+ 
} http_auth_challenge_t;
+
+ 
static void
+ 
init_http_auth_challenge(http_auth_challenge_t *b)
+ 
{
+ 
	b->scheme = HTTPAS_UNKNOWN;
+ 
	b->realm = b->qop = b->nonce = b->opaque = b->algo = NULL;
+ 
	b->stale = b->nc = 0;
+ 
}
+
+ 
static void
+ 
clean_http_auth_challenge(http_auth_challenge_t *b)
+ 
{
+ 
	if (b->realm)
+ 
		free(b->realm);
+ 
	if (b->qop)
+ 
		free(b->qop);
+ 
	if (b->nonce)
+ 
		free(b->nonce);
+ 
	if (b->opaque)
+ 
		free(b->opaque);
+ 
	if (b->algo)
+ 
		free(b->algo);
+ 
	init_http_auth_challenge(b);
+ 
}
+
+ 
/* Data holder for an array of challenges offered in an http response. */
+ 
#define MAX_CHALLENGES 10
+ 
typedef struct {
+ 
	http_auth_challenge_t *challenges[MAX_CHALLENGES];
+ 
	int	count; /* Number of parsed challenges in the array */
+ 
	int	valid; /* We did parse an authenticate header */
+ 
} http_auth_challenges_t;
+
+ 
static void
+ 
init_http_auth_challenges(http_auth_challenges_t *cs)
+ 
{
+ 
	int i;
+ 
	for (i = 0; i < MAX_CHALLENGES; i++)
+ 
		cs->challenges[i] = NULL;
+ 
	cs->count = cs->valid = 0;
+ 
}
+
+ 
static void
+ 
clean_http_auth_challenges(http_auth_challenges_t *cs)
+ 
{
+ 
	int i;
+ 
	/* We rely on non-zero pointers being allocated, not on the count */
+ 
	for (i = 0; i < MAX_CHALLENGES; i++) {
+ 
		if (cs->challenges[i] != NULL) {
+ 
			clean_http_auth_challenge(cs->challenges[i]);
+ 
			free(cs->challenges[i]);
+ 
		}
+ 
	}
+ 
	init_http_auth_challenges(cs);
+ 
}
+
+ 
/*
+ 
 * Enumeration for lexical elements. Separators will be returned as their own
+ 
 * ascii value
+ 
 */
+ 
typedef enum {HTTPHL_WORD=256, HTTPHL_STRING=257, HTTPHL_END=258,
+ 
	      HTTPHL_ERROR = 259} http_header_lex_t;
+
+ 
/*
+ 
 * Determine what kind of token comes next and return possible value
+ 
 * in buf, which is supposed to have been allocated big enough by
+ 
 * caller. Advance input pointer and return element type.
+ 
 */
+ 
static int
+ 
http_header_lex(const char **cpp, char *buf)
+ 
{
+ 
	size_t l;
+ 
	/* Eat initial whitespace */
+ 
	*cpp += strspn(*cpp, " \t");
+ 
	if (**cpp == 0)
+ 
		return (HTTPHL_END);
+
+ 
	/* Separator ? */
+ 
	if (**cpp == ',' || **cpp == '=')
+ 
		return (*((*cpp)++));
+
+ 
	/* String ? */
+ 
	if (**cpp == '"') {
+ 
		*cpp = http_parse_headerstring(++*cpp, buf);
+ 
		if (*cpp == NULL)
+ 
			return (HTTPHL_ERROR);
+ 
		return (HTTPHL_STRING);
+ 
	}
+
+ 
	/* Read other token, until separator or whitespace */
+ 
	l = strcspn(*cpp, " \t,=");
+ 
	memcpy(buf, *cpp, l);
+ 
	buf[l] = 0;
+ 
	*cpp += l;
+ 
	return (HTTPHL_WORD);
+ 
}
+
+ 
/*
+ 
 * Read challenges from http xxx-authenticate header and accumulate them
+ 
 * in the challenges list structure.
+ 
 *
+ 
 * Headers with multiple challenges are specified by rfc2617, but
+ 
 * servers (ie: squid) often send them in separate headers instead,
+ 
 * which in turn is forbidden by the http spec (multiple headers with
+ 
 * the same name are only allowed for pure comma-separated lists, see
+ 
 * rfc2616 sec 4.2).
+ 
 *
+ 
 * We support both approaches anyway
+ 
 */
+ 
static int
+ 
http_parse_authenticate(const char *cp, http_auth_challenges_t *cs)
+ 
{
+ 
	int ret = -1;
+ 
	http_header_lex_t lex;
+ 
	char *key = malloc(strlen(cp) + 1);
+ 
	char *value = malloc(strlen(cp) + 1);
+ 
	char *buf = malloc(strlen(cp) + 1);
+
+ 
	if (key == NULL || value == NULL || buf == NULL) {
+ 
		fetch_syserr();
+ 
		goto out;
+ 
	}
+
+ 
	/* In any case we've seen the header and we set the valid bit */
+ 
	cs->valid = 1;
+
+ 
	/* Need word first */
+ 
	lex = http_header_lex(&cp, key);
+ 
	if (lex != HTTPHL_WORD)
+ 
		goto out;
+
+ 
	/* Loop on challenges */
+ 
	for (; cs->count < MAX_CHALLENGES; cs->count++) {
+ 
		cs->challenges[cs->count] =
+ 
			malloc(sizeof(http_auth_challenge_t));
+ 
		if (cs->challenges[cs->count] == NULL) {
+ 
			fetch_syserr();
+ 
			goto out;
+ 
		}
+ 
		init_http_auth_challenge(cs->challenges[cs->count]);
+ 
		if (strcasecmp(key, "basic") == 0) {
+ 
			cs->challenges[cs->count]->scheme = HTTPAS_BASIC;
+ 
		} else if (strcasecmp(key, "digest") == 0) {
+ 
			cs->challenges[cs->count]->scheme = HTTPAS_DIGEST;
+ 
		} else {
+ 
			cs->challenges[cs->count]->scheme = HTTPAS_UNKNOWN;
+ 
			/*
+ 
			 * Continue parsing as basic or digest may
+ 
			 * follow, and the syntax is the same for
+ 
			 * all. We'll just ignore this one when
+ 
			 * looking at the list
+ 
			 */
+ 
		}
+
+ 
		/* Loop on attributes */
+ 
		for (;;) {
+ 
			/* Key */
+ 
			lex = http_header_lex(&cp, key);
+ 
			if (lex != HTTPHL_WORD)
+ 
				goto out;
+
+ 
			/* Equal sign */
+ 
			lex = http_header_lex(&cp, buf);
+ 
			if (lex != '=')
+ 
				goto out;
+
+ 
			/* Value */
+ 
			lex = http_header_lex(&cp, value);
+ 
			if (lex != HTTPHL_WORD && lex != HTTPHL_STRING)
+ 
				goto out;
+
+ 
			if (strcasecmp(key, "realm") == 0) {
+ 
				cs->challenges[cs->count]->realm =
+ 
				    strdup(value);
+ 
			} else if (strcasecmp(key, "qop") == 0) {
+ 
				cs->challenges[cs->count]->qop =
+ 
				    strdup(value);
+ 
			} else if (strcasecmp(key, "nonce") == 0) {
+ 
				cs->challenges[cs->count]->nonce =
+ 
				    strdup(value);
+ 
			} else if (strcasecmp(key, "opaque") == 0) {
+ 
				cs->challenges[cs->count]->opaque =
+ 
				    strdup(value);
+ 
			} else if (strcasecmp(key, "algorithm") == 0) {
+ 
				cs->challenges[cs->count]->algo =
+ 
				    strdup(value);
+ 
			} else if (strcasecmp(key, "stale") == 0) {
+ 
				cs->challenges[cs->count]->stale =
+ 
				    strcasecmp(value, "no");
+ 
			} else {
+ 
				/* ignore unknown attributes */
+ 
			}
+
+ 
			/* Comma or Next challenge or End */
+ 
			lex = http_header_lex(&cp, key);
+ 
			/*
+ 
			 * If we get a word here, this is the beginning of the
+ 
			 * next challenge. Break the attributes loop
+ 
			 */
+ 
			if (lex == HTTPHL_WORD)
+ 
				break;
+
+ 
			if (lex == HTTPHL_END) {
+ 
				/* End while looking for ',' is normal exit */
+ 
				cs->count++;
+ 
				ret = 0;
+ 
				goto out;
+ 
			}
+ 
			/* Anything else is an error */
+ 
			if (lex != ',')
+ 
				goto out;
+
+ 
		} /* End attributes loop */
+ 
	} /* End challenge loop */
+
+ 
	/*
+ 
	 * Challenges max count exceeded. This really can't happen
+ 
	 * with normal data, something's fishy -> error
+ 
	 */
+
+ 
out:
+ 
	if (key)
+ 
		free(key);
+ 
	if (value)
+ 
		free(value);
+ 
	if (buf)
+ 
		free(buf);
+ 
	return (ret);
+ 
}
+
+
+ 
/*
+ 
 * Parse a last-modified header
+ 
 */
+ 
static int
+ 
http_parse_mtime(const char *p, time_t *mtime)
+ 
{
+ 
	char locale[64], *r;
+ 
	struct tm tm;
+
+ 
	memset(&tm, 0, sizeof(tm));
+ 
	strlcpy(locale, setlocale(LC_TIME, NULL), sizeof(locale));
+ 
	setlocale(LC_TIME, "C");
+ 
	r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
+ 
	/*
+ 
	 * Some proxies use UTC in response, but it should still be
+ 
	 * parsed. RFC2616 states GMT and UTC are exactly equal for HTTP.
+ 
	 */
+ 
	if (r == NULL)
+ 
		r = strptime(p, "%a, %d %b %Y %H:%M:%S UTC", &tm);
+ 
	/* XXX should add support for date-2 and date-3 */
+ 
	setlocale(LC_TIME, locale);
+ 
	if (r == NULL)
+ 
		return (-1);
+ 
	DEBUGF("last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n",
+ 
	    tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+ 
	    tm.tm_hour, tm.tm_min, tm.tm_sec);
+ 
	*mtime = timegm(&tm);
+ 
	return (0);
+ 
}
+
+ 
/*
+ 
 * Parse a content-length header
+ 
 */
+ 
static int
+ 
http_parse_length(const char *p, off_t *length)
+ 
{
+ 
	off_t len;
+
+ 
	for (len = 0; *p && isdigit((unsigned char)*p); ++p)
+ 
		len = len * 10 + (*p - '0');
+ 
	if (*p)
+ 
		return (-1);
+ 
	DEBUGF("content length: [%lld]\n", (long long)len);
+ 
	*length = len;
+ 
	return (0);
+ 
}
+
+ 
/*
+ 
 * Parse a content-range header
+ 
 */
+ 
static int
+ 
http_parse_range(const char *p, off_t *offset, off_t *length, off_t *size)
+ 
{
+ 
	off_t first, last, len;
+
+ 
	if (strncasecmp(p, "bytes ", 6) != 0)
+ 
		return (-1);
+ 
	p += 6;
+ 
	if (*p == '*') {
+ 
		first = last = -1;
+ 
		++p;
+ 
	} else {
+ 
		for (first = 0; *p && isdigit((unsigned char)*p); ++p)
+ 
			first = first * 10 + *p - '0';
+ 
		if (*p != '-')
+ 
			return (-1);
+ 
		for (last = 0, ++p; *p && isdigit((unsigned char)*p); ++p)
+ 
			last = last * 10 + *p - '0';
+ 
	}
+ 
	if (first > last || *p != '/')
+ 
		return (-1);
+ 
	for (len = 0, ++p; *p && isdigit((unsigned char)*p); ++p)
+ 
		len = len * 10 + *p - '0';
+ 
	if (*p || len < last - first + 1)
+ 
		return (-1);
+ 
	if (first == -1) {
+ 
		DEBUGF("content range: [*/%lld]\n", (long long)len);
+ 
		*length = 0;
+ 
	} else {
+ 
		DEBUGF("content range: [%lld-%lld/%lld]\n",
+ 
		    (long long)first, (long long)last, (long long)len);
+ 
		*length = last - first + 1;
+ 
	}
+ 
	*offset = first;
+ 
	*size = len;
+ 
	return (0);
+ 
}
+
+
+ 
/*****************************************************************************
+ 
 * Helper functions for authorization
+ 
 */
+
+ 
/*
+ 
 * Base64 encoding
+ 
 */
+ 
static char *
+ 
http_base64(const char *src)
+ 
{
+ 
	static const char base64[] =
+ 
	    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ 
	    "abcdefghijklmnopqrstuvwxyz"
+ 
	    "0123456789+/";
+ 
	char *str, *dst;
+ 
	size_t l;
+ 
	int t;
+
+ 
	l = strlen(src);
+ 
	if ((str = malloc(((l + 2) / 3) * 4 + 1)) == NULL)
+ 
		return (NULL);
+ 
	dst = str;
+
+ 
	while (l >= 3) {
+ 
		t = (src[0] << 16) | (src[1] << 8) | src[2];
+ 
		dst[0] = base64[(t >> 18) & 0x3f];
+ 
		dst[1] = base64[(t >> 12) & 0x3f];
+ 
		dst[2] = base64[(t >> 6) & 0x3f];
+ 
		dst[3] = base64[(t >> 0) & 0x3f];
+ 
		src += 3; l -= 3;
+ 
		dst += 4;
+ 
	}
+
+ 
	switch (l) {
+ 
	case 2:
+ 
		t = (src[0] << 16) | (src[1] << 8);
+ 
		dst[0] = base64[(t >> 18) & 0x3f];
+ 
		dst[1] = base64[(t >> 12) & 0x3f];
+ 
		dst[2] = base64[(t >> 6) & 0x3f];
+ 
		dst[3] = '=';
+ 
		dst += 4;
+ 
		break;
+ 
	case 1:
+ 
		t = src[0] << 16;
+ 
		dst[0] = base64[(t >> 18) & 0x3f];
+ 
		dst[1] = base64[(t >> 12) & 0x3f];
+ 
		dst[2] = dst[3] = '=';
+ 
		dst += 4;
+ 
		break;
+ 
	case 0:
+ 
		break;
+ 
	}
+
+ 
	*dst = 0;
+ 
	return (str);
+ 
}
+
+
+ 
/*
+ 
 * Extract authorization parameters from environment value.
+ 
 * The value is like scheme:realm:user:pass
+ 
 */
+ 
typedef struct {
+ 
	char	*scheme;
+ 
	char	*realm;
+ 
	char	*user;
+ 
	char	*password;
+ 
} http_auth_params_t;
+
+ 
static void
+ 
init_http_auth_params(http_auth_params_t *s)
+ 
{
+ 
	s->scheme = s->realm = s->user = s->password = NULL;
+ 
}
+
+ 
static void
+ 
clean_http_auth_params(http_auth_params_t *s)
+ 
{
+ 
	if (s->scheme)
+ 
		free(s->scheme);
+ 
	if (s->realm)
+ 
		free(s->realm);
+ 
	if (s->user)
+ 
		free(s->user);
+ 
	if (s->password)
+ 
		free(s->password);
+ 
	init_http_auth_params(s);
+ 
}
+
+ 
static int
+ 
http_authfromenv(const char *p, http_auth_params_t *parms)
+ 
{
+ 
	int ret = -1;
+ 
	char *v, *ve;
+ 
	char *str = strdup(p);
+
+ 
	if (str == NULL) {
+ 
		fetch_syserr();
+ 
		return (-1);
+ 
	}
+ 
	v = str;
+
+ 
	if ((ve = strchr(v, ':')) == NULL)
+ 
		goto out;
+
+ 
	*ve = 0;
+ 
	if ((parms->scheme = strdup(v)) == NULL) {
+ 
		fetch_syserr();
+ 
		goto out;
+ 
	}
+ 
	v = ve + 1;
+
+ 
	if ((ve = strchr(v, ':')) == NULL)
+ 
		goto out;
+
+ 
	*ve = 0;
+ 
	if ((parms->realm = strdup(v)) == NULL) {
+ 
		fetch_syserr();
+ 
		goto out;
+ 
	}
+ 
	v = ve + 1;
+
+ 
	if ((ve = strchr(v, ':')) == NULL)
+ 
		goto out;
+
+ 
	*ve = 0;
+ 
	if ((parms->user = strdup(v)) == NULL) {
+ 
		fetch_syserr();
+ 
		goto out;
+ 
	}
+ 
	v = ve + 1;
+
+
+ 
	if ((parms->password = strdup(v)) == NULL) {
+ 
		fetch_syserr();
+ 
		goto out;
+ 
	}
+ 
	ret = 0;
+ 
out:
+ 
	if (ret == -1)
+ 
		clean_http_auth_params(parms);
+ 
	if (str)
+ 
		free(str);
+ 
	return (ret);
+ 
}
+
+
+ 
/*
+ 
 * Digest response: the code to compute the digest is taken from the
+ 
 * sample implementation in RFC2616
+ 
 */
+ 
#define IN const
+ 
#define OUT
+
+ 
#define HASHLEN 16
+ 
typedef char HASH[HASHLEN];
+ 
#define HASHHEXLEN 32
+ 
typedef char HASHHEX[HASHHEXLEN+1];
+
+ 
static const char *hexchars = "0123456789abcdef";
+ 
static void
+ 
CvtHex(IN HASH Bin, OUT HASHHEX Hex)
+ 
{
+ 
	unsigned short i;
+ 
	unsigned char j;
+
+ 
	for (i = 0; i < HASHLEN; i++) {
+ 
		j = (Bin[i] >> 4) & 0xf;
+ 
		Hex[i*2] = hexchars[j];
+ 
		j = Bin[i] & 0xf;
+ 
		Hex[i*2+1] = hexchars[j];
+ 
	}
+ 
	Hex[HASHHEXLEN] = '\0';
+ 
};
+
+ 
/* calculate H(A1) as per spec */
+ 
static void
+ 
DigestCalcHA1(
+ 
	IN char * pszAlg,
+ 
	IN char * pszUserName,
+ 
	IN char * pszRealm,
+ 
	IN char * pszPassword,
+ 
	IN char * pszNonce,
+ 
	IN char * pszCNonce,
+ 
	OUT HASHHEX SessionKey
+ 
	)
+ 
{
+ 
	MD5_CTX Md5Ctx;
+ 
	HASH HA1;
+
+ 
	MD5Init(&Md5Ctx);
+ 
	MD5Update(&Md5Ctx, pszUserName, strlen(pszUserName));
+ 
	MD5Update(&Md5Ctx, ":", 1);
+ 
	MD5Update(&Md5Ctx, pszRealm, strlen(pszRealm));
+ 
	MD5Update(&Md5Ctx, ":", 1);
+ 
	MD5Update(&Md5Ctx, pszPassword, strlen(pszPassword));
+ 
	MD5Final(HA1, &Md5Ctx);
+ 
	if (strcasecmp(pszAlg, "md5-sess") == 0) {
+
+ 
		MD5Init(&Md5Ctx);
+ 
		MD5Update(&Md5Ctx, HA1, HASHLEN);
+ 
		MD5Update(&Md5Ctx, ":", 1);
+ 
		MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce));
+ 
		MD5Update(&Md5Ctx, ":", 1);
+ 
		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
+ 
		MD5Final(HA1, &Md5Ctx);
+ 
	}
+ 
	CvtHex(HA1, SessionKey);
+ 
}
+
+ 
/* calculate request-digest/response-digest as per HTTP Digest spec */
+ 
static void
+ 
DigestCalcResponse(
+ 
	IN HASHHEX HA1,           /* H(A1) */
+ 
	IN char * pszNonce,       /* nonce from server */
+ 
	IN char * pszNonceCount,  /* 8 hex digits */
+ 
	IN char * pszCNonce,      /* client nonce */
+ 
	IN char * pszQop,         /* qop-value: "", "auth", "auth-int" */
+ 
	IN char * pszMethod,      /* method from the request */
+ 
	IN char * pszDigestUri,   /* requested URL */
+ 
	IN HASHHEX HEntity,       /* H(entity body) if qop="auth-int" */
+ 
	OUT HASHHEX Response      /* request-digest or response-digest */
+ 
	)
+ 
{
+ 
#if 0
+ 
	DEBUGF("Calc: HA1[%s] Nonce[%s] qop[%s] method[%s] URI[%s]\n",
+ 
	    HA1, pszNonce, pszQop, pszMethod, pszDigestUri);
+ 
#endif
+ 
	MD5_CTX Md5Ctx;
+ 
	HASH HA2;
+ 
	HASH RespHash;
+ 
	HASHHEX HA2Hex;
+
+ 
	// calculate H(A2)
+ 
	MD5Init(&Md5Ctx);
+ 
	MD5Update(&Md5Ctx, pszMethod, strlen(pszMethod));
+ 
	MD5Update(&Md5Ctx, ":", 1);
+ 
	MD5Update(&Md5Ctx, pszDigestUri, strlen(pszDigestUri));
+ 
	if (strcasecmp(pszQop, "auth-int") == 0) {
+ 
		MD5Update(&Md5Ctx, ":", 1);
+ 
		MD5Update(&Md5Ctx, HEntity, HASHHEXLEN);
+ 
	}
+ 
	MD5Final(HA2, &Md5Ctx);
+ 
	CvtHex(HA2, HA2Hex);
+
+ 
	// calculate response
+ 
	MD5Init(&Md5Ctx);
+ 
	MD5Update(&Md5Ctx, HA1, HASHHEXLEN);
+ 
	MD5Update(&Md5Ctx, ":", 1);
+ 
	MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce));
+ 
	MD5Update(&Md5Ctx, ":", 1);
+ 
	if (*pszQop) {
+ 
		MD5Update(&Md5Ctx, pszNonceCount, strlen(pszNonceCount));
+ 
		MD5Update(&Md5Ctx, ":", 1);
+ 
		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
+ 
		MD5Update(&Md5Ctx, ":", 1);
+ 
		MD5Update(&Md5Ctx, pszQop, strlen(pszQop));
+ 
		MD5Update(&Md5Ctx, ":", 1);
+ 
	}
+ 
	MD5Update(&Md5Ctx, HA2Hex, HASHHEXLEN);
+ 
	MD5Final(RespHash, &Md5Ctx);
+ 
	CvtHex(RespHash, Response);
+ 
}
+
+ 
/*
+ 
 * Generate/Send a Digest authorization header
+ 
 * This looks like: [Proxy-]Authorization: credentials
+ 
 *
+ 
 *  credentials      = "Digest" digest-response
+ 
 *  digest-response  = 1#( username | realm | nonce | digest-uri
+ 
 *                      | response | [ algorithm ] | [cnonce] |
+ 
 *                      [opaque] | [message-qop] |
+ 
 *                          [nonce-count]  | [auth-param] )
+ 
 *  username         = "username" "=" username-value
+ 
 *  username-value   = quoted-string
+ 
 *  digest-uri       = "uri" "=" digest-uri-value
+ 
 *  digest-uri-value = request-uri   ; As specified by HTTP/1.1
+ 
 *  message-qop      = "qop" "=" qop-value
+ 
 *  cnonce           = "cnonce" "=" cnonce-value
+ 
 *  cnonce-value     = nonce-value
+ 
 *  nonce-count      = "nc" "=" nc-value
+ 
 *  nc-value         = 8LHEX
+ 
 *  response         = "response" "=" request-digest
+ 
 *  request-digest = <"> 32LHEX <">
+ 
 */
+ 
static int
+ 
http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c,
+ 
		 http_auth_params_t *parms, struct url *url)
+ 
{
+ 
	int r;
+ 
	char noncecount[10];
+ 
	char cnonce[40];
+ 
	char *options = NULL;
+
+ 
	if (!c->realm || !c->nonce) {
+ 
		DEBUGF("realm/nonce not set in challenge\n");
+ 
		return(-1);
+ 
	}
+ 
	if (!c->algo)
+ 
		c->algo = strdup("");
+
+ 
	if (asprintf(&options, "%s%s%s%s",
+ 
	    *c->algo? ",algorithm=" : "", c->algo,
+ 
	    c->opaque? ",opaque=" : "", c->opaque?c->opaque:"") < 0)
+ 
		return (-1);
+
+ 
	if (!c->qop) {
+ 
		c->qop = strdup("");
+ 
		*noncecount = 0;
+ 
		*cnonce = 0;
+ 
	} else {
+ 
		c->nc++;
+ 
		sprintf(noncecount, "%08x", c->nc);
+ 
		/* We don't try very hard with the cnonce ... */
+ 
		sprintf(cnonce, "%x%lx", getpid(), (unsigned long)time(0));
+ 
	}
+
+ 
	HASHHEX HA1;
+ 
	DigestCalcHA1(c->algo, parms->user, c->realm,
+ 
		      parms->password, c->nonce, cnonce, HA1);
+ 
	DEBUGF("HA1: [%s]\n", HA1);
+ 
	HASHHEX digest, null;
+ 
	memset(null, 0, sizeof(null));
+ 
	DigestCalcResponse(HA1, c->nonce, noncecount, cnonce, c->qop,
+ 
			   "GET", url->doc, null, digest);
+
+ 
	if (c->qop[0]) {
+ 
		r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\","
+ 
			     "nonce=\"%s\",uri=\"%s\",response=\"%s\","
+ 
			     "qop=\"auth\", cnonce=\"%s\", nc=%s%s",
+ 
			     hdr, parms->user, c->realm,
+ 
			     c->nonce, url->doc, digest,
+ 
			     cnonce, noncecount, options);
+ 
	} else {
+ 
		r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\","
+ 
			     "nonce=\"%s\",uri=\"%s\",response=\"%s\"%s",
+ 
			     hdr, parms->user, c->realm,
+ 
			     c->nonce, url->doc, digest, options);
+ 
	}
+ 
	if (options)
+ 
		free(options);
+ 
	return (r);
+ 
}
+
+ 
/*
+ 
 * Encode username and password
+ 
 */
+ 
static int
+ 
http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd)
+ 
{
+ 
	char *upw, *auth;
+ 
	int r;
+
+ 
	DEBUGF("basic: usr: [%s]\n", usr);
+ 
	DEBUGF("basic: pwd: [%s]\n", pwd);
+ 
	if (asprintf(&upw, "%s:%s", usr, pwd) == -1)
+ 
		return (-1);
+ 
	auth = http_base64(upw);
+ 
	free(upw);
+ 
	if (auth == NULL)
+ 
		return (-1);
+ 
	r = http_cmd(conn, "%s: Basic %s", hdr, auth);
+ 
	free(auth);
+ 
	return (r);
+ 
}
+
+ 
/*
+ 
 * Chose the challenge to answer and call the appropriate routine to
+ 
 * produce the header.
+ 
 */
+ 
static int
+ 
http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
+ 
	       http_auth_params_t *parms, struct url *url)
+ 
{
+ 
	http_auth_challenge_t *digest = NULL;
+ 
	int i;
+
+ 
	/* If user or pass are null we're not happy */
+ 
	if (!parms->user || !parms->password) {
+ 
		DEBUGF("NULL usr or pass\n");
+ 
		return (-1);
+ 
	}
+
+ 
	/* Look for a Digest */
+ 
	for (i = 0; i < cs->count; i++) {
+ 
		if (cs->challenges[i]->scheme == HTTPAS_DIGEST)
+ 
			digest = cs->challenges[i];
+ 
	}
+
+ 
	/* Error if "Digest" was specified and there is no Digest challenge */
+ 
	if (!digest &&
+ 
	    (parms->scheme && strcasecmp(parms->scheme, "digest") == 0)) {
+ 
		DEBUGF("Digest auth in env, not supported by peer\n");
+ 
		return (-1);
+ 
	}
+ 
	/*
+ 
	 * If "basic" was specified in the environment, or there is no Digest
+ 
	 * challenge, do the basic thing. Don't need a challenge for this,
+ 
	 * so no need to check basic!=NULL
+ 
	 */
+ 
	if (!digest ||
+ 
	    (parms->scheme && strcasecmp(parms->scheme, "basic") == 0))
+ 
		return (http_basic_auth(conn,hdr,parms->user,parms->password));
+
+ 
	/* Else, prefer digest. We just checked that it's not NULL */
+ 
	return (http_digest_auth(conn, hdr, digest, parms, url));
+ 
}
+
+ 
/*
+ 
 * Send custom headers from a newline-separated string.
+ 
 * Each line should be a complete "Header-Name: value" pair.
+ 
 * Empty lines and lines without a colon are silently skipped.
+ 
 */
+ 
static void
+ 
http_custom_headers(conn_t *conn, const char *hdrs)
+ 
{
+ 
	const char *p, *eol;
+ 
	size_t len;
+
+ 
	for (p = hdrs; *p != '\0'; p = eol) {
+ 
		eol = strpbrk(p, "\r\n");
+ 
		if (eol == NULL)
+ 
			eol = p + strlen(p);
+ 
		len = eol - p;
+ 
		/* skip empty lines and lines without ':' */
+ 
		if (len > 0 && memchr(p, ':', len) != NULL)
+ 
			http_cmd(conn, "%.*s", (int)len, p);
+ 
		/* skip past \r\n or \n */
+ 
		if (*eol == '\r')
+ 
			eol++;
+ 
		if (*eol == '\n')
+ 
			eol++;
+ 
	}
+ 
}
+
+
+ 
/*****************************************************************************
+ 
 * Helper functions for connecting to a server or proxy
+ 
 */
+
+ 
/*
+ 
 * Connect to the correct HTTP server or proxy.
+ 
 */
+ 
static conn_t *
+ 
http_connect(struct url *URL, struct url *purl, const char *flags)
+ 
{
+ 
	struct url *curl;
+ 
	conn_t *conn;
+ 
	hdr_t h;
+ 
	http_headerbuf_t headerbuf;
+ 
	const char *p;
+ 
	int verbose;
+ 
	int af, val;
+ 
	int serrno;
+ 
	bool isproxyauth = false;
+ 
	http_auth_challenges_t proxy_challenges;
+
+ 
#ifdef INET6
+ 
	af = AF_UNSPEC;
+ 
#else
+ 
	af = AF_INET;
+ 
#endif
+
+ 
	verbose = CHECK_FLAG('v');
+ 
	if (CHECK_FLAG('4'))
+ 
		af = AF_INET;
+ 
#ifdef INET6
+ 
	else if (CHECK_FLAG('6'))
+ 
		af = AF_INET6;
+ 
#endif
+
+ 
	curl = (purl != NULL) ? purl : URL;
+
+ 
	/*
+ 
	 * Try to reuse a cached connection.  Only for direct (non-proxy)
+ 
	 * connections where host, port, and scheme match.
+ 
	 */
+ 
	if (purl == NULL && http_cached_conn != NULL) {
+ 
		int is_ssl = (strcmp(URL->scheme, SCHEME_HTTPS) == 0);
+ 
		if (strcmp(http_cached_conn->cache_host, URL->host) == 0 &&
+ 
		    http_cached_conn->cache_port == URL->port &&
+ 
		    http_cached_conn->cache_ssl == is_ssl) {
+ 
			conn = http_cached_conn;
+ 
			http_cached_conn = NULL;
+ 
			DEBUGF("reusing cached connection to %s:%d\n",
+ 
			    URL->host, URL->port);
+ 
			val = 1;
+ 
			setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH,
+ 
			    &val, sizeof(val));
+ 
			return (conn);
+ 
		}
+ 
		/* stale cache — different host */
+ 
		http_cache_flush();
+ 
	}
+
+ 
retry:
+ 
	if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL)
+ 
		/* fetch_connect() has already set an error code */
+ 
		return (NULL);
+
+ 
	/* record connection identity for later cache matching */
+ 
	strlcpy(conn->cache_host, URL->host, sizeof(conn->cache_host));
+ 
	conn->cache_port = URL->port;
+ 
	conn->cache_ssl = (strcmp(URL->scheme, SCHEME_HTTPS) == 0);
+ 
	init_http_headerbuf(&headerbuf);
+ 
	if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
+ 
		init_http_auth_challenges(&proxy_challenges);
+ 
		http_cmd(conn, "CONNECT %s:%d HTTP/1.1", URL->host, URL->port);
+ 
		http_cmd(conn, "Host: %s:%d", URL->host, URL->port);
+ 
		if (isproxyauth) {
+ 
			http_auth_params_t aparams;
+ 
			init_http_auth_params(&aparams);
+ 
			if (*purl->user || *purl->pwd) {
+ 
				aparams.user = strdup(purl->user);
+ 
				aparams.password = strdup(purl->pwd);
+ 
			} else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL &&
+ 
				    *p != '\0') {
+ 
				if (http_authfromenv(p, &aparams) < 0) {
+ 
					http_seterr(HTTP_NEED_PROXY_AUTH);
+ 
					fetch_syserr();
+ 
					goto ouch;
+ 
				}
+ 
			} else if (fetch_netrc_auth(purl) == 0) {
+ 
				aparams.user = strdup(purl->user);
+ 
				aparams.password = strdup(purl->pwd);
+ 
			} else {
+ 
				/*
+ 
				 * No auth information found in system - exiting
+ 
				 * with warning.
+ 
				 */
+ 
				warnx("Missing username and/or password set");
+ 
				fetch_syserr();
+ 
				goto ouch;
+ 
			}
+ 
			http_authorize(conn, "Proxy-Authorization",
+ 
			    &proxy_challenges, &aparams, purl);
+ 
			clean_http_auth_params(&aparams);
+ 
		}
+ 
		http_cmd(conn, "");
+ 
		/* Get reply from CONNECT Tunnel attempt */
+ 
		int httpreply = http_get_reply(conn);
+ 
		if (httpreply != HTTP_OK) {
+ 
			http_seterr(httpreply);
+ 
			/* If the error is a 407/HTTP_NEED_PROXY_AUTH */
+ 
			if (httpreply == HTTP_NEED_PROXY_AUTH &&
+ 
			    ! isproxyauth) {
+ 
				/* Try again with authentication. */
+ 
				clean_http_headerbuf(&headerbuf);
+ 
				fetch_close(conn);
+ 
				isproxyauth = true;
+ 
				goto retry;
+ 
			}
+ 
			goto ouch;
+ 
		}
+ 
		/* Read and discard the rest of the proxy response */
+ 
		if (fetch_getln(conn) < 0) {
+ 
			fetch_syserr();
+ 
			goto ouch;
+ 
		}
+ 
		do {
+ 
			switch ((h = http_next_header(conn, &headerbuf, &p))) {
+ 
			case hdr_syserror:
+ 
				fetch_syserr();
+ 
				goto ouch;
+ 
			case hdr_error:
+ 
				http_seterr(HTTP_PROTOCOL_ERROR);
+ 
				goto ouch;
+ 
			default:
+ 
				/* ignore */ ;
+ 
			}
+ 
		} while (h > hdr_end);
+ 
	}
+ 
	if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 &&
+ 
	    fetch_ssl(conn, URL, verbose) == -1) {
+ 
		/* grrr */
+ 
		errno = EAUTH;
+ 
		fetch_syserr();
+ 
		goto ouch;
+ 
	}
+
+ 
	val = 1;
+ 
	setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val));
+
+ 
	clean_http_headerbuf(&headerbuf);
+ 
	return (conn);
+ 
ouch:
+ 
	serrno = errno;
+ 
	clean_http_headerbuf(&headerbuf);
+ 
	fetch_close(conn);
+ 
	errno = serrno;
+ 
	return (NULL);
+ 
}
+
+ 
static struct url *
+ 
http_get_proxy(struct url * url, const char *flags)
+ 
{
+ 
	struct url *purl;
+ 
	char *p;
+
+ 
	if (flags != NULL && strchr(flags, 'd') != NULL)
+ 
		return (NULL);
+ 
	if (fetch_no_proxy_match(url->host))
+ 
		return (NULL);
+ 
	if (((p = getenv("HTTP_PROXY")) || (p = getenv("http_proxy"))) &&
+ 
	    *p && (purl = fetchParseURL(p))) {
+ 
		if (!*purl->scheme)
+ 
			strcpy(purl->scheme, SCHEME_HTTP);
+ 
		if (!purl->port)
+ 
			purl->port = fetch_default_proxy_port(purl->scheme);
+ 
		if (strcmp(purl->scheme, SCHEME_HTTP) == 0)
+ 
			return (purl);
+ 
		fetchFreeURL(purl);
+ 
	}
+ 
	return (NULL);
+ 
}
+
+ 
static void
+ 
http_print_html(FILE *out, FILE *in)
+ 
{
+ 
	ssize_t len = 0;
+ 
	size_t cap;
+ 
	char *line = NULL, *p, *q;
+ 
	int comment, tag;
+
+ 
	comment = tag = 0;
+ 
	while ((len = getline(&line, &cap, in)) >= 0) {
+ 
		while (len && isspace((unsigned char)line[len - 1]))
+ 
			--len;
+ 
		for (p = q = line; q < line + len; ++q) {
+ 
			if (comment && *q == '-') {
+ 
				if (q + 2 < line + len &&
+ 
				    strcmp(q, "-->") == 0) {
+ 
					tag = comment = 0;
+ 
					q += 2;
+ 
				}
+ 
			} else if (tag && !comment && *q == '>') {
+ 
				p = q + 1;
+ 
				tag = 0;
+ 
			} else if (!tag && *q == '<') {
+ 
				if (q > p)
+ 
					fwrite(p, q - p, 1, out);
+ 
				tag = 1;
+ 
				if (q + 3 < line + len &&
+ 
				    strcmp(q, "<!--") == 0) {
+ 
					comment = 1;
+ 
					q += 3;
+ 
				}
+ 
			}
+ 
		}
+ 
		if (!tag && q > p)
+ 
			fwrite(p, q - p, 1, out);
+ 
		fputc('\n', out);
+ 
	}
+
+ 
	free(line);
+ 
}
+
+
+ 
/*****************************************************************************
+ 
 * Core
+ 
 */
+
+ 
FILE *
+ 
http_request(struct url *URL, const char *op, struct url_stat *us,
+ 
	struct url *purl, const char *flags)
+ 
{
+
+ 
	return (http_request_body(URL, op, us, purl, flags, NULL, NULL));
+ 
}
+
+ 
/*
+ 
 * Send a request and process the reply
+ 
 *
+ 
 * XXX This function is way too long, the do..while loop should be split
+ 
 * XXX off into a separate function.
+ 
 */
+ 
FILE *
+ 
http_request_body(struct url *URL, const char *op, struct url_stat *us,
+ 
	struct url *purl, const char *flags, const char *content_type,
+ 
	const char *body)
+ 
{
+ 
	char timebuf[80];
+ 
	char hbuf[MAXHOSTNAMELEN + 7], *host;
+ 
	conn_t *conn;
+ 
	struct url *url, *new;
+ 
	int chunked, conn_close, direct, from_cache, ims, noredirect, verbose;
+ 
	int e, i, n, val;
+ 
	off_t offset, clength, length, size;
+ 
	time_t mtime;
+ 
	const char *p;
+ 
	FILE *f;
+ 
	hdr_t h;
+ 
	struct tm *timestruct;
+ 
	http_headerbuf_t headerbuf;
+ 
	http_auth_challenges_t server_challenges;
+ 
	http_auth_challenges_t proxy_challenges;
+ 
	size_t body_len;
+
+ 
	/* The following calls don't allocate anything */
+ 
	init_http_headerbuf(&headerbuf);
+ 
	init_http_auth_challenges(&server_challenges);
+ 
	init_http_auth_challenges(&proxy_challenges);
+
+ 
	direct = CHECK_FLAG('d');
+ 
	noredirect = CHECK_FLAG('A');
+ 
	verbose = CHECK_FLAG('v');
+ 
	ims = CHECK_FLAG('i');
+
+ 
	if (direct && purl) {
+ 
		fetchFreeURL(purl);
+ 
		purl = NULL;
+ 
	}
+
+ 
	/* try the provided URL first */
+ 
	url = URL;
+
+ 
	n = MAX_REDIRECT;
+ 
	i = 0;
+
+ 
	e = HTTP_PROTOCOL_ERROR;
+ 
	do {
+ 
		new = NULL;
+ 
		chunked = 0;
+ 
		conn_close = 0;
+ 
		offset = 0;
+ 
		clength = -1;
+ 
		length = -1;
+ 
		size = -1;
+ 
		mtime = 0;
+
+ 
		/* check port */
+ 
		if (!url->port)
+ 
			url->port = fetch_default_port(url->scheme);
+
+ 
		/* connect to server or proxy */
+ 
		from_cache = (http_cached_conn != NULL);
+ 
		if ((conn = http_connect(url, purl, flags)) == NULL)
+ 
			goto ouch;
+
+ 
		/* append port number only if necessary */
+ 
		host = url->host;
+ 
		if (url->port != fetch_default_port(url->scheme)) {
+ 
			snprintf(hbuf, sizeof(hbuf), "%s:%d", host, url->port);
+ 
			host = hbuf;
+ 
		}
+
+ 
		/* send request */
+ 
		if (verbose)
+ 
			fetch_info("requesting %s://%s%s",
+ 
			    url->scheme, host, url->doc);
+ 
		if (purl && strcmp(url->scheme, SCHEME_HTTPS) != 0) {
+ 
			http_cmd(conn, "%s %s://%s%s HTTP/1.1",
+ 
			    op, url->scheme, host, url->doc);
+ 
		} else {
+ 
			http_cmd(conn, "%s %s HTTP/1.1",
+ 
			    op, url->doc);
+ 
		}
+
+ 
		if (ims && url->ims_time) {
+ 
			timestruct = gmtime((time_t *)&url->ims_time);
+ 
			(void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT",
+ 
			    timestruct);
+ 
			if (verbose)
+ 
				fetch_info("If-Modified-Since: %s", timebuf);
+ 
			http_cmd(conn, "If-Modified-Since: %s", timebuf);
+ 
		}
+ 
		/* virtual host */
+ 
		http_cmd(conn, "Host: %s", host);
+
+ 
		/*
+ 
		 * Proxy authorization: we only send auth after we received
+ 
		 * a 407 error. We do not first try basic anyway (changed
+ 
		 * when support was added for digest-auth)
+ 
		 */
+ 
		if (purl && proxy_challenges.valid) {
+ 
			http_auth_params_t aparams;
+ 
			init_http_auth_params(&aparams);
+ 
			if (*purl->user || *purl->pwd) {
+ 
				aparams.user = strdup(purl->user);
+ 
				aparams.password = strdup(purl->pwd);
+ 
			} else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL &&
+ 
				   *p != '\0') {
+ 
				if (http_authfromenv(p, &aparams) < 0) {
+ 
					http_seterr(HTTP_NEED_PROXY_AUTH);
+ 
					goto ouch;
+ 
				}
+ 
			} else if (fetch_netrc_auth(purl) == 0) {
+ 
				aparams.user = strdup(purl->user);
+ 
				aparams.password = strdup(purl->pwd);
+ 
			}
+ 
			http_authorize(conn, "Proxy-Authorization",
+ 
				       &proxy_challenges, &aparams, url);
+ 
			clean_http_auth_params(&aparams);
+ 
		}
+
+ 
		/*
+ 
		 * Server authorization: we never send "a priori"
+ 
		 * Basic auth, which used to be done if user/pass were
+ 
		 * set in the url. This would be weird because we'd send the
+ 
		 * password in the clear even if Digest is finally to be
+ 
		 * used (it would have made more sense for the
+ 
		 * pre-digest version to do this when Basic was specified
+ 
		 * in the environment)
+ 
		 */
+ 
		if (server_challenges.valid) {
+ 
			http_auth_params_t aparams;
+ 
			init_http_auth_params(&aparams);
+ 
			if (*url->user || *url->pwd) {
+ 
				aparams.user = strdup(url->user);
+ 
				aparams.password = strdup(url->pwd);
+ 
			} else if ((p = getenv("HTTP_AUTH")) != NULL &&
+ 
				   *p != '\0') {
+ 
				if (http_authfromenv(p, &aparams) < 0) {
+ 
					http_seterr(HTTP_NEED_AUTH);
+ 
					goto ouch;
+ 
				}
+ 
			} else if (fetch_netrc_auth(url) == 0) {
+ 
				aparams.user = strdup(url->user);
+ 
				aparams.password = strdup(url->pwd);
+ 
			} else if (fetchAuthMethod &&
+ 
				   fetchAuthMethod(url) == 0) {
+ 
				aparams.user = strdup(url->user);
+ 
				aparams.password = strdup(url->pwd);
+ 
			} else {
+ 
				http_seterr(HTTP_NEED_AUTH);
+ 
				goto ouch;
+ 
			}
+ 
			http_authorize(conn, "Authorization",
+ 
				       &server_challenges, &aparams, url);
+ 
			clean_http_auth_params(&aparams);
+ 
		}
+
+ 
		/* other headers */
+ 
		if ((p = getenv("HTTP_ACCEPT")) != NULL) {
+ 
			if (*p != '\0')
+ 
				http_cmd(conn, "Accept: %s", p);
+ 
		} else {
+ 
			http_cmd(conn, "Accept: */*");
+ 
		}
+ 
		if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') {
+ 
			if (strcasecmp(p, "auto") == 0)
+ 
				http_cmd(conn, "Referer: %s://%s%s",
+ 
				    url->scheme, host, url->doc);
+ 
			else
+ 
				http_cmd(conn, "Referer: %s", p);
+ 
		}
+ 
		if ((p = getenv("HTTP_USER_AGENT")) != NULL) {
+ 
			/* no User-Agent if defined but empty */
+ 
			if  (*p != '\0')
+ 
				http_cmd(conn, "User-Agent: %s", p);
+ 
		} else {
+ 
			/* default User-Agent */
+ 
			http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER,
+ 
			    getprogname());
+ 
		}
+ 
		if (url->offset > 0)
+ 
			http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
+
+ 
		/* custom headers from the API and environment */
+ 
		if (fetchCustomHTTPHeaders != NULL)
+ 
			http_custom_headers(conn, fetchCustomHTTPHeaders);
+ 
		if ((p = getenv("HTTP_HEADERS")) != NULL && *p != '\0')
+ 
			http_custom_headers(conn, p);
+
+ 
		if (body) {
+ 
			body_len = strlen(body);
+ 
			http_cmd(conn, "Content-Length: %zu", body_len);
+ 
			if (content_type != NULL)
+ 
				http_cmd(conn, "Content-Type: %s", content_type);
+ 
		}
+
+ 
		http_cmd(conn, "");
+
+ 
		if (body)
+ 
			fetch_write(conn, body, body_len);
+
+ 
		/*
+ 
		 * Force the queued request to be dispatched.  Normally, one
+ 
		 * would do this with shutdown(2) but squid proxies can be
+ 
		 * configured to disallow such half-closed connections.  To
+ 
		 * be compatible with such configurations, fiddle with socket
+ 
		 * options to force the pending data to be written.
+ 
		 */
+ 
		val = 0;
+ 
		setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val,
+ 
			   sizeof(val));
+ 
		val = 1;
+ 
		setsockopt(conn->sd, IPPROTO_TCP, TCP_NODELAY, &val,
+ 
			   sizeof(val));
+
+ 
		/* get reply */
+ 
		switch (http_get_reply(conn)) {
+ 
		case HTTP_OK:
+ 
		case HTTP_PARTIAL:
+ 
		case HTTP_NOT_MODIFIED:
+ 
			/* fine */
+ 
			break;
+ 
		case HTTP_MOVED_PERM:
+ 
		case HTTP_MOVED_TEMP:
+ 
		case HTTP_TEMP_REDIRECT:
+ 
		case HTTP_PERM_REDIRECT:
+ 
		case HTTP_SEE_OTHER:
+ 
		case HTTP_USE_PROXY:
+ 
			/*
+ 
			 * Not so fine, but we still have to read the
+ 
			 * headers to get the new location.
+ 
			 */
+ 
			break;
+ 
		case HTTP_NEED_AUTH:
+ 
			if (server_challenges.valid) {
+ 
				/*
+ 
				 * We already sent out authorization code,
+ 
				 * so there's nothing more we can do.
+ 
				 */
+ 
				http_seterr(conn->err);
+ 
				goto ouch;
+ 
			}
+ 
			/* try again, but send the password this time */
+ 
			if (verbose)
+ 
				fetch_info("server requires authorization");
+ 
			break;
+ 
		case HTTP_NEED_PROXY_AUTH:
+ 
			if (proxy_challenges.valid) {
+ 
				/*
+ 
				 * We already sent our proxy
+ 
				 * authorization code, so there's
+ 
				 * nothing more we can do. */
+ 
				http_seterr(conn->err);
+ 
				goto ouch;
+ 
			}
+ 
			/* try again, but send the password this time */
+ 
			if (verbose)
+ 
				fetch_info("proxy requires authorization");
+ 
			break;
+ 
		case HTTP_BAD_RANGE:
+ 
			/*
+ 
			 * This can happen if we ask for 0 bytes because
+ 
			 * we already have the whole file.  Consider this
+ 
			 * a success for now, and check sizes later.
+ 
			 */
+ 
			break;
+ 
		case HTTP_PROTOCOL_ERROR:
+ 
			/* fall through */
+ 
		case -1:
+ 
			/*
+ 
			 * If we got this connection from the cache, the
+ 
			 * server may have closed it while idle.  Retry
+ 
			 * once with a fresh connection.
+ 
			 */
+ 
			if (from_cache) {
+ 
				if (verbose)
+ 
					fetch_info("stale cached connection, "
+ 
					    "retrying");
+ 
				fetch_close(conn);
+ 
				conn = NULL;
+ 
				from_cache = 0;
+ 
				continue;
+ 
			}
+ 
			fetch_syserr();
+ 
			goto ouch;
+ 
		default:
+ 
			http_seterr(conn->err);
+ 
			if (!verbose)
+ 
				goto ouch;
+ 
			/* fall through so we can get the full error message */
+ 
		}
+
+ 
		/* get headers. http_next_header expects one line readahead */
+ 
		if (fetch_getln(conn) == -1) {
+ 
			fetch_syserr();
+ 
			goto ouch;
+ 
		}
+ 
		do {
+ 
			switch ((h = http_next_header(conn, &headerbuf, &p))) {
+ 
			case hdr_syserror:
+ 
				fetch_syserr();
+ 
				goto ouch;
+ 
			case hdr_error:
+ 
				http_seterr(HTTP_PROTOCOL_ERROR);
+ 
				goto ouch;
+ 
			case hdr_content_length:
+ 
				http_parse_length(p, &clength);
+ 
				break;
+ 
			case hdr_content_range:
+ 
				http_parse_range(p, &offset, &length, &size);
+ 
				break;
+ 
			case hdr_last_modified:
+ 
				http_parse_mtime(p, &mtime);
+ 
				break;
+ 
			case hdr_location:
+ 
				if (!HTTP_REDIRECT(conn->err))
+ 
					break;
+ 
				/*
+ 
				 * if the A flag is set, we don't follow
+ 
				 * temporary redirects.
+ 
				 */
+ 
				if (noredirect &&
+ 
				    conn->err != HTTP_MOVED_PERM &&
+ 
				    conn->err != HTTP_PERM_REDIRECT &&
+ 
				    conn->err != HTTP_USE_PROXY) {
+ 
					n = 1;
+ 
					break;
+ 
				}
+ 
				if (new)
+ 
					free(new);
+ 
				if (verbose)
+ 
					fetch_info("%d redirect to %s",
+ 
					    conn->err, p);
+ 
				if (*p == '/')
+ 
					/* absolute path */
+ 
					new = fetchMakeURL(url->scheme, url->host,
+ 
					    url->port, p, url->user, url->pwd);
+ 
				else
+ 
					new = fetchParseURL(p);
+ 
				if (new == NULL) {
+ 
					/* XXX should set an error code */
+ 
					DEBUGF("failed to parse new URL\n");
+ 
					goto ouch;
+ 
				}
+
+ 
				/* Only copy credentials if the host matches */
+ 
				if (strcmp(new->host, url->host) == 0 &&
+ 
				    !*new->user && !*new->pwd) {
+ 
					strcpy(new->user, url->user);
+ 
					strcpy(new->pwd, url->pwd);
+ 
				}
+ 
				new->offset = url->offset;
+ 
				new->length = url->length;
+ 
				new->ims_time = url->ims_time;
+ 
				break;
+ 
			case hdr_transfer_encoding:
+ 
				/* XXX weak test*/
+ 
				chunked = (strcasecmp(p, "chunked") == 0);
+ 
				break;
+ 
			case hdr_www_authenticate:
+ 
				if (conn->err != HTTP_NEED_AUTH)
+ 
					break;
+ 
				if (http_parse_authenticate(p, &server_challenges) == 0)
+ 
					++n;
+ 
				break;
+ 
			case hdr_proxy_authenticate:
+ 
				if (conn->err != HTTP_NEED_PROXY_AUTH)
+ 
					break;
+ 
				if (http_parse_authenticate(p, &proxy_challenges) == 0)
+ 
					++n;
+ 
				break;
+ 
			case hdr_connection:
+ 
				if (strcasecmp(p, "close") == 0)
+ 
					conn_close = 1;
+ 
				break;
+ 
			case hdr_end:
+ 
				/* fall through */
+ 
			case hdr_unknown:
+ 
				/* ignore */
+ 
				break;
+ 
			}
+ 
		} while (h > hdr_end);
+
+ 
		/* we need to provide authentication */
+ 
		if (conn->err == HTTP_NEED_AUTH ||
+ 
		    conn->err == HTTP_NEED_PROXY_AUTH) {
+ 
			e = conn->err;
+ 
			if ((conn->err == HTTP_NEED_AUTH &&
+ 
			     !server_challenges.valid) ||
+ 
			    (conn->err == HTTP_NEED_PROXY_AUTH &&
+ 
			     !proxy_challenges.valid)) {
+ 
				/* 401/7 but no www/proxy-authenticate ?? */
+ 
				DEBUGF("%03d without auth header\n", conn->err);
+ 
				goto ouch;
+ 
			}
+ 
			fetch_close(conn);
+ 
			conn = NULL;
+ 
			continue;
+ 
		}
+
+ 
		/* requested range not satisfiable */
+ 
		if (conn->err == HTTP_BAD_RANGE) {
+ 
			if (url->offset > 0 && url->length == 0) {
+ 
				/* asked for 0 bytes; fake it */
+ 
				offset = url->offset;
+ 
				clength = -1;
+ 
				conn->err = HTTP_OK;
+ 
				break;
+ 
			} else {
+ 
				http_seterr(conn->err);
+ 
				goto ouch;
+ 
			}
+ 
		}
+
+ 
		/* we have a hit or an error */
+ 
		if (conn->err == HTTP_OK
+ 
		    || conn->err == HTTP_NOT_MODIFIED
+ 
		    || conn->err == HTTP_PARTIAL
+ 
		    || HTTP_ERROR(conn->err))
+ 
			break;
+
+ 
		/* all other cases: we got a redirect */
+ 
		e = conn->err;
+ 
		clean_http_auth_challenges(&server_challenges);
+ 
		fetch_close(conn);
+ 
		conn = NULL;
+ 
		if (!new) {
+ 
			DEBUGF("redirect with no new location\n");
+ 
			break;
+ 
		}
+ 
		if (url != URL)
+ 
			fetchFreeURL(url);
+ 
		url = new;
+ 
	} while (++i < n);
+
+ 
	/* we failed, or ran out of retries */
+ 
	if (conn == NULL) {
+ 
		http_seterr(e);
+ 
		goto ouch;
+ 
	}
+
+ 
	DEBUGF("offset %lld, length %lld, size %lld, clength %lld\n",
+ 
	    (long long)offset, (long long)length,
+ 
	    (long long)size, (long long)clength);
+
+ 
	if (conn->err == HTTP_NOT_MODIFIED) {
+ 
		http_seterr(HTTP_NOT_MODIFIED);
+ 
		return (NULL);
+ 
	}
+
+ 
	/* check for inconsistencies */
+ 
	if (clength != -1 && length != -1 && clength != length) {
+ 
		http_seterr(HTTP_PROTOCOL_ERROR);
+ 
		goto ouch;
+ 
	}
+ 
	if (clength == -1)
+ 
		clength = length;
+ 
	if (clength != -1)
+ 
		length = offset + clength;
+ 
	if (length != -1 && size != -1 && length != size) {
+ 
		http_seterr(HTTP_PROTOCOL_ERROR);
+ 
		goto ouch;
+ 
	}
+ 
	if (size == -1)
+ 
		size = length;
+
+ 
	/* fill in stats */
+ 
	if (us) {
+ 
		us->size = size;
+ 
		us->atime = us->mtime = mtime;
+ 
	}
+
+ 
	/* too far? */
+ 
	if (URL->offset > 0 && offset > URL->offset) {
+ 
		http_seterr(HTTP_PROTOCOL_ERROR);
+ 
		goto ouch;
+ 
	}
+
+ 
	/* report back real offset and size */
+ 
	URL->offset = offset;
+ 
	URL->length = clength;
+
+ 
	/* wrap it up in a FILE */
+ 
	if ((f = http_funopen(conn, chunked, !conn_close && purl == NULL, clength)) == NULL) {
+ 
		fetch_syserr();
+ 
		goto ouch;
+ 
	}
+
+ 
	if (url != URL)
+ 
		fetchFreeURL(url);
+ 
	if (purl)
+ 
		fetchFreeURL(purl);
+
+ 
	if (HTTP_ERROR(conn->err)) {
+ 
		http_print_html(stderr, f);
+ 
		fclose(f);
+ 
		f = NULL;
+ 
	}
+ 
	clean_http_headerbuf(&headerbuf);
+ 
	clean_http_auth_challenges(&server_challenges);
+ 
	clean_http_auth_challenges(&proxy_challenges);
+ 
	return (f);
+
+ 
ouch:
+ 
	if (url != URL)
+ 
		fetchFreeURL(url);
+ 
	if (purl)
+ 
		fetchFreeURL(purl);
+ 
	if (conn != NULL)
+ 
		fetch_close(conn);
+ 
	clean_http_headerbuf(&headerbuf);
+ 
	clean_http_auth_challenges(&server_challenges);
+ 
	clean_http_auth_challenges(&proxy_challenges);
+ 
	return (NULL);
+ 
}
+
+
+ 
/*****************************************************************************
+ 
 * Entry points
+ 
 */
+
+ 
/*
+ 
 * Retrieve and stat a file by HTTP
+ 
 */
+ 
FILE *
+ 
fetchXGetHTTP(struct url *URL, struct url_stat *us, const char *flags)
+ 
{
+ 
	return (http_request(URL, "GET", us, http_get_proxy(URL, flags), flags));
+ 
}
+
+ 
/*
+ 
 * Retrieve a file by HTTP
+ 
 */
+ 
FILE *
+ 
fetchGetHTTP(struct url *URL, const char *flags)
+ 
{
+ 
	return (fetchXGetHTTP(URL, NULL, flags));
+ 
}
+
+ 
/*
+ 
 * Store a file by HTTP
+ 
 */
+ 
FILE *
+ 
fetchPutHTTP(struct url *URL __unused, const char *flags __unused)
+ 
{
+ 
	warnx("fetchPutHTTP(): not implemented");
+ 
	return (NULL);
+ 
}
+
+ 
/*
+ 
 * Get an HTTP document's metadata
+ 
 */
+ 
int
+ 
fetchStatHTTP(struct url *URL, struct url_stat *us, const char *flags)
+ 
{
+ 
	FILE *f;
+
+ 
	f = http_request(URL, "HEAD", us, http_get_proxy(URL, flags), flags);
+ 
	if (f == NULL)
+ 
		return (-1);
+ 
	fclose(f);
+ 
	return (0);
+ 
}
+
+ 
/*
+ 
 * List a directory
+ 
 */
+ 
struct url_ent *
+ 
fetchListHTTP(struct url *url __unused, const char *flags __unused)
+ 
{
+ 
	warnx("fetchListHTTP(): not implemented");
+ 
	return (NULL);
+ 
}
+
+ 
/*
+ 
 * Arbitrary HTTP verb and content requests
+ 
 */
+ 
FILE *
+ 
fetchReqHTTP(struct url *URL, const char *method, const char *flags,
+ 
	const char *content_type, const char *body)
+ 
{
+
+ 
	return (http_request_body(URL, method, NULL, http_get_proxy(URL, flags),
+ 
	    flags, content_type, body));
+ 
}
added external/libfetch/http.errors
@@ -0,0 +1,45 @@
+ 
#
+ 
# This list is taken from RFC 2068.
+ 
#
+ 
100 OK		Continue
+ 
101 OK		Switching Protocols
+ 
200 OK		OK
+ 
201 OK		Created
+ 
202 OK		Accepted
+ 
203 INFO	Non-Authoritative Information
+ 
204 OK		No Content
+ 
205 OK		Reset Content
+ 
206 OK		Partial Content
+ 
300 MOVED	Multiple Choices
+ 
301 MOVED	Moved Permanently
+ 
302 MOVED	Moved Temporarily
+ 
303 MOVED	See Other
+ 
304 OK		Not Modified
+ 
305 INFO	Use Proxy
+ 
307 MOVED	Temporary Redirect
+ 
308 MOVED	Permanent Redirect
+ 
400 PROTO	Bad Request
+ 
401 AUTH	Unauthorized
+ 
402 AUTH	Payment Required
+ 
403 AUTH	Forbidden
+ 
404 UNAVAIL	Not Found
+ 
405 PROTO	Method Not Allowed
+ 
406 PROTO	Not Acceptable
+ 
407 AUTH	Proxy Authentication Required
+ 
408 TIMEOUT	Request Time-out
+ 
409 EXISTS	Conflict
+ 
410 UNAVAIL	Gone
+ 
411 PROTO	Length Required
+ 
412 SERVER	Precondition Failed
+ 
413 PROTO	Request Entity Too Large
+ 
414 PROTO	Request-URI Too Large
+ 
415 PROTO	Unsupported Media Type
+ 
416 UNAVAIL	Requested Range Not Satisfiable
+ 
417 SERVER	Expectation Failed
+ 
500 SERVER	Internal Server Error
+ 
501 PROTO	Not Implemented
+ 
502 SERVER	Bad Gateway
+ 
503 TEMP	Service Unavailable
+ 
504 TIMEOUT	Gateway Time-out
+ 
505 PROTO	HTTP Version not supported
+ 
999 PROTO	Protocol error
added external/libfetch/httperr.h
@@ -0,0 +1,45 @@
+ 
static struct fetcherr http_errlist[] = {
+ 
    { 100, FETCH_OK, "Continue" },
+ 
    { 101, FETCH_OK, "Switching Protocols" },
+ 
    { 200, FETCH_OK, "OK" },
+ 
    { 201, FETCH_OK, "Created" },
+ 
    { 202, FETCH_OK, "Accepted" },
+ 
    { 203, FETCH_INFO, "Non-Authoritative Information" },
+ 
    { 204, FETCH_OK, "No Content" },
+ 
    { 205, FETCH_OK, "Reset Content" },
+ 
    { 206, FETCH_OK, "Partial Content" },
+ 
    { 300, FETCH_MOVED, "Multiple Choices" },
+ 
    { 301, FETCH_MOVED, "Moved Permanently" },
+ 
    { 302, FETCH_MOVED, "Moved Temporarily" },
+ 
    { 303, FETCH_MOVED, "See Other" },
+ 
    { 304, FETCH_OK, "Not Modified" },
+ 
    { 305, FETCH_INFO, "Use Proxy" },
+ 
    { 307, FETCH_MOVED, "Temporary Redirect" },
+ 
    { 308, FETCH_MOVED, "Permanent Redirect" },
+ 
    { 400, FETCH_PROTO, "Bad Request" },
+ 
    { 401, FETCH_AUTH, "Unauthorized" },
+ 
    { 402, FETCH_AUTH, "Payment Required" },
+ 
    { 403, FETCH_AUTH, "Forbidden" },
+ 
    { 404, FETCH_UNAVAIL, "Not Found" },
+ 
    { 405, FETCH_PROTO, "Method Not Allowed" },
+ 
    { 406, FETCH_PROTO, "Not Acceptable" },
+ 
    { 407, FETCH_AUTH, "Proxy Authentication Required" },
+ 
    { 408, FETCH_TIMEOUT, "Request Time-out" },
+ 
    { 409, FETCH_EXISTS, "Conflict" },
+ 
    { 410, FETCH_UNAVAIL, "Gone" },
+ 
    { 411, FETCH_PROTO, "Length Required" },
+ 
    { 412, FETCH_SERVER, "Precondition Failed" },
+ 
    { 413, FETCH_PROTO, "Request Entity Too Large" },
+ 
    { 414, FETCH_PROTO, "Request-URI Too Large" },
+ 
    { 415, FETCH_PROTO, "Unsupported Media Type" },
+ 
    { 416, FETCH_UNAVAIL, "Requested Range Not Satisfiable" },
+ 
    { 417, FETCH_SERVER, "Expectation Failed" },
+ 
    { 500, FETCH_SERVER, "Internal Server Error" },
+ 
    { 501, FETCH_PROTO, "Not Implemented" },
+ 
    { 502, FETCH_SERVER, "Bad Gateway" },
+ 
    { 503, FETCH_TEMP, "Service Unavailable" },
+ 
    { 504, FETCH_TIMEOUT, "Gateway Time-out" },
+ 
    { 505, FETCH_PROTO, "HTTP Version not supported" },
+ 
    { 999, FETCH_PROTO, "Protocol error" },
+ 
    { -1, FETCH_UNKNOWN, "Unknown HTTP error" }
+ 
};
modified libpkg/Makefile.autosetup
@@ -57,6 +57,7 @@ SRCS= backup_lib.c \
	flags.c \

	fetch_ssh.c \

	fetch_libcurl.c \
+ 
	fetch_libfetch.c \

	fetch_file.c \

	triggers.c \

	pkghash.c \
@@ -69,12 +70,14 @@ LOCAL_CFLAGS= -I$(top_srcdir)/compat \
		-I$(top_srcdir)/external/uthash \

		-I$(top_srcdir)/external/picosat \

		-I$(top_srcdir)/external/yxml \
+ 
		-I$(top_srcdir)/external/uthash \

		-I$(top_srcdir)/external/include \

		-I$(top_srcdir)/external/libucl/include \

		-I$(top_srcdir)/external/lua/src \

		-I$(top_srcdir)/external/liblua/ \

		-I$(top_srcdir)/external/libder/libder \

		-I$(top_srcdir)/external/libecc/include \
+ 
		-I$(top_srcdir)/external/libfetch \

		-I$(top_srcdir)/libpkg/repo \

		-I$(top_builddir)/libpkg/repo \

		-I$(top_srcdir)/libpkg \
@@ -100,6 +103,7 @@ LOCAL_LDFLAGS= @waflags@ \
		-L$(top_builddir)/external/liblua -llua_pic \

		-L$(top_builddir)/external/libder -lder_pic \

		-L$(top_builddir)/external/libecc -lecc_pic \
+ 
		-L$(top_builddir)/external/libfetch -lfetch_pic \

		@nowaflags@ \

		-lm
@@ -114,6 +118,7 @@ STATIC_LIBS= @REPOS_STATIC_LIBS@ \
		$(top_builddir)/compat/libbsd_compat.a \

		$(top_builddir)/external/libder/libder.a \

		$(top_builddir)/external/libecc/libecc.a \
+ 
		$(top_builddir)/external/libfetch/libfetch.a \

		lib$(LIB).a



@if HAVE_LIBUTIL
added libpkg/fetch_libfetch.c
@@ -0,0 +1,294 @@
+ 
/*-
+ 
 * Copyright (c) 2020-2026 Baptiste Daroussin <bapt@FreeBSD.org>
+ 
 *
+ 
 * Redistribution and use in source and binary forms, with or without
+ 
 * modification, are permitted provided that the following conditions
+ 
 * are met:
+ 
 * 1. Redistributions of source code must retain the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer
+ 
 *    in this position and unchanged.
+ 
 * 2. Redistributions in binary form must reproduce the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer in the
+ 
 *    documentation and/or other materials provided with the distribution.
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ 
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ 
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ 
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ 
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ 
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ 
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ 
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
 */
+
+ 
#include <sys/param.h>
+ 
#include <sys/wait.h>
+ 
#include <sys/socket.h>
+ 
#include <sys/time.h>
+
+ 
#include <ctype.h>
+ 
#include <fcntl.h>
+ 
#include <errno.h>
+ 
#include <stdio.h>
+ 
#include <string.h>
+ 
#include <fetch.h>
+ 
#include <paths.h>
+ 
#include <poll.h>
+
+ 
#include <bsd_compat.h>
+
+ 
#include "pkg.h"
+ 
#include "private/event.h"
+ 
#include "private/pkg.h"
+ 
#include "private/fetch.h"
+ 
#include "private/utils.h"
+
+ 
struct http_mirror {
+ 
	struct url *url;
+ 
	bool reldoc;
+ 
	struct http_mirror *next;
+ 
};
+
+ 
static void
+ 
gethttpmirrors(struct pkg_repo *repo, const char *url, bool withdoc) {
+ 
	FILE *f;
+ 
	char *line = NULL, *walk;
+ 
	size_t linecap = 0;
+ 
	ssize_t linelen;
+ 
	struct http_mirror *m;
+ 
	struct url *u;
+
+ 
	if ((f = fetchGetURL(url, "")) == NULL)
+ 
		return;
+
+ 
	while ((linelen = getline(&line, &linecap, f)) > 0) {
+ 
		if (strncmp(line, "URL:", 4) == 0) {
+ 
			walk = line;
+ 
			/* trim '\n' */
+ 
			if (walk[linelen - 1] == '\n')
+ 
				walk[linelen - 1 ] = '\0';
+
+ 
			walk += 4;
+ 
			while (isspace(*walk)) {
+ 
				walk++;
+ 
			}
+ 
			if (*walk == '\0')
+ 
				continue;
+
+ 
			if ((u = fetchParseURL(walk)) != NULL) {
+ 
				m = xmalloc(sizeof(struct http_mirror));
+ 
				m->reldoc = withdoc;
+ 
				m->url = u;
+ 
				m->next = NULL;
+ 
				LL_APPEND(repo->http, m);
+ 
			}
+ 
		}
+ 
	}
+
+ 
	free(line);
+ 
	fclose(f);
+ 
}
+
+ 
int
+ 
libfetch_open(struct pkg_repo *repo, struct fetch_item *fi)
+ 
{
+ 
	struct url *u;
+ 
	struct url *repourl;
+ 
	int64_t max_retry, retry;
+ 
	int64_t fetch_timeout;
+ 
	char docpath[MAXPATHLEN];
+ 
	char zone[MAXHOSTNAMELEN + 24];
+ 
	char *doc, *reldoc, *opts;
+ 
	struct dns_srvinfo *srv_current = NULL;
+ 
	struct http_mirror *http_current = NULL;
+ 
	struct url_stat st;
+ 
	xstring *fetchOpts = NULL;
+
+ 
	max_retry = pkg_object_int(pkg_config_get("FETCH_RETRY"));
+ 
	fetch_timeout = pkg_object_int(pkg_config_get("FETCH_TIMEOUT"));
+
+ 
	fetchTimeout = (int)MIN(fetch_timeout, INT_MAX);
+ 
	if (fetch_timeout > 0) {
+ 
		fetchSpeedLimit = 2 * 1024;	/* 2KB/s, same as curl fetcher */
+ 
		fetchSpeedTime = (int)MIN(fetch_timeout, INT_MAX);
+ 
	}
+
+ 
	u = fetchParseURL(fi->url);
+ 
	if (u == NULL) {
+ 
		pkg_emit_error("%s: parse error", fi->url);
+ 
		return (EPKG_FATAL);
+ 
	}
+
+ 
	repourl = fetchParseURL(repo->url);
+ 
	if (repourl == NULL) {
+ 
		pkg_emit_error("%s: parse error", repo->url);
+ 
		fetchFreeURL(u);
+ 
		return (EPKG_FATAL);
+ 
	}
+ 
	retry = max_retry;
+ 
	doc = u->doc;
+ 
	reldoc = doc + strlen(repourl->doc);
+ 
	fetchFreeURL(repourl);
+
+ 
	u->ims_time = fi->mtime;
+ 
	if (fi->offset > 0)
+ 
		u->offset = fi->offset;
+
+ 
	/* HTTP authentication */
+ 
	const char *userpasswd = get_http_auth();
+ 
	if (userpasswd != NULL) {
+ 
		const char *colon = strchr(userpasswd, ':');
+ 
		if (colon != NULL) {
+ 
			size_t ulen = colon - userpasswd;
+ 
			if (ulen < sizeof(u->user))
+ 
				strlcpy(u->user, userpasswd, ulen + 1);
+ 
			strlcpy(u->pwd, colon + 1, sizeof(u->pwd));
+ 
		}
+ 
	}
+
+ 
	pkg_dbg(PKG_DBG_FETCH, 1, "libfetch> connecting");
+
+ 
	while (repo->fh == NULL) {
+ 
		if (repo->mirror_type == SRV &&
+ 
		    (strncmp(u->scheme, "http", 4) == 0)) {
+ 
			if (repo->srv == NULL) {
+ 
				snprintf(zone, sizeof(zone),
+ 
				    "_%s._tcp.%s", u->scheme, u->host);
+ 
				repo->srv = dns_getsrvinfo(zone);
+ 
			}
+ 
			srv_current = repo->srv;
+ 
		} else if (repo->mirror_type == HTTP &&
+ 
		    strncmp(u->scheme, "http", 4) == 0) {
+ 
			if (u->port == 0) {
+ 
				if (strcmp(u->scheme, "https") == 0)
+ 
					u->port = 443;
+ 
				else
+ 
					u->port = 80;
+ 
			}
+ 
			snprintf(zone, sizeof(zone),
+ 
			    "%s://%s:%d", u->scheme, u->host, u->port);
+ 
			if (repo->http == NULL)
+ 
				gethttpmirrors(repo, zone, false);
+ 
			if (repo->http == NULL)
+ 
				gethttpmirrors(repo, repo->url, true);
+ 
			http_current = repo->http;
+ 
		}
+ 
		if (repo->mirror_type == SRV && repo->srv != NULL) {
+ 
			strlcpy(u->host, srv_current->host, sizeof(u->host));
+ 
			u->port = srv_current->port;
+ 
		} else if (repo->mirror_type == HTTP &&
+ 
		    http_current != NULL) {
+ 
			strlcpy(u->scheme, http_current->url->scheme, sizeof(u->scheme));
+ 
			strlcpy(u->host, http_current->url->host, sizeof(u->host));
+ 
			snprintf(docpath, sizeof(docpath), "%s%s",
+ 
			    http_current->url->doc, http_current->reldoc ? reldoc : doc);
+ 
			u->doc = docpath;
+ 
			u->port = http_current->url->port;
+ 
		}
+ 
		fetchOpts = xstring_new();
+ 
		fputs("i", fetchOpts->fp);
+ 
		if (repo->ip == IPV4)
+ 
			fputs("4", fetchOpts->fp);
+ 
		else if (repo->ip == IPV6)
+ 
			fputs("6", fetchOpts->fp);
+
+ 
		if (ctx.debug_level >= 4)
+ 
			fputs("v", fetchOpts->fp);
+
+ 
		opts = xstring_get(fetchOpts);
+ 
		pkg_dbg(PKG_DBG_FETCH, 1,
+ 
		    "libfetch> fetching from: %s://%s%s%s%s with opts \"%s\"",
+ 
		    u->scheme,
+ 
		    u->user,
+ 
		    u->user[0] != '\0' ? "@" : "",
+ 
		    u->host,
+ 
		    u->doc,
+ 
		    opts);
+
+ 
		repo->fh = fetchXGet(u, &st, opts);
+ 
		if (repo->fh == NULL) {
+ 
			if (fetchLastErrCode == FETCH_OK) {
+ 
				fetchFreeURL(u);
+ 
				return (EPKG_UPTODATE);
+ 
			}
+ 
			if (fetchLastErrCode == FETCH_ABORT) {
+ 
				fetchFreeURL(u);
+ 
				return (EPKG_CANCEL);
+ 
			}
+ 
			if (fetchLastErrCode == FETCH_UNAVAIL) {
+ 
				if (!repo->silent)
+ 
					pkg_emit_error("%s://%s%s%s%s: %s",
+ 
					    u->scheme,
+ 
					    u->user,
+ 
					    u->user[0] != '\0' ? "@" : "",
+ 
					    u->host,
+ 
					    u->doc,
+ 
					    fetchLastErrString);
+ 
				fetchFreeURL(u);
+ 
				return (EPKG_ENOENT);
+ 
			}
+ 
			if (fetchLastErrCode == FETCH_NETWORK ||
+ 
			    fetchLastErrCode == FETCH_RESOLV ||
+ 
			    fetchLastErrCode == FETCH_DOWN) {
+ 
				pkg_emit_pkg_errno(EPKG_NONETWORK,
+ 
				    "libfetch_open", NULL);
+ 
			}
+ 
			--retry;
+ 
			if (retry <= 0) {
+ 
				if (!repo->silent)
+ 
					pkg_emit_error("%s://%s%s%s%s: %s",
+ 
					    u->scheme,
+ 
					    u->user,
+ 
					    u->user[0] != '\0' ? "@" : "",
+ 
					    u->host,
+ 
					    u->doc,
+ 
					    fetchLastErrString);
+ 
				fetchFreeURL(u);
+ 
				return (EPKG_FATAL);
+ 
			}
+ 
			if (repo->mirror_type == SRV && repo->srv != NULL) {
+ 
				srv_current = srv_current->next;
+ 
				if (srv_current == NULL)
+ 
					srv_current = repo->srv;
+ 
			} else if (repo->mirror_type == HTTP &&
+ 
			    http_current != NULL) {
+ 
				http_current = http_current->next;
+ 
				if (http_current == NULL)
+ 
					http_current = repo->http;
+ 
			}
+ 
		}
+ 
	}
+ 
	fi->size = st.size > 0 ? st.size : 0;
+ 
	fi->mtime = st.mtime;
+ 
	fetchFreeURL(u);
+ 
	return (EPKG_OK);
+ 
}
+
+ 
int
+ 
libfetch_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)
+ 
{
+ 
	int ret;
+
+ 
	ret = stdio_fetch(repo, dest, fi);
+
+ 
	if (ret == EPKG_OK && ferror(repo->fh)) {
+ 
		pkg_emit_error("%s: %s", fi->url, fetchLastErrString);
+ 
		return (EPKG_FATAL);
+ 
	}
+ 
	return (ret);
+ 
}
+
+ 
void
+ 
libfetch_cleanup(struct pkg_repo *repo)
+ 
{
+ 
	struct http_mirror *m, *tmp;
+
+ 
	LL_FOREACH_SAFE(repo->http, m, tmp) {
+ 
		fetchFreeURL(m->url);
+ 
		free(m);
+ 
	}
+ 
	repo->http = NULL;
+ 
	fh_close(repo);
+ 
}
modified libpkg/private/fetch.h
@@ -49,3 +49,6 @@ int stdio_fetch(struct pkg_repo *, int dest, struct fetch_item *);
int curl_open(struct pkg_repo *, struct fetch_item *);

int curl_fetch(struct pkg_repo *, int dest, struct fetch_item *);

void curl_cleanup(struct pkg_repo *);
+ 
int libfetch_open(struct pkg_repo *, struct fetch_item *);
+ 
int libfetch_fetch(struct pkg_repo *, int dest, struct fetch_item *);
+ 
void libfetch_cleanup(struct pkg_repo *);