c6c6c75 fetch: finish removing libfetch by handling timeout per fetcher — HardenedBSD-pkg

HardenedBSD-pkg

rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9

HardenedBSD Package Manager

fetch: finish removing libfetch by handling timeout per fetcher

Baptiste Daroussin committed 2 years ago

commit c6c6c75fc7a71f02c8249f3e7f4e5fabf6e8571e
parent 30bd39b

17 files changed +38 -5907

modified auto.def

@@ -353,7 +353,7 @@ make-template tests/frontend/test_environment.sh.in

 make-template Makefile.autosetup Makefile
 foreach dir [list external/blake2 external/picosat \
 	external/linenoise external/libfetch external/sqlite \
 	external/linenoise external/sqlite \
 	external compat libpkg libpkg/repo libpkg/repo/binary src \
 	external/libucl external/libelf external/libmachista tests docs \
 	external/liblua external/msgpuck external/yxml scripts external/libcurl] {

modified external/Makefile.autosetup

@@ -1,5 +1,5 @@

 include @builddir@/mk/defs.mk
 DIRS=	blake2 picosat linenoise libfetch sqlite libucl liblua msgpuck yxml libcurl
 DIRS=	blake2 picosat linenoise sqlite libucl liblua msgpuck yxml libcurl
 @if libmachista
 DIRS+=	libmachista
 @endif

deleted external/libfetch/Makefile.autosetup

@@ -1,17 +0,0 @@

 include @builddir@/mk/defs.mk
 LIB=	fetch
 SRCS=	common.c \
 	fetch.c \
 	http.c
 LOCAL_CFLAGS=	-I$(top_srcdir)/compat \
 		-I$(top_srcdir) \
 		-I$(top_builddir) \
 		-Wno-unused-parameter \
 		-Wno-pointer-sign \
 		-DWITH_SSL \
 		-DINET6
 VPATH=	$(top_srcdir)/external/libfetch
 include $(MK)/static-lib.mk

deleted external/libfetch/common.c

@@ -1,1819 +0,0 @@

 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 1998-2016 Dag-Erling Smørgrav
  * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "bsd_compat.h"
 __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/socket.h>
 #include <sys/time.h>
 #include <sys/uio.h>
 #include <netinet/in.h>
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>
 #include <netdb.h>
 #include <paths.h>
 #include <poll.h>
 #include <pwd.h>
 #include <stdarg.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 #ifdef WITH_SSL
 #include <openssl/x509v3.h>
 #endif
 #include "fetch.h"
 #include "common.h"
 #ifndef INFTIM
 #define INFTIM (-1)
 #endif
 /*** Local data **************************************************************/
 /*
  * Error messages for resolver errors
  */
 static struct fetcherr netdb_errlist[] = {
 #ifdef EAI_NODATA
 	{ EAI_NODATA,	FETCH_RESOLV,	"Host not found" },
 #endif
 	{ EAI_AGAIN,	FETCH_TEMP,	"Transient resolver failure" },
 	{ EAI_FAIL,	FETCH_RESOLV,	"Non-recoverable resolver failure" },
 	{ EAI_NONAME,	FETCH_RESOLV,	"No address record" },
 	{ -1,		FETCH_UNKNOWN,	"Unknown resolver error" }
 };
 /*
  * SOCKS5 error enumerations
  */
 enum SOCKS5_ERR {
 /* Protocol errors */
 	SOCKS5_ERR_SELECTION,
 	SOCKS5_ERR_READ_METHOD,
 	SOCKS5_ERR_VER5_ONLY,
 	SOCKS5_ERR_NOMETHODS,
 	SOCKS5_ERR_NOTIMPLEMENTED,
 	SOCKS5_ERR_HOSTNAME_SIZE,
 	SOCKS5_ERR_REQUEST,
 	SOCKS5_ERR_REPLY,
 	SOCKS5_ERR_NON_VER5_RESP,
 	SOCKS5_ERR_GENERAL,
 	SOCKS5_ERR_NOT_ALLOWED,
 	SOCKS5_ERR_NET_UNREACHABLE,
 	SOCKS5_ERR_HOST_UNREACHABLE,
 	SOCKS5_ERR_CONN_REFUSED,
 	SOCKS5_ERR_TTL_EXPIRED,
 	SOCKS5_ERR_COM_UNSUPPORTED,
 	SOCKS5_ERR_ADDR_UNSUPPORTED,
 	SOCKS5_ERR_UNSPECIFIED,
 /* Configuration errors */
 	SOCKS5_ERR_BAD_HOST,
 	SOCKS5_ERR_BAD_PROXY_FORMAT,
 	SOCKS5_ERR_BAD_PORT
 };
 /*
  * Error messages for SOCKS5 errors
  */
 static struct fetcherr socks5_errlist[] = {
 /* SOCKS5 protocol errors */
 	{ SOCKS5_ERR_SELECTION,		FETCH_ABORT,	"SOCKS5: Failed to send selection method" },
 	{ SOCKS5_ERR_READ_METHOD,	FETCH_ABORT,	"SOCKS5: Failed to read method" },
 	{ SOCKS5_ERR_VER5_ONLY,		FETCH_PROTO,	"SOCKS5: Only version 5 is implemented" },
 	{ SOCKS5_ERR_NOMETHODS,		FETCH_PROTO,	"SOCKS5: No acceptable methods" },
 	{ SOCKS5_ERR_NOTIMPLEMENTED,	FETCH_PROTO,	"SOCKS5: Method currently not implemented" },
 	{ SOCKS5_ERR_HOSTNAME_SIZE,	FETCH_PROTO,	"SOCKS5: Hostname size is above 256 bytes" },
 	{ SOCKS5_ERR_REQUEST,		FETCH_PROTO,	"SOCKS5: Failed to request" },
 	{ SOCKS5_ERR_REPLY,		FETCH_PROTO,	"SOCKS5: Failed to receive reply" },
 	{ SOCKS5_ERR_NON_VER5_RESP,	FETCH_PROTO,	"SOCKS5: Server responded with a non-version 5 response" },
 	{ SOCKS5_ERR_GENERAL,		FETCH_ABORT,	"SOCKS5: General server failure" },
 	{ SOCKS5_ERR_NOT_ALLOWED,	FETCH_AUTH,	"SOCKS5: Connection not allowed by ruleset" },
 	{ SOCKS5_ERR_NET_UNREACHABLE,	FETCH_NETWORK,	"SOCKS5: Network unreachable" },
 	{ SOCKS5_ERR_HOST_UNREACHABLE,	FETCH_ABORT,	"SOCKS5: Host unreachable" },
 	{ SOCKS5_ERR_CONN_REFUSED,	FETCH_ABORT,	"SOCKS5: Connection refused" },
 	{ SOCKS5_ERR_TTL_EXPIRED,	FETCH_TIMEOUT,	"SOCKS5: TTL expired" },
 	{ SOCKS5_ERR_COM_UNSUPPORTED,	FETCH_PROTO,	"SOCKS5: Command not supported" },
 	{ SOCKS5_ERR_ADDR_UNSUPPORTED,	FETCH_ABORT,	"SOCKS5: Address type not supported" },
 	{ SOCKS5_ERR_UNSPECIFIED,	FETCH_UNKNOWN,	"SOCKS5: Unspecified error" },
 /* Configuration error */
 	{ SOCKS5_ERR_BAD_HOST,		FETCH_ABORT,	"SOCKS5: Bad proxy host" },
 	{ SOCKS5_ERR_BAD_PROXY_FORMAT,	FETCH_ABORT,	"SOCKS5: Bad proxy format" },
 	{ SOCKS5_ERR_BAD_PORT,		FETCH_ABORT,	"SOCKS5: Bad port" }
 };
 /* End-of-Line */
 static const char ENDL[2] = "\r\n";
 /*** Error-reporting functions ***********************************************/
 /*
  * Map error code to string
  */
 static struct fetcherr *
 fetch_finderr(struct fetcherr *p, int e)
 {
 	while (p->num != -1 && p->num != e)
 		p++;
 	return (p);
 }
 /*
  * Set error code
  */
 void
 fetch_seterr(struct fetcherr *p, int e)
 {
 	p = fetch_finderr(p, e);
 	fetchLastErrCode = p->cat;
 	snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string);
 }
 /*
  * Set error code according to errno
  */
 void
 fetch_syserr(void)
 {
 	switch (errno) {
 	case 0:
 		fetchLastErrCode = FETCH_OK;
 		break;
 	case EPERM:
 	case EACCES:
 	case EROFS:
 	case EAUTH:
 	case ENEEDAUTH:
 		fetchLastErrCode = FETCH_AUTH;
 		break;
 	case ENOENT:
 	case EISDIR: /* XXX */
 		fetchLastErrCode = FETCH_UNAVAIL;
 		break;
 	case ENOMEM:
 		fetchLastErrCode = FETCH_MEMORY;
 		break;
 	case EBUSY:
 	case EAGAIN:
 		fetchLastErrCode = FETCH_TEMP;
 		break;
 	case EEXIST:
 		fetchLastErrCode = FETCH_EXISTS;
 		break;
 	case ENOSPC:
 		fetchLastErrCode = FETCH_FULL;
 		break;
 	case EADDRINUSE:
 	case EADDRNOTAVAIL:
 	case ENETDOWN:
 	case ENETUNREACH:
 	case ENETRESET:
 	case EHOSTUNREACH:
 		fetchLastErrCode = FETCH_NETWORK;
 		break;
 	case ECONNABORTED:
 	case ECONNRESET:
 		fetchLastErrCode = FETCH_ABORT;
 		break;
 	case ETIMEDOUT:
 		fetchLastErrCode = FETCH_TIMEOUT;
 		break;
 	case ECONNREFUSED:
 	case EHOSTDOWN:
 		fetchLastErrCode = FETCH_DOWN;
 		break;
 	default:
 		fetchLastErrCode = FETCH_UNKNOWN;
 	}
 	snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno));
 }
 /*
  * Emit status message
  */
 void
 fetch_info(const char *fmt, ...)
 {
 	va_list ap;
 	va_start(ap, fmt);
 	vfprintf(stderr, fmt, ap);
 	va_end(ap);
 	fputc('\n', stderr);
 }
 /*** Network-related utility functions ***************************************/
 /*
  * Return the default port for a scheme
  */
 int
 fetch_default_port(const char *scheme)
 {
 	struct servent *se;
 	if ((se = getservbyname(scheme, "tcp")) != NULL)
 		return (ntohs(se->s_port));
 	if (strcmp(scheme, SCHEME_FTP) == 0)
 		return (FTP_DEFAULT_PORT);
 	if (strcmp(scheme, SCHEME_HTTP) == 0)
 		return (HTTP_DEFAULT_PORT);
 	return (0);
 }
 /*
  * Return the default proxy port for a scheme
  */
 int
 fetch_default_proxy_port(const char *scheme)
 {
 	if (strcmp(scheme, SCHEME_FTP) == 0)
 		return (FTP_DEFAULT_PROXY_PORT);
 	if (strcmp(scheme, SCHEME_HTTP) == 0)
 		return (HTTP_DEFAULT_PROXY_PORT);
 	return (0);
 }
 /*
  * Create a connection for an existing descriptor.
  */
 conn_t *
 fetch_reopen(int sd)
 {
 	conn_t *conn;
 #ifdef SO_NOSIGPIPE
 	int opt = 1;
 #endif
 	/* allocate and fill connection structure */
 	if ((conn = calloc(1, sizeof(*conn))) == NULL)
 		return (NULL);
 	fcntl(sd, F_SETFD, FD_CLOEXEC);
 #ifdef SO_NOSIGPIPE
 	setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, &opt, sizeof opt);
 #endif
 	conn->sd = sd;
 	++conn->ref;
 	return (conn);
 }
 /*
  * Bump a connection's reference count.
  */
 conn_t *
 fetch_ref(conn_t *conn)
 {
 	++conn->ref;
 	return (conn);
 }
 /*
  * Resolve an address
  */
 struct addrinfo *
 fetch_resolve(const char *addr, int port, int af)
 {
 	char hbuf[256], sbuf[8];
 	struct addrinfo hints, *res;
 	const char *hb, *he, *sep;
 	const char *host, *service;
 	int err, len;
 	/* first, check for a bracketed IPv6 address */
 	if (*addr == '[') {
 		hb = addr + 1;
 		if ((sep = strchr(hb, ']')) == NULL) {
 			errno = EINVAL;
 			goto syserr;
 		}
 		he = sep++;
 	} else {
 		hb = addr;
 #if HAVE_STRCHRNUL
 		sep = strchrnul(hb, ':');
 #else
 		sep = hb;
 		do { sep++; } while (*sep && *sep != ':');
 #endif
 		he = sep;
 	}
 	/* see if we need to copy the host name */
 	if (*he != '\0') {
 		len = snprintf(hbuf, sizeof(hbuf),
 		    "%.*s", (int)(he - hb), hb);
 		if (len < 0)
 			goto syserr;
 		if (len >= (int)sizeof(hbuf)) {
 			errno = ENAMETOOLONG;
 			goto syserr;
 		}
 		host = hbuf;
 	} else {
 		host = hb;
 	}
 	/* was it followed by a service name? */
 	if (*sep == '\0' && port != 0) {
 		if (port < 1 || port > 65535) {
 			errno = EINVAL;
 			goto syserr;
 		}
 		if (snprintf(sbuf, sizeof(sbuf), "%d", port) < 0)
 			goto syserr;
 		service = sbuf;
 	} else if (*sep != '\0') {
 		service = sep + 1;
 	} else {
 		service = NULL;
 	}
 	/* resolve */
 	memset(&hints, 0, sizeof(hints));
 	hints.ai_family = af;
 	hints.ai_socktype = SOCK_STREAM;
 	hints.ai_flags = AI_ADDRCONFIG;
 	if ((err = getaddrinfo(host, service, &hints, &res)) != 0) {
 		netdb_seterr(err);
 		return (NULL);
 	}
 	return (res);
 syserr:
 	fetch_syserr();
 	return (NULL);
 }
 /*
  * Bind a socket to a specific local address
  */
 int
 fetch_bind(int sd, int af, const char *addr)
 {
 	struct addrinfo *cliai, *ai;
 	int err;
 	if ((cliai = fetch_resolve(addr, 0, af)) == NULL)
 		return (-1);
 	for (ai = cliai; ai != NULL; ai = ai->ai_next)
 		if ((err = bind(sd, ai->ai_addr, ai->ai_addrlen)) == 0)
 			break;
 	if (err != 0)
 		fetch_syserr();
 	freeaddrinfo(cliai);
 	return (err == 0 ? 0 : -1);
 }
 /*
  * SOCKS5 connection initiation, based on RFC 1928
  * Default DNS resolution over SOCKS5
  */
 int
 fetch_socks5_init(conn_t *conn, const char *host, int port, int verbose)
 {
 	/*
 	 * Size is based on largest packet prefix (4 bytes) +
 	 * Largest FQDN (256) + one byte size (1) +
 	 * Port (2)
 	 */
 	unsigned char buf[BUFF_SIZE];
 	unsigned char *ptr;
 	int ret = 1;
 	if (verbose)
 		fetch_info("Initializing SOCKS5 connection: %s:%d", host, port);
 	/* Connection initialization */
 	ptr = buf;
 	*ptr++ = SOCKS_VERSION_5;
 	*ptr++ = SOCKS_CONNECTION;
 	*ptr++ = SOCKS_RSV;
 	if (fetch_write(conn, buf, 3) != 3) {
 		ret = SOCKS5_ERR_SELECTION;
 		goto fail;
 	}
 	/* Verify response from SOCKS5 server */
 	if (fetch_read(conn, buf, 2) != 2) {
 		ret = SOCKS5_ERR_READ_METHOD;
 		goto fail;
 	}
 	ptr = buf;
 	if (ptr[0] != SOCKS_VERSION_5) {
 		ret = SOCKS5_ERR_VER5_ONLY;
 		goto fail;
 	}
 	if (ptr[1] == SOCKS_NOMETHODS) {
 		ret = SOCKS5_ERR_NOMETHODS;
 		goto fail;
 	}
 	else if (ptr[1] != SOCKS5_NOTIMPLEMENTED) {
 		ret = SOCKS5_ERR_NOTIMPLEMENTED;
 		goto fail;
 	}
 	/* Send Request */
 	*ptr++ = SOCKS_VERSION_5;
 	*ptr++ = SOCKS_CONNECTION;
 	*ptr++ = SOCKS_RSV;
 	/* Encode all targets as a hostname to avoid DNS leaks */
 	*ptr++ = SOCKS_ATYP_DOMAINNAME;
 	if (strlen(host) > FQDN_SIZE) {
 		ret = SOCKS5_ERR_HOSTNAME_SIZE;
 		goto fail;
 	}
 	*ptr++ = strlen(host);
 	strncpy(ptr, host, strlen(host));
 	ptr = ptr + strlen(host);
 	port = htons(port);
 	*ptr++ = port & 0x00ff;
 	*ptr++ = (port & 0xff00) >> 8;
 	if (fetch_write(conn, buf, ptr - buf) != ptr - buf) {
 		ret = SOCKS5_ERR_REQUEST;
 		goto fail;
 	}
 	/* BND.ADDR is variable length, read the largest on non-blocking socket */
 	if (!fetch_read(conn, buf, BUFF_SIZE)) {
 		ret = SOCKS5_ERR_REPLY;
 		goto fail;
 	}
 	ptr = buf;
 	if (*ptr++ != SOCKS_VERSION_5) {
 		ret = SOCKS5_ERR_NON_VER5_RESP;
 		goto fail;
 	}
 	switch(*ptr++) {
 	case SOCKS_SUCCESS:
 		break;
 	case SOCKS_GENERAL_FAILURE:
 		ret = SOCKS5_ERR_GENERAL;
 		goto fail;
 	case SOCKS_CONNECTION_NOT_ALLOWED:
 		ret = SOCKS5_ERR_NOT_ALLOWED;
 		goto fail;
 	case SOCKS_NETWORK_UNREACHABLE:
 		ret = SOCKS5_ERR_NET_UNREACHABLE;
 		goto fail;
 	case SOCKS_HOST_UNREACHABLE:
 		ret = SOCKS5_ERR_HOST_UNREACHABLE;
 		goto fail;
 	case SOCKS_CONNECTION_REFUSED:
 		ret = SOCKS5_ERR_CONN_REFUSED;
 		goto fail;
 	case SOCKS_TTL_EXPIRED:
 		ret = SOCKS5_ERR_TTL_EXPIRED;
 		goto fail;
 	case SOCKS_COMMAND_NOT_SUPPORTED:
 		ret = SOCKS5_ERR_COM_UNSUPPORTED;
 		goto fail;
 	case SOCKS_ADDRESS_NOT_SUPPORTED:
 		ret = SOCKS5_ERR_ADDR_UNSUPPORTED;
 		goto fail;
 	default:
 		ret = SOCKS5_ERR_UNSPECIFIED;
 		goto fail;
 	}
 	return (ret);
 fail:
 	socks5_seterr(ret);
 	return (0);
 }
 /*
  * Perform SOCKS5 initialization
  */
 int
 fetch_socks5_getenv(char **host, int *port)
 {
 	char *socks5env, *endptr, *ext;
 	const char *portDelim;
 	size_t slen;
 	portDelim = ":";
 	if ((socks5env = getenv("SOCKS5_PROXY")) == NULL || *socks5env == '\0') {
 		*host = NULL;
 		*port = -1;
 		return (-1);
 	}
 	/*
 	 * IPv6 addresses begin and end in brackets.  Set the port delimiter
 	 * accordingly and search for it so we can do appropriate validation.
 	 */
 	if (socks5env[0] == '[')
 		portDelim = "]:";
 	slen = strlen(socks5env);
 	ext = strstr(socks5env, portDelim);
 	if (socks5env[0] == '[') {
 		if (socks5env[slen - 1] == ']') {
 			*host = strndup(socks5env, slen);
 		} else if (ext != NULL) {
 			*host = strndup(socks5env, ext - socks5env + 1);
 		} else {
 			socks5_seterr(SOCKS5_ERR_BAD_PROXY_FORMAT);
 			return (0);
 		}
 	} else {
 		*host = strndup(socks5env, ext - socks5env);
 	}
 	if (*host == NULL) {
 		fprintf(stderr, "Failure to allocate memory, exiting.\n");
 		return (-1);
 	}
 	if (ext == NULL) {
 		*port = 1080; /* Default port as defined in RFC1928 */
 	} else {
 		ext += strlen(portDelim);
 		errno = 0;
 		*port = strtoimax(ext, (char **)&endptr, 10);
 		if (*endptr != '\0' || errno != 0 || *port < 0 ||
 		    *port > 65535) {
 			free(*host);
 			*host = NULL;
 			socks5_seterr(SOCKS5_ERR_BAD_PORT);
 			return (0);
 		}
 	}
 	return (2);
 }
 /*
  * Establish a TCP connection to the specified port on the specified host.
  */
 conn_t *
 fetch_connect(const char *host, int port, int af, int verbose)
 {
 	struct addrinfo *cais = NULL, *sais = NULL, *cai, *sai;
 	const char *bindaddr;
 	conn_t *conn = NULL;
 	int err = 0, sd = -1;
 	char *sockshost;
 	int socksport;
 	DEBUGF("---> %s:%d\n", host, port);
 	/*
 	 * Check if SOCKS5_PROXY env variable is set.  fetch_socks5_getenv
 	 * will either set sockshost = NULL or allocate memory in all cases.
 	 */
 	sockshost = NULL;
 	if (!fetch_socks5_getenv(&sockshost, &socksport))
 		goto fail;
 	/* Not using SOCKS5 proxy */
 	if (sockshost == NULL) {
 		/* resolve server address */
 		if (verbose)
 			fetch_info("resolving server address: %s:%d", host,
 			    port);
 		if ((sais = fetch_resolve(host, port, af)) == NULL)
 			goto fail;
 		/* resolve client address */
 		bindaddr = getenv("FETCH_BIND_ADDRESS");
 		if (bindaddr != NULL && *bindaddr != '\0') {
 			if (verbose)
 				fetch_info("resolving client address: %s",
 				    bindaddr);
 			if ((cais = fetch_resolve(bindaddr, 0, af)) == NULL)
 				goto fail;
 		}
 	} else {
 		/* resolve socks5 proxy address */
 		if (verbose)
 			fetch_info("resolving SOCKS5 server address: %s:%d",
 			    sockshost, socksport);
 		if ((sais = fetch_resolve(sockshost, socksport, af)) == NULL) {
 			socks5_seterr(SOCKS5_ERR_BAD_HOST);
 			goto fail;
 		}
 	}
 	/* try each server address in turn */
 	for (err = 0, sai = sais; sai != NULL; sai = sai->ai_next) {
 		/* open socket */
 		if ((sd = socket(sai->ai_family, SOCK_STREAM, 0)) < 0)
 			goto syserr;
 		/* attempt to bind to client address */
 		for (err = 0, cai = cais; cai != NULL; cai = cai->ai_next) {
 			if (cai->ai_family != sai->ai_family)
 				continue;
 			if ((err = bind(sd, cai->ai_addr, cai->ai_addrlen)) == 0)
 				break;
 		}
 		if (err != 0) {
 			if (verbose)
 				fetch_info("failed to bind to %s", bindaddr);
 			goto syserr;
 		}
 		/* attempt to connect to server address */
 		if ((err = connect(sd, sai->ai_addr, sai->ai_addrlen)) == 0)
 			break;
 		/* clean up before next attempt */
 		close(sd);
 		sd = -1;
 	}
 	if (err != 0) {
 		if (verbose && sockshost == NULL) {
 			fetch_info("failed to connect to %s:%d", host, port);
 			goto syserr;
 		} else if (sockshost != NULL) {
 			if (verbose)
 				fetch_info(
 				    "failed to connect to SOCKS5 server %s:%d",
 				    sockshost, socksport);
 			socks5_seterr(SOCKS5_ERR_CONN_REFUSED);
 			goto fail;
 		}
 		goto syserr;
 	}
 	if ((conn = fetch_reopen(sd)) == NULL)
 		goto syserr;
 	if (sockshost)
 		if (!fetch_socks5_init(conn, host, port, verbose))
 			goto fail;
 	free(sockshost);
 	if (cais != NULL)
 		freeaddrinfo(cais);
 	if (sais != NULL)
 		freeaddrinfo(sais);
 	return (conn);
 syserr:
 	fetch_syserr();
 fail:
 	free(sockshost);
 	/* Fully close if it was opened; otherwise just don't leak the fd. */
 	if (conn != NULL)
 		fetch_close(conn);
 	else if (sd >= 0)
 		close(sd);
 	if (cais != NULL)
 		freeaddrinfo(cais);
 	if (sais != NULL)
 		freeaddrinfo(sais);
 	return (NULL);
 }
 #ifdef WITH_SSL
 /*
  * Convert characters A-Z to lowercase (intentionally avoid any locale
  * specific conversions).
  */
 static char
 fetch_ssl_tolower(char in)
 {
 	if (in >= 'A' && in <= 'Z')
 		return (in + 32);
 	else
 		return (in);
 }
 /*
  * isalpha implementation that intentionally avoids any locale specific
  * conversions.
  */
 static int
 fetch_ssl_isalpha(char in)
 {
 	return ((in >= 'A' && in <= 'Z') || (in >= 'a' && in <= 'z'));
 }
 /*
  * Check if passed hostnames a and b are equal.
  */
 static int
 fetch_ssl_hname_equal(const char *a, size_t alen, const char *b,
     size_t blen)
 {
 	size_t i;
 	if (alen != blen)
 		return (0);
 	for (i = 0; i < alen; ++i) {
 		if (fetch_ssl_tolower(a[i]) != fetch_ssl_tolower(b[i]))
 			return (0);
 	}
 	return (1);
 }
 /*
  * Check if domain label is traditional, meaning that only A-Z, a-z, 0-9
  * and '-' (hyphen) are allowed. Hyphens have to be surrounded by alpha-
  * numeric characters. Double hyphens (like they're found in IDN a-labels
  * 'xn--') are not allowed. Empty labels are invalid.
  */
 static int
 fetch_ssl_is_trad_domain_label(const char *l, size_t len, int wcok)
 {
 	size_t i;
 	if (!len || l[0] == '-' || l[len-1] == '-')
 		return (0);
 	for (i = 0; i < len; ++i) {
 		if (!isdigit(l[i]) &&
 		    !fetch_ssl_isalpha(l[i]) &&
 		    !(l[i] == '*' && wcok) &&
 		    !(l[i] == '-' && l[i - 1] != '-'))
 			return (0);
 	}
 	return (1);
 }
 /*
  * Check if host name consists only of numbers. This might indicate an IP
  * address, which is not a good idea for CN wildcard comparison.
  */
 static int
 fetch_ssl_hname_is_only_numbers(const char *hostname, size_t len)
 {
 	size_t i;
 	for (i = 0; i < len; ++i) {
 		if (!((hostname[i] >= '0' && hostname[i] <= '9') ||
 		    hostname[i] == '.'))
 			return (0);
 	}
 	return (1);
 }
 /*
  * Check if the host name h passed matches the pattern passed in m which
  * is usually part of subjectAltName or CN of a certificate presented to
  * the client. This includes wildcard matching. The algorithm is based on
  * RFC6125, sections 6.4.3 and 7.2, which clarifies RFC2818 and RFC3280.
  */
 static int
 fetch_ssl_hname_match(const char *h, size_t hlen, const char *m,
     size_t mlen)
 {
 	int delta, hdotidx, mdot1idx, wcidx;
 	const char *hdot, *mdot1, *mdot2;
 	const char *wc; /* wildcard */
 	if (!(h && *h && m && *m))
 		return (0);
 	if ((wc = strnstr(m, "*", mlen)) == NULL)
 		return (fetch_ssl_hname_equal(h, hlen, m, mlen));
 	wcidx = wc - m;
 	/* hostname should not be just dots and numbers */
 	if (fetch_ssl_hname_is_only_numbers(h, hlen))
 		return (0);
 	/* only one wildcard allowed in pattern */
 	if (strnstr(wc + 1, "*", mlen - wcidx - 1) != NULL)
 		return (0);
 	/*
 	 * there must be at least two more domain labels and
 	 * wildcard has to be in the leftmost label (RFC6125)
 	 */
 	mdot1 = strnstr(m, ".", mlen);
 	if (mdot1 == NULL || mdot1 < wc || (mlen - (mdot1 - m)) < 4)
 		return (0);
 	mdot1idx = mdot1 - m;
 	mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1);
 	if (mdot2 == NULL || (mlen - (mdot2 - m)) < 2)
 		return (0);
 	/* hostname must contain a dot and not be the 1st char */
 	hdot = strnstr(h, ".", hlen);
 	if (hdot == NULL || hdot == h)
 		return (0);
 	hdotidx = hdot - h;
 	/*
 	 * host part of hostname must be at least as long as
 	 * pattern it's supposed to match
 	 */
 	if (hdotidx < mdot1idx)
 		return (0);
 	/*
 	 * don't allow wildcards in non-traditional domain names
 	 * (IDN, A-label, U-label...)
 	 */
 	if (!fetch_ssl_is_trad_domain_label(h, hdotidx, 0) ||
 	    !fetch_ssl_is_trad_domain_label(m, mdot1idx, 1))
 		return (0);
 	/* match domain part (part after first dot) */
 	if (!fetch_ssl_hname_equal(hdot, hlen - hdotidx, mdot1,
 	    mlen - mdot1idx))
 		return (0);
 	/* match part left of wildcard */
 	if (!fetch_ssl_hname_equal(h, wcidx, m, wcidx))
 		return (0);
 	/* match part right of wildcard */
 	delta = mdot1idx - wcidx - 1;
 	if (!fetch_ssl_hname_equal(hdot - delta, delta,
 	    mdot1 - delta, delta))
 		return (0);
 	/* all tests succeeded, it's a match */
 	return (1);
 }
 /*
  * Get numeric host address info - returns NULL if host was not an IP
  * address. The caller is responsible for deallocation using
  * freeaddrinfo(3).
  */
 static struct addrinfo *
 fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len)
 {
 	struct addrinfo hints, *res;
 	char *host;
 	host = (char *)malloc(len + 1);
 	memcpy(host, hostname, len);
 	host[len] = '\0';
 	memset(&hints, 0, sizeof(hints));
 	hints.ai_family = PF_UNSPEC;
 	hints.ai_socktype = SOCK_STREAM;
 	hints.ai_protocol = 0;
 	hints.ai_flags = AI_NUMERICHOST;
 	/* port is not relevant for this purpose */
 	if (getaddrinfo(host, "443", &hints, &res) != 0)
 		res = NULL;
 	free(host);
 	return res;
 }
 /*
  * Compare ip address in addrinfo with address passes.
  */
 static int
 fetch_ssl_ipaddr_match_bin(const struct addrinfo *lhost, const char *rhost,
     size_t rhostlen)
 {
 	const void *left;
 	if (lhost->ai_family == AF_INET && rhostlen == 4) {
 		left = (void *)&((struct sockaddr_in*)(void *)
 		    lhost->ai_addr)->sin_addr.s_addr;
 #ifdef INET6
 	} else if (lhost->ai_family == AF_INET6 && rhostlen == 16) {
 		left = (void *)&((struct sockaddr_in6 *)(void *)
 		    lhost->ai_addr)->sin6_addr;
 #endif
 	} else
 		return (0);
 	return (!memcmp(left, (const void *)rhost, rhostlen) ? 1 : 0);
 }
 /*
  * Compare ip address in addrinfo with host passed. If host is not an IP
  * address, comparison will fail.
  */
 static int
 fetch_ssl_ipaddr_match(const struct addrinfo *laddr, const char *r,
     size_t rlen)
 {
 	struct addrinfo *raddr;
 	int ret;
 	char *rip;
 	ret = 0;
 	if ((raddr = fetch_ssl_get_numeric_addrinfo(r, rlen)) == NULL)
 		return 0; /* not a numeric host */
 	if (laddr->ai_family == raddr->ai_family) {
 		if (laddr->ai_family == AF_INET) {
 			rip = (char *)&((struct sockaddr_in *)(void *)
 			    raddr->ai_addr)->sin_addr.s_addr;
 			ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 4);
 #ifdef INET6
 		} else if (laddr->ai_family == AF_INET6) {
 			rip = (char *)&((struct sockaddr_in6 *)(void *)
 			    raddr->ai_addr)->sin6_addr;
 			ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 16);
 #endif
 		}
 	}
 	freeaddrinfo(raddr);
 	return (ret);
 }
 /*
  * Verify server certificate by subjectAltName.
  */
 static int
 fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames,
     const char *host, struct addrinfo *ip)
 {
 	const GENERAL_NAME *name;
 	size_t nslen;
 	int i;
 	const char *ns;
 	for (i = 0; i < sk_GENERAL_NAME_num(altnames); ++i) {
 #if OPENSSL_VERSION_NUMBER < 0x10000000L
 		/*
 		 * This is a workaround, since the following line causes
 		 * alignment issues in clang:
 		 * name = sk_GENERAL_NAME_value(altnames, i);
 		 * OpenSSL explicitly warns not to use those macros
 		 * directly, but there isn't much choice (and there
 		 * shouldn't be any ill side effects)
 		 */
 		name = (GENERAL_NAME *)SKM_sk_value(void, altnames, i);
 #else
 		name = sk_GENERAL_NAME_value(altnames, i);
 #endif
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
 		ns = (const char *)ASN1_STRING_data(name->d.ia5);
 #else
 		ns = (const char *)ASN1_STRING_get0_data(name->d.ia5);
 #endif
 		nslen = (size_t)ASN1_STRING_length(name->d.ia5);
 		if (name->type == GEN_DNS && ip == NULL &&
 		    fetch_ssl_hname_match(host, strlen(host), ns, nslen))
 			return (1);
 		else if (name->type == GEN_IPADD && ip != NULL &&
 		    fetch_ssl_ipaddr_match_bin(ip, ns, nslen))
 			return (1);
 	}
 	return (0);
 }
 /*
  * Verify server certificate by CN.
  */
 static int
 fetch_ssl_verify_cn(X509_NAME *subject, const char *host,
     struct addrinfo *ip)
 {
 	ASN1_STRING *namedata;
 	X509_NAME_ENTRY *nameentry;
 	int cnlen, lastpos, loc, ret;
 	unsigned char *cn;
 	ret = 0;
 	lastpos = -1;
 	loc = -1;
 	cn = NULL;
 	/* get most specific CN (last entry in list) and compare */
 	while ((lastpos = X509_NAME_get_index_by_NID(subject,
 	    NID_commonName, lastpos)) != -1)
 		loc = lastpos;
 	if (loc > -1) {
 		nameentry = X509_NAME_get_entry(subject, loc);
 		namedata = X509_NAME_ENTRY_get_data(nameentry);
 		cnlen = ASN1_STRING_to_UTF8(&cn, namedata);
 		if (ip == NULL &&
 		    fetch_ssl_hname_match(host, strlen(host), cn, cnlen))
 			ret = 1;
 		else if (ip != NULL && fetch_ssl_ipaddr_match(ip, cn, cnlen))
 			ret = 1;
 		OPENSSL_free(cn);
 	}
 	return (ret);
 }
 /*
  * Verify that server certificate subjectAltName/CN matches
  * hostname. First check, if there are alternative subject names. If yes,
  * those have to match. Only if those don't exist it falls back to
  * checking the subject's CN.
  */
 static int
 fetch_ssl_verify_hname(X509 *cert, const char *host)
 {
 	struct addrinfo *ip;
 	STACK_OF(GENERAL_NAME) *altnames;
 	X509_NAME *subject;
 	int ret;
 	ret = 0;
 	ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host));
 	altnames = X509_get_ext_d2i(cert, NID_subject_alt_name,
 	    NULL, NULL);
 	if (altnames != NULL) {
 		ret = fetch_ssl_verify_altname(altnames, host, ip);
 	} else {
 		subject = X509_get_subject_name(cert);
 		if (subject != NULL)
 			ret = fetch_ssl_verify_cn(subject, host, ip);
 	}
 	if (ip != NULL)
 		freeaddrinfo(ip);
 	if (altnames != NULL)
 		GENERAL_NAMES_free(altnames);
 	return (ret);
 }
 /*
  * Configure transport security layer based on environment.
  */
 static void
 fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose)
 {
 	long ssl_ctx_options;
 	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_SSLv3 | SSL_OP_NO_TICKET;
 	if (getenv("SSL_NO_TLS1") != NULL)
 		ssl_ctx_options |= SSL_OP_NO_TLSv1;
 	if (getenv("SSL_NO_TLS1_1") != NULL)
 		ssl_ctx_options |= SSL_OP_NO_TLSv1_1;
 	if (getenv("SSL_NO_TLS1_2") != NULL)
 		ssl_ctx_options |= SSL_OP_NO_TLSv1_2;
 	if (verbose)
 		fetch_info("SSL options: %lx", ssl_ctx_options);
 	SSL_CTX_set_options(ctx, ssl_ctx_options);
 }
 /*
  * Configure peer verification based on environment.
  */
 #define LOCAL_CERT_FILE	"/usr/local/etc/ssl/cert.pem"
 #define BASE_CERT_FILE	"/etc/ssl/cert.pem"
 static int
 fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose)
 {
 	X509_LOOKUP *crl_lookup;
 	X509_STORE *crl_store;
 	const char *ca_cert_file, *ca_cert_path, *crl_file;
 	if (getenv("SSL_NO_VERIFY_PEER") == NULL) {
 		ca_cert_file = getenv("SSL_CA_CERT_FILE");
 		if (ca_cert_file == NULL &&
 		    access(LOCAL_CERT_FILE, R_OK) == 0)
 			ca_cert_file = LOCAL_CERT_FILE;
 		if (ca_cert_file == NULL &&
 		    access(BASE_CERT_FILE, R_OK) == 0)
 			ca_cert_file = BASE_CERT_FILE;
 		ca_cert_path = getenv("SSL_CA_CERT_PATH");
 		if (verbose) {
 			fetch_info("Peer verification enabled");
 			if (ca_cert_file != NULL)
 				fetch_info("Using CA cert file: %s",
 				    ca_cert_file);
 			if (ca_cert_path != NULL)
 				fetch_info("Using CA cert path: %s",
 				    ca_cert_path);
 			if (ca_cert_file == NULL && ca_cert_path == NULL)
 				fetch_info("Using OpenSSL default "
 				    "CA cert file and path");
 		}
 		SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER,
 		    fetch_ssl_cb_verify_crt);
 		if (ca_cert_file != NULL || ca_cert_path != NULL)
 			SSL_CTX_load_verify_locations(ctx, ca_cert_file,
 			    ca_cert_path);
 		else
 			SSL_CTX_set_default_verify_paths(ctx);
 		if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) {
 			if (verbose)
 				fetch_info("Using CRL file: %s", crl_file);
 			crl_store = SSL_CTX_get_cert_store(ctx);
 			crl_lookup = X509_STORE_add_lookup(crl_store,
 			    X509_LOOKUP_file());
 			if (crl_lookup == NULL ||
 			    !X509_load_crl_file(crl_lookup, crl_file,
 				X509_FILETYPE_PEM)) {
 				fprintf(stderr,
 				    "Could not load CRL file %s\n",
 				    crl_file);
 				return (0);
 			}
 			X509_STORE_set_flags(crl_store,
 			    X509_V_FLAG_CRL_CHECK |
 			    X509_V_FLAG_CRL_CHECK_ALL);
 		}
 	}
 	return (1);
 }
 /*
  * Configure client certificate based on environment.
  */
 static int
 fetch_ssl_setup_client_certificate(SSL_CTX *ctx, int verbose)
 {
 	const char *client_cert_file, *client_key_file;
 	if ((client_cert_file = getenv("SSL_CLIENT_CERT_FILE")) != NULL) {
 		client_key_file = getenv("SSL_CLIENT_KEY_FILE") != NULL ?
 		    getenv("SSL_CLIENT_KEY_FILE") : client_cert_file;
 		if (verbose) {
 			fetch_info("Using client cert file: %s",
 			    client_cert_file);
 			fetch_info("Using client key file: %s",
 			    client_key_file);
 		}
 		if (SSL_CTX_use_certificate_chain_file(ctx,
 			client_cert_file) != 1) {
 			fprintf(stderr,
 			    "Could not load client certificate %s\n",
 			    client_cert_file);
 			return (0);
 		}
 		if (SSL_CTX_use_PrivateKey_file(ctx, client_key_file,
 			SSL_FILETYPE_PEM) != 1) {
 			fprintf(stderr,
 			    "Could not load client key %s\n",
 			    client_key_file);
 			return (0);
 		}
 	}
 	return (1);
 }
 /*
  * Callback for SSL certificate verification, this is called on server
  * cert verification. It takes no decision, but informs the user in case
  * verification failed.
  */
 int
 fetch_ssl_cb_verify_crt(int verified, X509_STORE_CTX *ctx)
 {
 	X509 *crt;
 	X509_NAME *name;
 	char *str;
 	str = NULL;
 	if (!verified) {
 		if ((crt = X509_STORE_CTX_get_current_cert(ctx)) != NULL &&
 		    (name = X509_get_subject_name(crt)) != NULL)
 			str = X509_NAME_oneline(name, 0, 0);
 		fprintf(stderr, "Certificate verification failed for %s\n",
 		    str != NULL ? str : "no relevant certificate");
 		OPENSSL_free(str);
 	}
 	return (verified);
 }
 #endif
 /*
  * Enable SSL on a connection.
  */
 int
 fetch_ssl(conn_t *conn, const struct url *URL, int verbose)
 {
 #ifdef WITH_SSL
 	int ret, ssl_err;
 	X509_NAME *name;
 	char *str;
 	/* Init the SSL library and context */
 	if (!SSL_library_init()){
 		fprintf(stderr, "SSL library init failed\n");
 		return (-1);
 	}
 	SSL_load_error_strings();
 	conn->ssl_meth = SSLv23_client_method();
 	conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);
 	SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
 	fetch_ssl_setup_transport_layer(conn->ssl_ctx, verbose);
 	if (!fetch_ssl_setup_peer_verification(conn->ssl_ctx, verbose))
 		return (-1);
 	if (!fetch_ssl_setup_client_certificate(conn->ssl_ctx, verbose))
 		return (-1);
 	conn->ssl = SSL_new(conn->ssl_ctx);
 	if (conn->ssl == NULL) {
 		fprintf(stderr, "SSL context creation failed\n");
 		return (-1);
 	}
 	SSL_set_fd(conn->ssl, conn->sd);
 #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
 	if (!SSL_set_tlsext_host_name(conn->ssl,
 	    __DECONST(struct url *, URL)->host)) {
 		fprintf(stderr,
 		    "TLS server name indication extension failed for host %s\n",
 		    URL->host);
 		return (-1);
 	}
 #endif
 	while ((ret = SSL_connect(conn->ssl)) == -1) {
 		ssl_err = SSL_get_error(conn->ssl, ret);
 		if (ssl_err != SSL_ERROR_WANT_READ &&
 		    ssl_err != SSL_ERROR_WANT_WRITE) {
 			ERR_print_errors_fp(stderr);
 			return (-1);
 		}
 	}
 	conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
 	if (conn->ssl_cert == NULL) {
 		fprintf(stderr, "No server SSL certificate\n");
 		return (-1);
 	}
 	if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) {
 		if (verbose)
 			fetch_info("Verify hostname");
 		if (!fetch_ssl_verify_hname(conn->ssl_cert, URL->host)) {
 			fprintf(stderr,
 			    "SSL certificate subject doesn't match host %s\n",
 			    URL->host);
 			return (-1);
 		}
 	}
 	if (verbose) {
 		fetch_info("%s connection established using %s",
 		    SSL_get_version(conn->ssl), SSL_get_cipher(conn->ssl));
 		name = X509_get_subject_name(conn->ssl_cert);
 		str = X509_NAME_oneline(name, 0, 0);
 		fetch_info("Certificate subject: %s", str);
 		OPENSSL_free(str);
 		name = X509_get_issuer_name(conn->ssl_cert);
 		str = X509_NAME_oneline(name, 0, 0);
 		fetch_info("Certificate issuer: %s", str);
 		OPENSSL_free(str);
 	}
 	return (0);
 #else
 	(void)conn;
 	(void)verbose;
 	(void)URL;
 	fprintf(stderr, "SSL support disabled\n");
 	return (-1);
 #endif
 }
 #define FETCH_READ_WAIT		-2
 #define FETCH_READ_ERROR	-1
 #define FETCH_READ_DONE		 0
 #ifdef WITH_SSL
 static ssize_t
 fetch_ssl_read(SSL *ssl, char *buf, size_t len)
 {
 	ssize_t rlen;
 	int ssl_err;
 	rlen = SSL_read(ssl, buf, len);
 	if (rlen < 0) {
 		ssl_err = SSL_get_error(ssl, rlen);
 		if (ssl_err == SSL_ERROR_WANT_READ ||
 		    ssl_err == SSL_ERROR_WANT_WRITE) {
 			return (FETCH_READ_WAIT);
 		} else {
 			ERR_print_errors_fp(stderr);
 			return (FETCH_READ_ERROR);
 		}
 	}
 	return (rlen);
 }
 #endif
 static ssize_t
 fetch_socket_read(int sd, char *buf, size_t len)
 {
 	ssize_t rlen;
 	rlen = read(sd, buf, len);
 	if (rlen < 0) {
 		if (errno == EAGAIN || (errno == EINTR && fetchRestartCalls))
 			return (FETCH_READ_WAIT);
 		else
 			return (FETCH_READ_ERROR);
 	}
 	return (rlen);
 }
 /*
  * Read a character from a connection w/ timeout
  */
 ssize_t
 fetch_read(conn_t *conn, char *buf, size_t len)
 {
 	struct timeval now, timeout, delta;
 	struct pollfd pfd;
 	ssize_t rlen;
 	int deltams;
 	if (fetchTimeout > 0) {
 		gettimeofday(&timeout, NULL);
 		timeout.tv_sec += fetchTimeout;
 	}
 	deltams = INFTIM;
 	memset(&pfd, 0, sizeof pfd);
 	pfd.fd = conn->sd;
 	pfd.events = POLLIN | POLLERR;
 	for (;;) {
 		/*
 		 * The socket is non-blocking.  Instead of the canonical
 		 * poll() -> read(), we do the following:
 		 *
 		 * 1) call read() or SSL_read().
 		 * 2) if we received some data, return it.
 		 * 3) if an error occurred, return -1.
 		 * 4) if read() or SSL_read() signaled EOF, return.
 		 * 5) if we did not receive any data but we're not at EOF,
 		 *    call poll().
 		 *
 		 * In the SSL case, this is necessary because if we
 		 * receive a close notification, we have to call
 		 * SSL_read() one additional time after we've read
 		 * everything we received.
 		 *
 		 * In the non-SSL case, it may improve performance (very
 		 * slightly) when reading small amounts of data.
 		 */
 #ifdef WITH_SSL
 		if (conn->ssl != NULL)
 			rlen = fetch_ssl_read(conn->ssl, buf, len);
 		else
 #endif
 			rlen = fetch_socket_read(conn->sd, buf, len);
 		if (rlen >= 0) {
 			break;
 		} else if (rlen == FETCH_READ_ERROR) {
 			fetch_syserr();
 			return (-1);
 		}
 		// assert(rlen == FETCH_READ_WAIT);
 		if (fetchTimeout > 0) {
 			gettimeofday(&now, NULL);
 			if (!timercmp(&timeout, &now, >)) {
 				errno = ETIMEDOUT;
 				fetch_syserr();
 				return (-1);
 			}
 			timersub(&timeout, &now, &delta);
 			deltams = delta.tv_sec * 1000 +
 			    delta.tv_usec / 1000;;
 		}
 		errno = 0;
 		pfd.revents = 0;
 		if (poll(&pfd, 1, deltams) < 0) {
 			if (errno == EINTR && fetchRestartCalls)
 				continue;
 			fetch_syserr();
 			return (-1);
 		}
 	}
 	return (rlen);
 }
 /*
  * Read a line of text from a connection w/ timeout
  */
 #define MIN_BUF_SIZE 1024
 int
 fetch_getln(conn_t *conn)
 {
 	char *tmp;
 	size_t tmpsize;
 	ssize_t len;
 	char c;
 	if (conn->buf == NULL) {
 		if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
 			errno = ENOMEM;
 			return (-1);
 		}
 		conn->bufsize = MIN_BUF_SIZE;
 	}
 	conn->buf[0] = '\0';
 	conn->buflen = 0;
 	do {
 		len = fetch_read(conn, &c, 1);
 		if (len == -1)
 			return (-1);
 		if (len == 0)
 			break;
 		conn->buf[conn->buflen++] = c;
 		if (conn->buflen == conn->bufsize) {
 			tmp = conn->buf;
 			tmpsize = conn->bufsize * 2 + 1;
 			if ((tmp = realloc(tmp, tmpsize)) == NULL) {
 				errno = ENOMEM;
 				return (-1);
 			}
 			conn->buf = tmp;
 			conn->bufsize = tmpsize;
 		}
 	} while (c != '\n');
 	conn->buf[conn->buflen] = '\0';
 	DEBUGF("<<< %s", conn->buf);
 	return (0);
 }
 /*
  * Write to a connection w/ timeout
  */
 ssize_t
 fetch_write(conn_t *conn, const char *buf, size_t len)
 {
 	struct iovec iov;
 	iov.iov_base = __DECONST(char *, buf);
 	iov.iov_len = len;
 	return fetch_writev(conn, &iov, 1);
 }
 /*
  * Write a vector to a connection w/ timeout
  * Note: can modify the iovec.
  */
 ssize_t
 fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
 {
 	struct timeval now, timeout, delta;
 	struct pollfd pfd;
 	ssize_t wlen, total;
 	int deltams;
 	memset(&pfd, 0, sizeof pfd);
 	if (fetchTimeout) {
 		pfd.fd = conn->sd;
 		pfd.events = POLLOUT | POLLERR;
 		gettimeofday(&timeout, NULL);
 		timeout.tv_sec += fetchTimeout;
 	}
 	total = 0;
 	while (iovcnt > 0) {
 		while (fetchTimeout && pfd.revents == 0) {
 			gettimeofday(&now, NULL);
 			if (!timercmp(&timeout, &now, >)) {
 				errno = ETIMEDOUT;
 				fetch_syserr();
 				return (-1);
 			}
 			timersub(&timeout, &now, &delta);
 			deltams = delta.tv_sec * 1000 +
 			    delta.tv_usec / 1000;
 			errno = 0;
 			pfd.revents = 0;
 			if (poll(&pfd, 1, deltams) < 0) {
 				/* POSIX compliance */
 				if (errno == EAGAIN)
 					continue;
 				if (errno == EINTR && fetchRestartCalls)
 					continue;
 				return (-1);
 			}
 		}
 		errno = 0;
 #ifdef WITH_SSL
 		if (conn->ssl != NULL)
 			wlen = SSL_write(conn->ssl,
 			    iov->iov_base, iov->iov_len);
 		else
 #endif
 			wlen = writev(conn->sd, iov, iovcnt);
 		if (wlen == 0) {
 			/* we consider a short write a failure */
 			/* XXX perhaps we shouldn't in the SSL case */
 			errno = EPIPE;
 			fetch_syserr();
 			return (-1);
 		}
 		if (wlen < 0) {
 			if (errno == EINTR && fetchRestartCalls)
 				continue;
 			return (-1);
 		}
 		total += wlen;
 		while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
 			wlen -= iov->iov_len;
 			iov++;
 			iovcnt--;
 		}
 		if (iovcnt > 0) {
 			iov->iov_len -= wlen;
 			iov->iov_base = __DECONST(char *, iov->iov_base) + wlen;
 		}
 	}
 	return (total);
 }
 /*
  * Write a line of text to a connection w/ timeout
  */
 int
 fetch_putln(conn_t *conn, const char *str, size_t len)
 {
 	struct iovec iov[2];
 	int ret;
 	DEBUGF(">>> %s\n", str);
 	iov[0].iov_base = __DECONST(char *, str);
 	iov[0].iov_len = len;
 	iov[1].iov_base = __DECONST(char *, ENDL);
 	iov[1].iov_len = sizeof(ENDL);
 	if (len == 0)
 		ret = fetch_writev(conn, &iov[1], 1);
 	else
 		ret = fetch_writev(conn, iov, 2);
 	if (ret == -1)
 		return (-1);
 	return (0);
 }
 /*
  * Close connection
  */
 int
 fetch_close(conn_t *conn)
 {
 	int ret;
 	if (--conn->ref > 0)
 		return (0);
 #ifdef WITH_SSL
 	if (conn->ssl) {
 		SSL_shutdown(conn->ssl);
 		SSL_set_connect_state(conn->ssl);
 		SSL_free(conn->ssl);
 		conn->ssl = NULL;
 	}
 	if (conn->ssl_ctx) {
 		SSL_CTX_free(conn->ssl_ctx);
 		conn->ssl_ctx = NULL;
 	}
 	if (conn->ssl_cert) {
 		X509_free(conn->ssl_cert);
 		conn->ssl_cert = NULL;
 	}
 #endif
 	ret = close(conn->sd);
 	free(conn->buf);
 	free(conn);
 	return (ret);
 }
 /*** Directory-related utility functions *************************************/
 int
 fetch_add_entry(struct url_ent **p, int *size, int *len,
     const char *name, struct url_stat *us)
 {
 	struct url_ent *tmp;
 	if (*p == NULL) {
 		*size = 0;
 		*len = 0;
 	}
 	if (*len >= *size - 1) {
 #if !HAVE_REALLOCARRAY
 		tmp = realloc(*p, (*size * 2 + 1) * sizeof(**p));
 #else
 		tmp = reallocarray(*p, *size * 2 + 1, sizeof(**p));
 #endif
 		if (tmp == NULL) {
 			errno = ENOMEM;
 			fetch_syserr();
 			return (-1);
 		}
 		*size = (*size * 2 + 1);
 		*p = tmp;
 	}
 	tmp = *p + *len;
 	snprintf(tmp->name, PATH_MAX, "%s", name);
 	memcpy(&tmp->stat, us, sizeof(*us));
 	(*len)++;
 	(++tmp)->name[0] = 0;
 	return (0);
 }
 /*** Authentication-related utility functions ********************************/
 static const char *
 fetch_read_word(FILE *f)
 {
 	static char word[1024];
 	if (fscanf(f, " %1023s ", word) != 1)
 		return (NULL);
 	return (word);
 }
 static int
 fetch_netrc_open(void)
 {
 	struct passwd *pwd;
 	char fn[PATH_MAX];
 	const char *p;
 	int fd, serrno;
 	if ((p = getenv("NETRC")) != NULL) {
 		DEBUGF("NETRC=%s\n", p);
 		if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
 			fetch_info("$NETRC specifies a file name "
 			    "longer than PATH_MAX");
 			return (-1);
 		}
 	} else {
 		if ((p = getenv("HOME")) == NULL) {
 			if ((pwd = getpwuid(getuid())) == NULL ||
 			    (p = pwd->pw_dir) == NULL)
 				return (-1);
 		}
 		if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))
 			return (-1);
 	}
 	if ((fd = open(fn, O_RDONLY)) < 0) {
 		serrno = errno;
 		DEBUGF("%s: %s\n", fn, strerror(serrno));
 		errno = serrno;
 	}
 	return (fd);
 }
 /*
  * Get authentication data for a URL from .netrc
  */
 int
 fetch_netrc_auth(struct url *url)
 {
 	const char *word;
 	int serrno;
 	FILE *f;
 	if (url->netrcfd < 0)
 		url->netrcfd = fetch_netrc_open();
 	if (url->netrcfd < 0)
 		return (-1);
 	if ((f = fdopen(url->netrcfd, "r")) == NULL) {
 		serrno = errno;
 		DEBUGF("fdopen(netrcfd): %s", strerror(errno));
 		close(url->netrcfd);
 		url->netrcfd = -1;
 		errno = serrno;
 		return (-1);
 	}
 	rewind(f);
 	DEBUGF("searching netrc for %s\n", url->host);
 	while ((word = fetch_read_word(f)) != NULL) {
 		if (strcmp(word, "default") == 0) {
 			DEBUGF("using default netrc settings\n");
 			break;
 		}
 		if (strcmp(word, "machine") == 0 &&
 		    (word = fetch_read_word(f)) != NULL &&
 		    strcasecmp(word, url->host) == 0) {
 			DEBUGF("using netrc settings for %s\n", word);
 			break;
 		}
 	}
 	if (word == NULL)
 		goto ferr;
 	while ((word = fetch_read_word(f)) != NULL) {
 		if (strcmp(word, "login") == 0) {
 			if ((word = fetch_read_word(f)) == NULL)
 				goto ferr;
 			if (snprintf(url->user, sizeof(url->user),
 				"%s", word) > (int)sizeof(url->user)) {
 				fetch_info("login name in .netrc is too long");
 				url->user[0] = '\0';
 			}
 		} else if (strcmp(word, "password") == 0) {
 			if ((word = fetch_read_word(f)) == NULL)
 				goto ferr;
 			if (snprintf(url->pwd, sizeof(url->pwd),
 				"%s", word) > (int)sizeof(url->pwd)) {
 				fetch_info("password in .netrc is too long");
 				url->pwd[0] = '\0';
 			}
 		} else if (strcmp(word, "account") == 0) {
 			if ((word = fetch_read_word(f)) == NULL)
 				goto ferr;
 			/* XXX not supported! */
 		} else {
 			break;
 		}
 	}
 	fclose(f);
 	url->netrcfd = -1;
 	return (0);
 ferr:
 	serrno = errno;
 	fclose(f);
 	url->netrcfd = -1;
 	errno = serrno;
 	return (-1);
 }
 /*
  * The no_proxy environment variable specifies a set of domains for
  * which the proxy should not be consulted; the contents is a comma-,
  * or space-separated list of domain names.  A single asterisk will
  * override all proxy variables and no transactions will be proxied
  * (for compatibility with lynx and curl, see the discussion at
  * <http://curl.haxx.se/mail/archive_pre_oct_99/0009.html>).
  */
 int
 fetch_no_proxy_match(const char *host)
 {
 	const char *no_proxy, *p, *q;
 	size_t h_len, d_len;
 	if ((no_proxy = getenv("NO_PROXY")) == NULL &&
 	    (no_proxy = getenv("no_proxy")) == NULL)
 		return (0);
 	/* asterisk matches any hostname */
 	if (strcmp(no_proxy, "*") == 0)
 		return (1);
 	h_len = strlen(host);
 	p = no_proxy;
 	do {
 		/* position p at the beginning of a domain suffix */
 		while (*p == ',' || isspace((unsigned char)*p))
 			p++;
 		/* position q at the first separator character */
 		for (q = p; *q; ++q)
 			if (*q == ',' || isspace((unsigned char)*q))
 				break;
 		d_len = q - p;
 		if (d_len > 0 && h_len >= d_len &&
 		    strncasecmp(host + h_len - d_len,
 			p, d_len) == 0) {
 			/* domain name matches */
 			return (1);
 		}
 		p = q + 1;
 	} while (*q);
 	return (0);
 }

deleted external/libfetch/common.h

@@ -1,175 +0,0 @@

 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 1998-2014 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD: head/lib/libfetch/common.h 334317 2018-05-29 10:28:20Z des $
  */
 #ifndef _COMMON_H_INCLUDED
 #define _COMMON_H_INCLUDED
 #define FTP_DEFAULT_PORT	21
 #define HTTP_DEFAULT_PORT	80
 #define FTP_DEFAULT_PROXY_PORT	21
 #define HTTP_DEFAULT_PROXY_PORT	3128
 #ifdef WITH_SSL
 #include <openssl/crypto.h>
 #include <openssl/x509.h>
 #include <openssl/pem.h>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 #endif
 /* Connection */
 typedef struct fetchconn conn_t;
 struct fetchconn {
 	int		 sd;		/* socket descriptor */
 	char		*buf;		/* buffer */
 	size_t		 bufsize;	/* buffer size */
 	size_t		 buflen;	/* length of buffer contents */
 	int		 err;		/* last protocol reply code */
 #ifdef WITH_SSL
 	SSL		*ssl;		/* SSL handle */
 	SSL_CTX		*ssl_ctx;	/* SSL context */
 	X509		*ssl_cert;	/* server certificate */
 	const SSL_METHOD *ssl_meth;	/* SSL method */
 #endif
 	int		 ref;		/* reference count */
 };
 /* Structure used for error message lists */
 struct fetcherr {
 	const int	 num;
 	const int	 cat;
 	const char	*string;
 };
 /* For SOCKS header size */
 #define HEAD_SIZE	4
 #define FQDN_SIZE	256
 #define PACK_SIZE	1
 #define PORT_SIZE	2
 #define BUFF_SIZE	HEAD_SIZE + FQDN_SIZE + PACK_SIZE + PORT_SIZE
 /* SOCKS5 Request Header */
 #define SOCKS_VERSION_5		0x05
 /* SOCKS5 CMD */
 #define SOCKS_CONNECTION	0x01
 #define SOCKS_BIND		0x02
 #define SOCKS_UDP		0x03
 #define SOCKS_NOMETHODS		0xFF
 #define SOCKS5_NOTIMPLEMENTED	0x00
 /* SOCKS5 Reserved */
 #define SOCKS_RSV		0x00
 /* SOCKS5 Address Type */
 #define SOCKS_ATYP_IPV4		0x01
 #define SOCKS_ATYP_DOMAINNAME	0x03
 #define SOCKS_ATYP_IPV6		0x04
 /* SOCKS5 Reply Field */
 #define SOCKS_SUCCESS			0x00
 #define SOCKS_GENERAL_FAILURE		0x01
 #define SOCKS_CONNECTION_NOT_ALLOWED	0x02
 #define SOCKS_NETWORK_UNREACHABLE	0x03
 #define SOCKS_HOST_UNREACHABLE		0x04
 #define SOCKS_CONNECTION_REFUSED	0x05
 #define SOCKS_TTL_EXPIRED		0x06
 #define SOCKS_COMMAND_NOT_SUPPORTED	0x07
 #define SOCKS_ADDRESS_NOT_SUPPORTED	0x08
 /* for fetch_writev */
 struct iovec;
 void		 fetch_seterr(struct fetcherr *, int);
 void		 fetch_syserr(void);
 void		 fetch_info(const char *, ...) __printflike(1, 2);
 int		 fetch_socks5_getenv(char **host, int *port);
 int		 fetch_socks5_init(conn_t *conn, const char *host,
 		     int port, int verbose);
 int		 fetch_default_port(const char *);
 int		 fetch_default_proxy_port(const char *);
 struct addrinfo *fetch_resolve(const char *, int, int);
 int		 fetch_bind(int, int, const char *);
 conn_t		*fetch_connect(const char *, int, int, int);
 conn_t		*fetch_reopen(int);
 conn_t		*fetch_ref(conn_t *);
 #ifdef WITH_SSL
 int		 fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*);
 #endif
 int		 fetch_ssl(conn_t *, const struct url *, int);
 ssize_t		 fetch_read(conn_t *, char *, size_t);
 int		 fetch_getln(conn_t *);
 ssize_t		 fetch_write(conn_t *, const char *, size_t);
 ssize_t		 fetch_writev(conn_t *, struct iovec *, int);
 int		 fetch_putln(conn_t *, const char *, size_t);
 int		 fetch_close(conn_t *);
 int		 fetch_add_entry(struct url_ent **, int *, int *,
 		     const char *, struct url_stat *);
 int		 fetch_netrc_auth(struct url *url);
 int		 fetch_no_proxy_match(const char *);
 #define ftp_seterr(n)	 fetch_seterr(ftp_errlist, n)
 #define http_seterr(n)	 fetch_seterr(http_errlist, n)
 #define netdb_seterr(n)	 fetch_seterr(netdb_errlist, n)
 #define url_seterr(n)	 fetch_seterr(url_errlist, n)
 #define socks5_seterr(n) fetch_seterr(socks5_errlist, n)
 #ifndef NDEBUG
 #define DEBUGF(...)							\
 	do {								\
 		if (fetchDebug)						\
 			fprintf(stderr, __VA_ARGS__);			\
 	} while (0)
 #else
 #define DEBUGF(...)							\
 	do {								\
 		/* nothing */						\
 	} while (0)
 #endif
 /*
  * I don't really like exporting http_request()
  * but the HTTP and FTP code occasionally needs to cross-call
  * eachother, and this saves me from adding a lot of special-case code
  * to handle those cases.
  *
  * Note that _*_request() free purl, which is way ugly but saves us a
  * whole lot of trouble.
  */
 FILE		*http_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
 FILE		*http_request_body(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *,
 		     const char *, const char *);
 /*
  * Check whether a particular flag is set
  */
 #define CHECK_FLAG(x)	(flags && strchr(flags, (x)))
 #endif

deleted external/libfetch/fetch.3

@@ -1,841 +0,0 @@

 .\"-
 .\" Copyright (c) 1998-2013 Dag-Erling Smørgrav
 .\" Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
 .\" 1. Redistributions of source code must retain the above copyright
 .\"    notice, this list of conditions and the following disclaimer.
 .\" 2. Redistributions in binary form must reproduce the above copyright
 .\"    notice, this list of conditions and the following disclaimer in the
 .\"    documentation and/or other materials provided with the distribution.
 .\"
 .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 .\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD: head/lib/libfetch/fetch.3 273124 2014-10-15 07:35:50Z des $
 .\"
 .Dd October 15, 2014
 .Dt FETCH 3
 .Os
 .Sh NAME
 .Nm fetchMakeURL ,
 .Nm fetchParseURL ,
 .Nm fetchFreeURL ,
 .Nm fetchXGetURL ,
 .Nm fetchGetURL ,
 .Nm fetchPutURL ,
 .Nm fetchStatURL ,
 .Nm fetchListURL ,
 .Nm fetchXGet ,
 .Nm fetchGet ,
 .Nm fetchPut ,
 .Nm fetchStat ,
 .Nm fetchList ,
 .Nm fetchXGetFile ,
 .Nm fetchGetFile ,
 .Nm fetchPutFile ,
 .Nm fetchStatFile ,
 .Nm fetchListFile ,
 .Nm fetchXGetHTTP ,
 .Nm fetchGetHTTP ,
 .Nm fetchPutHTTP ,
 .Nm fetchStatHTTP ,
 .Nm fetchListHTTP ,
 .Nm fetchXGetFTP ,
 .Nm fetchGetFTP ,
 .Nm fetchPutFTP ,
 .Nm fetchStatFTP ,
 .Nm fetchListFTP
 .Nd file transfer functions
 .Sh LIBRARY
 .Lb libfetch
 .Sh SYNOPSIS
 .In sys/param.h
 .In stdio.h
 .In fetch.h
 .Ft struct url *
 .Fn fetchMakeURL "const char *scheme" "const char *host" "int port" "const char *doc" "const char *user" "const char *pwd"
 .Ft struct url *
 .Fn fetchParseURL "const char *URL"
 .Ft void
 .Fn fetchFreeURL "struct url *u"
 .Ft FILE *
 .Fn fetchXGetURL "const char *URL" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGetURL "const char *URL" "const char *flags"
 .Ft FILE *
 .Fn fetchPutURL "const char *URL" "const char *flags"
 .Ft int
 .Fn fetchStatURL "const char *URL" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchListURL "const char *URL" "const char *flags"
 .Ft FILE *
 .Fn fetchXGet "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGet "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchPut "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStat "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchList "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchXGetFile "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGetFile "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchPutFile "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatFile "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchListFile "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGetHTTP "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchPutHTTP "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatHTTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchListHTTP "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
 .Fn fetchGetFTP "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchPutFTP "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatFTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft struct url_ent *
 .Fn fetchListFTP "struct url *u" "const char *flags"
 .Sh DESCRIPTION
 These functions implement a high-level library for retrieving and
 uploading files using Uniform Resource Locators (URLs).
 .Pp
 .Fn fetchParseURL
 takes a URL in the form of a null-terminated string and splits it into
 its components function according to the Common Internet Scheme Syntax
 detailed in RFC1738.
 A regular expression which produces this syntax is:
 .Bd -literal
     <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
 .Ed
 .Pp
 If the URL does not seem to begin with a scheme name, the following
 syntax is assumed:
 .Bd -literal
     ((<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
 .Ed
 .Pp
 Note that some components of the URL are not necessarily relevant to
 all URL schemes.
 For instance, the file scheme only needs the <scheme> and <document>
 components.
 .Pp
 .Fn fetchMakeURL
 and
 .Fn fetchParseURL
 return a pointer to a
 .Vt url
 structure, which is defined as follows in
 .In fetch.h :
 .Bd -literal
 #define URL_SCHEMELEN 16
 #define URL_USERLEN 256
 #define URL_PWDLEN 256
 struct url {
     char	 scheme[URL_SCHEMELEN+1];
     char	 user[URL_USERLEN+1];
     char	 pwd[URL_PWDLEN+1];
     char	 host[MAXHOSTNAMELEN+1];
     int		 port;
     char	*doc;
     off_t	 offset;
     size_t	 length;
     time_t	 ims_time;
 };
 .Ed
 .Pp
 The
 .Va ims_time
 field stores the time value for
 .Li If-Modified-Since
 HTTP requests.
 .Pp
 The pointer returned by
 .Fn fetchMakeURL
 or
 .Fn fetchParseURL
 should be freed using
 .Fn fetchFreeURL .
 .Pp
 .Fn fetchXGetURL ,
 .Fn fetchGetURL ,
 and
 .Fn fetchPutURL
 constitute the recommended interface to the
 .Nm fetch
 library.
 They examine the URL passed to them to determine the transfer
 method, and call the appropriate lower-level functions to perform the
 actual transfer.
 .Fn fetchXGetURL
 also returns the remote document's metadata in the
 .Vt url_stat
 structure pointed to by the
 .Fa us
 argument.
 .Pp
 The
 .Fa flags
 argument is a string of characters which specify transfer options.
 The
 meaning of the individual flags is scheme-dependent, and is detailed
 in the appropriate section below.
 .Pp
 .Fn fetchStatURL
 attempts to obtain the requested document's metadata and fill in the
 structure pointed to by its second argument.
 The
 .Vt url_stat
 structure is defined as follows in
 .In fetch.h :
 .Bd -literal
 struct url_stat {
     off_t	 size;
     time_t	 atime;
     time_t	 mtime;
 };
 .Ed
 .Pp
 If the size could not be obtained from the server, the
 .Fa size
 field is set to -1.
 If the modification time could not be obtained from the server, the
 .Fa mtime
 field is set to the epoch.
 If the access time could not be obtained from the server, the
 .Fa atime
 field is set to the modification time.
 .Pp
 .Fn fetchListURL
 attempts to list the contents of the directory pointed to by the URL
 provided.
 If successful, it returns a malloced array of
 .Vt url_ent
 structures.
 The
 .Vt url_ent
 structure is defined as follows in
 .In fetch.h :
 .Bd -literal
 struct url_ent {
     char         name[PATH_MAX];
     struct url_stat stat;
 };
 .Ed
 .Pp
 The list is terminated by an entry with an empty name.
 .Pp
 The pointer returned by
 .Fn fetchListURL
 should be freed using
 .Fn free .
 .Pp
 .Fn fetchXGet ,
 .Fn fetchGet ,
 .Fn fetchPut
 and
 .Fn fetchStat
 are similar to
 .Fn fetchXGetURL ,
 .Fn fetchGetURL ,
 .Fn fetchPutURL
 and
 .Fn fetchStatURL ,
 except that they expect a pre-parsed URL in the form of a pointer to
 a
 .Vt struct url
 rather than a string.
 .Pp
 All of the
 .Fn fetchXGetXXX ,
 .Fn fetchGetXXX
 and
 .Fn fetchPutXXX
 functions return a pointer to a stream which can be used to read or
 write data from or to the requested document, respectively.
 Note that
 although the implementation details of the individual access methods
 vary, it can generally be assumed that a stream returned by one of the
 .Fn fetchXGetXXX
 or
 .Fn fetchGetXXX
 functions is read-only, and that a stream returned by one of the
 .Fn fetchPutXXX
 functions is write-only.
 .Sh FILE SCHEME
 .Fn fetchXGetFile ,
 .Fn fetchGetFile
 and
 .Fn fetchPutFile
 provide access to documents which are files in a locally mounted file
 system.
 Only the <document> component of the URL is used.
 .Pp
 .Fn fetchXGetFile
 and
 .Fn fetchGetFile
 do not accept any flags.
 .Pp
 .Fn fetchPutFile
 accepts the
 .Ql a
 (append to file) flag.
 If that flag is specified, the data written to
 the stream returned by
 .Fn fetchPutFile
 will be appended to the previous contents of the file, instead of
 replacing them.
 .Sh FTP SCHEME
 .Fn fetchXGetFTP ,
 .Fn fetchGetFTP
 and
 .Fn fetchPutFTP
 implement the FTP protocol as described in RFC959.
 .Pp
 If the
 .Ql P
 (not passive) flag is specified, an active (rather than passive)
 connection will be attempted.
 .Pp
 The
 .Ql p
 flag is supported for compatibility with earlier versions where active
 connections were the default.
 It has precedence over the
 .Ql P
 flag, so if both are specified,
 .Nm
 will use a passive connection.
 .Pp
 If the
 .Ql l
 (low) flag is specified, data sockets will be allocated in the low (or
 default) port range instead of the high port range (see
 .Xr ip 4 ) .
 .Pp
 If the
 .Ql d
 (direct) flag is specified,
 .Fn fetchXGetFTP ,
 .Fn fetchGetFTP
 and
 .Fn fetchPutFTP
 will use a direct connection even if a proxy server is defined.
 .Pp
 If no user name or password is given, the
 .Nm fetch
 library will attempt an anonymous login, with user name "anonymous"
 and password "anonymous@<hostname>".
 .Sh HTTP SCHEME
 The
 .Fn fetchXGetHTTP ,
 .Fn fetchGetHTTP
 and
 .Fn fetchPutHTTP
 functions implement the HTTP/1.1 protocol.
 With a little luck, there is
 even a chance that they comply with RFC2616 and RFC2617.
 .Pp
 If the
 .Ql d
 (direct) flag is specified,
 .Fn fetchXGetHTTP ,
 .Fn fetchGetHTTP
 and
 .Fn fetchPutHTTP
 will use a direct connection even if a proxy server is defined.
 .Pp
 If the
 .Ql i
 (if-modified-since) flag is specified, and
 the
 .Va ims_time
 field is set in
 .Vt "struct url" ,
 then
 .Fn fetchXGetHTTP
 and
 .Fn fetchGetHTTP
 will send a conditional
 .Li If-Modified-Since
 HTTP header to only fetch the content if it is newer than
 .Va ims_time .
 .Pp
 Since there seems to be no good way of implementing the HTTP PUT
 method in a manner consistent with the rest of the
 .Nm fetch
 library,
 .Fn fetchPutHTTP
 is currently unimplemented.
 .Sh HTTPS SCHEME
 Based on HTTP SCHEME.
 By default the peer is verified using the CA bundle located in
 .Pa /etc/ssl/cert.pem .
 The file may contain multiple CA certificates.
 A common source of a current CA bundle is
 .Pa \%security/ca_root_nss .
 .Pp
 The CA bundle used for peer verification can be changed by setting the
 environment variables
 .Ev SSL_CA_CERT_FILE
 to point to a concatenated bundle of trusted certificates and
 .Ev SSL_CA_CERT_PATH
 to point to a directory containing hashes of trusted CAs (see
 .Xr verify 1 ) .
 .Pp
 A certificate revocation list (CRL) can be used by setting the
 environment variable
 .Ev SSL_CRL_FILE
 (see
 .Xr crl 1 ) .
 .Pp
 Peer verification can be disabled by setting the environment variable
 .Ev SSL_NO_VERIFY_PEER .
 Note that this also disables CRL checking.
 .Pp
 By default the service identity is verified according to the rules
 detailed in RFC6125 (also known as hostname verification).
 This feature can be disabled by setting the environment variable
 .Ev SSL_NO_VERIFY_HOSTNAME .
 .Pp
 Client certificate based authentication is supported.
 The environment variable
 .Ev SSL_CLIENT_CERT_FILE
 should be set to point to a file containing key and client certificate
 to be used in PEM format. In case the key is stored in a separate
 file, the environment variable
 .Ev SSL_CLIENT_KEY_FILE
 can be set to point to the key in PEM format.
 In case the key uses a password, the user will be prompted on standard
 input (see
 .Xr PEM 3 ) .
 .Pp
 By default
 .Nm libfetch
 allows TLSv1 and newer when negotiating the connecting with the remote
 peer.
 You can change this behavior by setting the
 .Ev SSL_ALLOW_SSL2
 and
 .Ev SSL_ALLOW_SSL3
 environment variables to allow SSLv2 and SSLv3, respectively, and
 .Ev SSL_NO_TLS1 ,
 .Ev SSL_NO_TLS1_1 and
 .Ev SSL_NO_TLS1_2
 to disable TLS 1.0, 1.1 and 1.2 respectively.
 .Sh AUTHENTICATION
 Apart from setting the appropriate environment variables and
 specifying the user name and password in the URL or the
 .Vt struct url ,
 the calling program has the option of defining an authentication
 function with the following prototype:
 .Pp
 .Ft int
 .Fn myAuthMethod "struct url *u"
 .Pp
 The callback function should fill in the
 .Fa user
 and
 .Fa pwd
 fields in the provided
 .Vt struct url
 and return 0 on success, or any other value to indicate failure.
 .Pp
 To register the authentication callback, simply set
 .Va fetchAuthMethod
 to point at it.
 The callback will be used whenever a site requires authentication and
 the appropriate environment variables are not set.
 .Pp
 This interface is experimental and may be subject to change.
 .Sh RETURN VALUES
 .Fn fetchParseURL
 returns a pointer to a
 .Vt struct url
 containing the individual components of the URL.
 If it is
 unable to allocate memory, or the URL is syntactically incorrect,
 .Fn fetchParseURL
 returns a NULL pointer.
 .Pp
 The
 .Fn fetchStat
 functions return 0 on success and -1 on failure.
 .Pp
 All other functions return a stream pointer which may be used to
 access the requested document, or NULL if an error occurred.
 .Pp
 The following error codes are defined in
 .In fetch.h :
 .Bl -tag -width 18n
 .It Bq Er FETCH_ABORT
 Operation aborted
 .It Bq Er FETCH_AUTH
 Authentication failed
 .It Bq Er FETCH_DOWN
 Service unavailable
 .It Bq Er FETCH_EXISTS
 File exists
 .It Bq Er FETCH_FULL
 File system full
 .It Bq Er FETCH_INFO
 Informational response
 .It Bq Er FETCH_MEMORY
 Insufficient memory
 .It Bq Er FETCH_MOVED
 File has moved
 .It Bq Er FETCH_NETWORK
 Network error
 .It Bq Er FETCH_OK
 No error
 .It Bq Er FETCH_PROTO
 Protocol error
 .It Bq Er FETCH_RESOLV
 Resolver error
 .It Bq Er FETCH_SERVER
 Server error
 .It Bq Er FETCH_TEMP
 Temporary error
 .It Bq Er FETCH_TIMEOUT
 Operation timed out
 .It Bq Er FETCH_UNAVAIL
 File is not available
 .It Bq Er FETCH_UNKNOWN
 Unknown error
 .It Bq Er FETCH_URL
 Invalid URL
 .El
 .Pp
 The accompanying error message includes a protocol-specific error code
 and message, e.g.\& "File is not available (404 Not Found)"
 .Sh ENVIRONMENT
 .Bl -tag -width ".Ev FETCH_BIND_ADDRESS"
 .It Ev FETCH_BIND_ADDRESS
 Specifies a hostname or IP address to which sockets used for outgoing
 connections will be bound.
 .It Ev FTP_LOGIN
 Default FTP login if none was provided in the URL.
 .It Ev FTP_PASSIVE_MODE
 If set to
 .Ql no ,
 forces the FTP code to use active mode.
 If set to any other value, forces passive mode even if the application
 requested active mode.
 .It Ev FTP_PASSWORD
 Default FTP password if the remote server requests one and none was
 provided in the URL.
 .It Ev FTP_PROXY
 URL of the proxy to use for FTP requests.
 The document part is ignored.
 FTP and HTTP proxies are supported; if no scheme is specified, FTP is
 assumed.
 If the proxy is an FTP proxy,
 .Nm libfetch
 will send
 .Ql user@host
 as user name to the proxy, where
 .Ql user
 is the real user name, and
 .Ql host
 is the name of the FTP server.
 .Pp
 If this variable is set to an empty string, no proxy will be used for
 FTP requests, even if the
 .Ev HTTP_PROXY
 variable is set.
 .It Ev ftp_proxy
 Same as
 .Ev FTP_PROXY ,
 for compatibility.
 .It Ev HTTP_ACCEPT
 Specifies the value of the
 .Va Accept
 header for HTTP requests.
 If empty, no
 .Va Accept
 header is sent.
 The default is
 .Dq */* .
 .It Ev HTTP_AUTH
 Specifies HTTP authorization parameters as a colon-separated list of
 items.
 The first and second item are the authorization scheme and realm
 respectively; further items are scheme-dependent.
 Currently, the
 .Dq basic
 and
 .Dq digest
 authorization methods are supported.
 .Pp
 Both methods require two parameters: the user name and
 password, in that order.
 .Pp
 This variable is only used if the server requires authorization and
 no user name or password was specified in the URL.
 .It Ev HTTP_PROXY
 URL of the proxy to use for HTTP requests.
 The document part is ignored.
 Only HTTP proxies are supported for HTTP requests.
 If no port number is specified, the default is 3128.
 .Pp
 Note that this proxy will also be used for FTP documents, unless the
 .Ev FTP_PROXY
 variable is set.
 .It Ev http_proxy
 Same as
 .Ev HTTP_PROXY ,
 for compatibility.
 .It Ev HTTP_PROXY_AUTH
 Specifies authorization parameters for the HTTP proxy in the same
 format as the
 .Ev HTTP_AUTH
 variable.
 .Pp
 This variable is used if and only if connected to an HTTP proxy, and
 is ignored if a user and/or a password were specified in the proxy
 URL.
 .It Ev HTTP_REFERER
 Specifies the referrer URL to use for HTTP requests.
 If set to
 .Dq auto ,
 the document URL will be used as referrer URL.
 .It Ev HTTP_USER_AGENT
 Specifies the User-Agent string to use for HTTP requests.
 This can be useful when working with HTTP origin or proxy servers that
 differentiate between user agents.
 If defined but empty, no User-Agent header is sent.
 .It Ev NETRC
 Specifies a file to use instead of
 .Pa ~/.netrc
 to look up login names and passwords for FTP sites.
 See
 .Xr ftp 1
 for a description of the file format.
 This feature is experimental.
 .It Ev NO_PROXY
 Either a single asterisk, which disables the use of proxies
 altogether, or a comma- or whitespace-separated list of hosts for
 which proxies should not be used.
 .It Ev no_proxy
 Same as
 .Ev NO_PROXY ,
 for compatibility.
 .It Ev SSL_ALLOW_SSL2
 Allow SSL version 2 when negotiating the connection (not recommended).
 .It Ev SSL_ALLOW_SSL3
 Allow SSL version 3 when negotiating the connection (not recommended).
 .It Ev SSL_CA_CERT_FILE
 CA certificate bundle containing trusted CA certificates.
 Default value:
 .Pa /etc/ssl/cert.pem .
 .It Ev SSL_CA_CERT_PATH
 Path containing trusted CA hashes.
 .It Ev SSL_CLIENT_CERT_FILE
 PEM encoded client certificate/key which will be used in
 client certificate authentication.
 .It Ev SSL_CLIENT_KEY_FILE
 PEM encoded client key in case key and client certificate
 are stored separately.
 .It Ev SSL_CRL_FILE
 File containing certificate revocation list.
 .It Ev SSL_NO_TLS1
 Do not allow TLS version 1.0 when negotiating the connection.
 .It Ev SSL_NO_TLS1_1
 Do not allow TLS version 1.1 when negotiating the connection.
 .It Ev SSL_NO_TLS1_2
 Do not allow TLS version 1.2 when negotiating the connection.
 .It Ev SSL_NO_VERIFY_HOSTNAME
 If set, do not verify that the hostname matches the subject of the
 certificate presented by the server.
 .It Ev SSL_NO_VERIFY_PEER
 If set, do not verify the peer certificate against trusted CAs.
 .El
 .Sh EXAMPLES
 To access a proxy server on
 .Pa proxy.example.com
 port 8080, set the
 .Ev HTTP_PROXY
 environment variable in a manner similar to this:
 .Pp
 .Dl HTTP_PROXY=http://proxy.example.com:8080
 .Pp
 If the proxy server requires authentication, there are
 two options available for passing the authentication data.
 The first method is by using the proxy URL:
 .Pp
 .Dl HTTP_PROXY=http://<user>:<pwd>@proxy.example.com:8080
 .Pp
 The second method is by using the
 .Ev HTTP_PROXY_AUTH
 environment variable:
 .Bd -literal -offset indent
 HTTP_PROXY=http://proxy.example.com:8080
 HTTP_PROXY_AUTH=basic:*:<user>:<pwd>
 .Ed
 .Pp
 To disable the use of a proxy for an HTTP server running on the local
 host, define
 .Ev NO_PROXY
 as follows:
 .Bd -literal -offset indent
 NO_PROXY=localhost,127.0.0.1
 .Ed
 .Pp
 Access HTTPS website without any certificate verification whatsoever:
 .Bd -literal -offset indent
 SSL_NO_VERIFY_PEER=1
 SSL_NO_VERIFY_HOSTNAME=1
 .Ed
 .Pp
 Access HTTPS website using client certificate based authentication
 and a private CA:
 .Bd -literal -offset indent
 SSL_CLIENT_CERT_FILE=/path/to/client.pem
 SSL_CA_CERT_FILE=/path/to/myca.pem
 .Ed
 .Sh SEE ALSO
 .Xr fetch 1 ,
 .Xr ftpio 3 ,
 .Xr ip 4
 .Rs
 .%A J. Postel
 .%A J. K. Reynolds
 .%D October 1985
 .%B File Transfer Protocol
 .%O RFC959
 .Re
 .Rs
 .%A P. Deutsch
 .%A A. Emtage
 .%A A. Marine.
 .%D May 1994
 .%T How to Use Anonymous FTP
 .%O RFC1635
 .Re
 .Rs
 .%A T. Berners-Lee
 .%A L. Masinter
 .%A M. McCahill
 .%D December 1994
 .%T Uniform Resource Locators (URL)
 .%O RFC1738
 .Re
 .Rs
 .%A R. Fielding
 .%A J. Gettys
 .%A J. Mogul
 .%A H. Frystyk
 .%A L. Masinter
 .%A P. Leach
 .%A T. Berners-Lee
 .%D January 1999
 .%B Hypertext Transfer Protocol -- HTTP/1.1
 .%O RFC2616
 .Re
 .Rs
 .%A J. Franks
 .%A P. Hallam-Baker
 .%A J. Hostetler
 .%A S. Lawrence
 .%A P. Leach
 .%A A. Luotonen
 .%A L. Stewart
 .%D June 1999
 .%B HTTP Authentication: Basic and Digest Access Authentication
 .%O RFC2617
 .Re
 .Sh HISTORY
 The
 .Nm fetch
 library first appeared in
 .Fx 3.0 .
 .Sh AUTHORS
 .An -nosplit
 The
 .Nm fetch
 library was mostly written by
 .An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org
 with numerous suggestions and contributions from
 .An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org ,
 .An Eugene Skepner Aq Mt eu@qub.com ,
 .An Hajimu Umemoto Aq Mt ume@FreeBSD.org ,
 .An Henry Whincup Aq Mt henry@techiebod.com ,
 .An Jukka A. Ukkonen Aq Mt jau@iki.fi ,
 .An Jean-Fran\(,cois Dockes Aq Mt jf@dockes.org ,
 .An Michael Gmelin Aq Mt freebsd@grem.de
 and others.
 It replaces the older
 .Nm ftpio
 library written by
 .An Poul-Henning Kamp Aq Mt phk@FreeBSD.org
 and
 .An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org .
 .Pp
 This manual page was written by
 .An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org
 and
 .An Michael Gmelin Aq Mt freebsd@grem.de .
 .Sh BUGS
 Some parts of the library are not yet implemented.
 The most notable
 examples of this are
 .Fn fetchPutHTTP ,
 .Fn fetchListHTTP ,
 .Fn fetchListFTP
 and FTP proxy support.
 .Pp
 There is no way to select a proxy at run-time other than setting the
 .Ev HTTP_PROXY
 or
 .Ev FTP_PROXY
 environment variables as appropriate.
 .Pp
 .Nm libfetch
 does not understand or obey 305 (Use Proxy) replies.
 .Pp
 Error numbers are unique only within a certain context; the error
 codes used for FTP and HTTP overlap, as do those used for resolver and
 system errors.
 For instance, error code 202 means "Command not
 implemented, superfluous at this site" in an FTP context and
 "Accepted" in an HTTP context.
 .Pp
 .Fn fetchStatFTP
 does not check that the result of an MDTM command is a valid date.
 .Pp
 In case password protected keys are used for client certificate based
 authentication the user is prompted for the password on each and every
 fetch operation.
 .Pp
 The man page is incomplete, poorly written and produces badly
 formatted text.
 .Pp
 The error reporting mechanism is unsatisfactory.
 .Pp
 Some parts of the code are not fully reentrant.

deleted external/libfetch/fetch.c

@@ -1,479 +0,0 @@

 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 1998-2004 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "bsd_compat.h"
 __FBSDID("$FreeBSD: head/lib/libfetch/fetch.c 357212 2020-01-28 18:37:18Z gordon $");
 #include <sys/param.h>
 #include <netinet/in.h>
 #include <errno.h>
 #include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "fetch.h"
 #include "common.h"
 auth_t	 fetchAuthMethod;
 int	 fetchLastErrCode;
 char	 fetchLastErrString[MAXERRSTRING];
 int	 fetchTimeout;
 int	 fetchRestartCalls = 1;
 int	 fetchDebug;
 /*** Local data **************************************************************/
 /*
  * Error messages for parser errors
  */
 #define URL_MALFORMED		1
 #define URL_BAD_SCHEME		2
 #define URL_BAD_PORT		3
 static struct fetcherr url_errlist[] = {
 	{ URL_MALFORMED,	FETCH_URL,	"Malformed URL" },
 	{ URL_BAD_SCHEME,	FETCH_URL,	"Invalid URL scheme" },
 	{ URL_BAD_PORT,		FETCH_URL,	"Invalid server port" },
 	{ -1,			FETCH_UNKNOWN,	"Unknown parser error" }
 };
 /*** Public API **************************************************************/
 /*
  * Select the appropriate protocol for the URL scheme, and return a
  * read-only stream connected to the document referenced by the URL.
  * Also fill out the struct url_stat.
  */
 FILE *
 fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
 {
 	if (us != NULL) {
 		us->size = -1;
 		us->atime = us->mtime = 0;
 	}
 	if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchXGetHTTP(URL, us, flags));
 	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchXGetHTTP(URL, us, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
 }
 /*
  * Select the appropriate protocol for the URL scheme, and return a
  * read-only stream connected to the document referenced by the URL.
  */
 FILE *
 fetchGet(struct url *URL, const char *flags)
 {
 	return (fetchXGet(URL, NULL, flags));
 }
 /*
  * Select the appropriate protocol for the URL scheme, and return a
  * write-only stream connected to the document referenced by the URL.
  */
 FILE *
 fetchPut(struct url *URL, const char *flags)
 {
 	if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchPutHTTP(URL, flags));
 	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchPutHTTP(URL, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
 }
 /*
  * Select the appropriate protocol for the URL scheme, and return the
  * size of the document referenced by the URL if it exists.
  */
 int
 fetchStat(struct url *URL, struct url_stat *us, const char *flags)
 {
 	if (us != NULL) {
 		us->size = -1;
 		us->atime = us->mtime = 0;
 	}
 	if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchStatHTTP(URL, us, flags));
 	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchStatHTTP(URL, us, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (-1);
 }
 /*
  * Select the appropriate protocol for the URL scheme, and return a
  * list of files in the directory pointed to by the URL.
  */
 struct url_ent *
 fetchList(struct url *URL, const char *flags)
 {
 	if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchListHTTP(URL, flags));
 	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchListHTTP(URL, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchXGet().
  */
 FILE *
 fetchXGetURL(const char *URL, struct url_stat *us, const char *flags)
 {
 	struct url *u;
 	FILE *f;
 	if ((u = fetchParseURL(URL)) == NULL)
 		return (NULL);
 	f = fetchXGet(u, us, flags);
 	fetchFreeURL(u);
 	return (f);
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchGet().
  */
 FILE *
 fetchGetURL(const char *URL, const char *flags)
 {
 	return (fetchXGetURL(URL, NULL, flags));
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchPut().
  */
 FILE *
 fetchPutURL(const char *URL, const char *flags)
 {
 	struct url *u;
 	FILE *f;
 	if ((u = fetchParseURL(URL)) == NULL)
 		return (NULL);
 	f = fetchPut(u, flags);
 	fetchFreeURL(u);
 	return (f);
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchStat().
  */
 int
 fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
 {
 	struct url *u;
 	int s;
 	if ((u = fetchParseURL(URL)) == NULL)
 		return (-1);
 	s = fetchStat(u, us, flags);
 	fetchFreeURL(u);
 	return (s);
 }
 /*
  * Attempt to parse the given URL; if successful, call fetchList().
  */
 struct url_ent *
 fetchListURL(const char *URL, const char *flags)
 {
 	struct url *u;
 	struct url_ent *ue;
 	if ((u = fetchParseURL(URL)) == NULL)
 		return (NULL);
 	ue = fetchList(u, flags);
 	fetchFreeURL(u);
 	return (ue);
 }
 /*
  * Make a URL
  */
 struct url *
 fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
     const char *user, const char *pwd)
 {
 	struct url *u;
 	if (!scheme || (!host && !doc)) {
 		url_seterr(URL_MALFORMED);
 		return (NULL);
 	}
 	if (port < 0 || port > 65535) {
 		url_seterr(URL_BAD_PORT);
 		return (NULL);
 	}
 	/* allocate struct url */
 	if ((u = calloc(1, sizeof(*u))) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	u->netrcfd = -1;
 	if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
 		fetch_syserr();
 		free(u);
 		return (NULL);
 	}
 #define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)
 	seturl(scheme);
 	seturl(host);
 	seturl(user);
 	seturl(pwd);
 #undef seturl
 	u->port = port;
 	return (u);
 }
 /*
  * Return value of the given hex digit.
  */
 static int
 fetch_hexval(char ch)
 {
 	if (ch >= '0' && ch <= '9')
 		return (ch - '0');
 	else if (ch >= 'a' && ch <= 'f')
 		return (ch - 'a' + 10);
 	else if (ch >= 'A' && ch <= 'F')
 		return (ch - 'A' + 10);
 	return (-1);
 }
 /*
  * Decode percent-encoded URL component from src into dst, stopping at end
  * of string, or at @ or : separators.  Returns a pointer to the unhandled
  * part of the input string (null terminator, @, or :).  No terminator is
  * written to dst (it is the caller's responsibility).
  */
 static const char *
 fetch_pctdecode(char *dst, const char *src, size_t dlen)
 {
 	int d1, d2;
 	char c;
 	const char *s;
 	for (s = src; *s != '\0' && *s != '@' && *s != ':'; s++) {
 		if (s[0] == '%' && (d1 = fetch_hexval(s[1])) >= 0 &&
 		    (d2 = fetch_hexval(s[2])) >= 0 && (d1 > 0 || d2 > 0)) {
 			c = d1 << 4 | d2;
 			s += 2;
 		} else {
 			c = *s;
 		}
 		if (dlen-- > 0)
 			*dst++ = c;
 		else
 			return (NULL);
 	}
 	return (s);
 }
 /*
  * Split an URL into components. URL syntax is:
  * [method:/][/[user[:pwd]@]host[:port]/][document]
  * This almost, but not quite, RFC1738 URL syntax.
  */
 struct url *
 fetchParseURL(const char *URL)
 {
 	char *doc;
 	const char *p, *q;
 	struct url *u;
 	int i, n;
 	/* allocate struct url */
 	if ((u = calloc(1, sizeof(*u))) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	u->netrcfd = -1;
 	/* scheme name */
 	if ((p = strstr(URL, ":/"))) {
                 if (p - URL > URL_SCHEMELEN)
                         goto ouch;
                 for (i = 0; URL + i < p; i++)
                         u->scheme[i] = tolower((unsigned char)URL[i]);
 		URL = ++p;
 		/*
 		 * Only one slash: no host, leave slash as part of document
 		 * Two slashes: host follows, strip slashes
 		 */
 		if (URL[1] == '/')
 			URL = (p += 2);
 	} else {
 		p = URL;
 	}
 	if (!*URL || *URL == '/' || *URL == '.' ||
 	    (u->scheme[0] == '\0' &&
 		strchr(URL, '/') == NULL && strchr(URL, ':') == NULL))
 		goto nohost;
 	p = strpbrk(URL, "/@");
 	if (p && *p == '@') {
 		/* username */
 		q = fetch_pctdecode(u->user, URL, URL_USERLEN);
 		if (q == NULL)
 			goto ouch;
 		/* password */
 		if (*q == ':') {
 			q = fetch_pctdecode(u->pwd, q + 1, URL_PWDLEN);
 			if (q == NULL)
 				goto ouch;
 		}
 		p++;
 	} else {
 		p = URL;
 	}
 	/* hostname */
 	if (*p == '[') {
 		q = p + 1 + strspn(p + 1, ":0123456789ABCDEFabcdef");
 		if (*q++ != ']')
 			goto ouch;
 	} else {
 		/* valid characters in a DNS name */
 		q = p + strspn(p, "-." "0123456789"
 		    "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "_"
 		    "abcdefghijklmnopqrstuvwxyz");
 	}
 	if ((*q != '\0' && *q != '/' && *q != ':') || q - p > MAXHOSTNAMELEN)
 		goto ouch;
 	for (i = 0; p + i < q; i++)
 		u->host[i] = tolower((unsigned char)p[i]);
 	u->host[i] = '\0';
 	p = q;
 	/* port */
 	if (*p == ':') {
 		for (n = 0, q = ++p; *q && (*q != '/'); q++) {
 			if (*q >= '0' && *q <= '9' && n < INT_MAX / 10) {
 				n = n * 10 + (*q - '0');
 			} else {
 				/* invalid port */
 				url_seterr(URL_BAD_PORT);
 				goto ouch;
 			}
 		}
 		/* pkg extension allow for ssh compat */
 		/*if (n < 1 || n > IPPORT_MAX) */
 #ifndef IPPORT_MAX
 #define IPPORT_MAX 65535
 #endif
 		if (n < 0 || n > IPPORT_MAX)
 			goto ouch;
 		u->port = n;
 		p = q;
 	}
 nohost:
 	/* document */
 	if (!*p)
 		p = "/";
 	if (strcmp(u->scheme, SCHEME_HTTP) == 0 ||
 	    strcmp(u->scheme, SCHEME_HTTPS) == 0) {
 		const char hexnums[] = "0123456789abcdef";
 		/* percent-escape whitespace. */
 		if ((doc = malloc(strlen(p) * 3 + 1)) == NULL) {
 			fetch_syserr();
 			goto ouch;
 		}
 		u->doc = doc;
 		while (*p != '\0') {
 			if (!isspace((unsigned char)*p)) {
 				*doc++ = *p++;
 			} else {
 				*doc++ = '%';
 				*doc++ = hexnums[((unsigned int)*p) >> 4];
 				*doc++ = hexnums[((unsigned int)*p) & 0xf];
 				p++;
 			}
 		}
 		*doc = '\0';
 	} else if ((u->doc = strdup(p)) == NULL) {
 		fetch_syserr();
 		goto ouch;
 	}
 	DEBUGF("scheme:   \"%s\"\n"
 	    "user:     \"%s\"\n"
 	    "password: \"%s\"\n"
 	    "host:     \"%s\"\n"
 	    "port:     \"%d\"\n"
 	    "document: \"%s\"\n",
 	    u->scheme, u->user, u->pwd,
 	    u->host, u->port, u->doc);
 	return (u);
 ouch:
 	free(u);
 	return (NULL);
 }
 /*
  * Free a URL
  */
 void
 fetchFreeURL(struct url *u)
 {
 	free(u->doc);
 	free(u);
 }

deleted external/libfetch/fetch.h

@@ -1,160 +0,0 @@

 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 1998-2004 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD: head/lib/libfetch/fetch.h 326219 2017-11-26 02:00:33Z pfg $
  */
 #ifndef _FETCH_H_INCLUDED
 #define _FETCH_H_INCLUDED
 #define _LIBFETCH_VER "libfetch/2.0"
 #define URL_SCHEMELEN 16
 #define URL_USERLEN 256
 #define URL_PWDLEN 256
 struct url {
 	char		 scheme[URL_SCHEMELEN+1];
 	char		 user[URL_USERLEN+1];
 	char		 pwd[URL_PWDLEN+1];
 	char		 host[MAXHOSTNAMELEN+1];
 	int		 port;
 	char		*doc;
 	off_t		 offset;
 	size_t		 length;
 	time_t		 ims_time;
 	int		 netrcfd;
 };
 struct url_stat {
 	off_t		 size;
 	time_t		 atime;
 	time_t		 mtime;
 };
 struct url_ent {
 	char		 name[PATH_MAX];
 	struct url_stat	 stat;
 };
 /* Recognized schemes */
 #define SCHEME_FTP	"ftp"
 #define SCHEME_HTTP	"http"
 #define SCHEME_HTTPS	"https"
 #define SCHEME_FILE	"file"
 /* Error codes */
 #define	FETCH_ABORT	 1
 #define	FETCH_AUTH	 2
 #define	FETCH_DOWN	 3
 #define	FETCH_EXISTS	 4
 #define	FETCH_FULL	 5
 #define	FETCH_INFO	 6
 #define	FETCH_MEMORY	 7
 #define	FETCH_MOVED	 8
 #define	FETCH_NETWORK	 9
 #define	FETCH_OK	10
 #define	FETCH_PROTO	11
 #define	FETCH_RESOLV	12
 #define	FETCH_SERVER	13
 #define	FETCH_TEMP	14
 #define	FETCH_TIMEOUT	15
 #define	FETCH_UNAVAIL	16
 #define	FETCH_UNKNOWN	17
 #define	FETCH_URL	18
 #define	FETCH_VERBOSE	19
 #ifdef __cplusplus
 extern "C" {
 #endif
 /* FILE-specific functions */
 FILE		*fetchXGetFile(struct url *, struct url_stat *, const char *);
 FILE		*fetchGetFile(struct url *, const char *);
 FILE		*fetchPutFile(struct url *, const char *);
 int		 fetchStatFile(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListFile(struct url *, const char *);
 /* HTTP-specific functions */
 FILE		*fetchXGetHTTP(struct url *, struct url_stat *, const char *);
 FILE		*fetchGetHTTP(struct url *, const char *);
 FILE		*fetchPutHTTP(struct url *, const char *);
 int		 fetchStatHTTP(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListHTTP(struct url *, const char *);
 FILE		*fetchReqHTTP(struct url *, const char *, const char *,
 		    const char *, const char *);
 /* FTP-specific functions */
 FILE		*fetchXGetFTP(struct url *, struct url_stat *, const char *);
 FILE		*fetchGetFTP(struct url *, const char *);
 FILE		*fetchPutFTP(struct url *, const char *);
 int		 fetchStatFTP(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListFTP(struct url *, const char *);
 /* Generic functions */
 FILE		*fetchXGetURL(const char *, struct url_stat *, const char *);
 FILE		*fetchGetURL(const char *, const char *);
 FILE		*fetchPutURL(const char *, const char *);
 int		 fetchStatURL(const char *, struct url_stat *, const char *);
 struct url_ent	*fetchListURL(const char *, const char *);
 FILE		*fetchXGet(struct url *, struct url_stat *, const char *);
 FILE		*fetchGet(struct url *, const char *);
 FILE		*fetchPut(struct url *, const char *);
 int		 fetchStat(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchList(struct url *, const char *);
 /* URL parsing */
 struct url	*fetchMakeURL(const char *, const char *, int,
 		     const char *, const char *, const char *);
 struct url	*fetchParseURL(const char *);
 void		 fetchFreeURL(struct url *);
 #ifdef __cplusplus
 }
 #endif
 /* Authentication */
 typedef int (*auth_t)(struct url *);
 extern auth_t		 fetchAuthMethod;
 /* Last error code */
 extern int		 fetchLastErrCode;
 #define MAXERRSTRING 256
 extern char		 fetchLastErrString[MAXERRSTRING];
 /* I/O timeout */
 extern int		 fetchTimeout;
 /* Restart interrupted syscalls */
 extern int		 fetchRestartCalls;
 /* Extra verbosity */
 extern int		 fetchDebug;
 #endif

deleted external/libfetch/file.c

@@ -1,156 +0,0 @@

 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "bsd_compat.h"
 __FBSDID("$FreeBSD: head/lib/libfetch/file.c 326219 2017-11-26 02:00:33Z pfg $");
 #include <sys/param.h>
 #include <sys/stat.h>
 #include <dirent.h>
 #include <fcntl.h>
 #include <stdio.h>
 #include <string.h>
 #include "fetch.h"
 #include "common.h"
 FILE *
 fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 {
 	FILE *f;
 	if (us && fetchStatFile(u, us, flags) == -1)
 		return (NULL);
 	f = fopen(u->doc, "re");
 	if (f == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	if (u->offset && fseeko(f, u->offset, SEEK_SET) == -1) {
 		fclose(f);
 		fetch_syserr();
 		return (NULL);
 	}
 	return (f);
 }
 FILE *
 fetchGetFile(struct url *u, const char *flags)
 {
 	return (fetchXGetFile(u, NULL, flags));
 }
 FILE *
 fetchPutFile(struct url *u, const char *flags)
 {
 	FILE *f;
 	if (CHECK_FLAG('a'))
 		f = fopen(u->doc, "ae");
 	else
 		f = fopen(u->doc, "w+e");
 	if (f == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	if (u->offset && fseeko(f, u->offset, SEEK_SET) == -1) {
 		fclose(f);
 		fetch_syserr();
 		return (NULL);
 	}
 	return (f);
 }
 static int
 fetch_stat_file(const char *fn, struct url_stat *us)
 {
 	struct stat sb;
 	us->size = -1;
 	us->atime = us->mtime = 0;
 	if (stat(fn, &sb) == -1) {
 		fetch_syserr();
 		return (-1);
 	}
 	us->size = sb.st_size;
 	us->atime = sb.st_atime;
 	us->mtime = sb.st_mtime;
 	return (0);
 }
 int
 fetchStatFile(struct url *u, struct url_stat *us, const char *flags __unused)
 {
 	return (fetch_stat_file(u->doc, us));
 }
 struct url_ent *
 fetchListFile(struct url *u, const char *flags __unused)
 {
 	struct dirent *de;
 	struct url_stat us;
 	struct url_ent *ue;
 	int size, len;
 	char fn[PATH_MAX], *p;
 	DIR *dir;
 	int l;
 	if ((dir = opendir(u->doc)) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	ue = NULL;
 	strncpy(fn, u->doc, sizeof(fn) - 2);
 	fn[sizeof(fn) - 2] = 0;
 	strcat(fn, "/");
 	p = strchr(fn, 0);
 	l = sizeof(fn) - strlen(fn) - 1;
 	while ((de = readdir(dir)) != NULL) {
 		strncpy(p, de->d_name, l - 1);
 		p[l - 1] = 0;
 		if (fetch_stat_file(fn, &us) == -1)
 			/* should I return a partial result, or abort? */
 			break;
 		fetch_add_entry(&ue, &size, &len, de->d_name, &us);
 	}
 	closedir(dir);
 	return (ue);
 }

deleted external/libfetch/http.c

@@ -1,2145 +0,0 @@

 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 2000-2014 Dag-Erling Smørgrav
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "bsd_compat.h"
 __FBSDID("$FreeBSD: head/lib/libfetch/http.c 351573 2019-08-28 17:01:28Z markj $");
 /*
  * The following copyright applies to the base64 code:
  *
  *-
  * Copyright 1997 Massachusetts Institute of Technology
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
  * granted, provided that both the above copyright notice and this
  * permission notice appear in all copies, that both the above
  * copyright notice and this permission notice appear in all
  * supporting documentation, and that the name of M.I.T. not be used
  * in advertising or publicity pertaining to distribution of the
  * software without specific, written prior permission.  M.I.T. makes
  * no representations about the suitability of this software for any
  * purpose.  It is provided "as is" without express or implied
  * warranty.
  *
  * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
  * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
  * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 #include <sys/param.h>
 #include <sys/socket.h>
 #include <sys/time.h>
 #include <ctype.h>
 #include <err.h>
 #include <errno.h>
 #include <locale.h>
 #include <netdb.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
 #ifdef WITH_SSL
 #include <openssl/md5.h>
 #define MD5Init(c) MD5_Init(c)
 #define MD5Update(c, data, len) MD5_Update(c, data, len)
 #define MD5Final(md, c) MD5_Final(md, c)
 #else
 #include <md5.h>
 #endif
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include "fetch.h"
 #include "common.h"
 #include "httperr.h"
 /* Maximum number of redirects to follow */
 #define MAX_REDIRECT 20
 /* Symbolic names for reply codes we care about */
 #define HTTP_OK			200
 #define HTTP_PARTIAL		206
 #define HTTP_MOVED_PERM		301
 #define HTTP_MOVED_TEMP		302
 #define HTTP_SEE_OTHER		303
 #define HTTP_NOT_MODIFIED	304
 #define HTTP_USE_PROXY		305
 #define HTTP_TEMP_REDIRECT	307
 #define HTTP_PERM_REDIRECT	308
 #define HTTP_NEED_AUTH		401
 #define HTTP_NEED_PROXY_AUTH	407
 #define HTTP_BAD_RANGE		416
 #define HTTP_PROTOCOL_ERROR	999
 #define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \
 			    || (xyz) == HTTP_MOVED_TEMP \
 			    || (xyz) == HTTP_TEMP_REDIRECT \
 			    || (xyz) == HTTP_PERM_REDIRECT \
 			    || (xyz) == HTTP_USE_PROXY \
 			    || (xyz) == HTTP_SEE_OTHER)
 #define HTTP_ERROR(xyz) ((xyz) >= 400 && (xyz) <= 599)
 /*****************************************************************************
  * I/O functions for decoding chunked streams
  */
 struct httpio
 {
 	conn_t		*conn;		/* connection */
 	int		 chunked;	/* chunked mode */
 	char		*buf;		/* chunk buffer */
 	size_t		 bufsize;	/* size of chunk buffer */
 	size_t		 buflen;	/* amount of data currently in buffer */
 	size_t		 bufpos;	/* current read offset in buffer */
 	int		 eof;		/* end-of-file flag */
 	int		 error;		/* error flag */
 	size_t		 chunksize;	/* remaining size of current chunk */
 #ifndef NDEBUG
 	size_t		 total;
 #endif
 };
 /*
  * Get next chunk header
  */
 static int
 http_new_chunk(struct httpio *io)
 {
 	char *p;
 	if (fetch_getln(io->conn) == -1)
 		return (-1);
 	if (io->conn->buflen < 2 || !isxdigit((unsigned char)*io->conn->buf))
 		return (-1);
 	for (p = io->conn->buf; *p && !isspace((unsigned char)*p); ++p) {
 		if (*p == ';')
 			break;
 		if (!isxdigit((unsigned char)*p))
 			return (-1);
 		if (isdigit((unsigned char)*p)) {
 			io->chunksize = io->chunksize * 16 +
 			    *p - '0';
 		} else {
 			io->chunksize = io->chunksize * 16 +
 + tolower((unsigned char)*p) - 'a';
 		}
 	}
 #ifndef NDEBUG
 	if (fetchDebug) {
 		io->total += io->chunksize;
 		if (io->chunksize == 0)
 			fprintf(stderr, "%s(): end of last chunk\n", __func__);
 		else
 			fprintf(stderr, "%s(): new chunk: %lu (%lu)\n",
 			    __func__, (unsigned long)io->chunksize,
 			    (unsigned long)io->total);
 	}
 #endif
 	return (io->chunksize);
 }
 /*
  * Grow the input buffer to at least len bytes
  */
 static inline int
 http_growbuf(struct httpio *io, size_t len)
 {
 	char *tmp;
 	if (io->bufsize >= len)
 		return (0);
 	if ((tmp = realloc(io->buf, len)) == NULL)
 		return (-1);
 	io->buf = tmp;
 	io->bufsize = len;
 	return (0);
 }
 /*
  * Fill the input buffer, do chunk decoding on the fly
  */
 static ssize_t
 http_fillbuf(struct httpio *io, size_t len)
 {
 	ssize_t nbytes;
 	char ch;
 	if (io->error)
 		return (-1);
 	if (io->eof)
 		return (0);
 	/* not chunked: just fetch the requested amount */
 	if (io->chunked == 0) {
 		if (http_growbuf(io, len) == -1)
 			return (-1);
 		if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) {
 			io->error = errno;
 			return (-1);
 		}
 		io->buflen = nbytes;
 		io->bufpos = 0;
 		return (io->buflen);
 	}
 	/* chunked, but we ran out: get the next chunk header */
 	if (io->chunksize == 0) {
 		switch (http_new_chunk(io)) {
 		case -1:
 			io->error = EPROTO;
 			return (-1);
 		case 0:
 			io->eof = 1;
 			return (0);
 		}
 	}
 	/* fetch the requested amount, but no more than the current chunk */
 	if (len > io->chunksize)
 		len = io->chunksize;
 	if (http_growbuf(io, len) == -1)
 		return (-1);
 	if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) {
 		io->error = errno;
 		return (-1);
 	}
 	io->bufpos = 0;
 	io->buflen = nbytes;
 	io->chunksize -= nbytes;
 	if (io->chunksize == 0) {
 		if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' ||
 		    fetch_read(io->conn, &ch, 1) != 1 || ch != '\n')
 			return (-1);
 	}
 	return (io->buflen);
 }
 /*
  * Read function
  */
 static int
 http_readfn(void *v, char *buf, int len)
 {
 	struct httpio *io = (struct httpio *)v;
 	int rlen;
 	if (io->error)
 		return (-1);
 	if (io->eof)
 		return (0);
 	/* empty buffer */
 	if (!io->buf || io->bufpos == io->buflen) {
 		if ((rlen = http_fillbuf(io, len)) < 0) {
 			if ((errno = io->error) == EINTR)
 				io->error = 0;
 			return (-1);
 		} else if (rlen == 0) {
 			return (0);
 		}
 	}
 	rlen = io->buflen - io->bufpos;
 	if (len < rlen)
 		rlen = len;
 	memcpy(buf, io->buf + io->bufpos, rlen);
 	io->bufpos += rlen;
 	return (rlen);
 }
 /*
  * Write function
  */
 static int
 http_writefn(void *v, const char *buf, int len)
 {
 	struct httpio *io = (struct httpio *)v;
 	return (fetch_write(io->conn, buf, len));
 }
 /*
  * Close function
  */
 static int
 http_closefn(void *v)
 {
 	struct httpio *io = (struct httpio *)v;
 	int r;
 	r = fetch_close(io->conn);
 	if (io->buf)
 		free(io->buf);
 	free(io);
 	return (r);
 }
 /*
  * Wrap a file descriptor up
  */
 static FILE *
 http_funopen(conn_t *conn, int chunked)
 {
 	struct httpio *io;
 	FILE *f;
 	if ((io = calloc(1, sizeof(*io))) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
 	io->conn = conn;
 	io->chunked = chunked;
 	f = funopen(io, http_readfn, http_writefn, NULL, http_closefn);
 	if (f == NULL) {
 		fetch_syserr();
 		free(io);
 		return (NULL);
 	}
 	return (f);
 }
 /*****************************************************************************
  * Helper functions for talking to the server and parsing its replies
  */
 /* Header types */
 typedef enum {
 	hdr_syserror = -2,
 	hdr_error = -1,
 	hdr_end = 0,
 	hdr_unknown = 1,
 	hdr_content_length,
 	hdr_content_range,
 	hdr_last_modified,
 	hdr_location,
 	hdr_transfer_encoding,
 	hdr_www_authenticate,
 	hdr_proxy_authenticate,
 } hdr_t;
 /* Names of interesting headers */
 static struct {
 	hdr_t		 num;
 	const char	*name;
 } hdr_names[] = {
 	{ hdr_content_length,		"Content-Length" },
 	{ hdr_content_range,		"Content-Range" },
 	{ hdr_last_modified,		"Last-Modified" },
 	{ hdr_location,			"Location" },
 	{ hdr_transfer_encoding,	"Transfer-Encoding" },
 	{ hdr_www_authenticate,		"WWW-Authenticate" },
 	{ hdr_proxy_authenticate,	"Proxy-Authenticate" },
 	{ hdr_unknown,			NULL },
 };
 /*
  * Send a formatted line; optionally echo to terminal
  */
 static int
 http_cmd(conn_t *conn, const char *fmt, ...)
 {
 	va_list ap;
 	size_t len;
 	char *msg;
 	int r;
 	va_start(ap, fmt);
 	len = vasprintf(&msg, fmt, ap);
 	va_end(ap);
 	if (msg == NULL) {
 		errno = ENOMEM;
 		fetch_syserr();
 		return (-1);
 	}
 	r = fetch_putln(conn, msg, len);
 	free(msg);
 	if (r == -1) {
 		fetch_syserr();
 		return (-1);
 	}
 	return (0);
 }
 /*
  * Get and parse status line
  */
 static int
 http_get_reply(conn_t *conn)
 {
 	char *p;
 	if (fetch_getln(conn) == -1)
 		return (-1);
 	/*
 	 * A valid status line looks like "HTTP/m.n xyz reason" where m
 	 * and n are the major and minor protocol version numbers and xyz
 	 * is the reply code.
 	 * Unfortunately, there are servers out there (NCSA 1.5.1, to name
 	 * just one) that do not send a version number, so we can't rely
 	 * on finding one, but if we do, insist on it being 1.0 or 1.1.
 	 * We don't care about the reason phrase.
 	 */
 	if (strncmp(conn->buf, "HTTP", 4) != 0)
 		return (HTTP_PROTOCOL_ERROR);
 	p = conn->buf + 4;
 	if (*p == '/') {
 		if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1'))
 			return (HTTP_PROTOCOL_ERROR);
 		p += 4;
 	}
 	if (*p != ' ' ||
 	    !isdigit((unsigned char)p[1]) ||
 	    !isdigit((unsigned char)p[2]) ||
 	    !isdigit((unsigned char)p[3]))
 		return (HTTP_PROTOCOL_ERROR);
 	conn->err = (p[1] - '0') * 100 + (p[2] - '0') * 10 + (p[3] - '0');
 	return (conn->err);
 }
 /*
  * Check a header; if the type matches the given string, return a pointer
  * to the beginning of the value.
  */
 static const char *
 http_match(const char *str, const char *hdr)
 {
 	while (*str && *hdr &&
 	    tolower((unsigned char)*str++) == tolower((unsigned char)*hdr++))
 		/* nothing */;
 	if (*str || *hdr != ':')
 		return (NULL);
 	while (*hdr && isspace((unsigned char)*++hdr))
 		/* nothing */;
 	return (hdr);
 }
 /*
  * Get the next header and return the appropriate symbolic code.  We
  * need to read one line ahead for checking for a continuation line
  * belonging to the current header (continuation lines start with
  * white space).
  *
  * We get called with a fresh line already in the conn buffer, either
  * from the previous http_next_header() invocation, or, the first
  * time, from a fetch_getln() performed by our caller.
  *
  * This stops when we encounter an empty line (we dont read beyond the header
  * area).
  *
  * Note that the "headerbuf" is just a place to return the result. Its
  * contents are not used for the next call. This means that no cleanup
  * is needed when ie doing another connection, just call the cleanup when
  * fully done to deallocate memory.
  */
 /* Limit the max number of continuation lines to some reasonable value */
 #define HTTP_MAX_CONT_LINES 10
 /* Place into which to build a header from one or several lines */
 typedef struct {
 	char	*buf;		/* buffer */
 	size_t	 bufsize;	/* buffer size */
 	size_t	 buflen;	/* length of buffer contents */
 } http_headerbuf_t;
 static void
 init_http_headerbuf(http_headerbuf_t *buf)
 {
 	buf->buf = NULL;
 	buf->bufsize = 0;
 	buf->buflen = 0;
 }
 static void
 clean_http_headerbuf(http_headerbuf_t *buf)
 {
 	if (buf->buf)
 		free(buf->buf);
 	init_http_headerbuf(buf);
 }
 /* Remove whitespace at the end of the buffer */
 static void
 http_conn_trimright(conn_t *conn)
 {
 	while (conn->buflen &&
 	       isspace((unsigned char)conn->buf[conn->buflen - 1]))
 		conn->buflen--;
 	conn->buf[conn->buflen] = '\0';
 }
 static hdr_t
 http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p)
 {
 	unsigned int i, len;
 	/*
 	 * Have to do the stripping here because of the first line. So
 	 * it's done twice for the subsequent lines. No big deal
 	 */
 	http_conn_trimright(conn);
 	if (conn->buflen == 0)
 		return (hdr_end);
 	/* Copy the line to the headerbuf */
 	if (hbuf->bufsize < conn->buflen + 1) {
 		if ((hbuf->buf = realloc(hbuf->buf, conn->buflen + 1)) == NULL)
 			return (hdr_syserror);
 		hbuf->bufsize = conn->buflen + 1;
 	}
 	strcpy(hbuf->buf, conn->buf);
 	hbuf->buflen = conn->buflen;
 	/*
 	 * Fetch possible continuation lines. Stop at 1st non-continuation
 	 * and leave it in the conn buffer
 	 */
 	for (i = 0; i < HTTP_MAX_CONT_LINES; i++) {
 		if (fetch_getln(conn) == -1)
 			return (hdr_syserror);
 		/*
 		 * Note: we carry on the idea from the previous version
 		 * that a pure whitespace line is equivalent to an empty
 		 * one (so it's not continuation and will be handled when
 		 * we are called next)
 		 */
 		http_conn_trimright(conn);
 		if (conn->buf[0] != ' ' && conn->buf[0] != "\t"[0])
 			break;
 		/* Got a continuation line. Concatenate to previous */
 		len = hbuf->buflen + conn->buflen;
 		if (hbuf->bufsize < len + 1) {
 			len *= 2;
 			if ((hbuf->buf = realloc(hbuf->buf, len + 1)) == NULL)
 				return (hdr_syserror);
 			hbuf->bufsize = len + 1;
 		}
 		strcpy(hbuf->buf + hbuf->buflen, conn->buf);
 		hbuf->buflen += conn->buflen;
 	}
 	/*
 	 * We could check for malformed headers but we don't really care.
 	 * A valid header starts with a token immediately followed by a
 	 * colon; a token is any sequence of non-control, non-whitespace
 	 * characters except "()<>@,;:\\\"{}".
 	 */
 	for (i = 0; hdr_names[i].num != hdr_unknown; i++)
 		if ((*p = http_match(hdr_names[i].name, hbuf->buf)) != NULL)
 			return (hdr_names[i].num);
 	return (hdr_unknown);
 }
 /**************************
  * [Proxy-]Authenticate header parsing
  */
 /*
  * Read doublequote-delimited string into output buffer obuf (allocated
  * by caller, whose responsibility it is to ensure that it's big enough)
  * cp points to the first char after the initial '"'
  * Handles \ quoting
  * Returns pointer to the first char after the terminating double quote, or
  * NULL for error.
  */
 static const char *
 http_parse_headerstring(const char *cp, char *obuf)
 {
 	for (;;) {
 		switch (*cp) {
 		case 0: /* Unterminated string */
 			*obuf = 0;
 			return (NULL);
 		case '"': /* Ending quote */
 			*obuf = 0;
 			return (++cp);
 		case '\\':
 			if (*++cp == 0) {
 				*obuf = 0;
 				return (NULL);
 			}
 			/* FALLTHROUGH */
 		default:
 			*obuf++ = *cp++;
 		}
 	}
 }
 /* Http auth challenge schemes */
 typedef enum {HTTPAS_UNKNOWN, HTTPAS_BASIC,HTTPAS_DIGEST} http_auth_schemes_t;
 /* Data holder for a Basic or Digest challenge. */
 typedef struct {
 	http_auth_schemes_t scheme;
 	char	*realm;
 	char	*qop;
 	char	*nonce;
 	char	*opaque;
 	char	*algo;
 	int	 stale;
 	int	 nc; /* Nonce count */
 } http_auth_challenge_t;
 static void
 init_http_auth_challenge(http_auth_challenge_t *b)
 {
 	b->scheme = HTTPAS_UNKNOWN;
 	b->realm = b->qop = b->nonce = b->opaque = b->algo = NULL;
 	b->stale = b->nc = 0;
 }
 static void
 clean_http_auth_challenge(http_auth_challenge_t *b)
 {
 	if (b->realm)
 		free(b->realm);
 	if (b->qop)
 		free(b->qop);
 	if (b->nonce)
 		free(b->nonce);
 	if (b->opaque)
 		free(b->opaque);
 	if (b->algo)
 		free(b->algo);
 	init_http_auth_challenge(b);
 }
 /* Data holder for an array of challenges offered in an http response. */
 #define MAX_CHALLENGES 10
 typedef struct {
 	http_auth_challenge_t *challenges[MAX_CHALLENGES];
 	int	count; /* Number of parsed challenges in the array */
 	int	valid; /* We did parse an authenticate header */
 } http_auth_challenges_t;
 static void
 init_http_auth_challenges(http_auth_challenges_t *cs)
 {
 	int i;
 	for (i = 0; i < MAX_CHALLENGES; i++)
 		cs->challenges[i] = NULL;
 	cs->count = cs->valid = 0;
 }
 static void
 clean_http_auth_challenges(http_auth_challenges_t *cs)
 {
 	int i;
 	/* We rely on non-zero pointers being allocated, not on the count */
 	for (i = 0; i < MAX_CHALLENGES; i++) {
 		if (cs->challenges[i] != NULL) {
 			clean_http_auth_challenge(cs->challenges[i]);
 			free(cs->challenges[i]);
 		}
 	}
 	init_http_auth_challenges(cs);
 }
 /*
  * Enumeration for lexical elements. Separators will be returned as their own
  * ascii value
  */
 typedef enum {HTTPHL_WORD=256, HTTPHL_STRING=257, HTTPHL_END=258,
 	      HTTPHL_ERROR = 259} http_header_lex_t;
 /*
  * Determine what kind of token comes next and return possible value
  * in buf, which is supposed to have been allocated big enough by
  * caller. Advance input pointer and return element type.
  */
 static int
 http_header_lex(const char **cpp, char *buf)
 {
 	size_t l;
 	/* Eat initial whitespace */
 	*cpp += strspn(*cpp, " \t");
 	if (**cpp == 0)
 		return (HTTPHL_END);
 	/* Separator ? */
 	if (**cpp == ',' || **cpp == '=')
 		return (*((*cpp)++));
 	/* String ? */
 	if (**cpp == '"') {
 		*cpp = http_parse_headerstring(++*cpp, buf);
 		if (*cpp == NULL)
 			return (HTTPHL_ERROR);
 		return (HTTPHL_STRING);
 	}
 	/* Read other token, until separator or whitespace */
 	l = strcspn(*cpp, " \t,=");
 	memcpy(buf, *cpp, l);
 	buf[l] = 0;
 	*cpp += l;
 	return (HTTPHL_WORD);
 }
 /*
  * Read challenges from http xxx-authenticate header and accumulate them
  * in the challenges list structure.
  *
  * Headers with multiple challenges are specified by rfc2617, but
  * servers (ie: squid) often send them in separate headers instead,
  * which in turn is forbidden by the http spec (multiple headers with
  * the same name are only allowed for pure comma-separated lists, see
  * rfc2616 sec 4.2).
  *
  * We support both approaches anyway
  */
 static int
 http_parse_authenticate(const char *cp, http_auth_challenges_t *cs)
 {
 	int ret = -1;
 	http_header_lex_t lex;
 	char *key = malloc(strlen(cp) + 1);
 	char *value = malloc(strlen(cp) + 1);
 	char *buf = malloc(strlen(cp) + 1);
 	if (key == NULL || value == NULL || buf == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	/* In any case we've seen the header and we set the valid bit */
 	cs->valid = 1;
 	/* Need word first */
 	lex = http_header_lex(&cp, key);
 	if (lex != HTTPHL_WORD)
 		goto out;
 	/* Loop on challenges */
 	for (; cs->count < MAX_CHALLENGES; cs->count++) {
 		cs->challenges[cs->count] =
 			malloc(sizeof(http_auth_challenge_t));
 		if (cs->challenges[cs->count] == NULL) {
 			fetch_syserr();
 			goto out;
 		}
 		init_http_auth_challenge(cs->challenges[cs->count]);
 		if (strcasecmp(key, "basic") == 0) {
 			cs->challenges[cs->count]->scheme = HTTPAS_BASIC;
 		} else if (strcasecmp(key, "digest") == 0) {
 			cs->challenges[cs->count]->scheme = HTTPAS_DIGEST;
 		} else {
 			cs->challenges[cs->count]->scheme = HTTPAS_UNKNOWN;
 			/*
 			 * Continue parsing as basic or digest may
 			 * follow, and the syntax is the same for
 			 * all. We'll just ignore this one when
 			 * looking at the list
 			 */
 		}
 		/* Loop on attributes */
 		for (;;) {
 			/* Key */
 			lex = http_header_lex(&cp, key);
 			if (lex != HTTPHL_WORD)
 				goto out;
 			/* Equal sign */
 			lex = http_header_lex(&cp, buf);
 			if (lex != '=')
 				goto out;
 			/* Value */
 			lex = http_header_lex(&cp, value);
 			if (lex != HTTPHL_WORD && lex != HTTPHL_STRING)
 				goto out;
 			if (strcasecmp(key, "realm") == 0) {
 				cs->challenges[cs->count]->realm =
 				    strdup(value);
 			} else if (strcasecmp(key, "qop") == 0) {
 				cs->challenges[cs->count]->qop =
 				    strdup(value);
 			} else if (strcasecmp(key, "nonce") == 0) {
 				cs->challenges[cs->count]->nonce =
 				    strdup(value);
 			} else if (strcasecmp(key, "opaque") == 0) {
 				cs->challenges[cs->count]->opaque =
 				    strdup(value);
 			} else if (strcasecmp(key, "algorithm") == 0) {
 				cs->challenges[cs->count]->algo =
 				    strdup(value);
 			} else if (strcasecmp(key, "stale") == 0) {
 				cs->challenges[cs->count]->stale =
 				    strcasecmp(value, "no");
 			} else {
 				/* ignore unknown attributes */
 			}
 			/* Comma or Next challenge or End */
 			lex = http_header_lex(&cp, key);
 			/*
 			 * If we get a word here, this is the beginning of the
 			 * next challenge. Break the attributes loop
 			 */
 			if (lex == HTTPHL_WORD)
 				break;
 			if (lex == HTTPHL_END) {
 				/* End while looking for ',' is normal exit */
 				cs->count++;
 				ret = 0;
 				goto out;
 			}
 			/* Anything else is an error */
 			if (lex != ',')
 				goto out;
 		} /* End attributes loop */
 	} /* End challenge loop */
 	/*
 	 * Challenges max count exceeded. This really can't happen
 	 * with normal data, something's fishy -> error
 	 */
 out:
 	if (key)
 		free(key);
 	if (value)
 		free(value);
 	if (buf)
 		free(buf);
 	return (ret);
 }
 /*
  * Parse a last-modified header
  */
 static int
 http_parse_mtime(const char *p, time_t *mtime)
 {
 	char locale[64], *r;
 	struct tm tm;
 	strlcpy(locale, setlocale(LC_TIME, NULL), sizeof(locale));
 	setlocale(LC_TIME, "C");
 	r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
 	/*
 	 * Some proxies use UTC in response, but it should still be
 	 * parsed. RFC2616 states GMT and UTC are exactly equal for HTTP.
 	 */
 	if (r == NULL)
 		r = strptime(p, "%a, %d %b %Y %H:%M:%S UTC", &tm);
 	/* XXX should add support for date-2 and date-3 */
 	setlocale(LC_TIME, locale);
 	if (r == NULL)
 		return (-1);
 	DEBUGF("last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n",
 	    tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
 	    tm.tm_hour, tm.tm_min, tm.tm_sec);
 	*mtime = timegm(&tm);
 	return (0);
 }
 /*
  * Parse a content-length header
  */
 static int
 http_parse_length(const char *p, off_t *length)
 {
 	off_t len;
 	for (len = 0; *p && isdigit((unsigned char)*p); ++p)
 		len = len * 10 + (*p - '0');
 	if (*p)
 		return (-1);
 	DEBUGF("content length: [%lld]\n", (long long)len);
 	*length = len;
 	return (0);
 }
 /*
  * Parse a content-range header
  */
 static int
 http_parse_range(const char *p, off_t *offset, off_t *length, off_t *size)
 {
 	off_t first, last, len;
 	if (strncasecmp(p, "bytes ", 6) != 0)
 		return (-1);
 	p += 6;
 	if (*p == '*') {
 		first = last = -1;
 		++p;
 	} else {
 		for (first = 0; *p && isdigit((unsigned char)*p); ++p)
 			first = first * 10 + *p - '0';
 		if (*p != '-')
 			return (-1);
 		for (last = 0, ++p; *p && isdigit((unsigned char)*p); ++p)
 			last = last * 10 + *p - '0';
 	}
 	if (first > last || *p != '/')
 		return (-1);
 	for (len = 0, ++p; *p && isdigit((unsigned char)*p); ++p)
 		len = len * 10 + *p - '0';
 	if (*p || len < last - first + 1)
 		return (-1);
 	if (first == -1) {
 		DEBUGF("content range: [*/%lld]\n", (long long)len);
 		*length = 0;
 	} else {
 		DEBUGF("content range: [%lld-%lld/%lld]\n",
 		    (long long)first, (long long)last, (long long)len);
 		*length = last - first + 1;
 	}
 	*offset = first;
 	*size = len;
 	return (0);
 }
 /*****************************************************************************
  * Helper functions for authorization
  */
 /*
  * Base64 encoding
  */
 static char *
 http_base64(const char *src)
 {
 	static const char base64[] =
 	    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 	    "abcdefghijklmnopqrstuvwxyz"
 	    "0123456789+/";
 	char *str, *dst;
 	size_t l;
 	int t;
 	l = strlen(src);
 	if ((str = malloc(((l + 2) / 3) * 4 + 1)) == NULL)
 		return (NULL);
 	dst = str;
 	while (l >= 3) {
 		t = (src[0] << 16) | (src[1] << 8) | src[2];
 		dst[0] = base64[(t >> 18) & 0x3f];
 		dst[1] = base64[(t >> 12) & 0x3f];
 		dst[2] = base64[(t >> 6) & 0x3f];
 		dst[3] = base64[(t >> 0) & 0x3f];
 		src += 3; l -= 3;
 		dst += 4;
 	}
 	switch (l) {
 	case 2:
 		t = (src[0] << 16) | (src[1] << 8);
 		dst[0] = base64[(t >> 18) & 0x3f];
 		dst[1] = base64[(t >> 12) & 0x3f];
 		dst[2] = base64[(t >> 6) & 0x3f];
 		dst[3] = '=';
 		dst += 4;
 		break;
 	case 1:
 		t = src[0] << 16;
 		dst[0] = base64[(t >> 18) & 0x3f];
 		dst[1] = base64[(t >> 12) & 0x3f];
 		dst[2] = dst[3] = '=';
 		dst += 4;
 		break;
 	case 0:
 		break;
 	}
 	*dst = 0;
 	return (str);
 }
 /*
  * Extract authorization parameters from environment value.
  * The value is like scheme:realm:user:pass
  */
 typedef struct {
 	char	*scheme;
 	char	*realm;
 	char	*user;
 	char	*password;
 } http_auth_params_t;
 static void
 init_http_auth_params(http_auth_params_t *s)
 {
 	s->scheme = s->realm = s->user = s->password = NULL;
 }
 static void
 clean_http_auth_params(http_auth_params_t *s)
 {
 	if (s->scheme)
 		free(s->scheme);
 	if (s->realm)
 		free(s->realm);
 	if (s->user)
 		free(s->user);
 	if (s->password)
 		free(s->password);
 	init_http_auth_params(s);
 }
 static int
 http_authfromenv(const char *p, http_auth_params_t *parms)
 {
 	int ret = -1;
 	char *v, *ve;
 	char *str = strdup(p);
 	if (str == NULL) {
 		fetch_syserr();
 		return (-1);
 	}
 	v = str;
 	if ((ve = strchr(v, ':')) == NULL)
 		goto out;
 	*ve = 0;
 	if ((parms->scheme = strdup(v)) == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	v = ve + 1;
 	if ((ve = strchr(v, ':')) == NULL)
 		goto out;
 	*ve = 0;
 	if ((parms->realm = strdup(v)) == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	v = ve + 1;
 	if ((ve = strchr(v, ':')) == NULL)
 		goto out;
 	*ve = 0;
 	if ((parms->user = strdup(v)) == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	v = ve + 1;
 	if ((parms->password = strdup(v)) == NULL) {
 		fetch_syserr();
 		goto out;
 	}
 	ret = 0;
 out:
 	if (ret == -1)
 		clean_http_auth_params(parms);
 	if (str)
 		free(str);
 	return (ret);
 }
 /*
  * Digest response: the code to compute the digest is taken from the
  * sample implementation in RFC2616
  */
 #define IN const
 #define OUT
 #define HASHLEN 16
 typedef char HASH[HASHLEN];
 #define HASHHEXLEN 32
 typedef char HASHHEX[HASHHEXLEN+1];
 static const char *hexchars = "0123456789abcdef";
 static void
 CvtHex(IN HASH Bin, OUT HASHHEX Hex)
 {
 	unsigned short i;
 	unsigned char j;
 	for (i = 0; i < HASHLEN; i++) {
 		j = (Bin[i] >> 4) & 0xf;
 		Hex[i*2] = hexchars[j];
 		j = Bin[i] & 0xf;
 		Hex[i*2+1] = hexchars[j];
 	}
 	Hex[HASHHEXLEN] = '\0';
 };
 /* calculate H(A1) as per spec */
 static void
 DigestCalcHA1(
 	IN char * pszAlg,
 	IN char * pszUserName,
 	IN char * pszRealm,
 	IN char * pszPassword,
 	IN char * pszNonce,
 	IN char * pszCNonce,
 	OUT HASHHEX SessionKey
 	)
 {
 	MD5_CTX Md5Ctx;
 	HASH HA1;
 	MD5Init(&Md5Ctx);
 	MD5Update(&Md5Ctx, pszUserName, strlen(pszUserName));
 	MD5Update(&Md5Ctx, ":", 1);
 	MD5Update(&Md5Ctx, pszRealm, strlen(pszRealm));
 	MD5Update(&Md5Ctx, ":", 1);
 	MD5Update(&Md5Ctx, pszPassword, strlen(pszPassword));
 	MD5Final(HA1, &Md5Ctx);
 	if (strcasecmp(pszAlg, "md5-sess") == 0) {
 		MD5Init(&Md5Ctx);
 		MD5Update(&Md5Ctx, HA1, HASHLEN);
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce));
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
 		MD5Final(HA1, &Md5Ctx);
 	}
 	CvtHex(HA1, SessionKey);
 }
 /* calculate request-digest/response-digest as per HTTP Digest spec */
 static void
 DigestCalcResponse(
 	IN HASHHEX HA1,           /* H(A1) */
 	IN char * pszNonce,       /* nonce from server */
 	IN char * pszNonceCount,  /* 8 hex digits */
 	IN char * pszCNonce,      /* client nonce */
 	IN char * pszQop,         /* qop-value: "", "auth", "auth-int" */
 	IN char * pszMethod,      /* method from the request */
 	IN char * pszDigestUri,   /* requested URL */
 	IN HASHHEX HEntity,       /* H(entity body) if qop="auth-int" */
 	OUT HASHHEX Response      /* request-digest or response-digest */
 	)
 {
 #if 0
 	DEBUGF("Calc: HA1[%s] Nonce[%s] qop[%s] method[%s] URI[%s]\n",
 	    HA1, pszNonce, pszQop, pszMethod, pszDigestUri);
 #endif
 	MD5_CTX Md5Ctx;
 	HASH HA2;
 	HASH RespHash;
 	HASHHEX HA2Hex;
 	// calculate H(A2)
 	MD5Init(&Md5Ctx);
 	MD5Update(&Md5Ctx, pszMethod, strlen(pszMethod));
 	MD5Update(&Md5Ctx, ":", 1);
 	MD5Update(&Md5Ctx, pszDigestUri, strlen(pszDigestUri));
 	if (strcasecmp(pszQop, "auth-int") == 0) {
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, HEntity, HASHHEXLEN);
 	}
 	MD5Final(HA2, &Md5Ctx);
 	CvtHex(HA2, HA2Hex);
 	// calculate response
 	MD5Init(&Md5Ctx);
 	MD5Update(&Md5Ctx, HA1, HASHHEXLEN);
 	MD5Update(&Md5Ctx, ":", 1);
 	MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce));
 	MD5Update(&Md5Ctx, ":", 1);
 	if (*pszQop) {
 		MD5Update(&Md5Ctx, pszNonceCount, strlen(pszNonceCount));
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
 		MD5Update(&Md5Ctx, ":", 1);
 		MD5Update(&Md5Ctx, pszQop, strlen(pszQop));
 		MD5Update(&Md5Ctx, ":", 1);
 	}
 	MD5Update(&Md5Ctx, HA2Hex, HASHHEXLEN);
 	MD5Final(RespHash, &Md5Ctx);
 	CvtHex(RespHash, Response);
 }
 /*
  * Generate/Send a Digest authorization header
  * This looks like: [Proxy-]Authorization: credentials
  *
  *  credentials      = "Digest" digest-response
  *  digest-response  = 1#( username | realm | nonce | digest-uri
  *                      | response | [ algorithm ] | [cnonce] |
  *                      [opaque] | [message-qop] |
  *                          [nonce-count]  | [auth-param] )
  *  username         = "username" "=" username-value
  *  username-value   = quoted-string
  *  digest-uri       = "uri" "=" digest-uri-value
  *  digest-uri-value = request-uri   ; As specified by HTTP/1.1
  *  message-qop      = "qop" "=" qop-value
  *  cnonce           = "cnonce" "=" cnonce-value
  *  cnonce-value     = nonce-value
  *  nonce-count      = "nc" "=" nc-value
  *  nc-value         = 8LHEX
  *  response         = "response" "=" request-digest
  *  request-digest = <"> 32LHEX <">
  */
 static int
 http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c,
 		 http_auth_params_t *parms, struct url *url)
 {
 	int r;
 	char noncecount[10];
 	char cnonce[40];
 	char *options = NULL;
 	if (!c->realm || !c->nonce) {
 		DEBUGF("realm/nonce not set in challenge\n");
 		return(-1);
 	}
 	if (!c->algo)
 		c->algo = strdup("");
 	if (asprintf(&options, "%s%s%s%s",
 	    *c->algo? ",algorithm=" : "", c->algo,
 	    c->opaque? ",opaque=" : "", c->opaque?c->opaque:"") < 0)
 		return (-1);
 	if (!c->qop) {
 		c->qop = strdup("");
 		*noncecount = 0;
 		*cnonce = 0;
 	} else {
 		c->nc++;
 		sprintf(noncecount, "%08x", c->nc);
 		/* We don't try very hard with the cnonce ... */
 		sprintf(cnonce, "%x%lx", getpid(), (unsigned long)time(0));
 	}
 	HASHHEX HA1;
 	DigestCalcHA1(c->algo, parms->user, c->realm,
 		      parms->password, c->nonce, cnonce, HA1);
 	DEBUGF("HA1: [%s]\n", HA1);
 	HASHHEX digest;
 	DigestCalcResponse(HA1, c->nonce, noncecount, cnonce, c->qop,
 			   "GET", url->doc, "", digest);
 	if (c->qop[0]) {
 		r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\","
 			     "nonce=\"%s\",uri=\"%s\",response=\"%s\","
 			     "qop=\"auth\", cnonce=\"%s\", nc=%s%s",
 			     hdr, parms->user, c->realm,
 			     c->nonce, url->doc, digest,
 			     cnonce, noncecount, options);
 	} else {
 		r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\","
 			     "nonce=\"%s\",uri=\"%s\",response=\"%s\"%s",
 			     hdr, parms->user, c->realm,
 			     c->nonce, url->doc, digest, options);
 	}
 	if (options)
 		free(options);
 	return (r);
 }
 /*
  * Encode username and password
  */
 static int
 http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd)
 {
 	char *upw, *auth;
 	int r;
 	DEBUGF("basic: usr: [%s]\n", usr);
 	DEBUGF("basic: pwd: [%s]\n", pwd);
 	if (asprintf(&upw, "%s:%s", usr, pwd) == -1)
 		return (-1);
 	auth = http_base64(upw);
 	free(upw);
 	if (auth == NULL)
 		return (-1);
 	r = http_cmd(conn, "%s: Basic %s", hdr, auth);
 	free(auth);
 	return (r);
 }
 /*
  * Chose the challenge to answer and call the appropriate routine to
  * produce the header.
  */
 static int
 http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 	       http_auth_params_t *parms, struct url *url)
 {
 	http_auth_challenge_t *digest = NULL;
 	int i;
 	/* If user or pass are null we're not happy */
 	if (!parms->user || !parms->password) {
 		DEBUGF("NULL usr or pass\n");
 		return (-1);
 	}
 	/* Look for a Digest */
 	for (i = 0; i < cs->count; i++) {
 		if (cs->challenges[i]->scheme == HTTPAS_DIGEST)
 			digest = cs->challenges[i];
 	}
 	/* Error if "Digest" was specified and there is no Digest challenge */
 	if (!digest &&
 	    (parms->scheme && strcasecmp(parms->scheme, "digest") == 0)) {
 		DEBUGF("Digest auth in env, not supported by peer\n");
 		return (-1);
 	}
 	/*
 	 * If "basic" was specified in the environment, or there is no Digest
 	 * challenge, do the basic thing. Don't need a challenge for this,
 	 * so no need to check basic!=NULL
 	 */
 	if (!digest ||
 	    (parms->scheme && strcasecmp(parms->scheme, "basic") == 0))
 		return (http_basic_auth(conn,hdr,parms->user,parms->password));
 	/* Else, prefer digest. We just checked that it's not NULL */
 	return (http_digest_auth(conn, hdr, digest, parms, url));
 }
 /*****************************************************************************
  * Helper functions for connecting to a server or proxy
  */
 /*
  * Connect to the correct HTTP server or proxy.
  */
 static conn_t *
 http_connect(struct url *URL, struct url *purl, const char *flags)
 {
 	struct url *curl;
 	conn_t *conn;
 	hdr_t h;
 	http_headerbuf_t headerbuf;
 	const char *p;
 	int verbose;
 	int af, val;
 	int serrno;
 	bool isproxyauth = false;
 	http_auth_challenges_t proxy_challenges;
 #ifdef INET6
 	af = AF_UNSPEC;
 #else
 	af = AF_INET;
 #endif
 	verbose = CHECK_FLAG('v');
 	if (CHECK_FLAG('4'))
 		af = AF_INET;
 #ifdef INET6
 	else if (CHECK_FLAG('6'))
 		af = AF_INET6;
 #endif
 	curl = (purl != NULL) ? purl : URL;
 retry:
 	if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
 	init_http_headerbuf(&headerbuf);
 	if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
 		init_http_auth_challenges(&proxy_challenges);
 		http_cmd(conn, "CONNECT %s:%d HTTP/1.1", URL->host, URL->port);
 		http_cmd(conn, "Host: %s:%d", URL->host, URL->port);
 		if (isproxyauth) {
 			http_auth_params_t aparams;
 			init_http_auth_params(&aparams);
 			if (*purl->user || *purl->pwd) {
 				aparams.user = strdup(purl->user);
 				aparams.password = strdup(purl->pwd);
 			} else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL &&
 				    *p != '\0') {
 				if (http_authfromenv(p, &aparams) < 0) {
 					http_seterr(HTTP_NEED_PROXY_AUTH);
 					fetch_syserr();
 					goto ouch;
 				}
 			} else if (fetch_netrc_auth(purl) == 0) {
 				aparams.user = strdup(purl->user);
 				aparams.password = strdup(purl->pwd);
 			} else {
 				/*
 				 * No auth information found in system - exiting
 				 * with warning.
 				 */
 				warnx("Missing username and/or password set");
 				fetch_syserr();
 				goto ouch;
 			}
 			http_authorize(conn, "Proxy-Authorization",
 			    &proxy_challenges, &aparams, purl);
 			clean_http_auth_params(&aparams);
 		}
 		http_cmd(conn, "");
 		/* Get reply from CONNECT Tunnel attempt */
 		int httpreply = http_get_reply(conn);
 		if (httpreply != HTTP_OK) {
 			http_seterr(httpreply);
 			/* If the error is a 407/HTTP_NEED_PROXY_AUTH */
 			if (httpreply == HTTP_NEED_PROXY_AUTH &&
 			    ! isproxyauth) {
 				/* Try again with authentication. */
 				clean_http_headerbuf(&headerbuf);
 				fetch_close(conn);
 				isproxyauth = true;
 				goto retry;
 			}
 			goto ouch;
 		}
 		/* Read and discard the rest of the proxy response */
 		if (fetch_getln(conn) < 0) {
 			fetch_syserr();
 			goto ouch;
 		}
 		do {
 			switch ((h = http_next_header(conn, &headerbuf, &p))) {
 			case hdr_syserror:
 				fetch_syserr();
 				goto ouch;
 			case hdr_error:
 				http_seterr(HTTP_PROTOCOL_ERROR);
 				goto ouch;
 			default:
 				/* ignore */ ;
 			}
 		} while (h > hdr_end);
 	}
 	if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 &&
 	    fetch_ssl(conn, URL, verbose) == -1) {
 		/* grrr */
 		errno = EAUTH;
 		fetch_syserr();
 		goto ouch;
 	}
 #ifdef TCP_NOPUSH
 	val = 1;
 	setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val));
 #endif
 	clean_http_headerbuf(&headerbuf);
 	return (conn);
 ouch:
 	serrno = errno;
 	clean_http_headerbuf(&headerbuf);
 	fetch_close(conn);
 	errno = serrno;
 	return (NULL);
 }
 static struct url *
 http_get_proxy(struct url * url, const char *flags)
 {
 	struct url *purl;
 	char *p;
 	if (flags != NULL && strchr(flags, 'd') != NULL)
 		return (NULL);
 	if (fetch_no_proxy_match(url->host))
 		return (NULL);
 	if (((p = getenv("HTTP_PROXY")) || (p = getenv("http_proxy"))) &&
 	    *p && (purl = fetchParseURL(p))) {
 		if (!*purl->scheme)
 			strcpy(purl->scheme, SCHEME_HTTP);
 		if (!purl->port)
 			purl->port = fetch_default_proxy_port(purl->scheme);
 		if (strcmp(purl->scheme, SCHEME_HTTP) == 0)
 			return (purl);
 		fetchFreeURL(purl);
 	}
 	return (NULL);
 }
 static void
 http_print_html(FILE *out, FILE *in)
 {
 	ssize_t len = 0;
 	size_t cap;
 	char *line = NULL, *p, *q;
 	int comment, tag;
 	comment = tag = 0;
 	while ((len = getline(&line, &cap, in)) >= 0) {
 		while (len && isspace((unsigned char)line[len - 1]))
 			--len;
 		for (p = q = line; q < line + len; ++q) {
 			if (comment && *q == '-') {
 				if (q + 2 < line + len &&
 				    strcmp(q, "-->") == 0) {
 					tag = comment = 0;
 					q += 2;
 				}
 			} else if (tag && !comment && *q == '>') {
 				p = q + 1;
 				tag = 0;
 			} else if (!tag && *q == '<') {
 				if (q > p)
 					fwrite(p, q - p, 1, out);
 				tag = 1;
 				if (q + 3 < line + len &&
 				    strcmp(q, "<!--") == 0) {
 					comment = 1;
 					q += 3;
 				}
 			}
 		}
 		if (!tag && q > p)
 			fwrite(p, q - p, 1, out);
 		fputc('\n', out);
 	}
 	free(line);
 }
 /*****************************************************************************
  * Core
  */
 FILE *
 http_request(struct url *URL, const char *op, struct url_stat *us,
 	struct url *purl, const char *flags)
 {
 	return (http_request_body(URL, op, us, purl, flags, NULL, NULL));
 }
 /*
  * Send a request and process the reply
  *
  * XXX This function is way too long, the do..while loop should be split
  * XXX off into a separate function.
  */
 FILE *
 http_request_body(struct url *URL, const char *op, struct url_stat *us,
 	struct url *purl, const char *flags, const char *content_type,
 	const char *body)
 {
 	char timebuf[80];
 	char hbuf[MAXHOSTNAMELEN + 7], *host;
 	conn_t *conn;
 	struct url *url, *new;
 	int chunked, direct, ims, noredirect, verbose;
 	int e, i, n, val;
 	off_t offset, clength, length, size;
 	time_t mtime;
 	const char *p;
 	FILE *f;
 	hdr_t h;
 	struct tm *timestruct;
 	http_headerbuf_t headerbuf;
 	http_auth_challenges_t server_challenges;
 	http_auth_challenges_t proxy_challenges;
 	size_t body_len;
 	/* The following calls don't allocate anything */
 	init_http_headerbuf(&headerbuf);
 	init_http_auth_challenges(&server_challenges);
 	init_http_auth_challenges(&proxy_challenges);
 	direct = CHECK_FLAG('d');
 	noredirect = CHECK_FLAG('A');
 	verbose = CHECK_FLAG('v');
 	ims = CHECK_FLAG('i');
 	if (direct && purl) {
 		fetchFreeURL(purl);
 		purl = NULL;
 	}
 	/* try the provided URL first */
 	url = URL;
 	n = MAX_REDIRECT;
 	i = 0;
 	e = HTTP_PROTOCOL_ERROR;
 	do {
 		new = NULL;
 		chunked = 0;
 		offset = 0;
 		clength = -1;
 		length = -1;
 		size = -1;
 		mtime = 0;
 		/* check port */
 		if (!url->port)
 			url->port = fetch_default_port(url->scheme);
 		/* connect to server or proxy */
 		if ((conn = http_connect(url, purl, flags)) == NULL)
 			goto ouch;
 		/* append port number only if necessary */
 		host = url->host;
 		if (url->port != fetch_default_port(url->scheme)) {
 			snprintf(hbuf, sizeof(hbuf), "%s:%d", host, url->port);
 			host = hbuf;
 		}
 		/* send request */
 		if (verbose)
 			fetch_info("requesting %s://%s%s",
 			    url->scheme, host, url->doc);
 		if (purl && strcmp(url->scheme, SCHEME_HTTPS) != 0) {
 			http_cmd(conn, "%s %s://%s%s HTTP/1.1",
 			    op, url->scheme, host, url->doc);
 		} else {
 			http_cmd(conn, "%s %s HTTP/1.1",
 			    op, url->doc);
 		}
 		if (ims && url->ims_time) {
 			timestruct = gmtime((time_t *)&url->ims_time);
 			(void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT",
 			    timestruct);
 			if (verbose)
 				fetch_info("If-Modified-Since: %s", timebuf);
 			http_cmd(conn, "If-Modified-Since: %s", timebuf);
 		}
 		/* virtual host */
 		http_cmd(conn, "Host: %s", host);
 		/*
 		 * Proxy authorization: we only send auth after we received
 		 * a 407 error. We do not first try basic anyway (changed
 		 * when support was added for digest-auth)
 		 */
 		if (purl && proxy_challenges.valid) {
 			http_auth_params_t aparams;
 			init_http_auth_params(&aparams);
 			if (*purl->user || *purl->pwd) {
 				aparams.user = strdup(purl->user);
 				aparams.password = strdup(purl->pwd);
 			} else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL &&
 				   *p != '\0') {
 				if (http_authfromenv(p, &aparams) < 0) {
 					http_seterr(HTTP_NEED_PROXY_AUTH);
 					goto ouch;
 				}
 			} else if (fetch_netrc_auth(purl) == 0) {
 				aparams.user = strdup(purl->user);
 				aparams.password = strdup(purl->pwd);
 			}
 			http_authorize(conn, "Proxy-Authorization",
 				       &proxy_challenges, &aparams, url);
 			clean_http_auth_params(&aparams);
 		}
 		/*
 		 * Server authorization: we never send "a priori"
 		 * Basic auth, which used to be done if user/pass were
 		 * set in the url. This would be weird because we'd send the
 		 * password in the clear even if Digest is finally to be
 		 * used (it would have made more sense for the
 		 * pre-digest version to do this when Basic was specified
 		 * in the environment)
 		 */
 		if (server_challenges.valid) {
 			http_auth_params_t aparams;
 			init_http_auth_params(&aparams);
 			if (*url->user || *url->pwd) {
 				aparams.user = strdup(url->user);
 				aparams.password = strdup(url->pwd);
 			} else if ((p = getenv("HTTP_AUTH")) != NULL &&
 				   *p != '\0') {
 				if (http_authfromenv(p, &aparams) < 0) {
 					http_seterr(HTTP_NEED_AUTH);
 					goto ouch;
 				}
 			} else if (fetch_netrc_auth(url) == 0) {
 				aparams.user = strdup(url->user);
 				aparams.password = strdup(url->pwd);
 			} else if (fetchAuthMethod &&
 				   fetchAuthMethod(url) == 0) {
 				aparams.user = strdup(url->user);
 				aparams.password = strdup(url->pwd);
 			} else {
 				http_seterr(HTTP_NEED_AUTH);
 				goto ouch;
 			}
 			http_authorize(conn, "Authorization",
 				       &server_challenges, &aparams, url);
 			clean_http_auth_params(&aparams);
 		}
 		/* other headers */
 		if ((p = getenv("HTTP_ACCEPT")) != NULL) {
 			if (*p != '\0')
 				http_cmd(conn, "Accept: %s", p);
 		} else {
 			http_cmd(conn, "Accept: */*");
 		}
 		if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') {
 			if (strcasecmp(p, "auto") == 0)
 				http_cmd(conn, "Referer: %s://%s%s",
 				    url->scheme, host, url->doc);
 			else
 				http_cmd(conn, "Referer: %s", p);
 		}
 		if ((p = getenv("HTTP_USER_AGENT")) != NULL) {
 			/* no User-Agent if defined but empty */
 			if  (*p != '\0')
 				http_cmd(conn, "User-Agent: %s", p);
 		} else {
 			/* default User-Agent */
 			http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER,
 			    getprogname());
 		}
 		if (url->offset > 0)
 			http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
 		http_cmd(conn, "Connection: close");
 		if (body) {
 			body_len = strlen(body);
 			http_cmd(conn, "Content-Length: %zu", body_len);
 			if (content_type != NULL)
 				http_cmd(conn, "Content-Type: %s", content_type);
 		}
 		http_cmd(conn, "");
 		if (body)
 			fetch_write(conn, body, body_len);
 		/*
 		 * Force the queued request to be dispatched.  Normally, one
 		 * would do this with shutdown(2) but squid proxies can be
 		 * configured to disallow such half-closed connections.  To
 		 * be compatible with such configurations, fiddle with socket
 		 * options to force the pending data to be written.
 		 */
 #ifdef TCP_NOPUSH
 		val = 0;
 		setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val,
 			   sizeof(val));
 #endif
 		val = 1;
 		setsockopt(conn->sd, IPPROTO_TCP, TCP_NODELAY, &val,
 			   sizeof(val));
 		/* get reply */
 		switch (http_get_reply(conn)) {
 		case HTTP_OK:
 		case HTTP_PARTIAL:
 		case HTTP_NOT_MODIFIED:
 			/* fine */
 			break;
 		case HTTP_MOVED_PERM:
 		case HTTP_MOVED_TEMP:
 		case HTTP_TEMP_REDIRECT:
 		case HTTP_PERM_REDIRECT:
 		case HTTP_SEE_OTHER:
 		case HTTP_USE_PROXY:
 			/*
 			 * Not so fine, but we still have to read the
 			 * headers to get the new location.
 			 */
 			break;
 		case HTTP_NEED_AUTH:
 			if (server_challenges.valid) {
 				/*
 				 * We already sent out authorization code,
 				 * so there's nothing more we can do.
 				 */
 				http_seterr(conn->err);
 				goto ouch;
 			}
 			/* try again, but send the password this time */
 			if (verbose)
 				fetch_info("server requires authorization");
 			break;
 		case HTTP_NEED_PROXY_AUTH:
 			if (proxy_challenges.valid) {
 				/*
 				 * We already sent our proxy
 				 * authorization code, so there's
 				 * nothing more we can do. */
 				http_seterr(conn->err);
 				goto ouch;
 			}
 			/* try again, but send the password this time */
 			if (verbose)
 				fetch_info("proxy requires authorization");
 			break;
 		case HTTP_BAD_RANGE:
 			/*
 			 * This can happen if we ask for 0 bytes because
 			 * we already have the whole file.  Consider this
 			 * a success for now, and check sizes later.
 			 */
 			break;
 		case HTTP_PROTOCOL_ERROR:
 			/* fall through */
 		case -1:
 			fetch_syserr();
 			goto ouch;
 		default:
 			http_seterr(conn->err);
 			if (!verbose)
 				goto ouch;
 			/* fall through so we can get the full error message */
 		}
 		/* get headers. http_next_header expects one line readahead */
 		if (fetch_getln(conn) == -1) {
 			fetch_syserr();
 			goto ouch;
 		}
 		do {
 			switch ((h = http_next_header(conn, &headerbuf, &p))) {
 			case hdr_syserror:
 				fetch_syserr();
 				goto ouch;
 			case hdr_error:
 				http_seterr(HTTP_PROTOCOL_ERROR);
 				goto ouch;
 			case hdr_content_length:
 				http_parse_length(p, &clength);
 				break;
 			case hdr_content_range:
 				http_parse_range(p, &offset, &length, &size);
 				break;
 			case hdr_last_modified:
 				http_parse_mtime(p, &mtime);
 				break;
 			case hdr_location:
 				if (!HTTP_REDIRECT(conn->err))
 					break;
 				/*
 				 * if the A flag is set, we don't follow
 				 * temporary redirects.
 				 */
 				if (noredirect &&
 				    conn->err != HTTP_MOVED_PERM &&
 				    conn->err != HTTP_PERM_REDIRECT &&
 				    conn->err != HTTP_USE_PROXY) {
 					n = 1;
 					break;
 				}
 				if (new)
 					free(new);
 				if (verbose)
 					fetch_info("%d redirect to %s",
 					    conn->err, p);
 				if (*p == '/')
 					/* absolute path */
 					new = fetchMakeURL(url->scheme, url->host,
 					    url->port, p, url->user, url->pwd);
 				else
 					new = fetchParseURL(p);
 				if (new == NULL) {
 					/* XXX should set an error code */
 					DEBUGF("failed to parse new URL\n");
 					goto ouch;
 				}
 				/* Only copy credentials if the host matches */
 				if (strcmp(new->host, url->host) == 0 &&
 				    !*new->user && !*new->pwd) {
 					strcpy(new->user, url->user);
 					strcpy(new->pwd, url->pwd);
 				}
 				new->offset = url->offset;
 				new->length = url->length;
 				new->ims_time = url->ims_time;
 				break;
 			case hdr_transfer_encoding:
 				/* XXX weak test*/
 				chunked = (strcasecmp(p, "chunked") == 0);
 				break;
 			case hdr_www_authenticate:
 				if (conn->err != HTTP_NEED_AUTH)
 					break;
 				if (http_parse_authenticate(p, &server_challenges) == 0)
 					++n;
 				break;
 			case hdr_proxy_authenticate:
 				if (conn->err != HTTP_NEED_PROXY_AUTH)
 					break;
 				if (http_parse_authenticate(p, &proxy_challenges) == 0)
 					++n;
 				break;
 			case hdr_end:
 				/* fall through */
 			case hdr_unknown:
 				/* ignore */
 				break;
 			}
 		} while (h > hdr_end);
 		/* we need to provide authentication */
 		if (conn->err == HTTP_NEED_AUTH ||
 		    conn->err == HTTP_NEED_PROXY_AUTH) {
 			e = conn->err;
 			if ((conn->err == HTTP_NEED_AUTH &&
 			     !server_challenges.valid) ||
 			    (conn->err == HTTP_NEED_PROXY_AUTH &&
 			     !proxy_challenges.valid)) {
 				/* 401/7 but no www/proxy-authenticate ?? */
 				DEBUGF("%03d without auth header\n", conn->err);
 				goto ouch;
 			}
 			fetch_close(conn);
 			conn = NULL;
 			continue;
 		}
 		/* requested range not satisfiable */
 		if (conn->err == HTTP_BAD_RANGE) {
 			if (url->offset > 0 && url->length == 0) {
 				/* asked for 0 bytes; fake it */
 				offset = url->offset;
 				clength = -1;
 				conn->err = HTTP_OK;
 				break;
 			} else {
 				http_seterr(conn->err);
 				goto ouch;
 			}
 		}
 		/* we have a hit or an error */
 		if (conn->err == HTTP_OK
 		    || conn->err == HTTP_NOT_MODIFIED
 		    || conn->err == HTTP_PARTIAL
 		    || HTTP_ERROR(conn->err))
 			break;
 		/* all other cases: we got a redirect */
 		e = conn->err;
 		clean_http_auth_challenges(&server_challenges);
 		fetch_close(conn);
 		conn = NULL;
 		if (!new) {
 			DEBUGF("redirect with no new location\n");
 			break;
 		}
 		if (url != URL)
 			fetchFreeURL(url);
 		url = new;
 	} while (++i < n);
 	/* we failed, or ran out of retries */
 	if (conn == NULL) {
 		http_seterr(e);
 		goto ouch;
 	}
 	DEBUGF("offset %lld, length %lld, size %lld, clength %lld\n",
 	    (long long)offset, (long long)length,
 	    (long long)size, (long long)clength);
 	if (conn->err == HTTP_NOT_MODIFIED) {
 		http_seterr(HTTP_NOT_MODIFIED);
 		return (NULL);
 	}
 	/* check for inconsistencies */
 	if (clength != -1 && length != -1 && clength != length) {
 		http_seterr(HTTP_PROTOCOL_ERROR);
 		goto ouch;
 	}
 	if (clength == -1)
 		clength = length;
 	if (clength != -1)
 		length = offset + clength;
 	if (length != -1 && size != -1 && length != size) {
 		http_seterr(HTTP_PROTOCOL_ERROR);
 		goto ouch;
 	}
 	if (size == -1)
 		size = length;
 	/* fill in stats */
 	if (us) {
 		us->size = size;
 		us->atime = us->mtime = mtime;
 	}
 	/* too far? */
 	if (URL->offset > 0 && offset > URL->offset) {
 		http_seterr(HTTP_PROTOCOL_ERROR);
 		goto ouch;
 	}
 	/* report back real offset and size */
 	URL->offset = offset;
 	URL->length = clength;
 	/* wrap it up in a FILE */
 	if ((f = http_funopen(conn, chunked)) == NULL) {
 		fetch_syserr();
 		goto ouch;
 	}
 	if (url != URL)
 		fetchFreeURL(url);
 	if (purl)
 		fetchFreeURL(purl);
 	if (HTTP_ERROR(conn->err)) {
 		http_print_html(stderr, f);
 		fclose(f);
 		f = NULL;
 	}
 	clean_http_headerbuf(&headerbuf);
 	clean_http_auth_challenges(&server_challenges);
 	clean_http_auth_challenges(&proxy_challenges);
 	return (f);
 ouch:
 	if (url != URL)
 		fetchFreeURL(url);
 	if (purl)
 		fetchFreeURL(purl);
 	if (conn != NULL)
 		fetch_close(conn);
 	clean_http_headerbuf(&headerbuf);
 	clean_http_auth_challenges(&server_challenges);
 	clean_http_auth_challenges(&proxy_challenges);
 	return (NULL);
 }
 /*****************************************************************************
  * Entry points
  */
 /*
  * Retrieve and stat a file by HTTP
  */
 FILE *
 fetchXGetHTTP(struct url *URL, struct url_stat *us, const char *flags)
 {
 	return (http_request(URL, "GET", us, http_get_proxy(URL, flags), flags));
 }
 /*
  * Retrieve a file by HTTP
  */
 FILE *
 fetchGetHTTP(struct url *URL, const char *flags)
 {
 	return (fetchXGetHTTP(URL, NULL, flags));
 }
 /*
  * Store a file by HTTP
  */
 FILE *
 fetchPutHTTP(struct url *URL __unused, const char *flags __unused)
 {
 	warnx("fetchPutHTTP(): not implemented");
 	return (NULL);
 }
 /*
  * Get an HTTP document's metadata
  */
 int
 fetchStatHTTP(struct url *URL, struct url_stat *us, const char *flags)
 {
 	FILE *f;
 	f = http_request(URL, "HEAD", us, http_get_proxy(URL, flags), flags);
 	if (f == NULL)
 		return (-1);
 	fclose(f);
 	return (0);
 }
 /*
  * List a directory
  */
 struct url_ent *
 fetchListHTTP(struct url *url __unused, const char *flags __unused)
 {
 	warnx("fetchListHTTP(): not implemented");
 	return (NULL);
 }
 /*
  * Arbitrary HTTP verb and content requests
  */
 FILE *
 fetchReqHTTP(struct url *URL, const char *method, const char *flags,
 	const char *content_type, const char *body)
 {
 	return (http_request_body(URL, method, NULL, http_get_proxy(URL, flags),
 	    flags, content_type, body));
 }

deleted external/libfetch/http.errors

@@ -1,46 +0,0 @@

 # $FreeBSD: head/lib/libfetch/http.errors 241840 2012-10-22 03:00:10Z eadler $
 #
 # This list is taken from RFC 2068.
 #
 OK		Continue
 OK		Switching Protocols
 OK		OK
 OK		Created
 OK		Accepted
 INFO	Non-Authoritative Information
 OK		No Content
 OK		Reset Content
 OK		Partial Content
 MOVED	Multiple Choices
 MOVED	Moved Permanently
 MOVED	Moved Temporarily
 MOVED	See Other
 OK		Not Modified
 INFO	Use Proxy
 MOVED	Temporary Redirect
 MOVED	Permanent Redirect
 PROTO	Bad Request
 AUTH	Unauthorized
 AUTH	Payment Required
 AUTH	Forbidden
 UNAVAIL	Not Found
 PROTO	Method Not Allowed
 PROTO	Not Acceptable
 AUTH	Proxy Authentication Required
 TIMEOUT	Request Time-out
 EXISTS	Conflict
 UNAVAIL	Gone
 PROTO	Length Required
 SERVER	Precondition Failed
 PROTO	Request Entity Too Large
 PROTO	Request-URI Too Large
 PROTO	Unsupported Media Type
 UNAVAIL	Requested Range Not Satisfiable
 SERVER	Expectation Failed
 SERVER	Internal Server Error
 PROTO	Not Implemented
 SERVER	Bad Gateway
 TEMP	Service Unavailable
 TIMEOUT	Gateway Time-out
 PROTO	HTTP Version not supported
 PROTO	Protocol error

deleted external/libfetch/httperr.h

@@ -1,45 +0,0 @@

 static struct fetcherr http_errlist[] = {
     { 100, FETCH_OK, "Continue" },
     { 101, FETCH_OK, "Switching Protocols" },
     { 200, FETCH_OK, "OK" },
     { 201, FETCH_OK, "Created" },
     { 202, FETCH_OK, "Accepted" },
     { 203, FETCH_INFO, "Non-Authoritative Information" },
     { 204, FETCH_OK, "No Content" },
     { 205, FETCH_OK, "Reset Content" },
     { 206, FETCH_OK, "Partial Content" },
     { 300, FETCH_MOVED, "Multiple Choices" },
     { 301, FETCH_MOVED, "Moved Permanently" },
     { 302, FETCH_MOVED, "Moved Temporarily" },
     { 303, FETCH_MOVED, "See Other" },
     { 304, FETCH_OK, "Not Modified" },
     { 305, FETCH_INFO, "Use Proxy" },
     { 307, FETCH_MOVED, "Temporary Redirect" },
     { 308, FETCH_MOVED, "Permanent Redirect" },
     { 400, FETCH_PROTO, "Bad Request" },
     { 401, FETCH_AUTH, "Unauthorized" },
     { 402, FETCH_AUTH, "Payment Required" },
     { 403, FETCH_AUTH, "Forbidden" },
     { 404, FETCH_UNAVAIL, "Not Found" },
     { 405, FETCH_PROTO, "Method Not Allowed" },
     { 406, FETCH_PROTO, "Not Acceptable" },
     { 407, FETCH_AUTH, "Proxy Authentication Required" },
     { 408, FETCH_TIMEOUT, "Request Time-out" },
     { 409, FETCH_EXISTS, "Conflict" },
     { 410, FETCH_UNAVAIL, "Gone" },
     { 411, FETCH_PROTO, "Length Required" },
     { 412, FETCH_SERVER, "Precondition Failed" },
     { 413, FETCH_PROTO, "Request Entity Too Large" },
     { 414, FETCH_PROTO, "Request-URI Too Large" },
     { 415, FETCH_PROTO, "Unsupported Media Type" },
     { 416, FETCH_UNAVAIL, "Requested Range Not Satisfiable" },
     { 417, FETCH_SERVER, "Expectation Failed" },
     { 500, FETCH_SERVER, "Internal Server Error" },
     { 501, FETCH_PROTO, "Not Implemented" },
     { 502, FETCH_SERVER, "Bad Gateway" },
     { 503, FETCH_TEMP, "Service Unavailable" },
     { 504, FETCH_TIMEOUT, "Gateway Time-out" },
     { 505, FETCH_PROTO, "HTTP Version not supported" },
     { 999, FETCH_PROTO, "Protocol error" },
     { -1, FETCH_UNKNOWN, "Unknown HTTP error" }
 };

modified libpkg/Makefile.autosetup

@@ -58,7 +58,6 @@ LOCAL_CFLAGS= -I$(top_srcdir)/compat \

 		-I$(top_srcdir)/external/yxml \
 		-I$(top_srcdir)/external/include \
 		-I$(top_srcdir)/external/libucl/include \
 		-I$(top_srcdir)/external/libfetch \
 		-I$(top_srcdir)/external/curl/include \
 		-I$(top_srcdir)/external/lua/src \
 		-I$(top_srcdir)/external/liblua/ \

@@ -84,7 +83,6 @@ LOCAL_LDFLAGS= @waflags@ \

 		-L$(top_builddir)/external/yxml -lyxml_pic \
 		-L$(top_builddir)/external/blake2 -lblake2_pic \
 		-L$(top_builddir)/compat -lbsd_compat_pic \
 		-L$(top_builddir)/external/libfetch -lfetch_pic \
 		-L$(top_builddir)/external/libcurl -lcurl_pic \
 		-L$(top_builddir)/external/liblua -llua_pic \
 		-L$(top_builddir)/external/msgpuck -lmsgpuck_pic \

@@ -102,7 +100,6 @@ STATIC_LIBS= @REPOS_STATIC_LIBS@ \

 		$(top_builddir)/external/blake2/libblake2.a \
 		$(top_builddir)/external/liblua/liblua.a \
 		$(top_builddir)/compat/libbsd_compat.a \
 		$(top_builddir)/external/libfetch/libfetch.a \
 		$(top_builddir)/external/libcurl/libcurl.a \
 		$(top_builddir)/external/msgpuck/libmsgpuck.a \
 		lib$(LIB).a

modified libpkg/fetch.c

@@ -50,6 +50,7 @@

 static struct fetcher fetchers [] = {
 	{
 		"tcp",
 ,
 		tcp_open,
 		NULL,
 		fh_close,

@@ -57,6 +58,7 @@ static struct fetcher fetchers [] = {

 	},
 	{
 		"ssh",
 ,
 		ssh_open,
 		NULL,
 		fh_close,

@@ -64,6 +66,7 @@ static struct fetcher fetchers [] = {

 	},
 	{
 		"pkg+https",
 ,
 		curl_open,
 		NULL,
 		curl_cleanup,

@@ -71,6 +74,7 @@ static struct fetcher fetchers [] = {

 	},
 	{
 		"pkg+http",
 ,
 		curl_open,
 		NULL,
 		curl_cleanup,

@@ -78,6 +82,7 @@ static struct fetcher fetchers [] = {

 	},
 	{
 		"https",
 ,
 		curl_open,
 		NULL,
 		curl_cleanup,

@@ -85,6 +90,7 @@ static struct fetcher fetchers [] = {

 	},
 	{
 		"http",
 ,
 		curl_open,
 		NULL,
 		curl_cleanup,

@@ -92,6 +98,7 @@ static struct fetcher fetchers [] = {

 	},
 	{
 		"file",
 ,
 		file_open,
 		fh_close,
 		NULL,

@@ -178,6 +185,24 @@ pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t,

 #define URL_SCHEME_PREFIX	"pkg+"
 static struct fetcher *
 select_fetcher(const char *url)
 {
 	struct fetcher *f;
 	for (size_t i = 0; i < nitems(fetchers); i++) {
 		if ((strncasecmp(url, fetchers[i].scheme,
 		    strlen(fetchers[i].scheme)) == 0) &&
 		    url[strlen(fetchers[i].scheme)] == ':') {
 			f = &fetchers[i];
 			f->timeout =
 			    pkg_object_int(pkg_config_get("FETCH_TIMEOUT"));
 			return (f);
 		}
 	}
 	return (NULL);
 }
 int
 pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest,
     time_t *t, ssize_t offset, int64_t size, bool silent)

@@ -211,15 +236,8 @@ pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest,

 		repo = fakerepo;
 	}
 	if (repo->fetcher == NULL) {
 		for (int i = 0; i < nitems(fetchers); i++) {
 			if (strncasecmp(url, fetchers[i].scheme,
 			    strlen(fetchers[i].scheme)) == 0) {
 				repo->fetcher = &fetchers[i];
 				break;
 			}
 		}
 	}
 	if (repo->fetcher == NULL)
 		repo->fetcher = select_fetcher(url);
 	if (repo->fetcher == NULL) {
 		pkg_emit_error("Unknown scheme: %s", url);
 		return (EPKG_FATAL);

modified libpkg/fetch_file.c

@@ -27,7 +27,6 @@

 #include <sys/param.h>
 #include <stdio.h>
 #include <fetch.h>
 #include <errno.h>
 #include "pkg.h"

modified libpkg/fetch_ssh.c

@@ -34,7 +34,6 @@

 #include <errno.h>
 #include <stdio.h>
 #include <string.h>
 #include <fetch.h>
 #include <paths.h>
 #include <poll.h>
 #include <netdb.h>

@@ -335,7 +334,7 @@ ssh_close(void *data)

 }
 static int
 ssh_writev(int fd, struct iovec *iov, int iovcnt)
 ssh_writev(int fd, struct iovec *iov, int iovcnt, int64_t tmout)
 {
 	struct timeval now, timeout, delta;
 	struct pollfd pfd;

@@ -345,16 +344,16 @@ ssh_writev(int fd, struct iovec *iov, int iovcnt)

 	memset(&pfd, 0, sizeof pfd);
 	if (fetchTimeout) {
 	if (tmout > 0) {
 		pfd.fd = fd;
 		pfd.events = POLLOUT | POLLERR;
 		gettimeofday(&timeout, NULL);
 		timeout.tv_sec += fetchTimeout;
 		timeout.tv_sec += tmout;
 	}
 	total = 0;
 	while (iovcnt > 0) {
 		while (fetchTimeout && pfd.revents == 0) {
 		while (tmout && pfd.revents == 0) {
 			gettimeofday(&now, NULL);
 			if (!timercmp(&timeout, &now, >)) {
 				errno = ETIMEDOUT;

@@ -412,7 +411,7 @@ ssh_write(void *data, const char *buf, int l)

 	pkg_debug(1, "writing data");
 	return (ssh_writev(repo->sshio.out, &iov, 1));
 	return (ssh_writev(repo->sshio.out, &iov, 1, repo->fetcher->timeout));
 }
 static int

@@ -426,9 +425,9 @@ ssh_read(void *data, char *buf, int len)

 	pkg_debug(2, "ssh: start reading");
 	if (fetchTimeout > 0) {
 	if (repo->fetcher->timeout > 0) {
 		gettimeofday(&timeout, NULL);
 		timeout.tv_sec += fetchTimeout;
 		timeout.tv_sec += repo->fetcher->timeout;
 	}
 	deltams = -1;

@@ -451,7 +450,7 @@ ssh_read(void *data, char *buf, int len)

 		}
 		/* only EAGAIN should get here */
 		if (fetchTimeout > 0) {
 		if (repo->fetcher->timeout > 0) {
 			gettimeofday(&now, NULL);
 			if (!timercmp(&timeout, &now, >)) {
 				errno = ETIMEDOUT;

modified libpkg/private/pkg.h

@@ -180,6 +180,7 @@ struct pkg_repo;

 struct url;
 struct fetcher {
 	const char *scheme;
 	int64_t timeout;
 	int (*open)(struct pkg_repo *, const char *url, off_t *, time_t *t);
 	int (*close)(struct pkg_repo *);
 	int (*cleanup)(struct pkg_repo *);