Radish alpha
H
HardenedBSD-pkg
rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9
HardenedBSD Package Manager
Radicle
Git
Commits › 03cc302
libfetch: implement http keep-alive
Baptiste Daroussin committed 6 years ago
commit 03cc3021f18ec3d0d9ec71019d1fd5b78743a4db
parent 789aa2f
5 files changed +165 -19
modified external/libfetch/common.c
@@ -348,19 +348,19 @@ fetch_bind(int sd, int af, const char *addr)
 * Establish a TCP connection to the specified port on the specified host.

 */

conn_t *
- 
fetch_connect(const char *host, int port, int af, int verbose)
+ 
fetch_connect(struct url *u, int af, int verbose)

{

	struct addrinfo *cais = NULL, *sais = NULL, *cai, *sai;

	const char *bindaddr;

	conn_t *conn = NULL;

	int err = 0, sd = -1;
- 
	DEBUGF("---> %s:%d\n", host, port);
+ 
	DEBUGF("---> %s:%d\n", u->host, u->port);



	/* resolve server address */

	if (verbose)
- 
		fetch_info("resolving server address: %s:%d", host, port);
- 
	if ((sais = fetch_resolve(host, port, af)) == NULL)
+ 
		fetch_info("resolving server address: %s:%d", u->host, u->port);
+ 
	if ((sais = fetch_resolve(u->host, u->port, af)) == NULL)

		goto fail;



	/* resolve client address */
@@ -398,12 +398,20 @@ fetch_connect(const char *host, int port, int af, int verbose)
	}

	if (err != 0) {

		if (verbose)
- 
			fetch_info("failed to connect to %s:%d", host, port);
+ 
			fetch_info("failed to connect to %s:%d", u->host, u->port);

		goto syserr;

	}



	if ((conn = fetch_reopen(sd)) == NULL)

		goto syserr;
+
+ 
	strlcpy(conn->scheme, u->scheme, sizeof(conn->scheme));
+ 
	strlcpy(conn->host, u->host, sizeof(conn->host));
+ 
	strlcpy(conn->user, u->user, sizeof(conn->user));
+ 
	strlcpy(conn->pwd, u->pwd, sizeof(conn->pwd));
+ 
	conn->port = u->port;
+ 
	conn->af = af;
+ 


	if (cais != NULL)

		freeaddrinfo(cais);

	if (sais != NULL)
@@ -421,6 +429,103 @@ fail:
		freeaddrinfo(sais);

	return (NULL);

}
+ 
static conn_t *connection_cache;
+ 
static int cache_global_limit = 0;
+ 
static int cache_per_host_limit = 0;
+
+ 
/*
+ 
 * Initialise cache with the given limits.
+ 
 */
+ 
void
+ 
fetchConnectionCacheInit(int global_limit, int per_host_limit)
+ 
{
+
+ 
	if (global_limit < 0)
+ 
		cache_global_limit = INT_MAX;
+ 
	else if (per_host_limit > global_limit)
+ 
		cache_global_limit = per_host_limit;
+ 
	else
+ 
		cache_global_limit = global_limit;
+ 
	if (per_host_limit < 0)
+ 
		cache_per_host_limit = INT_MAX;
+ 
	else
+ 
		cache_per_host_limit = per_host_limit;
+ 
}
+
+ 
/*
+ 
 * Flush cache and free all associated resources.
+ 
 */
+ 
void
+ 
fetchConnectionCacheClose(void)
+ 
{
+ 
	conn_t *conn;
+
+ 
	while ((conn = connection_cache) != NULL) {
+ 
		connection_cache = conn->next;
+ 
		(*conn->close)(conn);
+ 
	}
+ 
}
+
+ 
/*
+ 
 * Check connection cache for an existing entry matching
+ 
 * protocol/host/port/user/password/family.
+ 
 */
+ 
conn_t *
+ 
fetch_cache_get(const struct url *url, int af)
+ 
{
+ 
	conn_t *conn, *last_conn = NULL;
+
+ 
	for (conn = connection_cache; conn; conn = conn->next) {
+ 
		if (conn->port == url->port &&
+ 
		    strcmp(conn->scheme, url->scheme) == 0 &&
+ 
		    strcmp(conn->host, url->host) == 0 &&
+ 
		    strcmp(conn->user, url->user) == 0 &&
+ 
		    strcmp(conn->pwd, url->pwd) == 0 &&
+ 
		    (conn->af == AF_UNSPEC || af == AF_UNSPEC ||
+ 
		     conn->af == af)) {
+ 
			if (last_conn != NULL)
+ 
				last_conn->next = conn->next;
+ 
			else
+ 
				connection_cache = conn->next;
+
+ 
			return conn;
+ 
		}
+ 
	}
+
+ 
	return NULL;
+ 
}
+
+ 
/*
+ 
 * Put the connection back into the cache for reuse.
+ 
 */
+ 
void
+ 
fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *))
+ 
{
+ 
	conn_t *iter, *last;
+ 
	int global_count, host_count;
+
+ 
	global_count = host_count = 0;
+ 
	last = NULL;
+ 
	for (iter = connection_cache; iter;
+ 
	    last = iter, iter = iter->next) {
+ 
		++global_count;
+ 
		if (strcmp(conn->host, iter->host) == 0)
+ 
			++host_count;
+ 
		if (global_count < cache_global_limit &&
+ 
		    host_count < cache_per_host_limit)
+ 
			continue;
+ 
		--global_count;
+ 
		if (last != NULL)
+ 
			last->next = iter->next;
+ 
		else
+ 
			connection_cache = iter->next;
+ 
		(*iter->close)(iter);
+ 
	}
+
+ 
	conn->close = closecb;
+ 
	conn->next = connection_cache;
+ 
	connection_cache = conn;
+ 
}



#ifdef WITH_SSL

/*
modified external/libfetch/common.h
@@ -61,6 +61,14 @@ struct fetchconn {
	const SSL_METHOD *ssl_meth;	/* SSL method */

#endif

	int		 ref;		/* reference count */
+ 
	char		 scheme[URL_SCHEMELEN+1];
+ 
	char		 user[URL_USERLEN+1];
+ 
	char		 pwd[URL_PWDLEN+1];
+ 
	char		 host[MAXHOSTNAMELEN+1];
+ 
	int		 port;
+ 
	int		 af;
+ 
	int		(*close)(conn_t *);
+ 
	conn_t		*next;

};



/* Structure used for error message lists */
@@ -80,7 +88,7 @@ int fetch_default_port(const char *);
int		 fetch_default_proxy_port(const char *);

struct addrinfo *fetch_resolve(const char *, int, int);

int		 fetch_bind(int, int, const char *);
- 
conn_t		*fetch_connect(const char *, int, int, int);
+ 
conn_t		*fetch_connect(struct url *, int, int);

conn_t		*fetch_reopen(int);

conn_t		*fetch_ref(conn_t *);

#ifdef WITH_SSL
@@ -97,6 +105,8 @@ int fetch_add_entry(struct url_ent **, int *, int *,
		     const char *, struct url_stat *);

int		 fetch_netrc_auth(struct url *url);

int		 fetch_no_proxy_match(const char *);
+ 
conn_t		*fetch_cache_get(const struct url *, int);
+ 
void		 fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *));



#define ftp_seterr(n)	 fetch_seterr(ftp_errlist, n)

#define http_seterr(n)	 fetch_seterr(http_errlist, n)
modified external/libfetch/fetch.h
@@ -131,10 +131,15 @@ struct url_ent *fetchList(struct url *, const char *);
struct url	*fetchMakeURL(const char *, const char *, int,

		     const char *, const char *, const char *);

struct url	*fetchParseURL(const char *);
+ 
struct url	*fetchDupURL(struct url *);

void		 fetchFreeURL(struct url *);



__END_DECLS
+ 
/* Connection caching */
+ 
void fetchConnectionCacheInit(int, int);
+ 
void fetchConnectionCacheClose(void);
+ 


/* Authentication */

typedef int (*auth_t)(struct url *);

extern auth_t		 fetchAuthMethod;
modified external/libfetch/ftp.c
@@ -976,10 +976,10 @@ ftp_connect(struct url *url, struct url *purl, const char *flags)
	/* check for proxy */

	if (purl) {

		/* XXX proxy authentication! */
- 
		conn = fetch_connect(purl->host, purl->port, af, verbose);
+ 
		conn = fetch_connect(purl, af, verbose);

	} else {

		/* no proxy, go straight to target */
- 
		conn = fetch_connect(url->host, url->port, af, verbose);
+ 
		conn = fetch_connect(url, af, verbose);

		purl = NULL;

	}
modified external/libfetch/http.c
@@ -131,6 +131,7 @@ struct httpio
{

	conn_t		*conn;		/* connection */

	int		 chunked;	/* chunked mode */
+ 
	int		 keep_alive;	/* keep-alive mode */

	char		*buf;		/* chunk buffer */

	size_t		 bufsize;	/* size of chunk buffer */

	size_t		 buflen;	/* amount of data currently in buffer */
@@ -316,11 +317,22 @@ static int
http_closefn(void *v)

{

	struct httpio *io = (struct httpio *)v;
- 
	int r;
+ 
	int r, val;
- 
	r = fetch_close(io->conn);
- 
	if (io->buf)
- 
		free(io->buf);
+ 
	if (io->keep_alive) {
+ 
		val = 0;
+ 
		setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NODELAY, &val,
+ 
		    sizeof(val));
+ 
		fetch_cache_put(io->conn, fetch_close);
+ 
		val = 1;
+ 
		setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val,
+ 
		    sizeof(val));
+ 
		r = 0;
+ 
	} else {
+ 
		r = fetch_close(io->conn);
+ 
	}
+
+ 
	free(io->buf);

	free(io);

	return (r);

}
@@ -329,7 +341,7 @@ http_closefn(void *v)
 * Wrap a file descriptor up

 */

static FILE *
- 
http_funopen(conn_t *conn, int chunked)
+ 
http_funopen(conn_t *conn, int chunked, int keep_alive)

{

	struct httpio *io;

	FILE *f;
@@ -340,6 +352,7 @@ http_funopen(conn_t *conn, int chunked)
	}

	io->conn = conn;

	io->chunked = chunked;
+ 
	io->keep_alive = keep_alive;

	f = funopen(io, http_readfn, http_writefn, NULL, http_closefn);

	if (f == NULL) {

		fetch_syserr();
@@ -360,6 +373,7 @@ typedef enum {
	hdr_error = -1,

	hdr_end = 0,

	hdr_unknown = 1,
+ 
	hdr_connection,

	hdr_content_length,

	hdr_content_range,

	hdr_last_modified,
@@ -374,6 +388,7 @@ static struct {
	hdr_t		 num;

	const char	*name;

} hdr_names[] = {
+ 
	{ hdr_connection,		"Connection" },

	{ hdr_content_length,		"Content-Length" },

	{ hdr_content_range,		"Content-Range" },

	{ hdr_last_modified,		"Last-Modified" },
@@ -1377,7 +1392,7 @@ http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 * Connect to the correct HTTP server or proxy.

 */

static conn_t *
- 
http_connect(struct url *URL, struct url *purl, const char *flags)
+ 
http_connect(struct url *URL, struct url *purl, const char *flags, int *cached)

{

	struct url *curl;

	conn_t *conn;
@@ -1404,7 +1419,12 @@ http_connect(struct url *URL, struct url *purl, const char *flags)


	curl = (purl != NULL) ? purl : URL;
- 
	if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL)
+ 
	if ((conn = fetch_cache_get(curl, af)) != NULL) {
+ 
		*cached = 1;
+ 
		return (conn);
+ 
	}
+
+ 
	if ((conn = fetch_connect(curl, af, verbose)) == NULL)

		/* fetch_connect() has already set an error code */

		return (NULL);

	init_http_headerbuf(&headerbuf);
@@ -1546,7 +1566,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
	char hbuf[MAXHOSTNAMELEN + 7], *host;

	conn_t *conn;

	struct url *url, *new;
- 
	int chunked, direct, ims, noredirect, verbose;
+ 
	int chunked, direct, ims, keep_alive, noredirect, verbose, cached;

	int e, i, n, val;

	off_t offset, clength, length, size;

	time_t mtime;
@@ -1568,6 +1588,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
	noredirect = CHECK_FLAG('A');

	verbose = CHECK_FLAG('v');

	ims = CHECK_FLAG('i');
+ 
	keep_alive = 0;



	if (direct && purl) {

		fetchFreeURL(purl);
@@ -1589,6 +1610,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
		length = -1;

		size = -1;

		mtime = 0;
+ 
		cached = 0;



		/* check port */

		if (!url->port)
@@ -1603,7 +1625,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
		}



		/* connect to server or proxy */
- 
		if ((conn = http_connect(url, purl, flags)) == NULL)
+ 
		if ((conn = http_connect(url, purl, flags, &cached)) == NULL)

			goto ouch;



		/* append port number only if necessary */
@@ -1724,7 +1746,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
		}

		if (url->offset > 0)

			http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
- 
		http_cmd(conn, "Connection: close");
+ 
		http_cmd(conn, "Connection: keep-alive");



		if (body) {

			body_len = strlen(body);
@@ -1828,6 +1850,10 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
			case hdr_error:

				http_seterr(HTTP_PROTOCOL_ERROR);

				goto ouch;
+ 
			case hdr_connection:
+ 
				/* XXX too weak? */
+ 
				keep_alive = (strcasecmp(p, "keep-alive") == 0);
+ 
				break;

			case hdr_content_length:

				http_parse_length(p, &clength);

				break;
@@ -2002,7 +2028,7 @@ http_request_body(struct url *URL, const char *op, struct url_stat *us,
	URL->length = clength;



	/* wrap it up in a FILE */
- 
	if ((f = http_funopen(conn, chunked)) == NULL) {
+ 
	if ((f = http_funopen(conn, chunked, keep_alive)) == NULL) {

		fetch_syserr();

		goto ouch;

	}