Radish alpha
H
HardenedBSD-pkg
rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9
HardenedBSD Package Manager
Radicle
Git
Commits › d85fcfd
fetch: switch from libfetch to libcurl
Baptiste Daroussin committed 2 years ago
commit d85fcfd1b0ec1ef8f119f5174d5eebe0471f28e9
parent 593573e
5 files changed +179 -9
modified libpkg/Makefile.autosetup
@@ -45,6 +45,7 @@ SRCS= backup_lib.c \
	flags.c \

	fetch_ssh.c \

	fetch_libfetch.c \
+ 
	fetch_libcurl.c \

	fetch_file.c \

	triggers.c \

	pkghash.c
@@ -58,6 +59,7 @@ LOCAL_CFLAGS= -I$(top_srcdir)/compat \
		-I$(top_srcdir)/external/include \

		-I$(top_srcdir)/external/libucl/include \

		-I$(top_srcdir)/external/libfetch \
+ 
		-I$(top_srcdir)/external/curl/include \

		-I$(top_srcdir)/external/lua/src \

		-I$(top_srcdir)/external/liblua/ \

		-I$(top_srcdir)/external/msgpuck \
@@ -83,6 +85,7 @@ LOCAL_LDFLAGS= @waflags@ \
		-L$(top_builddir)/external/blake2 -lblake2_pic \

		-L$(top_builddir)/compat -lbsd_compat_pic \

		-L$(top_builddir)/external/libfetch -lfetch_pic \
+ 
		-L$(top_builddir)/external/libcurl -lcurl_pic \

		-L$(top_builddir)/external/liblua -llua_pic \

		-L$(top_builddir)/external/msgpuck -lmsgpuck_pic \

		@nowaflags@ \
@@ -100,6 +103,7 @@ STATIC_LIBS= @REPOS_STATIC_LIBS@ \
		$(top_builddir)/external/liblua/liblua.a \

		$(top_builddir)/compat/libbsd_compat.a \

		$(top_builddir)/external/libfetch/libfetch.a \
+ 
		$(top_builddir)/external/libcurl/libcurl.a \

		$(top_builddir)/external/msgpuck/libmsgpuck.a \

		lib$(LIB).a
modified libpkg/fetch.c
@@ -65,17 +65,17 @@ static struct fetcher fetchers [] = {
	},

	{

		"pkg+https",
- 
		fetch_open,
- 
		fh_close,
+ 
		curl_open,

		NULL,
- 
		libfetch_fetch,
+ 
		curl_cleanup,
+ 
		curl_fetch,

	},

	{

		"pkg+http",
- 
		fetch_open,
- 
		fh_close,
+ 
		curl_open,

		NULL,
- 
		libfetch_fetch,
+ 
		curl_cleanup,
+ 
		curl_fetch,

	},

	{

		"https",
@@ -86,10 +86,10 @@ static struct fetcher fetchers [] = {
	},

	{

		"http",
- 
		fetch_open,
- 
		fh_close,
+ 
		curl_open,

		NULL,
- 
		libfetch_fetch,
+ 
		curl_cleanup,
+ 
		curl_fetch,

	},

	{

		"file",
added libpkg/fetch_libcurl.c
@@ -0,0 +1,162 @@
+ 
/*-
+ 
 * Copyright (c) 2023 Baptiste Daroussin <bapt@FreeBSD.org>
+ 
 *
+ 
 * Redistribution and use in source and binary forms, with or without
+ 
 * modification, are permitted provided that the following conditions
+ 
 * are met:
+ 
 * 1. Redistributions of source code must retain the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer
+ 
 *    in this position and unchanged.
+ 
 * 2. Redistributions in binary form must reproduce the above copyright
+ 
 *    notice, this list of conditions and the following disclaimer in the
+ 
 *    documentation and/or other materials provided with the distribution.
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ 
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ 
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ 
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ 
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ 
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ 
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ 
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
 */
+
+ 
#include <sys/param.h>
+ 
#include <stdlib.h>
+ 
#include <curl/curl.h>
+
+ 
#include "pkg.h"
+ 
#include "private/pkg.h"
+ 
#include "private/event.h"
+
+ 
struct curl_priv {
+ 
	CURLM *cm;
+ 
};
+ 
struct curl_userdata {
+ 
	int fd;
+ 
	FILE *fh;
+ 
	size_t size;
+ 
	size_t totalsize;
+ 
	bool started;
+ 
};
+
+ 
static size_t
+ 
curl_write_cb(char *data, size_t size, size_t nmemb, void *userdata)
+ 
{
+ 
	struct curl_userdata *d = (struct curl_userdata *)userdata;
+ 
	size_t written;
+
+ 
	if (!d->started) {
+ 
		pkg_emit_progress_start(NULL);
+ 
		d->started = true;
+ 
	}
+ 
	written = fwrite(data, size, nmemb, d->fh);
+ 
	d->size += written;
+ 
	pkg_emit_progress_tick(d->size, d->totalsize);
+ 
	return (written);
+ 
}
+
+ 
int
+ 
curl_open(struct pkg_repo *repo, struct url *u __unused, off_t *sz __unused)
+ 
{
+ 
	CURLM *cm;
+ 
	pkg_debug(1, "curl_open");
+
+ 
	if (repo->fetch_priv != NULL)
+ 
		return (EPKG_OK);
+
+ 
	curl_global_init(CURL_GLOBAL_ALL);
+ 
	cm = curl_multi_init();
+ 
	curl_multi_setopt(cm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
+ 
	curl_multi_setopt(cm, CURLMOPT_MAX_HOST_CONNECTIONS, 1);
+ 
	/* TODO: Later for parallel fetching */
+ 
	/*curl_multi_setopt(cm, CURLMOPT_MAX_HOST_TOTAL_CONNECTIONS, 4);*/
+ 
	repo->fetch_priv = cm;
+
+ 
	return (EPKG_OK);
+ 
}
+
+ 
int
+ 
curl_fetch(struct pkg_repo *repo, int dest, const char *url, struct url *u, off_t sz, time_t *t)
+ 
{
+ 
	CURL *cl;
+ 
	CURLM *cm = NULL;
+ 
	struct curl_userdata data = { 0 };
+ 
	int still_running = 1;
+ 
	CURLMsg *msg;
+ 
	int msgs_left;
+ 
	int retcode = EPKG_OK;
+
+ 
	cm = (CURLM *)repo->fetch_priv;
+
+ 
	data.fh = fdopen(dup(dest), "w");
+ 
	if (data.fh == NULL)
+ 
		return (EPKG_FATAL);
+ 
	data.totalsize = sz;
+
+ 
	pkg_debug(1, "curl> fetching %s\n", url);
+ 
	cl = curl_easy_init();
+ 
	curl_easy_setopt(cl, CURLOPT_FOLLOWLOCATION, 1L);
+ 
	curl_easy_setopt(cl, CURLOPT_WRITEFUNCTION, curl_write_cb);
+ 
	curl_easy_setopt(cl, CURLOPT_WRITEDATA, &data);
+ 
	curl_easy_setopt(cl, CURLOPT_NOPROGRESS, 1L);
+ 
	curl_easy_setopt(cl, CURLOPT_PRIVATE, &data);
+ 
	curl_easy_setopt(cl, CURLOPT_URL, url); /* TODO handle mirrors */
+ 
	curl_easy_setopt(cl, CURLOPT_TIMEVALUE, (long)*t);
+ 
	curl_easy_setopt(cl, CURLOPT_TIMECONDITION, (long)CURL_TIMECOND_IFMODSINCE);
+ 
	//curl_easy_setopt(cl, CURLOPT_MAXFILESIZE_LARGE, *sz);
+ 
	/* compat with libfetch */
+ 
	if (getenv("SSL_NO_VERFIRY_PEER") != NULL)
+ 
		curl_easy_setopt(cl, CURLOPT_SSL_VERIFYPEER, 0L);
+ 
	if (getenv("SSL_NO_VERIFY_HOSTNAME") != NULL)
+ 
		curl_easy_setopt(cl, CURLOPT_SSL_VERIFYHOST, 0L);
+ 
	curl_multi_add_handle(cm, cl);
+
+ 
	pkg_emit_fetch_begin(url);
+ 
	while(still_running) {
+ 
		CURLMcode mc = curl_multi_perform(cm, &still_running);
+
+ 
		if(still_running)
+ 
			/* wait for activity, timeout or "nothing" */
+ 
			mc = curl_multi_poll(cm, NULL, 0, 1000, NULL);
+
+ 
		if(mc)
+ 
			break;
+ 
	}
+ 
	while((msg = curl_multi_info_read(cm, &msgs_left))) {
+ 
		fclose(data.fh);
+ 
		if(msg->msg == CURLMSG_DONE) {
+ 
			CURL *eh = msg->easy_handle;
+ 
			long rc = 0;
+ 
			curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, &rc);
+ 
			if (rc == 304)
+ 
				retcode = EPKG_UPTODATE;
+ 
			else if (rc != 200) {
+ 
				pkg_emit_error("An error occured while fetching package");
+ 
				retcode = EPKG_FATAL;
+ 
			}
+ 
		} else {
+ 
			pkg_emit_error("An error occured while fetching package");
+ 
		}
+ 
	}
+
+ 
	curl_multi_remove_handle(cm, cl);
+ 
	curl_easy_cleanup(cl);
+
+ 
	return (retcode);
+ 
}
+
+ 
int
+ 
curl_cleanup(struct pkg_repo *repo)
+ 
{
+ 
	CURLM *cm;
+
+ 
	if (repo->fetch_priv == NULL)
+ 
		return (EPKG_OK);
+ 
	cm = repo->fetch_priv;
+ 
	curl_multi_cleanup(cm);
+ 
	repo->fetch_priv = NULL;
+ 
	return (EPKG_OK);
+ 
}
modified libpkg/private/fetch.h
@@ -32,3 +32,6 @@ int fh_close(struct pkg_repo *);
int tcp_open(struct pkg_repo *, struct url *, off_t *);

int stdio_fetch(struct pkg_repo *, int dest, const char *url, struct url *u, off_t sz, time_t *t);

int libfetch_fetch(struct pkg_repo *, int dest, const char *url, struct url *u, off_t sz, time_t *t);
+ 
int curl_open(struct pkg_repo *, struct url *, off_t *);
+ 
int curl_fetch(struct pkg_repo *, int dest, const char *url, struct url *u, off_t sz, time_t *t);
+ 
int curl_cleanup(struct pkg_repo *);
modified libpkg/private/pkg.h
@@ -522,6 +522,7 @@ struct pkg_repo {
	signature_t signature_type;

	char *fingerprints;

	FILE *fh;
+ 
	void *fetch_priv;

	bool silent;



	pkghash *trusted_fp;