2027d7a fetch: introduce a new struct fetch_item — HardenedBSD-pkg

modified libpkg/fetch.c

@@ -112,6 +112,9 @@ pkg_fetch_file_tmp(struct pkg_repo *repo, const char *url, char *dest,

 {
 	int fd = -1;
 	int retcode = EPKG_FATAL;
 	struct fetch_item fi;
 	memset(&fi, 0, sizeof(struct fetch_item));
 	fd = mkstemp(dest);

@@ -120,16 +123,18 @@ pkg_fetch_file_tmp(struct pkg_repo *repo, const char *url, char *dest,

 		return(EPKG_FATAL);
 	}
 	retcode = pkg_fetch_file_to_fd(repo, url, fd, &t, 0, -1, false);
 	fi.url = url;
 	fi.mtime = t;
 	retcode = pkg_fetch_file_to_fd(repo, fd, &fi, false);
 	if (t != 0) {
 	if (fi.mtime != 0) {
 		struct timeval ftimes[2] = {
 			{
 			.tv_sec = t,
 			.tv_sec = fi.mtime,
 			.tv_usec = 0
 			},
 			{
 			.tv_sec = t,
 			.tv_sec = fi.mtime,
 			.tv_usec = 0
 			}
 		};

@@ -151,6 +156,7 @@ pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t,

 {
 	int fd = -1;
 	int retcode = EPKG_FATAL;
 	struct fetch_item fi;
 	fd = open(dest, O_CREAT|O_APPEND|O_WRONLY, 00644);
 	if (fd == -1) {

@@ -158,7 +164,12 @@ pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t,

 		return(EPKG_FATAL);
 	}
 	retcode = pkg_fetch_file_to_fd(repo, url, fd, &t, offset, size, false);
 	fi.url = url;
 	fi.offset = offset;
 	fi.size = size;
 	fi.mtime = t;
 	retcode = pkg_fetch_file_to_fd(repo, fd, &fi, false);
 	if (t != 0) {
 		struct timeval ftimes[2] = {

@@ -204,8 +215,8 @@ select_fetcher(const char *url)

 }
 int
 pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest,
     time_t *t, ssize_t offset, int64_t size, bool silent)
 pkg_fetch_file_to_fd(struct pkg_repo *repo, int dest, struct fetch_item *fi,
     bool silent)
 {
 	struct pkg_kv	*kv;
 	kvlist_t	 envtorestore = tll_init();

@@ -229,31 +240,31 @@ pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest,

 	 * Error if using plain http://, https:// etc with SRV
 	 */
 	pkg_debug(1, "Request to fetch %s", url);
 	pkg_debug(1, "Request to fetch %s", fi->url);
 	if (repo == NULL) {
 		fakerepo = xcalloc(1, sizeof(struct pkg_repo));
 		fakerepo->url = xstrdup(url);
 		fakerepo->url = xstrdup(fi->url);
 		repo = fakerepo;
 	}
 	if (repo->fetcher == NULL)
 		repo->fetcher = select_fetcher(url);
 		repo->fetcher = select_fetcher(fi->url);
 	if (repo->fetcher == NULL) {
 		pkg_emit_error("Unknown scheme: %s", url);
 		pkg_emit_error("Unknown scheme: %s", fi->url);
 		return (EPKG_FATAL);
 	}
 	if (strncasecmp(URL_SCHEME_PREFIX, url,
 	if (strncasecmp(URL_SCHEME_PREFIX, fi->url,
 	    strlen(URL_SCHEME_PREFIX)) == 0) {
 		if (repo->fetcher == NULL && repo->mirror_type != SRV) {
 			pkg_emit_error("packagesite URL error for %s -- "
 					URL_SCHEME_PREFIX
 					":// implies SRV mirror type", url);
 					":// implies SRV mirror type", fi->url);
 			/* Too early for there to be anything to cleanup */
 			return(EPKG_FATAL);
 		}
 		url += strlen(URL_SCHEME_PREFIX);
 		fi->url += strlen(URL_SCHEME_PREFIX);
 	}
 	repo->silent = silent;

@@ -269,16 +280,16 @@ pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest,

 		setenv(k->item->key, k->item->value, 1);
 	}
 	if ((retcode = repo->fetcher->open(repo, url, &sz, t)) != EPKG_OK)
 	if ((retcode = repo->fetcher->open(repo, fi)) != EPKG_OK)
 		goto cleanup;
 	pkg_debug(1, "Fetch: fetcher used: %s", repo->fetcher->scheme);
 	if (sz <= 0 && size > 0)
 		sz = size;
 	if (sz <= 0 && fi->size > 0)
 		sz = fi->size;
 	retcode = repo->fetcher->fetch(repo, dest, url, sz, offset, t);
 	retcode = repo->fetcher->fetch(repo, dest, fi);
 	if (retcode == EPKG_OK)
 		pkg_emit_fetch_finished(url);
 		pkg_emit_fetch_finished(fi->url);
 cleanup:
 	tll_foreach(envtorestore, k) {

@@ -299,11 +310,11 @@ cleanup:

 	if (retcode == EPKG_OK) {
 		struct timeval ftimes[2] = {
 			{
 			.tv_sec = *t,
 			.tv_sec = fi->mtime,
 			.tv_usec = 0
 			},
 			{
 			.tv_sec = *t,
 			.tv_sec = fi->mtime,
 			.tv_usec = 0
 			}
 		};

modified libpkg/fetch_file.c

@@ -33,28 +33,28 @@

 #include "private/pkg.h"
 #include "private/event.h"
 #include "private/utils.h"
 #include "private/fetch.h"
 int
 file_open(struct pkg_repo *repo, const char *url, size_t *sz,
     time_t *t)
 file_open(struct pkg_repo *repo, struct fetch_item *fi)
 {
 	struct stat st;
 	const char *u = url;
 	const char *u = fi->url;
 	if (strlen(u) > 5)
 		u += 5; /* file: */
 	if (*u != '/') {
 		pkg_emit_error("invalid url: '%s'\n", url);
 		pkg_emit_error("invalid url: '%s'\n", fi->url);
 		return (EPKG_FATAL);
 	}
 	if (stat(u, &st) == -1) {
 		if (!repo->silent)
 			pkg_emit_error("%s: %s", url,
 			pkg_emit_error("%s: %s", fi->url,
 			    strerror(errno));
 		return (EPKG_FATAL);
 	}
 	*sz = st.st_size;
 	if (st.st_mtime <= *t)
 	fi->size = st.st_size;
 	if (st.st_mtime <= fi->mtime)
 		return (EPKG_UPTODATE);
 	repo->fh = fopen(u, "re");

@@ -72,20 +72,20 @@ fh_close(struct pkg_repo *repo)

 }
 int
 stdio_fetch(struct pkg_repo *repo, int dest, const char *url, off_t sz, off_t offset, time_t *t __unused)
 stdio_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)
 {
 	char buf[8192];
 	size_t buflen = 0, left = 0;
 	off_t done = 0, r;
 	pkg_emit_fetch_begin(url);
 	pkg_emit_fetch_begin(fi->url);
 	pkg_emit_progress_start(NULL);
 	if (offset > 0)
 		done += offset;
 	if (fi->offset > 0)
 		done += fi->offset;
 	buflen = sizeof(buf);
 	left = sizeof(buf);
 	if (sz > 0)
 		left = sz - done;
 	if (fi->size > 0)
 		left = fi->size - done;
 	while ((r = fread(buf, 1, left < buflen ? left : buflen, repo->fh)) > 0) {
 		if (write(dest, buf, r) != r) {

@@ -93,13 +93,13 @@ stdio_fetch(struct pkg_repo *repo, int dest, const char *url, off_t sz, off_t of

 			return (EPKG_FATAL);
 		}
 		done += r;
 		if (sz > 0) {
 		if (fi->size > 0) {
 			left -= r;
 			pkg_debug(4, "Read status: %jd over %jd", (intmax_t)done, (intmax_t)sz);
 			pkg_debug(4, "Read status: %jd over %jd", (intmax_t)done, (intmax_t)fi->size);
 		} else
 			pkg_debug(4, "Read status: %jd", (intmax_t)done);
 		if (sz > 0)
 			pkg_emit_progress_tick(done, sz);
 		if (fi->size > 0)
 			pkg_emit_progress_tick(done, fi->size);
 	}
 	if (r != 0) {
 		pkg_emit_error("An error occurred while fetching package");

modified libpkg/fetch_libcurl.c

@@ -30,6 +30,7 @@

 #include "pkg.h"
 #include "private/pkg.h"
 #include "private/event.h"
 #include "private/fetch.h"
 struct curl_userdata {
 	int fd;

@@ -83,10 +84,8 @@ curl_progress_cb(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_

 	return (0);
 }
 int
 curl_open(struct pkg_repo *repo, const char *u __unused,
     size_t *sz __unused, time_t *t __unused)
 curl_open(struct pkg_repo *repo, struct fetch_item *fi __unused)
 {
 	CURLM *cm;
 	pkg_debug(1, "curl_open");

@@ -106,7 +105,7 @@ curl_open(struct pkg_repo *repo, const char *u __unused,

 }
 int
 curl_fetch(struct pkg_repo *repo, int dest, const char *url, off_t sz, off_t offset __unused, time_t *t)
 curl_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)
 {
 	CURL *cl;
 	CURLM *cm = NULL;

@@ -121,10 +120,10 @@ curl_fetch(struct pkg_repo *repo, int dest, const char *url, off_t sz, off_t off

 	data.fh = fdopen(dup(dest), "w");
 	if (data.fh == NULL)
 		return (EPKG_FATAL);
 	data.totalsize = sz;
 	data.url = url;
 	data.totalsize = fi->size;
 	data.url = fi->url;
 	pkg_debug(1, "curl> fetching %s\n", url);
 	pkg_debug(1, "curl> fetching %s\n", fi->url);
 	cl = curl_easy_init();
 	data.cl = cl;
 	curl_easy_setopt(cl, CURLOPT_FOLLOWLOCATION, 1L);

@@ -134,8 +133,8 @@ curl_fetch(struct pkg_repo *repo, int dest, const char *url, off_t sz, off_t off

 	curl_easy_setopt(cl, CURLOPT_PRIVATE, &data);
 	curl_easy_setopt(cl, CURLOPT_XFERINFOFUNCTION, curl_progress_cb);
 	curl_easy_setopt(cl, CURLOPT_XFERINFODATA, &data);
 	curl_easy_setopt(cl, CURLOPT_URL, url); /* TODO handle mirrors */
 	curl_easy_setopt(cl, CURLOPT_TIMEVALUE, (long)*t);
 	curl_easy_setopt(cl, CURLOPT_URL, fi->url); /* TODO handle mirrors */
 	curl_easy_setopt(cl, CURLOPT_TIMEVALUE, (long)fi->mtime);
 	curl_easy_setopt(cl, CURLOPT_TIMECONDITION, (long)CURL_TIMECOND_IFMODSINCE);
 	curl_easy_setopt(cl, CURLOPT_HEADERFUNCTION, curl_parseheader_cb);
 	curl_easy_setopt(cl, CURLOPT_HEADERDATA, &data);

@@ -183,15 +182,15 @@ curl_fetch(struct pkg_repo *repo, int dest, const char *url, off_t sz, off_t off

 	return (retcode);
 }
 int
 void
 curl_cleanup(struct pkg_repo *repo)
 {
 	CURLM *cm;
 	if (repo->fetch_priv == NULL)
 		return (EPKG_OK);
 		return;
 	cm = repo->fetch_priv;
 	curl_multi_cleanup(cm);
 	repo->fetch_priv = NULL;
 	return (EPKG_OK);
 	return;
 }

modified libpkg/fetch_ssh.c

@@ -230,8 +230,8 @@ ssh_cleanup:

 }
 static int
 pkgprotocol_open(struct pkg_repo *repo, const char *u, off_t *sz,
     int (*proto_connect)(struct pkg_repo *, struct yuarel *), time_t *t)
 pkgprotocol_open(struct pkg_repo *repo, struct fetch_item *fi,
     int (*proto_connect)(struct pkg_repo *, struct yuarel *))
 {
 	char *line = NULL;
 	size_t linecap = 0;

@@ -239,11 +239,11 @@ pkgprotocol_open(struct pkg_repo *repo, const char *u, off_t *sz,

 	const char *errstr;
 	int retcode = EPKG_FATAL;
 	struct yuarel url;
 	char *url_to_free = xstrdup(u);
 	char *url_to_free = xstrdup(fi->url);
 	if (yuarel_parse(&url, url_to_free) == -1) {
 		free(url_to_free);
 		pkg_emit_error("Invalid url: '%s'", u);
 		pkg_emit_error("Invalid url: '%s'", fi->url);
 		return (EPKG_FATAL);
 	}

@@ -256,20 +256,20 @@ pkgprotocol_open(struct pkg_repo *repo, const char *u, off_t *sz,

 	if (retcode != EPKG_OK)
 		return (retcode);
 	pkg_debug(1, "SSH> get %s %" PRIdMAX "", url.path, (intmax_t)*t);
 	fprintf(repo->fh, "get %s %" PRIdMAX "\n", url.path, (intmax_t)*t);
 	pkg_debug(1, "SSH> get %s %" PRIdMAX "", url.path, (intmax_t)fi->mtime);
 	fprintf(repo->fh, "get %s %" PRIdMAX "\n", url.path, (intmax_t)fi->mtime);
 	if ((linelen = getline(&line, &linecap, repo->fh)) > 0) {
 		if (line[linelen -1 ] == '\n')
 			line[linelen -1 ] = '\0';
 		pkg_debug(1, "SSH> recv: %s", line);
 		if (strncmp(line, "ok:", 3) == 0) {
 			*sz = strtonum(line + 4, 0, LONG_MAX, &errstr);
 			fi->size = strtonum(line + 4, 0, LONG_MAX, &errstr);
 			if (errstr) {
 				goto out;
 			}
 			if (*sz == 0) {
 			if (fi->size == 0) {
 				retcode = EPKG_UPTODATE;
 				goto out;
 			}

@@ -290,15 +290,15 @@ out:

 }
 int
 tcp_open(struct pkg_repo *repo, const char *u, off_t *sz, time_t *t)
 tcp_open(struct pkg_repo *repo, struct fetch_item *fi)
 {
 	return (pkgprotocol_open(repo, u, sz, tcp_connect, t));
 	return (pkgprotocol_open(repo, fi, tcp_connect));
 }
 int
 ssh_open(struct pkg_repo *repo, const char *u, off_t *sz, time_t *t)
 ssh_open(struct pkg_repo *repo, struct fetch_item *fi)
 {
 	return (pkgprotocol_open(repo, u, sz, ssh_connect, t));
 	return (pkgprotocol_open(repo, fi, ssh_connect));
 }
 static int

modified libpkg/pkg_config.c

@@ -45,6 +45,7 @@

 #include "pkg.h"
 #include "private/pkg.h"
 #include "private/event.h"
 #include "private/fetch.h"
 #include "pkg_repos.h"
 #ifndef PORTSDIR

modified libpkg/pkg_repo.c

@@ -55,6 +55,7 @@

 #include "private/utils.h"
 #include "private/pkg.h"
 #include "private/pkgdb.h"
 #include "private/fetch.h"
 struct sig_cert {
 	char name[MAXPATHLEN];

@@ -70,11 +71,14 @@ int

 pkg_repo_fetch_remote_tmp(struct pkg_repo *repo,
   const char *filename, const char *extension, time_t *t, int *rc, bool silent)
 {
 	struct fetch_item fi;
 	char url[MAXPATHLEN];
 	char tmp[MAXPATHLEN];
 	int fd;
 	const char *tmpdir, *dot;
 	memset(&fi, 0, sizeof(struct fetch_item));
 	/*
 	 * XXX: here we support old naming scheme, such as filename.yaml
 	 */

@@ -104,10 +108,13 @@ pkg_repo_fetch_remote_tmp(struct pkg_repo *repo,

 	}
 	(void)unlink(tmp);
 	if ((*rc = pkg_fetch_file_to_fd(repo, url, fd, t, -1, 0, silent)) != EPKG_OK) {
 	fi.url = url;
 	fi.mtime = *t;
 	if ((*rc = pkg_fetch_file_to_fd(repo, fd, &fi, silent)) != EPKG_OK) {
 		close(fd);
 		fd = -1;
 	}
 	*t = fi.mtime;
 	return (fd);
 }

modified libpkg/private/fetch.h

@@ -25,12 +25,28 @@

 #pragma once
 int fetch_open(struct pkg_repo *, const char *, off_t *, time_t *t);
 int ssh_open(struct pkg_repo *, const char *, off_t *, time_t *t);
 int file_open(struct pkg_repo *, const char *, off_t *, time_t *t);
 int fh_close(struct pkg_repo *);
 int tcp_open(struct pkg_repo *, const char *, off_t *, time_t *t);
 int stdio_fetch(struct pkg_repo *, int dest, const char *url, off_t sz, off_t offset, time_t *t);
 int curl_open(struct pkg_repo *, const char *, off_t *, time_t *t);
 int curl_fetch(struct pkg_repo *, int dest, const char *url, off_t sz, off_t offset, time_t *t);
 int curl_cleanup(struct pkg_repo *);
 struct fetch_item {
 	const char *url;
 	off_t size;
 	off_t offset;
 	time_t mtime;
 };
 struct fetcher {
 	const char *scheme;
 	int64_t timeout;
 	int (*open)(struct pkg_repo *, struct fetch_item *);
 	void (*close)(struct pkg_repo *);
 	void (*cleanup)(struct pkg_repo *);
 	int (*fetch)(struct pkg_repo *repo, int dest, struct fetch_item *);
 };
 int fetch_open(struct pkg_repo *, struct fetch_item *);
 int ssh_open(struct pkg_repo *, struct fetch_item *);
 int file_open(struct pkg_repo *, struct fetch_item *);
 void fh_close(struct pkg_repo *);
 int tcp_open(struct pkg_repo *, struct fetch_item *);
 int stdio_fetch(struct pkg_repo *, int dest, struct fetch_item *);
 int curl_open(struct pkg_repo *, struct fetch_item *);
 int curl_fetch(struct pkg_repo *, int dest, struct fetch_item *);
 void curl_cleanup(struct pkg_repo *);

modified libpkg/private/pkg.h

@@ -43,6 +43,7 @@

 #include "xmalloc.h"
 #include "private/utils.h"
 #include "private/fetch.h"
 #include "pkghash.h"
 #define UCL_COUNT(obj) ((obj)?((obj)->len):0)

@@ -178,14 +179,7 @@ extern struct pkg_ctx ctx;

 struct pkg_repo_it;
 struct pkg_repo;
 struct url;
 struct fetcher {
 	const char *scheme;
 	int64_t timeout;
 	int (*open)(struct pkg_repo *, const char *url, off_t *, time_t *t);
 	int (*close)(struct pkg_repo *);
 	int (*cleanup)(struct pkg_repo *);
 	int (*fetch)(struct pkg_repo *repo, int dest, const char *url, off_t sz, off_t offset, time_t *t);
 };
 struct fetcher;
 struct pkg_message;
 typedef tll(struct pkg_message *) messages_t;

@@ -630,8 +624,8 @@ int pkg_delete(struct pkg *pkg, struct pkg *rpkg, struct pkgdb *db, int flags,

 #define PKG_DELETE_UPGRADE	(1 << 1)	/* delete as a split upgrade */
 #define PKG_DELETE_NOSCRIPT	(1 << 2)	/* don't run delete scripts */
 int pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest,
     time_t *t, ssize_t offset, int64_t size, bool silent);
 int pkg_fetch_file_to_fd(struct pkg_repo *repo, int dest, struct fetch_item *,
     bool silent);
 int pkg_repo_fetch_package(struct pkg *pkg);
 int pkg_repo_mirror_package(struct pkg *pkg, const char *destdir);
 int pkg_repo_fetch_remote_extract_fd(struct pkg_repo *repo,