fd5fba3 Implement resume on fetch for http/ftp — HardenedBSD-pkg

modified libpkg/fetch.c

@@ -98,7 +98,7 @@ pkg_fetch_file_tmp(struct pkg_repo *repo, const char *url, char *dest,

 		return(EPKG_FATAL);
 	}
 	retcode = pkg_fetch_file_to_fd(repo, url, fd, &t);
 	retcode = pkg_fetch_file_to_fd(repo, url, fd, &t, 0, -1);
 	if (t != 0) {
 		struct timeval ftimes[2] = {

@@ -124,19 +124,19 @@ pkg_fetch_file_tmp(struct pkg_repo *repo, const char *url, char *dest,

 }
 int
 pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t)
 pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t,
     ssize_t offset, int64_t size)
 {
 	int fd = -1;
 	int retcode = EPKG_FATAL;
 	fd = creat(dest, 00644);
 	fd = open(dest, O_CREAT|O_APPEND|O_WRONLY, 00644);
 	if (fd == -1) {
 		pkg_emit_errno("creat", dest);
 		pkg_emit_errno("open", dest);
 		return(EPKG_FATAL);
 	}
 	retcode = pkg_fetch_file_to_fd(repo, url, fd, &t);
 	retcode = pkg_fetch_file_to_fd(repo, url, fd, &t, offset, size);
 	if (t != 0) {
 		struct timeval ftimes[2] = {

@@ -453,14 +453,14 @@ start_ssh(struct pkg_repo *repo, struct url *u, off_t *sz)

 #define URL_SCHEME_PREFIX	"pkg+"
 int
 pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest, time_t *t)
 pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest,
     time_t *t, ssize_t offset, int64_t size)
 {
 	FILE		*remote = NULL;
 	struct url	*u = NULL;
 	struct url_stat	 st;
 	off_t		 done = 0;
 	off_t		 r;
 	int64_t		 max_retry, retry;
 	int64_t		 fetch_timeout;
 	char		 buf[10240];

@@ -559,7 +559,6 @@ pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest, time_t *t

 			u->port = http_current->url->port;
 		}
 		fetchOpts = sbuf_new_auto();
 		sbuf_cat(fetchOpts, "i");
 		if (repo != NULL) {

@@ -580,7 +579,9 @@ pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest, time_t *t

 		    sbuf_data(fetchOpts));
 		sbuf_finish(fetchOpts);
 		if (offset > 0)
 			u->offset = offset;
 		remote = fetchXGet(u, &st, sbuf_data(fetchOpts));
 		if (remote == NULL) {
 			if (fetchLastErrCode == FETCH_OK) {

@@ -619,52 +620,34 @@ pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest, time_t *t

 		sz = st.size;
 	}
 	if (sz <= 0 && size > 0)
 		sz = size;
 	pkg_emit_fetch_begin(url);
 	pkg_emit_progress_start(NULL);
         if (sz > 0) {
 		while (done < sz) {
 			int to_read = MIN(sizeof(buf), sz - done);
 			pkg_debug(1, "Reading status: want read %d over %d, %d already done",
 				to_read, sz, done);
 			if ((r = fread(buf, 1, to_read, remote)) < 1)
 				break;
 			if (write(dest, buf, r) != r) {
 				pkg_emit_errno("write", "");
 				retcode = EPKG_FATAL;
 				goto cleanup;
 			}
 			done += r;
 			pkg_debug(1, "Read status: %d over %d", done, sz);
 			pkg_emit_progress_tick(done, sz);
 		}
 		if (done < sz) {
 			pkg_emit_error("An error occurred while fetching package");
 	if (offset > 0)
 		done += offset;
 	while ((r = fread(buf, 1, sizeof(buf), remote)) > 0) {
 		if (write(dest, buf, r) != r) {
 			pkg_emit_errno("write", "");
 			retcode = EPKG_FATAL;
 			goto cleanup;
 		}
         }
 	else {
 		while ((r = fread(buf, 1, sizeof(buf), remote)) > 0) {
 			if (write(dest, buf, r) != r) {
 				pkg_emit_errno("write", "");
 				retcode = EPKG_FATAL;
 				goto cleanup;
 			}
 			done += r;
 		}
 		if (r != 0) {
 			pkg_emit_error("An error occurred while fetching package");
 			retcode = EPKG_FATAL;
 			goto cleanup;
 		}
 		else {
 			pkg_emit_progress_tick(done, done);
 		}
 		done += r;
 		if (sz > 0) {
 			pkg_debug(1, "Read status: %d over %d", done, sz);
 		} else
 			pkg_debug(1, "Read status: %d", done);
 		if (sz > 0)
 			pkg_emit_progress_tick(done, sz);
 	}
 	if (r != 0) {
 		pkg_emit_error("An error occurred while fetching package");
 		retcode = EPKG_FATAL;
 		goto cleanup;
 	} else {
 		pkg_emit_progress_tick(done, done);
 	}
 	pkg_emit_fetch_finished(url);

@@ -674,8 +657,7 @@ pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest, time_t *t

 		goto cleanup;
 	}
 	cleanup:
 cleanup:
 	if (u != NULL) {
 		if (remote != NULL &&  repo != NULL && remote != repo->ssh)
 			fclose(remote);

modified libpkg/pkg.h.in

@@ -1199,7 +1199,8 @@ pkg_change_t pkg_version_change_between(const struct pkg * pkg1, const struct pk

  * Fetch a file.
  * @return An error code.
  */
 int pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t);
 int pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t,
     ssize_t offset, int64_t size);
 /**
  * Fetch a file to temporary destination
  */

modified libpkg/pkg_repo.c

@@ -105,7 +105,7 @@ pkg_repo_fetch_remote_tmp(struct pkg_repo *repo,

 	}
 	(void)unlink(tmp);
 	if ((*rc = pkg_fetch_file_to_fd(repo, url, fd, t)) != EPKG_OK) {
 	if ((*rc = pkg_fetch_file_to_fd(repo, url, fd, t, -1, 0)) != EPKG_OK) {
 		close(fd);
 		fd = -1;
 	}

modified libpkg/private/pkg.h

@@ -498,8 +498,8 @@ extern struct pkg_key {

 	int type;
 } pkg_keys[PKG_NUM_FIELDS];
 int pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url,
 		int dest, time_t *t);
 int pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest,
     time_t *t, ssize_t offset, int64_t size);
 int pkg_repo_fetch_package(struct pkg *pkg);
 int pkg_repo_mirror_package(struct pkg *pkg, const char *destdir);
 FILE* pkg_repo_fetch_remote_extract_tmp(struct pkg_repo *repo,

modified libpkg/repo/binary/fetch.c

@@ -136,6 +136,7 @@ pkg_repo_binary_try_fetch(struct pkg_repo *repo, struct pkg *pkg,

 	struct stat st;
 	char *path = NULL;
 	const char *packagesite = NULL;
 	ssize_t offset = -1;
 	int retcode = EPKG_OK;

@@ -156,8 +157,15 @@ pkg_repo_binary_try_fetch(struct pkg_repo *repo, struct pkg *pkg,

 	/* If it is already in the local cachedir, dont bother to
 	 * download it */
 	if (access(dest, F_OK) == 0)
 		goto checksum;
 	if (stat(dest, &st) == 0) {
 		/* try to resume */
 		if (pkg->pkgsize > st.st_size) {
 			offset = st.st_size;
 			pkg_debug(1, "Resuming fetch");
 		} else {
 			goto checksum;
 		}
 	}
 	/* Create the dirs in cachedir */
 	dir = strdup(dest);

@@ -193,7 +201,7 @@ pkg_repo_binary_try_fetch(struct pkg_repo *repo, struct pkg *pkg,

 		return (EPKG_OK);
 	}
 	retcode = pkg_fetch_file(repo, url, dest, 0);
 	retcode = pkg_fetch_file(repo, url, dest, 0, offset, pkg->pkgsize);
 	fetched = 1;
 	if (retcode != EPKG_OK)

modified src/add.c

@@ -146,7 +146,7 @@ exec_add(int argc, char **argv)

 		if (is_url(argv[i]) == EPKG_OK) {
 			snprintf(path, sizeof(path), "%s/%s.XXXXX",
 			    getenv("TMPDIR") != NULL ? getenv("TMPDIR") : "/tmp", basename(argv[i]));
 			if ((retcode = pkg_fetch_file(NULL, argv[i], path, 0)) != EPKG_OK)
 			if ((retcode = pkg_fetch_file(NULL, argv[i], path, 0, 0, 0)) != EPKG_OK)
 				break;
 			file = path;