f06399c curl: readd support for SRV mirroring type — HardenedBSD-pkg

modified libpkg/fetch.c

@@ -157,6 +157,7 @@ pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t,

 	int fd = -1;
 	int retcode = EPKG_FATAL;
 	struct fetch_item fi;
 	char *url_to_free = NULL;
 	fd = open(dest, O_CREAT|O_APPEND|O_WRONLY, 00644);
 	if (fd == -1) {

@@ -164,12 +165,19 @@ pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t,

 		return(EPKG_FATAL);
 	}
 	fi.url = url;
 	if (repo != NULL) {
 		xasprintf(&url_to_free, "%s/%s", repo->url, url);
 		fi.url = url_to_free;
 	} else {
 		fi.url = url;
 	}
 	fi.offset = offset;
 	fi.size = size;
 	fi.mtime = t;
 	retcode = pkg_fetch_file_to_fd(repo, fd, &fi, false);
 	free(url_to_free);
 	if (t != 0) {
 		struct timeval ftimes[2] = {

@@ -184,7 +192,6 @@ pkg_fetch_file(struct pkg_repo *repo, const char *url, char *dest, time_t t,

 		};
 		futimes(fd, ftimes);
 	}
 	close(fd);
 	/* Remove local file if fetch failed */

@@ -244,6 +251,7 @@ pkg_fetch_file_to_fd(struct pkg_repo *repo, int dest, struct fetch_item *fi,

 	if (repo == NULL) {
 		fakerepo = xcalloc(1, sizeof(struct pkg_repo));
 		fakerepo->url = xstrdup(fi->url);
 		fakerepo->mirror_type = NOMIRROR;
 		repo = fakerepo;
 	}

modified libpkg/fetch_libcurl.c

@@ -32,6 +32,12 @@

 #include "private/event.h"
 #include "private/fetch.h"
 struct curl_repodata {
 	CURLM *cm;
 	CURLU *url;
 };
 struct curl_userdata {
 	int fd;
 	CURL *cl;

@@ -60,13 +66,13 @@ curl_parseheader_cb(void *ptr __unused, size_t size, size_t nmemb, void *userdat

 {
 	struct curl_userdata *d = (struct curl_userdata *)userdata;
 	if (!d->started) {
 	curl_easy_getinfo(d->cl, CURLINFO_RESPONSE_CODE, &d->response);
 	if (d->response == 200 && !d->started) {
 		pkg_emit_fetch_begin(d->url);
 		pkg_emit_progress_start(NULL);
 		d->started = true;
 	}
 	curl_easy_getinfo(d->cl, CURLINFO_RESPONSE_CODE, &d->response);
 	return (size *nmemb);

@@ -87,19 +93,45 @@ curl_progress_cb(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_

 int
 curl_open(struct pkg_repo *repo, struct fetch_item *fi __unused)
 {
 	CURLM *cm;
 	struct curl_repodata *cr = xcalloc(1, sizeof(*cr));
 	pkg_debug(1, "curl_open");
 	if (repo->fetch_priv != NULL)
 		return (EPKG_OK);
 	curl_global_init(CURL_GLOBAL_ALL);
 	cm = curl_multi_init();
 	curl_multi_setopt(cm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
 	curl_multi_setopt(cm, CURLMOPT_MAX_HOST_CONNECTIONS, 1);
 	cr->cm = curl_multi_init();
 	curl_multi_setopt(cr->cm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
 	curl_multi_setopt(cr->cm, CURLMOPT_MAX_HOST_CONNECTIONS, 1);
 	/* TODO: Later for parallel fetching */
 	/*curl_multi_setopt(cm, CURLMOPT_MAX_HOST_TOTAL_CONNECTIONS, 4);*/
 	repo->fetch_priv = cm;
 	repo->fetch_priv = cr;
 	if (repo->mirror_type == SRV && repo->srv == NULL) {
 		int urloff = 0;
 		cr->url = curl_url();
 		if (strncasecmp(repo->url, "pkg+", 4) == 0)
 			urloff = 4;
 		CURLUcode c = curl_url_set(cr->url, CURLUPART_URL, repo->url + urloff, 0);
 		if (c) {
 			pkg_emit_error("impossible to parse url: '%s'", repo->url);
 			return (EPKG_FATAL);
 		}
 		if (repo->srv == NULL) {
 			char *zone;
 			char *host = NULL;
 			curl_url_get(cr->url, CURLUPART_HOST, &host, 0);
 			xasprintf(&zone, "_http._tcp.%s", host);
 			repo->srv = dns_getsrvinfo(zone);
 			free(zone);
 			free(host);
 			if (repo->srv == NULL) {
 				pkg_emit_error("No SRV record found for the "
 				    "repo '%s'\n", repo->name);
 				repo->mirror_type = NOMIRROR;
 			}
 		}
 	}
 	return (EPKG_OK);
 }

@@ -108,14 +140,17 @@ int

 curl_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)
 {
 	CURL *cl;
 	CURLM *cm = NULL;
 	struct curl_userdata data = { 0 };
 	int still_running = 1;
 	CURLMsg *msg;
 	int msgs_left;
 	int64_t retry;
 	int retcode = EPKG_OK;
 	struct dns_srvinfo *srv_current = NULL;
 	struct http_mirror *http_current = NULL;
 	char *urlpath;
 	cm = (CURLM *)repo->fetch_priv;
 	struct curl_repodata *cr = (struct curl_repodata *)repo->fetch_priv;
 	data.fh = fdopen(dup(dest), "w");
 	if (data.fh == NULL)

@@ -126,6 +161,33 @@ curl_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)

 	pkg_debug(1, "curl> fetching %s\n", fi->url);
 	cl = curl_easy_init();
 	data.cl = cl;
 	retry = pkg_object_int(pkg_config_get("FETCH_RETRY"));
 	if (repo->mirror_type == SRV) {
 		CURLU *cu = curl_url();
 		curl_url_set(cu, CURLUPART_URL, fi->url, 0);
 		curl_url_get(cu, CURLUPART_PATH, &urlpath, 0);
 		if (urlpath != NULL)
 			curl_url_set(cr->url, CURLUPART_PATH, urlpath, 0);
 		curl_url_cleanup(cu);
 	}
 retry:
 	if (repo->mirror_type == SRV) {
 		char *portstr;
 		if (srv_current == NULL)
 			srv_current = repo->srv;
 		else
 			srv_current = srv_current->next;
 		curl_url_set(cr->url, CURLUPART_HOST, srv_current->host, 0);
 		xasprintf(&portstr, "%d", srv_current->port);
 		curl_url_set(cr->url, CURLUPART_PORT, portstr, 0);
 		free(portstr);
 		curl_easy_setopt(cl, CURLOPT_CURLU, cr->url);
 	} else if (repo->mirror_type == HTTP) {
 		/* TODO http mirroring */
 	} else {
 		curl_easy_setopt(cl, CURLOPT_URL, fi->url);
 	}
 	curl_easy_setopt(cl, CURLOPT_FOLLOWLOCATION, 1L);
 	curl_easy_setopt(cl, CURLOPT_WRITEFUNCTION, curl_write_cb);
 	curl_easy_setopt(cl, CURLOPT_WRITEDATA, &data);

@@ -133,7 +195,6 @@ curl_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)

 	curl_easy_setopt(cl, CURLOPT_PRIVATE, &data);
 	curl_easy_setopt(cl, CURLOPT_XFERINFOFUNCTION, curl_progress_cb);
 	curl_easy_setopt(cl, CURLOPT_XFERINFODATA, &data);
 	curl_easy_setopt(cl, CURLOPT_URL, fi->url); /* TODO handle mirrors */
 	curl_easy_setopt(cl, CURLOPT_TIMEVALUE, (long)fi->mtime);
 	curl_easy_setopt(cl, CURLOPT_TIMECONDITION, (long)CURL_TIMECOND_IFMODSINCE);
 	curl_easy_setopt(cl, CURLOPT_HEADERFUNCTION, curl_parseheader_cb);

@@ -148,29 +209,34 @@ curl_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)

 		curl_easy_setopt(cl, CURLOPT_SSL_VERIFYPEER, 0L);
 	if (getenv("SSL_NO_VERIFY_HOSTNAME") != NULL)
 		curl_easy_setopt(cl, CURLOPT_SSL_VERIFYHOST, 0L);
 	curl_multi_add_handle(cm, cl);
 	curl_multi_add_handle(cr->cm, cl);
 	while(still_running) {
 		CURLMcode mc = curl_multi_perform(cm, &still_running);
 		CURLMcode mc = curl_multi_perform(cr->cm, &still_running);
 		if(still_running)
 			/* wait for activity, timeout or "nothing" */
 			mc = curl_multi_poll(cm, NULL, 0, 1000, NULL);
 			mc = curl_multi_poll(cr->cm, NULL, 0, 1000, NULL);
 		if(mc)
 			break;
 	}
 	while((msg = curl_multi_info_read(cm, &msgs_left))) {
 	while((msg = curl_multi_info_read(cr->cm, &msgs_left))) {
 		fclose(data.fh);
 		if(msg->msg == CURLMSG_DONE) {
 			CURL *eh = msg->easy_handle;
 			long rc = 0;
 			curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, &rc);
 			if (rc == 304)
 			if (rc == 304) {
 				retcode = EPKG_UPTODATE;
 			else if (rc != 200) {
 				pkg_emit_error("An error occured while fetching package");
 				retcode = EPKG_FATAL;
 			} else if (rc != 200) {
 				--retry;
 				if (retry <= 0 || rc == 404) {
 					pkg_emit_error("An error occured while "
 				    "fetching package");
 					retcode = EPKG_FATAL;
 				}
 				goto retry;
 			}
 		} else {
 			pkg_emit_error("An error occured while fetching package");

@@ -178,7 +244,7 @@ curl_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)

 	}
 	curl_easy_getinfo(cl, CURLINFO_FILETIME_T, &fi->mtime);
 	curl_multi_remove_handle(cm, cl);
 	curl_multi_remove_handle(cr->cm, cl);
 	curl_easy_cleanup(cl);
 	return (retcode);

@@ -187,12 +253,14 @@ curl_fetch(struct pkg_repo *repo, int dest, struct fetch_item *fi)

 void
 curl_cleanup(struct pkg_repo *repo)
 {
 	CURLM *cm;
 	struct curl_repodata *cr;
 	if (repo->fetch_priv == NULL)
 		return;
 	cm = repo->fetch_priv;
 	curl_multi_cleanup(cm);
 	cr = repo->fetch_priv;
 	curl_multi_cleanup(cr->cm);
 	if (cr->url != NULL)
 		curl_url_cleanup(cr->url);
 	repo->fetch_priv = NULL;
 	return;
 }

modified libpkg/repo/binary/fetch.c

@@ -200,7 +200,7 @@ pkg_repo_binary_try_fetch(struct pkg_repo *repo, struct pkg *pkg,

 		return EPKG_FATAL;
 	}
 	retcode = pkg_fetch_file(repo, url, dest, 0, offset, pkg->pkgsize);
 	retcode = pkg_fetch_file(repo, pkg->repopath, dest, 0, offset, pkg->pkgsize);
 	if (offset == -1)
 		fetched = true;