Radish alpha
H
HardenedBSD-pkg
rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9
HardenedBSD Package Manager
Radicle
Git
Commits › 250bddd
New HTTP_MIRRORS option default to off
Baptiste Daroussin committed 13 years ago
commit 250bdddc422fc4c6ca65db74392f14054389ba1b
parent 01b0be0
5 files changed +132 -17
modified libpkg/Makefile
@@ -69,6 +69,17 @@ SRCS+= gr_util.c
CFLAGS+=	-DHAVE_GRUTILS

.endif
+
+ 
#
+ 
# Set DEFAULT_MIRROR_TYPE default to SRV
+ 
# Possible values are:
+ 
# 0: none
+ 
# 1: srv
+ 
# 2: http
+ 
#
+ 
DEFAULT_MIRROR_TYPE?=	0
+
+ 
CFLAGS+=	-DDEFAULT_MIRROR_TYPE=${DEFAULT_MIRROR_TYPE}

CFLAGS+=	-std=c99

CFLAGS+=	-I${.CURDIR} \

		-I${.CURDIR}/../external/sqlite \
modified libpkg/fetch.c
@@ -27,18 +27,65 @@
#include <sys/param.h>

#include <sys/stat.h>
+ 
#include <ctype.h>

#include <fcntl.h>
+ 
#define _WITH_GETLINE

#include <stdio.h>

#include <string.h>

#include <time.h>

#include <unistd.h>

#include <fetch.h>
+ 
#include <pthread.h>



#include "pkg.h"

#include "private/event.h"

#include "private/pkg.h"

#include "private/utils.h"
+ 
struct http_mirror {
+ 
	struct url *url;
+ 
	STAILQ_ENTRY(http_mirror) next;
+ 
};
+
+ 
static struct dns_srvinfo *srv_mirrors = NULL;
+ 
static STAILQ_HEAD(,http_mirror) http_mirrors = STAILQ_HEAD_INITIALIZER(http_mirrors);
+
+ 
static void
+ 
gethttpmirrors(const char *url) {
+ 
	FILE *f;
+ 
	char *line = NULL;
+ 
	size_t linecap = 0;
+ 
	ssize_t linelen;
+ 
	struct http_mirror *m;
+ 
	struct url *u;
+
+ 
	if ((f = fetchGetURL(url, "")) == NULL)
+ 
		return;
+
+ 
	while ((linelen = getline(&line, &linecap, f)) > 0) {
+ 
		if (strncmp(line, "URL:", 4) == 0) {
+ 
			/* trim '\n' */
+ 
			if (line[linelen - 1] == '\n')
+ 
				line[linelen - 1 ] = '\0';
+
+ 
			line += 4;
+ 
			while (isspace(*line)) {
+ 
				line++;
+ 
			}
+ 
			if (*line == '\0')
+ 
				continue;
+
+ 
			if ((u = fetchParseURL(url)) != NULL) {
+ 
				m = malloc(sizeof(struct http_mirror));
+ 
				m->url = u;
+ 
				STAILQ_INSERT_TAIL(&http_mirrors, m, next);
+ 
			}
+ 
		}
+ 
	}
+ 
	fclose(f);
+ 
	return;
+ 
}
+ 


int

pkg_fetch_file(const char *url, const char *dest, time_t t)

{
@@ -75,12 +122,14 @@ pkg_fetch_file_to_fd(const char *url, int dest, time_t t)
	time_t now;

	time_t last = 0;

	char buf[10240];
+ 
	char *doc;
+ 
	char docpath[MAXPATHLEN];

	int retcode = EPKG_OK;

	bool srv = false;
+ 
	bool http = false;

	char zone[MAXHOSTNAMELEN + 13];
- 
	struct dns_srvinfo *mirrors, *current;
-
- 
	current = mirrors = NULL;
+ 
	struct dns_srvinfo *srv_current;
+ 
	struct http_mirror *http_current;



	fetchTimeout = 30;
@@ -90,21 +139,40 @@ pkg_fetch_file_to_fd(const char *url, int dest, time_t t)
	retry = max_retry;



	u = fetchParseURL(url);
+ 
	doc = u->doc;

	while (remote == NULL) {

		if (retry == max_retry) {

			pkg_config_bool(PKG_CONFIG_SRV_MIRROR, &srv);
- 
			if (srv) {
- 
				if (strcmp(u->scheme, "file") != 0) {
- 
					snprintf(zone, sizeof(zone),
- 
					    "_%s._tcp.%s", u->scheme, u->host);
- 
					mirrors = dns_getsrvinfo(zone);
- 
					current = mirrors;
- 
				}
+ 
			pkg_config_bool(PKG_CONFIG_HTTP_MIRROR, &http);
+ 
			if (srv && strcmp(u->scheme, "file") != 0) {
+ 
				snprintf(zone, sizeof(zone),
+ 
				    "_%s._tcp.%s", u->scheme, u->host);
+ 
				pthread_mutex_lock(&mirror_mtx);
+ 
				if (srv_mirrors != NULL)
+ 
					srv_mirrors = dns_getsrvinfo(zone);
+ 
				pthread_mutex_unlock(&mirror_mtx);
+ 
				srv_current = srv_mirrors;
+ 
			} else if (http && strcmp(u->scheme, "file") != 0 && \
+ 
			           strcmp(u->scheme, "ftp") != 0) {
+ 
				snprintf(zone, sizeof(zone),
+ 
				    "%s://%s", u->scheme, u->host);
+ 
				pthread_mutex_lock(&mirror_mtx);
+ 
				if (STAILQ_EMPTY(&http_mirrors))
+ 
					gethttpmirrors(zone);
+ 
				pthread_mutex_unlock(&mirror_mtx);
+ 
				http_current = STAILQ_FIRST(&http_mirrors);

			}

		}
- 
		if (mirrors != NULL)
- 
			strlcpy(u->host, current->host, sizeof(u->host));
+ 
		if (srv && srv_mirrors != NULL)
+ 
			strlcpy(u->host, srv_current->host, sizeof(u->host));
+ 
		else if (http && !STAILQ_EMPTY(&http_mirrors)) {
+ 
			strlcpy(u->scheme, http_current->url->scheme, sizeof(u->scheme));
+ 
			strlcpy(u->host, http_current->url->host, sizeof(u->host));
+ 
			snprintf(docpath, MAXPATHLEN, "%s%s", http_current->url->doc, doc);
+ 
			u->doc = docpath;
+ 
			u->port = http_current->url->port;
+ 
		}



		remote = fetchXGet(u, &st, "");

		if (remote == NULL) {
@@ -115,12 +183,16 @@ pkg_fetch_file_to_fd(const char *url, int dest, time_t t)
				retcode = EPKG_FATAL;

				goto cleanup;

			}
- 
			if (mirrors == NULL) {
- 
				sleep(1);
+ 
			if (srv && srv_mirrors != NULL) {
+ 
				srv_current = srv_current->next;
+ 
				if (srv_current == NULL)
+ 
					srv_current = srv_mirrors;
+ 
			} else if (http && !STAILQ_EMPTY(&http_mirrors)) {
+ 
				http_current = STAILQ_NEXT(http_current, next);
+ 
				if (http_current == NULL)
+ 
					http_current = STAILQ_FIRST(&http_mirrors);

			} else {
- 
				current = current->next;
- 
				if (current == NULL)
- 
					current = mirrors;
+ 
				sleep(1);

			}

		}

	}
@@ -162,6 +234,9 @@ pkg_fetch_file_to_fd(const char *url, int dest, time_t t)
	if (remote != NULL)

		fclose(remote);
+ 
	/* restore original doc */
+ 
	u->doc = doc;
+ 


	fetchFreeURL(u);



	return (retcode);
modified libpkg/pkg.h.in
@@ -312,6 +312,7 @@ typedef enum _pkg_config_key {
	PKG_CONFIG_PLUGINS_CONF_DIR,

	PKG_CONFIG_PERMISSIVE,

	PKG_CONFIG_REPO_AUTOUPDATE,
+ 
	PKG_CONFIG_HTTP_MIRROR,

} pkg_config_key;



typedef enum {
modified libpkg/pkg_config.c
@@ -34,6 +34,7 @@
#include <errno.h>

#include <stdlib.h>

#include <string.h>
+ 
#include <pthread.h>



#include <yaml.h>
@@ -43,6 +44,8 @@


#define ABI_VAR_STRING "${ABI}"
+ 
pthread_mutex_t mirror_mtx;
+ 


struct config_entry {

	uint8_t type;

	const char *key;
@@ -57,7 +60,11 @@ static struct config_entry c[] = {
	[PKG_CONFIG_REPO] = {

		PKG_CONFIG_STRING,

		"PACKAGESITE",
+ 
#ifdef DEFAULT_PACKAGESITE
+ 
		DEFAULT_PACKAGESITE,
+ 
#else

		NULL,
+ 
#endif

	},

	[PKG_CONFIG_DBDIR] = {

		PKG_CONFIG_STRING,
@@ -132,12 +139,20 @@ static struct config_entry c[] = {
	[PKG_CONFIG_PORTAUDIT_SITE] = {

		PKG_CONFIG_STRING,

		"PORTAUDIT_SITE",
+ 
#ifdef DEFAULT_AUDIT_URL
+ 
		DEFAULT_AUDIT_URL,
+ 
#else

		"http://portaudit.FreeBSD.org/auditfile.tbz",
+ 
#endif

	},

	[PKG_CONFIG_SRV_MIRROR] = {

		PKG_CONFIG_BOOL,

		"SRV_MIRRORS",
+ 
#if DEFAULT_MIRROR_TYPE == 1

		"YES",
+ 
#else
+ 
		"NO",
+ 
#endif

	},

	[PKG_CONFIG_FETCH_RETRY] = {

		PKG_CONFIG_INTEGER,
@@ -179,6 +194,15 @@ static struct config_entry c[] = {
		"REPO_AUTOUPDATE",

		"YES",

	},
+ 
	[PKG_CONFIG_HTTP_MIRROR] = {
+ 
		PKG_CONFIG_BOOL,
+ 
		"HTTP_MIRRORS",
+ 
#if DEFAULT_MIRROR_TYPE == 2
+ 
		"YES",
+ 
#else
+ 
		"NO",
+ 
#endif
+ 
	},

};



static bool parsed = false;
@@ -539,6 +563,8 @@ pkg_init(const char *path)
		return (EPKG_FATAL);

	}
+ 
	pthread_mutex_init(&mirror_mtx, NULL);
+ 


	for (i = 0; i < c_size; i++) {

		conf = malloc(sizeof(struct pkg_config));

		conf->id = i;
modified libpkg/private/pkg.h
@@ -79,6 +79,8 @@
	}  \

	} while (0)
+ 
extern pthread_mutex_t mirror_mtx;
+ 


struct pkg {

	struct sbuf	*fields[PKG_NUM_FIELDS];

	bool		 automatic;