Radish alpha
H
HardenedBSD-pkg
rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9
HardenedBSD Package Manager
Radicle
Git
Commits › a992cd2
repo: prefer data.pkg over packagesite.pkg
Baptiste Daroussin committed 2 years ago
commit a992cd2e3d37aa54274c444d48651ac487a542a3
parent ec16d25
13 files changed +175 -34
modified libpkg/fetch_libcurl.c
@@ -494,9 +494,13 @@ retry:
		retcode = EPKG_CANCEL;

	} else if (rc != 200) {

		--retry;
- 
		if (retry <= 0 || (rc == 404 && repo->mirror_type == NOMIRROR)) {
- 
			pkg_emit_error("An error occured while fetching package");
- 
			retcode = EPKG_FATAL;
+ 
		if (retry <= 0) {
+ 
			if (rc == 404) {
+ 
				retcode = EPKG_ENOENT;
+ 
			} else {
+ 
				pkg_emit_error("An error occured while fetching package");
+ 
				retcode = EPKG_FATAL;
+ 
			}

		} else

			goto retry;

	}
@@ -504,7 +508,7 @@ retry:
	if (res == CURLE_OK && t >= 0) {

		fi->mtime = t;

	} else if (rc != 304 && retcode != EPKG_FATAL &&
- 
	    retcode != EPKG_CANCEL) {
+ 
	    retcode != EPKG_CANCEL && retcode != EPKG_ENOENT) {

		pkg_emit_error("Impossible to get the value from Last-Modified"

		    " HTTP header");

		fi->mtime = 0;
modified libpkg/pkg.h.in
@@ -524,7 +524,8 @@ typedef enum {
	/**

	 * the operation has failed due to network down

	 */
- 
	EPKG_NONETWORK
+ 
	EPKG_NONETWORK,
+ 
	EPKG_ENOENT,

} pkg_error_t;



/**
modified libpkg/pkg_repo.c
@@ -1,5 +1,5 @@
/*-
- 
 * Copyright (c) 2011-2019 Baptiste Daroussin <bapt@FreeBSD.org>
+ 
 * Copyright (c) 2011-2024 Baptiste Daroussin <bapt@FreeBSD.org>

 * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org>

 * Copyright (c) 2011-2012 Marin Atanasov Nikolov <dnaeon@gmail.com>

 * Copyright (c) 2012-2015 Matthew Seaman <matthew@FreeBSD.org>
@@ -681,6 +681,55 @@ out:
}



int
+ 
pkg_repo_fetch_data_fd(struct pkg_repo *repo, struct pkg_repo_content *prc)
+ 
{
+ 
	int fd;
+ 
	const char *tmpdir;
+ 
	char tmp[MAXPATHLEN];
+ 
	struct stat st;
+ 
	int rc = EPKG_OK;
+
+ 
	fd = pkg_repo_fetch_remote_tmp(repo, repo->meta->data, "pkg", &prc->mtime, &rc, false);
+ 
	if (fd == -1) {
+ 
		if (rc == EPKG_UPTODATE)
+ 
			return (rc);
+ 
		fd = pkg_repo_fetch_remote_tmp(repo, repo->meta->data,
+ 
		    packing_format_to_string(repo->meta->packing_format), &prc->mtime, &rc, false);
+ 
	}
+ 
	if (fd == -1)
+ 
		return (EPKG_FATAL);
+
+ 
	tmpdir = getenv("TMMDIR");
+ 
	if (tmpdir == NULL)
+ 
		tmpdir = "/tmp";
+ 
	snprintf(tmp, sizeof(tmp), "%s/%s.XXXXXX", tmpdir, repo->meta->data);
+ 
	prc->data_fd = mkstemp(tmp);
+ 
	if (prc->data_fd == -1) {
+ 
		pkg_emit_error("Cound not create temporary file %s, "
+ 
		    "aborting update.\n", tmp);
+ 
		close(fd);
+ 
		return (EPKG_FATAL);
+ 
	}
+
+ 
	unlink(tmp);
+ 
	if (pkg_repo_archive_extract_check_archive(fd, repo->meta->data, repo, prc->data_fd) != EPKG_OK) {
+ 
		close(prc->data_fd);
+ 
		close(fd);
+ 
		return (EPKG_FATAL);
+ 
	}
+
+ 
	close(fd);
+ 
	if (fstat(prc->data_fd, &st) == -1) {
+ 
		close(prc->data_fd);
+ 
		return (EPKG_FATAL);
+ 
	}
+
+ 
	prc->data_len = st.st_size;
+
+ 
	return (EPKG_OK);
+ 
}
+
+ 
int

pkg_repo_fetch_remote_extract_fd(struct pkg_repo *repo, struct pkg_repo_content *prc)

{

	int fd;
modified libpkg/private/pkg.h
@@ -1,5 +1,5 @@
/*-
- 
 * Copyright (c) 2011-2023 Baptiste Daroussin <bapt@FreeBSD.org>
+ 
 * Copyright (c) 2011-2024 Baptiste Daroussin <bapt@FreeBSD.org>

 * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org>

 * Copyright (c) 2013 Matthew Seaman <matthew@FreeBSD.org>

 * Copyright (c) 2013-2017 Vsevolod Stakhov <vsevolod@FreeBSD.org>
@@ -181,6 +181,8 @@ struct pkg_repo_content {
	time_t mtime;

	int manifest_fd;

	size_t manifest_len;
+ 
	int data_fd;
+ 
	size_t data_len;

};



struct pkg_repo_it;
@@ -648,6 +650,8 @@ int pkg_repo_mirror_package(struct pkg *pkg, const char *destdir);
int pkg_repo_fetch_remote_extract_fd(struct pkg_repo *repo, struct pkg_repo_content *);

int pkg_repo_meta_dump_fd(struct pkg_repo_meta *target, const int fd);

int pkg_repo_fetch_meta(struct pkg_repo *repo, time_t *t);
+ 
int pkg_repo_fetch_remote_extract_fd(struct pkg_repo *repo, struct pkg_repo_content *);
+ 
int pkg_repo_fetch_data_fd(struct pkg_repo *repo, struct pkg_repo_content *);



struct pkg_repo_meta *pkg_repo_meta_default(void);

int pkg_repo_meta_load(const int fd, struct pkg_repo_meta **target);
@@ -849,4 +853,6 @@ void append_touched_dir(const char *path);
void append_touched_file(const char *path);

bool stringlist_contains(stringlist_t *l, const char *name);
+ 
int pkg_parse_manifest_ucl(struct pkg *pkg, ucl_object_t *o);
+ 


#endif
modified libpkg/repo/binary/update.c
@@ -362,6 +362,50 @@ pkg_repo_binary_register_conflicts(const char *origin, char **conflicts,
}



static int
+ 
pkg_repo_binary_add_from_ucl(sqlite3 *sqlite, ucl_object_t *o, struct pkg_repo *repo)
+ 
{
+ 
	int rc = EPKG_OK;
+ 
	struct pkg *pkg;
+ 
	const char *abi;
+
+ 
	rc = pkg_new(&pkg, PKG_REMOTE);
+ 
	if (rc != EPKG_OK)
+ 
		return (EPKG_FATAL);
+
+ 
	rc = pkg_parse_manifest_ucl(pkg, o);
+ 
	if (rc != EPKG_OK) {
+ 
		goto cleanup;
+ 
	}
+
+ 
	if (pkg->digest == NULL || !pkg_checksum_is_valid(pkg->digest, strlen(pkg->digest)))
+ 
		pkg_checksum_calculate(pkg, NULL, false, true, false);
+ 
	abi = pkg->abi != NULL ? pkg->abi : pkg->arch;
+ 
	if (abi == NULL || !is_valid_abi(abi, true)) {
+ 
		rc = EPKG_FATAL;
+ 
		pkg_emit_error("repository %s contains packages with wrong ABI: %s",
+ 
			repo->name, abi);
+ 
		goto cleanup;
+ 
	}
+ 
	if (!is_valid_os_version(pkg)) {
+ 
		rc = EPKG_FATAL;
+ 
		pkg_emit_error("repository %s contains packages for wrong OS "
+ 
		    "version: %s", repo->name, abi);
+ 
		goto cleanup;
+ 
	}
+
+ 
	free(pkg->reponame);
+ 
	pkg->reponame = xstrdup(repo->name);
+
+ 
	rc = pkg_repo_binary_add_pkg(pkg, NULL, sqlite, true);
+
+ 
cleanup:
+ 
	ucl_object_unref(o);
+ 
	pkg_free(pkg);
+
+ 
	return (rc);
+ 
}
+
+ 
static int

pkg_repo_binary_add_from_manifest(const char *buf, sqlite3 *sqlite, size_t len,

		struct pkg **p __unused,

		struct pkg_repo *repo)
@@ -454,6 +498,7 @@ rollback_repo(void *data)
	snprintf(path, sizeof(path), "%s-journal", name);

	unlink(path);

}
+ 


static int

pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,

	time_t *mtime, bool force)
@@ -470,6 +515,7 @@ pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,
	size_t linecap = 0;

	ssize_t linelen, totallen = 0;

	struct pkg_repo_content prc;
+ 
	ucl_object_t *data;



	pkg_debug(1, "Pkgrepo, begin update of '%s'", name);
@@ -488,12 +534,31 @@ pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,
	prc.manifest_fd = -1;

	prc.mtime = *mtime;

	prc.manifest_len = 0;
+ 
	prc.data_fd = -1;
+ 
	prc.data_len = 0;
- 
	rc = pkg_repo_fetch_remote_extract_fd(repo, &prc);
- 
	if (rc != EPKG_OK)
+ 
	rc = pkg_repo_fetch_data_fd(repo, &prc);
+ 
	if (rc == EPKG_UPTODATE)

		goto cleanup;
- 
	f = fdopen(prc.manifest_fd, "r");
- 
	rewind(f);
+
+ 
	if (rc == EPKG_OK) {
+ 
		struct ucl_parser *p = ucl_parser_new(0);
+ 
		if (!ucl_parser_add_fd(p, prc.data_fd)) {
+ 
			pkg_emit_error("Error parsing data file: %s'",
+ 
			    ucl_parser_get_error(p));
+ 
			ucl_parser_free(p);
+ 
			goto cleanup;
+ 
		}
+ 
		data = ucl_parser_get_object(p);
+ 
		ucl_parser_free(p);
+ 
		/* XXX TODO check against a schema */
+ 
	} else {
+ 
		rc = pkg_repo_fetch_remote_extract_fd(repo, &prc);
+ 
		if (rc != EPKG_OK)
+ 
			goto cleanup;
+ 
		f = fdopen(prc.manifest_fd, "r");
+ 
		rewind(f);
+ 
	}



	*mtime = prc.mtime;

	/*fconflicts = repo_fetch_remote_extract_tmp(repo,
@@ -513,7 +578,7 @@ pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,
	/* Here sqlite is initialized */

	sqlite = PRIV_GET(repo);
- 
	pkg_debug(1, "Pkgrepo, reading new %s for '%s'", repo->meta->manifests, name);
+ 
	pkg_debug(1, "Pkgrepo, reading new metadata");



	pkg_emit_incremental_update_begin(repo->name);

	pkg_emit_progress_start("Processing entries");
@@ -530,17 +595,33 @@ pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,
		goto cleanup;



	in_trans = true;
- 
	while ((linelen = getline(&line, &linecap, f)) > 0) {
- 
		cnt++;
- 
		totallen += linelen;
- 
		if ((cnt % 10 ) == 0)
- 
			cancel = pkg_emit_progress_tick(totallen, prc.manifest_len);
- 
		rc = pkg_repo_binary_add_from_manifest(line, sqlite, linelen,
- 
		    &pkg, repo);
- 
		if (rc != EPKG_OK || cancel != 0)
- 
			break;
+ 
	if (f != NULL) {
+ 
		while ((linelen = getline(&line, &linecap, f)) > 0) {
+ 
			cnt++;
+ 
			totallen += linelen;
+ 
			if ((cnt % 10 ) == 0)
+ 
				cancel = pkg_emit_progress_tick(totallen, prc.manifest_len);
+ 
			rc = pkg_repo_binary_add_from_manifest(line, sqlite, linelen,
+ 
			    &pkg, repo);
+ 
			if (rc != EPKG_OK || cancel != 0)
+ 
				break;
+ 
		}
+ 
		pkg_emit_progress_tick(prc.manifest_len, prc.manifest_len);
+ 
	}
+ 
	if (data != NULL) {
+ 
		ucl_object_t *pkgs = ucl_object_ref(ucl_object_find_key(data, "packages"));
+ 
		int nbel = ucl_array_size(pkgs);
+ 
		while (cnt < nbel) {
+ 
			ucl_object_t *o = ucl_array_pop_first(pkgs);
+ 
			cnt++;
+ 
			if ((cnt % 10 ) == 0)
+ 
				cancel = pkg_emit_progress_tick(nbel, cnt);
+ 
			rc = pkg_repo_binary_add_from_ucl(sqlite, o, repo);
+ 
			if (rc != EPKG_OK || cancel != 0)
+ 
				break;
+ 
		}
+ 
		pkg_emit_progress_tick(nbel, cnt);

	}
- 
	pkg_emit_progress_tick(prc.manifest_len, prc.manifest_len);



	if (rc == EPKG_OK)

		pkg_emit_incremental_update(repo->name, cnt);
modified tests/frontend/conflicts-multirepo.sh
@@ -146,12 +146,12 @@ EOF


OUTPUT="Updating local1 repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

local1 repository update completed. 2 packages processed.

Updating local2 repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

local2 repository update completed. 2 packages processed.

All repositories are up to date.
modified tests/frontend/conflicts.sh
@@ -434,7 +434,7 @@ EOF


OUTPUT="Updating local repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

local repository update completed. 1 packages processed.

All repositories are up to date.
modified tests/frontend/formula.sh
@@ -57,7 +57,7 @@ EOF


	OUTPUT="Updating local1 repository catalogue...

${JAILED}meta.conf                                 :  done
- 
${JAILED}packagesite.pkg                           :  done
+ 
${JAILED}data.pkg                           :  done

Processing entries:  done

local1 repository update completed. 2 packages processed.

All repositories are up to date.
modified tests/frontend/issue1425.sh
@@ -153,12 +153,12 @@ EOF


OUTPUT_CASE1="Updating repoA repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

repoA repository update completed. 4 packages processed.

Updating repoB repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

repoB repository update completed. 4 packages processed.

All repositories are up to date.
modified tests/frontend/issue1440.sh
@@ -188,12 +188,12 @@ EOF


OUTPUT_CASE1="Updating repoA repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

repoA repository update completed. 4 packages processed.

Updating repoB repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

repoB repository update completed. 4 packages processed.

All repositories are up to date.
modified tests/frontend/php-pr.sh
@@ -241,7 +241,7 @@ php53-gd-5.3.27


OUTPUT="Updating local repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

local repository update completed. 4 packages processed.

All repositories are up to date.
modified tests/frontend/requires.sh
@@ -41,7 +41,7 @@ EOF


	OUTPUT="Updating local1 repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

local1 repository update completed. 2 packages processed.

All repositories are up to date.
@@ -105,7 +105,7 @@ EOF


	OUTPUT="Updating local1 repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

local1 repository update completed. 2 packages processed.

All repositories are up to date.
modified tests/frontend/rubypuppet.sh
@@ -228,7 +228,7 @@ EOF


	OUTPUT="Updating local repository catalogue...

${JAILED}Fetching meta.conf:  done
- 
${JAILED}Fetching packagesite.pkg:  done
+ 
${JAILED}Fetching data.pkg:  done

Processing entries:  done

local repository update completed. 5 packages processed.

All repositories are up to date.