900dc14 repo: Add support for v2 repo — HardenedBSD-pkg

HardenedBSD-pkg

rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9

HardenedBSD Package Manager

repo: Add support for v2 repo

Emmanuel Vadot committed 6 years ago

commit 900dc145dce7c184056634d9cb7039fa01e38f32
parent bac3482

4 files changed +241 -81

modified libpkg/pkg_repo_create.c

@@ -177,6 +177,10 @@ pkg_create_repo_read_fts(struct pkg_fts_item **items, FTS *fts,

 		*ext = '\0';
 		if (pkg_repo_meta_is_old_file(fts_ent->fts_name, meta)) {
 			unlink(fts_ent->fts_path);
 			continue;
 		}
 		if (strcmp(fts_ent->fts_name, "meta") == 0 ||
 				pkg_repo_meta_is_special_file(fts_ent->fts_name, meta)) {
 			*ext = '.';

@@ -244,6 +248,7 @@ pkg_create_repo_worker(struct pkg_fts_item *start, size_t nelts,

 	else
 		flags = PKG_OPEN_MANIFEST_ONLY | PKG_OPEN_MANIFEST_COMPACT;
 	/* We are reading to digest buf but it's only to check the socketpair */
 	if (read(pip, digestbuf, 1) == -1) {
 		pkg_emit_errno("pkg_create_repo_worker", "read");
 		goto cleanup;

@@ -273,13 +278,16 @@ pkg_create_repo_worker(struct pkg_fts_item *start, size_t nelts,

 				mdigest = xmalloc(pkg_checksum_type_size(meta->digest_format));
 				pkg_emit_manifest_buf(pkg, b, PKG_MANIFEST_EMIT_COMPACT, NULL);
 				if (pkg_checksum_generate(pkg, mdigest,
 				     pkg_checksum_type_size(meta->digest_format),
 				     meta->digest_format) != EPKG_OK) {
 					pkg_emit_error("Cannot generate digest for a package");
 					ret = EPKG_FATAL;
 					goto cleanup;
 				/* Only version 1 needs the digest */
 				if (meta->version == 1) {
 					if (pkg_checksum_generate(pkg, mdigest,
 					    pkg_checksum_type_size(meta->digest_format),
 					    meta->digest_format) != EPKG_OK) {
 						pkg_emit_error("Cannot generate digest for a package");
 						ret = EPKG_FATAL;
 						goto cleanup;
 					}
 				}
 			}
 			mlen = utstring_len(b);

@@ -322,22 +330,24 @@ pkg_create_repo_worker(struct pkg_fts_item *start, size_t nelts,

 				flock(ffd, LOCK_UN);
 			}
 			r = snprintf(digestbuf, sizeof(digestbuf), "%s:%s:%ld:%ld:%ld:%s\n",
 				pkg->origin,
 				mdigest,
 				(long)mpos,
 				(long)fpos,
 				(long)mlen,
 				pkg->sum);
 			free(mdigest);
 			mdigest = NULL;
 			iov[0].iov_base = digestbuf;
 			iov[0].iov_len = r;
 			memset(&msg, 0, sizeof(msg));
 			msg.msg_iov = iov;
 			msg.msg_iovlen = 1;
 			sendmsg(pip, &msg, MSG_EOR);
 			if (meta->version == 1) {
 				r = snprintf(digestbuf, sizeof(digestbuf), "%s:%s:%ld:%ld:%ld:%s\n",
 				    pkg->origin,
 				    mdigest,
 				    (long)mpos,
 				    (long)fpos,
 				    (long)mlen,
 				    pkg->sum);
 				free(mdigest);
 				mdigest = NULL;
 				iov[0].iov_base = digestbuf;
 				iov[0].iov_len = r;
 				memset(&msg, 0, sizeof(msg));
 				msg.msg_iov = iov;
 				msg.msg_iovlen = 1;
 				sendmsg(pip, &msg, MSG_EOR);
 			}
 		}
 		cur_job ++;
 	}

@@ -355,7 +365,7 @@ cleanup:

 }
 static int
 pkg_create_repo_read_pipe(int fd, struct digest_list_entry **dlist)
 pkg_create_repo_read_pipe(int fd, struct digest_list_entry **dlist, struct pkg_repo_meta *meta)
 {
 	struct digest_list_entry *dig = NULL;
 	char buf[1024];

@@ -389,6 +399,13 @@ pkg_create_repo_read_pipe(int fd, struct digest_list_entry **dlist)

 		else if (r == 0)
 			return (EPKG_END);
 		/*
 		 * Don't bother adding to the digest list if we
 		 * aren't creating a repo v1
 		 */
 		if (meta->version != 1)
 			continue;
 		/*
 		 * XXX: can parse merely full lines
 		 */

@@ -555,11 +572,13 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 			goto cleanup;
 		}
 	}
 	if ((fd = openat(outputdir_fd, meta->digests, O_CREAT|O_TRUNC|O_RDWR, 00644)) == -1) {
 		goto cleanup;
 	}
 	if ((mandigests = fdopen(fd, "w")) == NULL) {
 		goto cleanup;
 	if (meta->version == 1) {
 		if ((fd = openat(outputdir_fd, meta->digests, O_CREAT|O_TRUNC|O_RDWR, 00644)) == -1) {
 			goto cleanup;
 		}
 		if ((mandigests = fdopen(fd, "w")) == NULL) {
 			goto cleanup;
 		}
 	}
 	len = 0;

@@ -655,7 +674,7 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 			for (i = 0; i < num_workers; i ++) {
 				if (pfd[i].fd != -1 &&
 								(pfd[i].revents & (POLLIN|POLLHUP|POLLERR))) {
 					if (pkg_create_repo_read_pipe(pfd[i].fd, &dlist) != EPKG_OK) {
 					if (pkg_create_repo_read_pipe(pfd[i].fd, &dlist, meta) != EPKG_OK) {
 						/*
 						 * Wait for the worker finished
 						 */

@@ -688,7 +707,8 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 	retcode = EPKG_OK;
 	/* Now sort all digests */
 	DL_SORT(dlist, pkg_digest_sort_compare_func);
 	if (meta->version == 1)
 		DL_SORT(dlist, pkg_digest_sort_compare_func);
 	/* Write metafile */
 	snprintf(repodb, sizeof(repodb), "%s/%s", output_dir,

@@ -723,25 +743,27 @@ cleanup:

 		fts_close(fts);
 	LL_FREE(fts_items, pkg_create_repo_fts_free);
 	LL_FOREACH_SAFE(dlist, cur_dig, dtmp) {
 		if (cur_dig->checksum != NULL)
 			fprintf(mandigests, "%s:%s:%ld:%ld:%ld:%s\n", cur_dig->origin,
 				cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos,
 				cur_dig->manifest_length, cur_dig->checksum);
 		else
 			fprintf(mandigests, "%s:%s:%ld:%ld:%ld\n", cur_dig->origin,
 				cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos,
 				cur_dig->manifest_length);
 		free(cur_dig->digest);
 		free(cur_dig->origin);
 		free(cur_dig);
 	}
 	if (meta->version == 1) {
 		LL_FOREACH_SAFE(dlist, cur_dig, dtmp) {
 			if (cur_dig->checksum != NULL)
 				fprintf(mandigests, "%s:%s:%ld:%ld:%ld:%s\n", cur_dig->origin,
 				    cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos,
 				    cur_dig->manifest_length, cur_dig->checksum);
 			else
 				fprintf(mandigests, "%s:%s:%ld:%ld:%ld\n", cur_dig->origin,
 				    cur_dig->digest, cur_dig->manifest_pos, cur_dig->files_pos,
 				    cur_dig->manifest_length);
 	pkg_repo_meta_free(meta);
 			free(cur_dig->digest);
 			free(cur_dig->origin);
 			free(cur_dig);
 		}
 	}
 	if (mandigests != NULL)
 	if (meta->version == 1 && mandigests != NULL)
 		fclose(mandigests);
 	pkg_repo_meta_free(meta);
 	return (retcode);
 }

@@ -921,8 +943,6 @@ pkg_finish_repo(const char *output_dir, pkg_password_cb *password_cb,

 			rsa_free(rsa);
 			close(fd);
 			return (EPKG_FATAL);
 		} else {
 			meta = pkg_repo_meta_default();
 		}
 		if (pkg_repo_pack_db(repo_meta_file, repo_path, repo_path, rsa, meta,
 			argv, argc) != EPKG_OK) {

@@ -961,14 +981,16 @@ pkg_finish_repo(const char *output_dir, pkg_password_cb *password_cb,

 	pkg_emit_progress_tick(nfile++, files_to_pack);
 	snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
 	    meta->digests);
 	snprintf(repo_archive, sizeof(repo_archive), "%s/%s", output_dir,
 	    meta->digests_archive);
 	if (pkg_repo_pack_db(meta->digests, repo_archive, repo_path, rsa, meta,
 		argv, argc) != EPKG_OK) {
 		ret = EPKG_FATAL;
 		goto cleanup;
 	if (meta->version == 1) {
 		snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
 		    meta->digests);
 		snprintf(repo_archive, sizeof(repo_archive), "%s/%s", output_dir,
 		    meta->digests_archive);
 		if (pkg_repo_pack_db(meta->digests, repo_archive, repo_path, rsa, meta,
 		    argv, argc) != EPKG_OK) {
 			ret = EPKG_FATAL;
 			goto cleanup;
 		}
 	}
 	pkg_emit_progress_tick(nfile++, files_to_pack);

@@ -1002,9 +1024,11 @@ pkg_finish_repo(const char *output_dir, pkg_password_cb *password_cb,

 		snprintf(repo_archive, sizeof(repo_archive), "%s/%s.txz",
 		    output_dir, meta->manifests_archive);
 		utimes(repo_archive, ftimes);
 		snprintf(repo_archive, sizeof(repo_archive), "%s/%s.txz",
 		    output_dir, meta->digests_archive);
 		utimes(repo_archive, ftimes);
 		if (meta->version == 1) {
 			snprintf(repo_archive, sizeof(repo_archive), "%s/%s.txz",
 			    output_dir, meta->digests_archive);
 			utimes(repo_archive, ftimes);
 		}
 		if (filelist) {
 			snprintf(repo_archive, sizeof(repo_archive),
 			    "%s/%s.txz", output_dir, meta->filesite_archive);

modified libpkg/pkg_repo_meta.c

@@ -30,7 +30,11 @@

 #include "private/event.h"
 #include "private/pkg.h"
 /* Default to repo v1 for now */
 #define	DEFAULT_META_VERSION	1
 static ucl_object_t *repo_meta_schema_v1 = NULL;
 static ucl_object_t *repo_meta_schema_v2 = NULL;
 static void
 pkg_repo_meta_set_default(struct pkg_repo_meta *meta)

@@ -43,14 +47,22 @@ pkg_repo_meta_set_default(struct pkg_repo_meta *meta)

 	meta->conflicts_archive = NULL;
 	meta->manifests = xstrdup("packagesite.yaml");
 	meta->manifests_archive = xstrdup("packagesite");
 	meta->digests = xstrdup("digests");
 	meta->digests_archive = xstrdup("digests");
 	meta->filesite = xstrdup("filesite.yaml");
 	meta->filesite_archive = xstrdup("filesite");
 	/* Not using fulldb */
 	meta->fulldb = NULL;
 	meta->fulldb_archive = NULL;
 	meta->version = 1;
 	/*
 	 * digest is only used on legacy v1 repository
 	 * but pkg_repo_meta_is_special_file depend on the
 	 * information in the pkg_repo_meta.
 	 * Leave digests here so pkg will not complain that
 	 * repodir/digest.txz isn't a valid package when switching
 	 * from version 1 to version 2
 	 */
 	meta->digests = xstrdup("digests");
 	meta->digests_archive = xstrdup("digests");
 }
 void

@@ -144,6 +156,61 @@ pkg_repo_meta_open_schema_v1()

 	return (repo_meta_schema_v1);
 }
 static ucl_object_t*
 pkg_repo_meta_open_schema_v2()
 {
 	struct ucl_parser *parser;
 	static const char meta_schema_str_v2[] = ""
 			"{"
 			"type = object;"
 			"properties {"
 			"version = {type = integer};\n"
 			"maintainer = {type = string};\n"
 			"source = {type = string};\n"
 			"packing_format = {enum = [txz, tbz, tgz, tar]};\n"
 			"manifests = {type = string};\n"
 			"conflicts = {type = string};\n"
 			"fulldb = {type = string};\n"
 			"filesite = {type = string};\n"
 			"manifests_archive = {type = string};\n"
 			"conflicts_archive = {type = string};\n"
 			"fulldb_archive = {type = string};\n"
 			"filesite_archive = {type = string};\n"
 			"source_identifier = {type = string};\n"
 			"revision = {type = integer};\n"
 			"eol = {type = integer};\n"
 			"cert = {"
 			"  type = object;\n"
 			"  properties {"
 			"    type = {enum = [rsa]};\n"
 			"    data = {type = string};\n"
 			"    name = {type = string};\n"
 			"  }"
 			"  required = [type, data, name];\n"
 			"};\n"
 			"}\n"
 			"required = [version]\n"
 			"}";
 	if (repo_meta_schema_v2 != NULL)
 		return (repo_meta_schema_v2);
 	parser = ucl_parser_new(UCL_PARSER_NO_FILEVARS);
 	if (!ucl_parser_add_chunk(parser, meta_schema_str_v2,
 			sizeof(meta_schema_str_v2) - 1)) {
 		pkg_emit_error("cannot parse schema for repo meta: %s",
 				ucl_parser_get_error(parser));
 		ucl_parser_free(parser);
 		return (NULL);
 	}
 	repo_meta_schema_v2 = ucl_parser_get_object(parser);
 	ucl_parser_free(parser);
 	return (repo_meta_schema_v2);
 }
 static struct pkg_repo_meta_key*
 pkg_repo_meta_parse_cert(const ucl_object_t *obj)
 {

@@ -273,23 +340,23 @@ pkg_repo_meta_load(const int fd, struct pkg_repo_meta **target)

 		return (EPKG_FATAL);
 	}
 	/* Now we support only v1 meta */
 	if (version == 1) {
 	/* Now we support only v1 and v2 meta */
 	if (version == 1)
 		schema = pkg_repo_meta_open_schema_v1();
 		if (schema != NULL) {
 			if (!ucl_object_validate(schema, top, &err)) {
 				pkg_emit_error("repository meta cannot be validated: %s", err.msg);
 				ucl_object_unref(top);
 				return (EPKG_FATAL);
 			}
 		}
 	}
 	else if (version == 2)
 		schema = pkg_repo_meta_open_schema_v2();
 	else {
 		pkg_emit_error("repository meta has wrong version %d", version);
 		ucl_object_unref(top);
 		return (EPKG_FATAL);
 	}
 	if (schema != NULL) {
 		if (!ucl_object_validate(schema, top, &err)) {
 			printf("repository meta cannot be validated: %s", err.msg);
 			ucl_object_unref(top);
 			return (EPKG_FATAL);
 		}
 	}
 	return (pkg_repo_meta_parse(top, target, version));
 }

@@ -300,6 +367,7 @@ pkg_repo_meta_default(void)

 	struct pkg_repo_meta *meta;
 	meta = xcalloc(1, sizeof(*meta));
 	meta->version = DEFAULT_META_VERSION;
 	pkg_repo_meta_set_default(meta);
 	return (meta);

@@ -329,15 +397,17 @@ pkg_repo_meta_to_ucl(struct pkg_repo_meta *meta)

 	META_EXPORT_FIELD_FUNC(result, meta, packing_format, string,
 		packing_format_to_string);
 	META_EXPORT_FIELD_FUNC(result, meta, digest_format, string,
 		pkg_checksum_type_to_string);
 	META_EXPORT_FIELD(result, meta, digests, string);
 	if (meta->version == 1) {
 		META_EXPORT_FIELD_FUNC(result, meta, digest_format, string,
 		    pkg_checksum_type_to_string);
 		META_EXPORT_FIELD(result, meta, digests, string);
 		META_EXPORT_FIELD(result, meta, digests_archive, string);
 	}
 	META_EXPORT_FIELD(result, meta, manifests, string);
 	META_EXPORT_FIELD(result, meta, conflicts, string);
 	META_EXPORT_FIELD(result, meta, fulldb, string);
 	META_EXPORT_FIELD(result, meta, filesite, string);
 	META_EXPORT_FIELD(result, meta, digests_archive, string);
 	META_EXPORT_FIELD(result, meta, manifests_archive, string);
 	META_EXPORT_FIELD(result, meta, conflicts_archive, string);
 	META_EXPORT_FIELD(result, meta, fulldb_archive, string);

@@ -371,3 +441,14 @@ pkg_repo_meta_is_special_file(const char *file, struct pkg_repo_meta *meta)

 	return (special);
 }
 bool
 pkg_repo_meta_is_old_file(const char *file, struct pkg_repo_meta *meta)
 {
 	bool special = false;
 	if (meta->version != 1)
 		special = META_SPECIAL_FILE(file, meta, digests_archive);
 	return (special);
 }

modified libpkg/private/pkg.h

@@ -663,6 +663,7 @@ int pkg_repo_meta_load(const int fd, struct pkg_repo_meta **target);

 void pkg_repo_meta_free(struct pkg_repo_meta *meta);
 ucl_object_t * pkg_repo_meta_to_ucl(struct pkg_repo_meta *meta);
 bool pkg_repo_meta_is_special_file(const char *file, struct pkg_repo_meta *meta);
 bool pkg_repo_meta_is_old_file(const char *file, struct pkg_repo_meta *meta);
 typedef enum {
 	HASH_UNKNOWN,

modified tests/frontend/repo.sh

@@ -3,10 +3,11 @@

 . $(atf_get_srcdir)/test_environment.sh
 tests_init \
 	repo \
 	repo_v1 \
 	repo_v2 \
 	repo_multiversion
 repo_body() {
 repo_v1_body() {
 	touch plop
 	touch bla
 	atf_check -s exit:0 sh ${RESOURCEDIR}/test_subr.sh new_pkg test test 1 "${TMPDIR}"

@@ -17,6 +18,9 @@ files: {

 }
 EOF
 	cat > meta.ucl << EOF
 version = 1
 EOF
 	atf_check \
 		-o empty \
 		-e empty \

@@ -27,7 +31,7 @@ EOF

 		-o inline:"Creating repository in .:  done\nPacking files for repository:  done\n" \
 		-e empty \
 		-s exit:0 \
 		pkg repo .
 		pkg repo --meta-file meta.ucl .
 	ln -s test-1.txz test.txz

@@ -35,7 +39,7 @@ EOF

 		-o inline:"Creating repository in .:  done\nPacking files for repository:  done\n" \
 		-e empty \
 		-s exit:0 \
 		pkg repo .
 		pkg repo --meta-file meta.ucl .
 	if [ `uname -s` = "Darwin" ]; then
 		atf_pass

@@ -57,6 +61,56 @@ EOF

 	atf_check_equal $nb 2
 }
 repo_v2_body() {
 	touch plop
 	touch bla
 	atf_check -s exit:0 sh ${RESOURCEDIR}/test_subr.sh new_pkg test test 1 "${TMPDIR}"
 	cat >> test.ucl << EOF
 files: {
 	"${TMPDIR}/plop": ""
 	"${TMPDIR}/bla": ""
 }
 EOF
 	cat > meta.ucl << EOF
 version = 2
 EOF
 	atf_check \
 		-o empty \
 		-e empty \
 		-s exit:0 \
 		pkg create -M test.ucl
 	atf_check \
 		-o inline:"Creating repository in .:  done\nPacking files for repository:  done\n" \
 		-e empty \
 		-s exit:0 \
 		pkg repo --meta-file meta.ucl .
 	ln -s test-1.txz test.txz
 	atf_check \
 		-o inline:"Creating repository in .:  done\nPacking files for repository:  done\n" \
 		-e empty \
 		-s exit:0 \
 		pkg repo --meta-file meta.ucl .
 	if [ `uname -s` = "Darwin" ]; then
 		atf_pass
 	fi
 	atf_check -s exit:127 -o ignore -e ignore "ls digest.txz"
 	mkdir Latest
 	ln -s test-1.txz Latest/test.txz
 	atf_check \
 		-o inline:"Creating repository in .:  done\nPacking files for repository:  done\n" \
 		-e empty \
 		-s exit:0 \
 		pkg repo .
 }
 repo_multiversion_body() {
 	atf_check -s exit:0 sh ${RESOURCEDIR}/test_subr.sh new_pkg test test 1.0 "${TMPDIR}"