efe25c6 repo_create: refactor the API to make it extensible — HardenedBSD-pkg

HardenedBSD-pkg

rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9

HardenedBSD Package Manager

repo_create: refactor the API to make it extensible

Baptiste Daroussin committed 2 years ago

commit efe25c69145781dfae9bf57b1d1eb010fd15d5e5
parent 4ce38f0

4 files changed +243 -197

modified libpkg/pkg.h.in

@@ -1,5 +1,5 @@

 /*-
  * Copyright (c) 2011-2016 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2011-2024 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org>
  * Copyright (c) 2011 Will Andrews <will@FreeBSD.org>
  * Copyright (c) 2011 Philippe Pepiot <phil@philpep.org>

@@ -101,6 +101,7 @@ struct pkg_option;

 struct pkg_license;
 struct pkg_config_file;
 struct pkg_create;
 struct pkg_repo_create;
 struct pkgdb;
 struct pkgdb_it;

@@ -808,6 +809,15 @@ int pkg_create_repo(char *path, const char *output_dir, bool filelist,

 int pkg_finish_repo(const char *output_dir, pkg_password_cb *cb, char **argv,
     int argc, bool filelist);
 struct pkg_repo_create *pkg_repo_create_new(void);
 void pkg_repo_create_free(struct pkg_repo_create *);
 void pkg_repo_create_set_create_filelist(struct pkg_repo_create *prc, bool);
 void pkg_repo_create_set_hash(struct pkg_repo_create *prc, bool);
 void pkg_repo_create_set_hash_symlink(struct pkg_repo_create *prc, bool);
 void pkg_repo_create_set_output_dir(struct pkg_repo_create *prc, const char *);
 void pkg_repo_create_set_metafile(struct pkg_repo_create *prc, const char *);
 void pkg_repo_create_set_sign(struct pkg_repo_create *prc, char **argv, int argc, pkg_password_cb *cb);
 int pkg_repo_create(struct pkg_repo_create *, char *path);
 /**
  * Test if the EUID has sufficient privilege to carry out some

modified libpkg/pkg_repo_create.c

@@ -1,5 +1,5 @@

 /*-
  * Copyright (c) 2011-2023 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2011-2024 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org>
  * Copyright (c) 2011-2012 Marin Atanasov Nikolov <dnaeon@gmail.com>
  * Copyright (c) 2012-2013 Matthew Seaman <matthew@FreeBSD.org>

@@ -61,6 +61,9 @@ enum {

 	MSG_PKG_READY,
 };
 static int pkg_repo_pack_db(const char *name, const char *archive, char *path,
     struct pkg_key *keyinfo, struct pkg_repo_meta *meta, char **argv, int argc);
 static int
 hash_file(struct pkg_repo_meta *meta, struct pkg *pkg, char *path)
 {

@@ -408,9 +411,183 @@ fts_compare(FTSENTP *a, FTSENTP *b)

 	return (strcmp((*a)->fts_name, (*b)->fts_name));
 }
 struct pkg_repo_create *
 pkg_repo_create_new(void)
 {
 	struct pkg_repo_create *prc;
 	prc = xcalloc(1, sizeof(*prc));
 	return (prc);
 }
 void
 pkg_repo_create_free(struct pkg_repo_create *prc)
 {
 	if (prc == NULL)
 		return;
 	pkg_repo_meta_free(prc->meta);
 	free(prc);
 }
 void
 pkg_repo_create_set_create_filelist(struct pkg_repo_create *prc, bool val)
 {
 	prc->filelist = val;
 }
 void
 pkg_repo_create_set_hash(struct pkg_repo_create *prc, bool val)
 {
 	prc->hash = val;
 }
 void
 pkg_repo_create_set_hash_symlink(struct pkg_repo_create *prc, bool val)
 {
 	prc->hash_symlink = val;
 }
 void
 pkg_repo_create_set_output_dir(struct pkg_repo_create *prc, const char *out)
 {
 	prc->outdir = out;
 }
 void
 pkg_repo_create_set_metafile(struct pkg_repo_create *prc, const char *metafile)
 {
 	prc->metafile = metafile;
 }
 void
 pkg_repo_create_set_sign(struct pkg_repo_create *prc, char **argv, int argc, pkg_password_cb *cb)
 {
 	prc->sign.argc = argc;
 	prc->sign.argv = argv;
 	prc->sign.cb = cb;
 }
 static int
 pkg_repo_create_pack_and_sign(struct pkg_repo_create *prc)
 {
 	char repo_path[MAXPATHLEN];
 	char repo_archive[MAXPATHLEN];
 	char *key_file;
 	const char *key_type;
 	struct pkg_key *keyinfo = NULL;
 	struct stat st;
 	int ret = EPKG_OK, nfile = 0, fd;
 	const int files_to_pack = 4;
 	if (prc->sign.argc == 1) {
 		key_type = key_file = prc->sign.argv[0];
 		if (strncmp(key_file, "rsa:", 4) == 0) {
 			key_file += 4;
 			*(key_file - 1) = '\0';
 		} else {
 			key_type = "rsa";
 		}
 		pkg_debug(1, "Loading %s key from '%s' for signing", key_type, key_file);
 		rsa_new(&keyinfo, prc->sign.cb, key_file);
 	}
 	if (prc->sign.argc > 1 && strcmp(prc->sign.argv[0], "signing_command:") != 0)
 		return (EPKG_FATAL);
 	if (prc->sign.argc > 1) {
 		prc->sign.argc--;
 		prc->sign.argv++;
 	}
 	pkg_emit_progress_start("Packing files for repository");
 	pkg_emit_progress_tick(nfile++, files_to_pack);
 	snprintf(repo_path, sizeof(repo_path), "%s/%s", prc->outdir,
 		repo_meta_file);
 	if (pkg_repo_pack_db(repo_meta_file, repo_path, repo_path, keyinfo,
 	    prc->meta, prc->sign.argv, prc->sign.argc) != EPKG_OK) {
 		ret = EPKG_FATAL;
 		goto cleanup;
 	}
 	snprintf(repo_path, sizeof(repo_path), "%s/%s", prc->outdir,
 	    prc->meta->manifests);
 	snprintf(repo_archive, sizeof(repo_archive), "%s/%s", prc->outdir,
 		prc->meta->manifests_archive);
 	if (pkg_repo_pack_db(prc->meta->manifests, repo_archive, repo_path, keyinfo,
 	    prc->meta, prc->sign.argv, prc->sign.argc) != EPKG_OK) {
 		ret = EPKG_FATAL;
 		goto cleanup;
 	}
 	pkg_emit_progress_tick(nfile++, files_to_pack);
 	if (prc->filelist) {
 		snprintf(repo_path, sizeof(repo_path), "%s/%s", prc->outdir,
 		    prc->meta->filesite);
 		snprintf(repo_archive, sizeof(repo_archive), "%s/%s",
 		    prc->outdir, prc->meta->filesite_archive);
 		if (pkg_repo_pack_db(prc->meta->filesite, repo_archive, repo_path, keyinfo,
 		    prc->meta, prc->sign.argv, prc->sign.argc) != EPKG_OK) {
 			ret = EPKG_FATAL;
 			goto cleanup;
 		}
 	}
 	pkg_emit_progress_tick(nfile++, files_to_pack);
 	snprintf(repo_path, sizeof(repo_path), "%s/%s", prc->outdir, prc->meta->data);
 	snprintf(repo_archive, sizeof(repo_archive), "%s/%s", prc->outdir,
 	    prc->meta->data_archive);
 	if (pkg_repo_pack_db(prc->meta->data, repo_archive, repo_path, keyinfo,
 	    prc->meta, prc->sign.argv, prc->sign.argc) != EPKG_OK) {
 		ret = EPKG_FATAL;
 		goto cleanup;
 	}
 	pkg_emit_progress_tick(nfile++, files_to_pack);
 	/* Now we need to set the equal mtime for all archives in the repo */
 	snprintf(repo_archive, sizeof(repo_archive), "%s/%s.pkg",
 	    prc->outdir, repo_meta_file);
 	if (stat(repo_archive, &st) == 0) {
 		struct timeval ftimes[2] = {
 			{
 			.tv_sec = st.st_mtime,
 			.tv_usec = 0
 			},
 			{
 			.tv_sec = st.st_mtime,
 			.tv_usec = 0
 			}
 		};
 		snprintf(repo_archive, sizeof(repo_archive), "%s/%s.pkg",
 		    prc->outdir, prc->meta->manifests_archive);
 		utimes(repo_archive, ftimes);
 		if (prc->filelist) {
 			snprintf(repo_archive, sizeof(repo_archive),
 			    "%s/%s.pkg", prc->outdir, prc->meta->filesite_archive);
 			utimes(repo_archive, ftimes);
 		}
 		snprintf(repo_archive, sizeof(repo_archive),
 			"%s/%s.pkg", prc->outdir, prc->meta->data_archive);
 		utimes(repo_archive, ftimes);
 		snprintf(repo_archive, sizeof(repo_archive),
 			"%s/%s.pkg", prc->outdir, repo_meta_file);
 		utimes(repo_archive, ftimes);
 	}
 cleanup:
 	pkg_emit_progress_tick(files_to_pack, files_to_pack);
 	rsa_free(keyinfo);
 	return (ret);
 }
 int
 pkg_create_repo(char *path, const char *output_dir, bool filelist,
 	const char *metafile, bool hash, bool hash_symlink)
 pkg_repo_create(struct pkg_repo_create *prc, char *path)
 {
 	FTS *fts = NULL;
 	int num_workers;

@@ -418,11 +595,13 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 	struct thr_env te = { 0 };
 	size_t len;
 	int fd, outputdir_fd;
 	struct pkg_repo_meta *meta = NULL;
 	int retcode = EPKG_FATAL;
 	ucl_object_t *meta_dump;
 	char *repopath[2];
 	if (prc->outdir == NULL)
 		prc->outdir = path;
 	te.mfd = te.ffd = te.dfd = -1;
 	if (!is_dir(path)) {

@@ -431,44 +610,44 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 	}
 	errno = 0;
 	if (!is_dir(output_dir)) {
 	if (!is_dir(prc->outdir)) {
 		/* Try to create dir */
 		if (errno == ENOENT) {
 			if (mkdir(output_dir, 00755) == -1) {
 			if (mkdir(prc->outdir, 00755) == -1) {
 				pkg_fatal_errno("cannot create output directory %s",
 					output_dir);
 					prc->outdir);
 			}
 		}
 		else {
 			pkg_emit_error("%s is not a directory", output_dir);
 			pkg_emit_error("%s is not a directory", prc->outdir);
 			return (EPKG_FATAL);
 		}
 	}
 	if ((outputdir_fd = open(output_dir, O_DIRECTORY)) == -1) {
 		pkg_emit_error("Cannot open %s", output_dir);
 	if ((outputdir_fd = open(prc->outdir, O_DIRECTORY)) == -1) {
 		pkg_emit_error("Cannot open %s", prc->outdir);
 		return (EPKG_FATAL);
 	}
 	if (metafile != NULL) {
 		fd = open(metafile, O_RDONLY);
 	if (prc->metafile != NULL) {
 		fd = open(prc->metafile, O_RDONLY);
 		if (fd == -1) {
 			pkg_emit_error("meta loading error while trying %s", metafile);
 			pkg_emit_error("meta loading error while trying %s", prc->metafile);
 			return (EPKG_FATAL);
 		}
 		if (pkg_repo_meta_load(fd, &meta) != EPKG_OK) {
 			pkg_emit_error("meta loading error while trying %s", metafile);
 		if (pkg_repo_meta_load(fd, &prc->meta) != EPKG_OK) {
 			pkg_emit_error("meta loading error while trying %s", prc->metafile);
 			close(fd);
 			return (EPKG_FATAL);
 		}
 		close(fd);
 	} else {
 		meta = pkg_repo_meta_default();
 		prc->meta = pkg_repo_meta_default();
 	}
 	meta->repopath = path;
 	meta->hash = hash;
 	meta->hash_symlink = hash_symlink;
 	prc->meta->repopath = path;
 	prc->meta->hash = prc->hash;
 	prc->meta->hash_symlink = prc->hash_symlink;
 	te.meta = meta;
 	te.meta = prc->meta;
 	repopath[0] = path;
 	repopath[1] = NULL;

@@ -485,16 +664,16 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 		goto cleanup;
 	}
 	if ((te.mfd = openat(outputdir_fd, meta->manifests,
 	if ((te.mfd = openat(outputdir_fd, prc->meta->manifests,
 	     O_CREAT|O_TRUNC|O_WRONLY, 00644)) == -1) {
 		goto cleanup;
 	}
 	if ((te.dfd = openat(outputdir_fd, meta->data,
 	if ((te.dfd = openat(outputdir_fd, prc->meta->data,
 	    O_CREAT|O_TRUNC|O_WRONLY, 00644)) == -1) {
 		goto cleanup;
 	}
 	if (filelist) {
 		if ((te.ffd = openat(outputdir_fd, meta->filesite,
 	if (prc->filelist) {
 		if ((te.ffd = openat(outputdir_fd, prc->meta->filesite,
 		        O_CREAT|O_TRUNC|O_WRONLY, 00644)) == -1) {
 			goto cleanup;
 		}

@@ -502,7 +681,7 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 	len = 0;
 	pkg_create_repo_read_fts(&te.fts_items, fts, path, &len, meta);
 	pkg_create_repo_read_fts(&te.fts_items, fts, path, &len, prc->meta);
 	if (len == 0) {
 		/* Nothing to do */

@@ -514,7 +693,7 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 	num_workers = MIN(num_workers, len);
 	/* Launch workers */
 	pkg_emit_progress_start("Creating repository in %s", output_dir);
 	pkg_emit_progress_start("Creating repository in %s", prc->outdir);
 	threads = xcalloc(num_workers, sizeof(pthread_t));

@@ -553,7 +732,7 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,

 	fd = openat(outputdir_fd, "meta", O_CREAT|O_TRUNC|O_CLOEXEC|O_WRONLY,
 );
 	if (fd != -1) {
 		meta_dump = pkg_repo_meta_to_ucl(meta);
 		meta_dump = pkg_repo_meta_to_ucl(prc->meta);
 		ucl_object_emit_fd(meta_dump, UCL_EMIT_CONFIG, fd);
 		close(fd);
 		fd = openat(outputdir_fd, "meta.conf",

@@ -584,9 +763,10 @@ cleanup:

 	tll_free_and_free(te.fts_items, pkg_create_repo_fts_free);
 	pkg_repo_meta_free(meta);
 	if (retcode != EPKG_OK)
 		return (retcode);
 	return (retcode);
 	return (pkg_repo_create_pack_and_sign(prc));
 }
 static int

@@ -743,152 +923,3 @@ pkg_repo_pack_db(const char *name, const char *archive, char *path,

 	return (ret);
 }
 int
 pkg_finish_repo(const char *output_dir, pkg_password_cb *password_cb,
     char **argv, int argc, bool filelist)
 {
 	char repo_path[MAXPATHLEN];
 	char repo_archive[MAXPATHLEN];
 	char *key_file;
 	const char *key_type;
 	struct pkg_key *keyinfo = NULL;
 	struct pkg_repo_meta *meta;
 	struct stat st;
 	int ret = EPKG_OK, nfile = 0, fd;
 	const int files_to_pack = 4;
 	if (!is_dir(output_dir)) {
 		pkg_emit_error("%s is not a directory", output_dir);
 		return (EPKG_FATAL);
 	}
 	if (argc == 1) {
 		key_type = key_file = argv[0];
 		if (strncmp(key_file, "rsa:", 4) == 0) {
 			key_file += 4;
 			*(key_file - 1) = '\0';
 		} else {
 			key_type = "rsa";
 		}
 		pkg_debug(1, "Loading %s key from '%s' for signing", key_type, key_file);
 		rsa_new(&keyinfo, password_cb, key_file);
 	}
 	if (argc > 1 && strcmp(argv[0], "signing_command:") != 0)
 		return (EPKG_FATAL);
 	if (argc > 1) {
 		argc--;
 		argv++;
 	}
 	pkg_emit_progress_start("Packing files for repository");
 	pkg_emit_progress_tick(nfile++, files_to_pack);
 	snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
 		repo_meta_file);
 	if ((fd = open(repo_path, O_RDONLY)) != -1) {
 		if (pkg_repo_meta_load(fd, &meta) != EPKG_OK) {
 			pkg_emit_error("meta loading error while trying %s", repo_path);
 			rsa_free(keyinfo);
 			close(fd);
 			return (EPKG_FATAL);
 		}
 		if (pkg_repo_pack_db(repo_meta_file, repo_path, repo_path, keyinfo,
 		    meta, argv, argc) != EPKG_OK) {
 			ret = EPKG_FATAL;
 			goto cleanup;
 		}
 	}
 	else {
 		meta = pkg_repo_meta_default();
 	}
 	snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
 	    meta->manifests);
 	snprintf(repo_archive, sizeof(repo_archive), "%s/%s", output_dir,
 		meta->manifests_archive);
 	if (pkg_repo_pack_db(meta->manifests, repo_archive, repo_path, keyinfo,
 	    meta, argv, argc) != EPKG_OK) {
 		ret = EPKG_FATAL;
 		goto cleanup;
 	}
 	pkg_emit_progress_tick(nfile++, files_to_pack);
 	if (filelist) {
 		snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
 		    meta->filesite);
 		snprintf(repo_archive, sizeof(repo_archive), "%s/%s",
 		    output_dir, meta->filesite_archive);
 		if (pkg_repo_pack_db(meta->filesite, repo_archive, repo_path, keyinfo,
 		    meta, argv, argc) != EPKG_OK) {
 			ret = EPKG_FATAL;
 			goto cleanup;
 		}
 	}
 	pkg_emit_progress_tick(nfile++, files_to_pack);
 	snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir, meta->data);
 	snprintf(repo_archive, sizeof(repo_archive), "%s/%s", output_dir,
 	    meta->data_archive);
 	if (pkg_repo_pack_db(meta->data, repo_archive, repo_path, keyinfo,
 	    meta, argv, argc) != EPKG_OK) {
 		ret = EPKG_FATAL;
 		goto cleanup;
 	}
 	pkg_emit_progress_tick(nfile++, files_to_pack);
 #if 0
 	snprintf(repo_path, sizeof(repo_path), "%s/%s", output_dir,
 		meta->conflicts);
 	snprintf(repo_archive, sizeof(repo_archive), "%s/%s", output_dir,
 		meta->conflicts_archive);
 	if (pkg_repo_pack_db(meta->conflicts, repo_archive, repo_path, keyinfo,
 	    meta, argv, argc) != EPKG_OK) {
 		ret = EPKG_FATAL;
 		goto cleanup;
 	}
 #endif
 	/* Now we need to set the equal mtime for all archives in the repo */
 	snprintf(repo_archive, sizeof(repo_archive), "%s/%s.pkg",
 	    output_dir, repo_meta_file);
 	if (stat(repo_archive, &st) == 0) {
 		struct timeval ftimes[2] = {
 			{
 			.tv_sec = st.st_mtime,
 			.tv_usec = 0
 			},
 			{
 			.tv_sec = st.st_mtime,
 			.tv_usec = 0
 			}
 		};
 		snprintf(repo_archive, sizeof(repo_archive), "%s/%s.pkg",
 		    output_dir, meta->manifests_archive);
 		utimes(repo_archive, ftimes);
 		if (filelist) {
 			snprintf(repo_archive, sizeof(repo_archive),
 			    "%s/%s.pkg", output_dir, meta->filesite_archive);
 			utimes(repo_archive, ftimes);
 		}
 		snprintf(repo_archive, sizeof(repo_archive),
 			"%s/%s.pkg", output_dir, meta->data_archive);
 		utimes(repo_archive, ftimes);
 		snprintf(repo_archive, sizeof(repo_archive),
 			"%s/%s.pkg", output_dir, repo_meta_file);
 		utimes(repo_archive, ftimes);
 	}
 cleanup:
 	pkg_emit_progress_tick(files_to_pack, files_to_pack);
 	pkg_repo_meta_free(meta);
 	rsa_free(keyinfo);
 	return (ret);
 }

modified libpkg/private/pkg.h

@@ -300,6 +300,20 @@ struct pkg_create {

 	const char *outdir;
 };
 struct pkg_repo_create {
 	bool filelist;
 	bool hash;
 	bool hash_symlink;
 	const char *outdir;
 	const char *metafile;
 	struct pkg_repo_meta *meta;
 	struct {
 		char **argv;
 		int argc;
 		pkg_password_cb *cb;
 	} sign;
 };
 struct pkg_dep {
 	char		*origin;
 	char		*name;

modified src/repo.c

@@ -90,13 +90,10 @@ password_cb(char *buf, int size, int rwflag, void *key)

 int
 exec_repo(int argc, char **argv)
 {
 	int	 ret;
 	int	 ch;
 	bool	 filelist = false;
 	const char *output_dir = NULL;
 	const char *meta_file = NULL;
 	bool	 hash = false;
 	bool	 hash_symlink = false;
 	struct pkg_repo_create *prc = pkg_repo_create_new();
 	hash = (getenv("PKG_REPO_HASH") != NULL);
 	hash_symlink = (getenv("PKG_REPO_SYMLINK") != NULL);

@@ -117,16 +114,16 @@ exec_repo(int argc, char **argv)

 			hash = true;
 			break;
 		case 'l':
 			filelist = true;
 			pkg_repo_create_set_create_filelist(prc, true);
 			break;
 		case 'o':
 			output_dir = optarg;
 			pkg_repo_create_set_output_dir(prc, optarg);
 			break;
 		case 'q':
 			quiet = true;
 			break;
 		case 'm':
 			meta_file = optarg;
 			pkg_repo_create_set_metafile(prc, optarg);
 			break;
 		case 's':
 			hash_symlink = true;

@@ -139,6 +136,10 @@ exec_repo(int argc, char **argv)

 	argc -= optind;
 	argv += optind;
 	pkg_repo_create_set_hash(prc, hash);
 	pkg_repo_create_set_hash_symlink(prc, hash_symlink);
 	pkg_repo_create_set_sign(prc, argv + 1, argc - 1, password_cb);
 	if (argc < 1) {
 		usage_repo();
 		return (EXIT_FAILURE);

@@ -149,20 +150,10 @@ exec_repo(int argc, char **argv)

 		return (EXIT_FAILURE);
 	}
 	if (output_dir == NULL)
 		output_dir = argv[0];
 	ret = pkg_create_repo(argv[0], output_dir, filelist, meta_file, hash,
 	    hash_symlink);
 	if (ret != EPKG_OK) {
 	if (pkg_repo_create(prc, argv[0]) != EPKG_OK) {
 		printf("Cannot create repository catalogue\n");
 		return (EXIT_FAILURE);
 	}
 	if (pkg_finish_repo(output_dir, password_cb, argv + 1, argc - 1,
 	    filelist) != EPKG_OK)
 		return (EXIT_FAILURE);
 	return (EXIT_SUCCESS);
 }