bd83d9c pkg: allow to define threads for zstd and xz — HardenedBSD-pkg

modified docs/pkg-create.8

@@ -14,7 +14,7 @@

 .\"
 .\"     @(#)pkg.8
 .\"
 .Dd October 13, 2020
 .Dd May 3, 2024
 .Dt PKG-CREATE 8
 .Os
 .\" ---------------------------------------------------------------------------

@@ -31,6 +31,7 @@

 .Op Fl p Ar plist
 .Op Fl r Ar rootdir
 .Op Fl t Ar timestamp
 .Op Fl T Ar threads
 .Fl m Ar metadatadir
 .Nm
 .Op Fl enqv

@@ -39,6 +40,7 @@

 .Op Fl o Ar outdir
 .Op Fl r Ar rootdir
 .Op Fl t Ar timestamp
 .Op Fl T Ar threads
 .Fl M Ar manifest
 .Nm
 .Op Fl egnqvx

@@ -47,6 +49,7 @@

 .Op Fl o Ar outdir
 .Op Fl r Ar rootdir
 .Op Fl t Ar timestamp
 .Op Fl T Ar threads
 .Ar pkg-name ...
 .Nm
 .Op Fl enqv

@@ -55,6 +58,7 @@

 .Op Fl o Ar outdir
 .Op Fl r Ar rootdir
 .Op Fl t Ar timestamp
 .Op Fl T Ar threads
 .Fl a
 .\" ---------------------------------------------------------------------------
 .Pp

@@ -274,6 +278,14 @@ disturbing similar content already on the system.

 If unspecified, the default is effectively
 .Pa / ,
 the actual root directory.
 .It Fl T Ar threads
 .Ar threads
 represent the number of threads to use during the compression of the archive.
 Set it to
 .Qq 0
 or
 .Qq auto
 to let detect the number of CPU and use it.
 .El
 .\" ---------------------------------------------------------------------------
 .Sh MANIFEST FILE DETAILS

modified docs/pkg.conf.5

@@ -127,6 +127,8 @@ Set the default compression level, special values are:

 .It 0 default value per libarchive developers
 .It -1 default value per pkg developers (default)
 .El
 .It Cm COMPRESSION_THREADS: integer
 Set the number of threads to use during compression (only functional for txz and tzst)
 .It Cm CONSERVATIVE_UPGRADE: boolean
 Ensure in multi repository mode that the priority is given as much as possible
 to the repository where a package was first installed from.

modified docs/pkg_create.3

@@ -1,4 +1,4 @@

 .Dd January 23, 2024
 .Dd May 3, 2024
 .Dt PKG_CREATE 3
 .Os
 .Sh NAME

@@ -6,7 +6,8 @@

 .Nm pkg_create_new , pkg_create_free ,
 .Nm pkg_create_set_format , pkg_create_set_overwrite ,
 .Nm pkg_create_set_compression_level , pkg_create_set_rootdir ,
 .Nm pkg_create_set_output_dir , pkg_create_set_timestamp
 .Nm pkg_create_set_output_dir , pkg_create_set_timestamp ,
 .Nm pkg_create_set_compression_threads
 .Nd create packages
 .Sh LIBRARY
 .Lb libpkg

@@ -21,6 +22,8 @@

 .Ft void
 .Fn pkg_create_set_compression_level "struct pkg_create *" "int"
 .Ft void
 .Fn pkg_create_set_compression_threads "struct pkg_create *" "int"
 .Ft void
 .Fn pkg_create_set_overwrite "struct pkg_create *" "bool"
 .Ft void
 .Fn pkg_create_set_rootdir "struct pkg_create *" "const char *"

@@ -63,6 +66,11 @@ Set the default (as specified in libarchive)

 Set the best compression ratio
 .El
 .Pp
 .Fn pkg_create_set_compression_threads
 take a
 .Ft int
 arguments which represents the expected numbers of threads used during
 compression.
 .Fn pkg_create_set_overwrite
 Accept a boolean to define the default behaviour when creating a package and
 a local file already exists.

modified libpkg/libpkg.ver

@@ -32,6 +32,7 @@ global:

 	pkg_create_installed;
 	pkg_create_new;
 	pkg_create_repo;
 	pkg_create_set_compression_threads;
 	pkg_create_set_compression_level;
 	pkg_create_set_expand_manifest;
 	pkg_create_set_format;

modified libpkg/packing.c

@@ -47,7 +47,7 @@

 int
 packing_init(struct packing **pack, const char *path, pkg_formats format, int clevel,
 	time_t timestamp, bool overwrite, bool compat_symlink)
 	int threads, time_t timestamp, bool overwrite, bool compat_symlink)
 {
 	char archive_path[MAXPATHLEN];
 	char archive_symlink[MAXPATHLEN];

@@ -79,7 +79,7 @@ packing_init(struct packing **pack, const char *path, pkg_formats format, int cl

 	(*pack)->awrite = archive_write_new();
 	archive_write_set_format_pax_restricted((*pack)->awrite);
 	ext = packing_set_format((*pack)->awrite, format, clevel);
 	ext = packing_set_format((*pack)->awrite, format, clevel, threads);
 	if (ext == NULL) {
 		archive_read_close((*pack)->aread);
 		archive_read_free((*pack)->aread);

@@ -312,7 +312,7 @@ packing_finish(struct packing *pack)

 }
 const char *
 packing_set_format(struct archive *a, pkg_formats format, int clevel)
 packing_set_format(struct archive *a, pkg_formats format, int clevel, int threads)
 {
 	const char *notsupp_fmt = "%s is not supported, trying %s";

@@ -370,44 +370,50 @@ out:

 	if (format == TAR && clevel != 0)
 		pkg_emit_error("Plain tar and a compression level does not make sense");
 	if (elected_format != TAR && clevel != 0) {
 	if (elected_format != TAR) {
 		char buf[16];
 		/*
 		 * A bit of a kludge but avoids dragging in headers for all of
 		 * these libraries.
 		 */
 		if (clevel == INT_MIN) {
 			switch (elected_format) {
 			case TZS:
 				clevel = -5;
 				break;
 			case TXZ:
 			case TBZ:
 			case TGZ:
 				clevel = 1;
 				break;
 			default:
 				__unreachable();
 			}
 		} else if (clevel == INT_MAX) {
 			switch (elected_format) {
 			case TZS:
 				clevel = 19;
 				break;
 			case TXZ:
 			case TBZ:
 			case TGZ:
 				clevel = 9;
 				break;
 			default:
 				__unreachable();
 		if (clevel != 0) {
 			/*
 			 * A bit of a kludge but avoids dragging in headers for all of
 			 * these libraries.
 			 */
 			if (clevel == INT_MIN) {
 				switch (elected_format) {
 					case TZS:
 						clevel = -5;
 						break;
 					case TXZ:
 					case TBZ:
 					case TGZ:
 						clevel = 1;
 						break;
 					default:
 						__unreachable();
 				}
 			} else if (clevel == INT_MAX) {
 				switch (elected_format) {
 					case TZS:
 						clevel = 19;
 						break;
 					case TXZ:
 					case TBZ:
 					case TGZ:
 						clevel = 9;
 						break;
 					default:
 						__unreachable();
 				}
 			}
 		}
 		snprintf(buf, sizeof(buf), "%d", clevel);
 		if (archive_write_set_filter_option(a, NULL, "compression-level", buf) != ARCHIVE_OK)
 			pkg_emit_error("bad compression-level %d", clevel);
 			snprintf(buf, sizeof(buf), "%d", clevel);
 			if (archive_write_set_filter_option(a, NULL, "compression-level", buf) != ARCHIVE_OK)
 				pkg_emit_error("bad compression-level %d", clevel);
 		}
 		if (threads >= 0) {
 			snprintf(buf, sizeof(buf), "%d", threads);
 			if (archive_write_set_filter_option(a, NULL, "threads", buf) != ARCHIVE_OK)
 				pkg_emit_error("bad threads value %d", threads);
 		}
 	}
 	return (packing_format_to_string(elected_format));

modified libpkg/pkg.h.in

@@ -1678,6 +1678,7 @@ void pkg_drop_privileges(void);

 struct pkg_create *pkg_create_new(void);
 void pkg_create_free(struct pkg_create *);
 bool pkg_create_set_format(struct pkg_create *, const char *);
 void pkg_create_set_compression_threads(struct pkg_create *, int);
 void pkg_create_set_compression_level(struct pkg_create *, int);
 void pkg_create_set_overwrite(struct pkg_create *, bool);
 void pkg_create_set_rootdir(struct pkg_create *, const char *);

modified libpkg/pkg_config.c

@@ -82,6 +82,7 @@ struct pkg_ctx ctx = {

 	.triggers = true,
 	.compression_format = NULL,
 	.compression_level = -1,
 	.compression_threads = -1,
 	.defer_triggers = false,
 };

@@ -515,6 +516,12 @@ static struct config_entry c[] = {

 		NULL,
 		"debug flags to activate",
 	},
 	{
 		PKG_INT,
 		"COMPRESSION_THREADS",
 		"-1",
 		"Set the default number of threads used for compression",
 	}
 };
 static bool parsed = false;

@@ -1399,6 +1406,7 @@ pkg_ini(const char *path, const char *reposdir, pkg_init_flags flags)

 	ctx.triggers_path = pkg_object_string(pkg_config_get("PKG_TRIGGERS_DIR"));
 	ctx.compression_format = pkg_object_string(pkg_config_get("COMPRESSION_FORMAT"));
 	ctx.compression_level = pkg_object_int(pkg_config_get("COMPRESSION_LEVEL"));
 	ctx.compression_threads = pkg_object_int(pkg_config_get("COMPRESSION_THREADS"));
 	ctx.archive_symlink = pkg_object_bool(pkg_config_get("ARCHIVE_SYMLINK"));
 	ctx.repo_accept_legacy_pkg = pkg_object_bool(pkg_config_get("REPO_ACCEPT_LEGACY_PKG"));

modified libpkg/pkg_create.c

@@ -216,7 +216,7 @@ pkg_create_archive(struct pkg *pkg, struct pkg_create *pc, unsigned required_fla

 	}
 	if (packing_init(&pkg_archive, pkg_path, pc->format,
 	    pc->compression_level, pc->timestamp, pc->overwrite, false) != EPKG_OK) {
 	    pc->compression_level, pc->compression_threads, pc->timestamp, pc->overwrite, false) != EPKG_OK) {
 		pkg_archive = NULL;
 	}

@@ -258,6 +258,7 @@ pkg_create_new(void)

 	pc = xcalloc(1, sizeof(*pc));
 	pc->format = packing_format_from_string(ctx.compression_format);
 	pc->compression_level = ctx.compression_level;
 	pc->compression_threads = ctx.compression_threads;
 	pc->timestamp = (time_t) -1;
 	pc->overwrite = true;
 	pc->expand_manifest = false;

@@ -296,6 +297,12 @@ pkg_create_set_compression_level(struct pkg_create *pc, int clevel)

 }
 void
 pkg_create_set_compression_threads(struct pkg_create *pc, int threads)
 {
 	pc->compression_threads = threads;
 }
 void
 pkg_create_set_expand_manifest(struct pkg_create *pc, bool expand)
 {
 	pc->expand_manifest = expand;

modified libpkg/pkg_repo_create.c

@@ -1099,7 +1099,7 @@ pkg_repo_pack_db(const char *name, const char *archive, char *path,

 	struct packing *pack;
 	int ret = EPKG_OK;
 	if (packing_init(&pack, archive, prc->meta->packing_format, 0, (time_t)-1, true, true) != EPKG_OK)
 	if (packing_init(&pack, archive, prc->meta->packing_format, 0, 0, (time_t)-1, true, true) != EPKG_OK)
 		return (EPKG_FATAL);
 	if (sctx != NULL) {

modified libpkg/private/packing.h

@@ -2,7 +2,7 @@

  * Copyright (c) 2011-2021 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2011 Will Andrews <will@FreeBSD.org>
  * All rights reserved.
  *
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:

@@ -12,7 +12,7 @@

  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

@@ -35,5 +35,5 @@ struct packing {

 	time_t timestamp;
 };
 const char *packing_set_format(struct archive *a, pkg_formats format, int clevel);
 const char *packing_set_format(struct archive *a, pkg_formats format, int clevel, int threads);
 #endif

modified libpkg/private/pkg.h

@@ -157,6 +157,7 @@ struct pkg_ctx {

 	const char *cachedir;
 	const char *compression_format;
 	int compression_level;
 	int compression_threads;
 	int rootfd;
 	int cachedirfd;
 	int devnullfd;

@@ -298,6 +299,7 @@ struct pkg_create {

 	bool overwrite;
 	bool expand_manifest;
 	int compression_level;
 	int compression_threads;
 	pkg_formats format;
 	time_t timestamp;
 	const char *rootdir;

@@ -713,7 +715,7 @@ void pkg_config_file_free(struct pkg_config_file *);

 struct iovec;
 struct packing;
 int packing_init(struct packing **pack, const char *path, pkg_formats format, int clevel, time_t timestamp, bool overwrite, bool archive_symlink);
 int packing_init(struct packing **pack, const char *path, pkg_formats format, int clevel, int threads, time_t timestamp, bool overwrite, bool archive_symlink);
 int packing_append_file_attr(struct packing *pack, const char *filepath,
      const char *newpath, const char *uname, const char *gname, mode_t perm,
      u_long fflags);

modified src/create.c

@@ -57,13 +57,13 @@ void

 usage_create(void)
 {
 	fprintf(stderr, "Usage: pkg create [-eOhnqv] [-f format] [-l level] "
 		"[-o outdir] [-p plist] [-r rootdir] -m metadatadir\n");
 		"[-T threads] [-o outdir] [-p plist] [-r rootdir] -m metadatadir\n");
 	fprintf(stderr, "Usage: pkg create [-eOhnqv] [-f format] [-l level] "
 		"[-o outdir] [-r rootdir] -M manifest\n");
 		"[-T threads] [-o outdir] [-r rootdir] -M manifest\n");
 	fprintf(stderr, "       pkg create [-eOhgnqvx] [-f format] [-l level] "
 		"[-o outdir] [-r rootdir] pkg-name ...\n");
 		"[-T threads] [-o outdir] [-r rootdir] pkg-name ...\n");
 	fprintf(stderr, "       pkg create [-eOhnqv] [-f format] [-l level] "
 		"[-o outdir] [-r rootdir] -a\n\n");
 		"[-T threads] [-o outdir] [-r rootdir] -a\n\n");
 	fprintf(stderr, "For more information see 'pkg help create'.\n");
 }

@@ -172,6 +172,8 @@ exec_create(int argc, char **argv)

 	int		 ch;
 	int		 level;
 	bool		 level_is_set = false;
 	int		 threads;
 	bool		 threads_is_set = false;
 	int		 ret;
 	bool		 hash = false;
 	bool		 overwrite = true;

@@ -208,7 +210,7 @@ exec_create(int argc, char **argv)

 		{ NULL,		0,			NULL,	0   },
 	};
 	while ((ch = getopt_long(argc, argv, "+aeghxf:l:r:m:M:no:p:qvt:", longopts, NULL)) != -1) {
 	while ((ch = getopt_long(argc, argv, "+aeghxf:l:r:m:M:no:p:qvt:T:", longopts, NULL)) != -1) {
 		switch (ch) {
 		case 'a':
 			match = MATCH_ALL;

@@ -272,6 +274,21 @@ exec_create(int argc, char **argv)

 				return (EXIT_FAILURE);
 			}
 			break;
 		case 'T':
 			{
 			const char *errstr;
 			threads_is_set = true;
 			threads = strtonum(optarg, 0, INT_MAX, &errstr);
 			if (errstr == NULL)
 				break;
 			if (strcasecmp(optarg, "auto") == 0) {
 				threads = 0;
 				break;
 			}
 			warnx("Invalid compression threads %s", optarg);
 			return (EXIT_FAILURE);
 			}
 		case 'v':
 			quiet = false;
 			break;

@@ -310,7 +327,9 @@ exec_create(int argc, char **argv)

 			warnx("unknown format %s, using the default", format);
 	}
 	if (level_is_set)
 	    pkg_create_set_compression_level(pc, level);
 		pkg_create_set_compression_level(pc, level);
 	if (threads_is_set)
 		pkg_create_set_compression_threads(pc, threads);
 	pkg_create_set_overwrite(pc, overwrite);
 	pkg_create_set_rootdir(pc, rootdir);
 	pkg_create_set_output_dir(pc, outdir);

modified tests/lib/packing.c

@@ -74,33 +74,33 @@ ATF_TC_BODY(packing_set_format, tc)

 	ATF_CHECK(a != NULL);
 #if defined(HAVE_ARCHIVE_WRITE_ADD_FILTER_ZSTD) && __FreeBSD_version >= 1300000
 	ATF_REQUIRE_STREQ(packing_set_format(a, TZS, -1), "tzst");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TZS, -1, -1), "tzst");
 #endif
 	ATF_REQUIRE_STREQ(packing_set_format(a, TXZ, -1), "txz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TBZ, -1), "tbz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TGZ, -1), "tgz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TAR, -1), "tar");
 	ATF_REQUIRE_EQ(packing_set_format(a, 28, -1), NULL);
 	ATF_REQUIRE_STREQ(packing_set_format(a, TXZ, -1, -1), "txz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TBZ, -1, -1), "tbz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TGZ, -1, -1), "tgz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TAR, -1, -1), "tar");
 	ATF_REQUIRE_EQ(packing_set_format(a, 28, -1, -1), NULL);
 	/* compression min */
 #if defined(HAVE_ARCHIVE_WRITE_ADD_FILTER_ZSTD) && __FreeBSD_version >= 1300000
 	ATF_REQUIRE_STREQ(packing_set_format(a, TZS, INT_MIN), "tzst");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TZS, INT_MIN, -1), "tzst");
 #endif
 	ATF_REQUIRE_STREQ(packing_set_format(a, TXZ, INT_MIN), "txz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TBZ, INT_MIN), "tbz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TGZ, INT_MIN), "tgz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TAR, INT_MIN), "tar");
 	ATF_REQUIRE_EQ(packing_set_format(a, 28, INT_MIN), NULL);
 	ATF_REQUIRE_STREQ(packing_set_format(a, TXZ, INT_MIN, -1), "txz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TBZ, INT_MIN, -1), "tbz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TGZ, INT_MIN, -1), "tgz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TAR, INT_MIN, -1), "tar");
 	ATF_REQUIRE_EQ(packing_set_format(a, 28, INT_MIN, -1), NULL);
 	/* compression max */
 #if defined(HAVE_ARCHIVE_WRITE_ADD_FILTER_ZSTD) && __FreeBSD_version >= 1300000
 	ATF_REQUIRE_STREQ(packing_set_format(a, TZS, INT_MAX), "tzst");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TZS, INT_MAX, -1), "tzst");
 #endif
 	ATF_REQUIRE_STREQ(packing_set_format(a, TXZ, INT_MAX), "txz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TBZ, INT_MAX), "tbz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TGZ, INT_MAX), "tgz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TAR, INT_MAX), "tar");
 	ATF_REQUIRE_EQ(packing_set_format(a, 28, INT_MAX), NULL);
 	ATF_REQUIRE_STREQ(packing_set_format(a, TXZ, INT_MAX, -1), "txz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TBZ, INT_MAX, -1), "tbz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TGZ, INT_MAX, -1), "tgz");
 	ATF_REQUIRE_STREQ(packing_set_format(a, TAR, INT_MAX, -1), "tar");
 	ATF_REQUIRE_EQ(packing_set_format(a, 28, INT_MAX, -1), NULL);
 }
 ATF_TP_ADD_TCS(tp)