83243c7 memory: drastically reduce memory usage — HardenedBSD-pkg

modified libpkg/pkg.c

@@ -75,6 +75,7 @@ pkg_free(struct pkg *pkg)

 	free(pkg->repourl);
 	free(pkg->reason);
 	free(pkg->dep_formula);
 	free(pkg->rootpath);
 	for (int i = 0; i < PKG_NUM_SCRIPTS; i++)
 		xstring_free(pkg->scripts[i]);

@@ -528,16 +529,16 @@ pkg_addfile_attr(struct pkg *pkg, const char *path, const char *sum,

 	}
 	f = xcalloc(1, sizeof(*f));
 	strlcpy(f->path, path, sizeof(f->path));
 	f->path = xstrdup(path);
 	if (sum != NULL)
 		f->sum = xstrdup(sum);
 	if (uname != NULL)
 		strlcpy(f->uname, uname, sizeof(f->uname));
 		f->uname = xstrdup(uname);
 	if (gname != NULL)
 		strlcpy(f->gname, gname, sizeof(f->gname));
 		f->gname = xstrdup(gname);
 	if (perm != 0)
 		f->perm = perm;

@@ -546,7 +547,7 @@ pkg_addfile_attr(struct pkg *pkg, const char *path, const char *sum,

 		f->fflags = fflags;
 	if (symlink_target != NULL)
 		strlcpy(f->symlink_target, symlink_target, sizeof(f->symlink_target));
 		f->symlink_target = xstrdup(symlink_target);
 	if (mtime > 0)
 		f->time[1].tv_sec = mtime;

@@ -571,7 +572,7 @@ pkg_addconfig_file(struct pkg *pkg, const char *path, const char *content)

 		return (EPKG_FATAL);
 	}
 	f = xcalloc(1, sizeof(*f));
 	strlcpy(f->path, path, sizeof(f->path));
 	f->path = xstrdup(path);
 	if (content != NULL)
 		f->content = xstrdup(content);

@@ -637,13 +638,13 @@ pkg_adddir_attr(struct pkg *pkg, const char *path, const char *uname,

 	}
 	d = xcalloc(1, sizeof(*d));
 	strlcpy(d->path, path, sizeof(d->path));
 	d->path = xstrdup(path);
 	if (uname != NULL)
 		strlcpy(d->uname, uname, sizeof(d->uname));
 		d->uname = xstrdup(uname);
 	if (gname != NULL)
 		strlcpy(d->gname, gname, sizeof(d->gname));
 		d->gname = xstrdup(gname);
 	if (perm != 0)
 		d->perm = perm;

@@ -1183,7 +1184,7 @@ pkg_list_free(struct pkg *pkg, pkg_list list) {

 		pkg->flags &= ~PKG_LOAD_FILES;
 		break;
 	case PKG_DIRS:
 		DL_FREE(pkg->dirs, free);
 		DL_FREE(pkg->dirs, pkg_dir_free);
 		pkghash_destroy(pkg->dirhash);
 		pkg->dirhash = NULL;
 		pkg->flags &= ~PKG_LOAD_DIRS;

@@ -1482,7 +1483,8 @@ pkg_check_meta(struct stat *st, const char *uname, const char *gname,

 	if (S_ISLNK(st->st_mode) != (fs_symlink_target[0] != '\0'))
 		file_status |= FILE_META_MISMATCH_SYMLINK;
 	else if (S_ISLNK(st->st_mode) && strcmp(db_symlink_target, fs_symlink_target) != 0)
 	else if (S_ISLNK(st->st_mode) &&
 	    strcmp(db_symlink_target ? db_symlink_target : "", fs_symlink_target) != 0)
 		file_status |= FILE_META_MISMATCH_SYMLINK;
 	return file_status;

@@ -1548,7 +1550,7 @@ emit_status:

 	if (file_status & FILE_META_MISMATCH_TYPE) {
 		pkg_emit_file_meta_mismatch(pkg, f, PKG_META_ATTR_TYPE,
 					    f->symlink_target[0] == '\0' ? "Regular File" : "Symbolic Link",
 					    f->symlink_target == NULL ? "Regular File" : "Symbolic Link",
 					    stat_type_tostring(st.st_mode));
 	}

@@ -1821,6 +1823,7 @@ int

 pkg_open_root_fd(struct pkg *pkg)
 {
 	const char *path;
 	char rootpath[MAXPATHLEN];
 	if (pkg->rootfd != -1)
 		return (EPKG_OK);

@@ -1834,12 +1837,13 @@ pkg_open_root_fd(struct pkg *pkg)

 		return (EPKG_OK);
 	}
 	pkg_absolutepath(path, pkg->rootpath, sizeof(pkg->rootpath), false);
 	pkg_absolutepath(path, rootpath, sizeof(rootpath), false);
 	if ((pkg->rootfd = openat(ctx.rootfd, pkg->rootpath + 1, O_DIRECTORY)) >= 0 )
 	if ((pkg->rootfd = openat(ctx.rootfd, RELATIVE_PATH(rootpath), O_DIRECTORY)) >= 0 ) {
 		pkg->rootpath = xstrdup(rootpath);
 		return (EPKG_OK);
 	}
 	pkg->rootpath[0] = '\0';
 	pkg_emit_errno("open", path);
 	return (EPKG_FATAL);

modified libpkg/pkg_add.c

@@ -465,8 +465,8 @@ do_extract_dir(struct pkg_add_context* context, struct archive *a __unused, stru

 	d->perm = aest->st_mode;
 	d->uid = get_uid_from_uname(archive_entry_uname(ae));
 	d->gid = get_gid_from_gname(archive_entry_gname(ae));
 	strlcpy(d->uname, archive_entry_uname(ae), sizeof(d->uname));
 	strlcpy(d->gname, archive_entry_gname(ae), sizeof(d->gname));
 	d->uname = xstrdup(archive_entry_uname(ae));
 	d->gname = xstrdup(archive_entry_uname(ae));
 	fill_timespec_buf(aest, d->time);
 	archive_entry_fflags(ae, &d->fflags, &clear);

@@ -504,8 +504,11 @@ create_symlinks(struct pkg_add_context *context, struct pkg_file *f, const char

 	bool tried_mkdir = false;
 	tmpdir = get_tempdir(context, f->path, tempdirs);
 	if (tmpdir == NULL && errno == 0)
 		hidden_tempfile(f->temppath, sizeof(f->temppath), f->path);
 	if (tmpdir == NULL && errno == 0) {
 		char temppath[MAXPATHLEN] = { 0 };
 		hidden_tempfile(temppath, sizeof(temppath), f->path);
 		f->temppath = xstrdup(temppath);
 	}
 	if (tmpdir == NULL) {
 		fd = context->rootfd;
 		path = f->temppath;

@@ -559,8 +562,8 @@ do_extract_symlink(struct pkg_add_context *context, struct archive *a __unused,

 	archive_entry_fflags(ae, &f->fflags, &clear);
 	f->uid = get_uid_from_uname(archive_entry_uname(ae));
 	f->gid = get_gid_from_gname(archive_entry_gname(ae));
 	strlcpy(f->uname, archive_entry_uname(ae), sizeof(f->uname));
 	strlcpy(f->gname, archive_entry_gname(ae), sizeof(f->gname));
 	f->uname = xstrdup(archive_entry_uname(ae));
 	f->gname = xstrdup(archive_entry_gname(ae));
 	f->perm = aest->st_mode;
 	fill_timespec_buf(aest, f->time);
 	archive_entry_fflags(ae, &f->fflags, &clear);

@@ -586,8 +589,11 @@ create_hardlink(struct pkg_add_context *context, struct pkg_file *f, const char

 	struct tempdir *tmphdir = NULL;
 	tmpdir = get_tempdir(context, f->path, tempdirs);
 	if (tmpdir == NULL && errno == 0)
 		hidden_tempfile(f->temppath, sizeof(f->temppath), f->path);
 	if (tmpdir == NULL && errno == 0) {
 		char temppath[MAXPATHLEN] = { 0 };
 		hidden_tempfile(temppath, sizeof(temppath), f->path);
 		f->temppath = xstrdup(temppath);
 	}
 	if (tmpdir != NULL) {
 		fd = tmpdir->fd;
 	} else {

@@ -600,7 +606,7 @@ create_hardlink(struct pkg_add_context *context, struct pkg_file *f, const char

 		    " hardlinked to %s", f->path, path);
 		return (EPKG_FATAL);
 	}
 	if (fh->temppath[0] == '\0') {
 	if (fh->temppath != NULL) {
 		vec_foreach(*tempdirs, i) {
 			if (strncmp(tempdirs->d[i]->name, fh->path, tempdirs->d[i]->len) == 0 &&
 			    fh->path[tempdirs->d[i]->len] == '/' ) {

@@ -706,8 +712,11 @@ create_regfile(struct pkg_add_context *context, struct pkg_file *f, struct archi

 	struct tempdir *tmpdir = NULL;
 	tmpdir = get_tempdir(context, f->path, tempdirs);
 	if (tmpdir == NULL && errno == 0)
 		hidden_tempfile(f->temppath, sizeof(f->temppath), f->path);
 	if (tmpdir == NULL && errno == 0) {
 		char temppath[MAXPATHLEN] = { 0 };
 		hidden_tempfile(temppath, sizeof(temppath), f->path);
 		f->temppath = xstrdup(temppath);
 	}
 	if (tmpdir != NULL) {
 		fd = open_tempfile(tmpdir->fd, f->path + tmpdir->len, f->perm);

@@ -807,8 +816,8 @@ do_extract_regfile(struct pkg_add_context *context, struct archive *a, struct ar

 	f->perm = aest->st_mode;
 	f->uid = get_uid_from_uname(archive_entry_uname(ae));
 	f->gid = get_gid_from_gname(archive_entry_gname(ae));
 	strlcpy(f->uname, archive_entry_uname(ae), sizeof(f->uname));
 	strlcpy(f->gname, archive_entry_gname(ae), sizeof(f->gname));
 	f->uname = xstrdup(archive_entry_uname(ae));
 	f->gname = xstrdup(archive_entry_gname(ae));
 	fill_timespec_buf(aest, f->time);
 	archive_entry_fflags(ae, &f->fflags, &clear);

@@ -985,7 +994,7 @@ pkg_extract_finalize(struct pkg *pkg, tempdirs_t *tempdirs)

 		    pkg_config_get("FILES_IGNORE_REGEX")))
 			continue;
 		append_touched_file(f->path);
 		if (*f->temppath == '\0')
 		if (f->temppath == NULL)
 			continue;
 		fto = f->path;
 		if (f->config && f->config->status == MERGE_FAILED &&

@@ -1282,7 +1291,7 @@ pkg_rollback_pkg(struct pkg *p)

 		    pkg_config_get("FILES_IGNORE_GLOB"),
 		    pkg_config_get("FILES_IGNORE_REGEX")))
 			continue;
 		if (*f->temppath != '\0') {
 		if (f->temppath  != NULL) {
 			unlinkat(p->rootfd, f->temppath, 0);
 		}
 	}

@@ -1651,7 +1660,7 @@ pkg_add_fromdir(struct pkg *pkg, const char *src, struct pkgdb *db __unused)

 		}
 		if (d->perm == 0)
 			d->perm = st.st_mode & ~S_IFMT;
 		if (d->uname[0] != '\0') {
 		if (d->uname != NULL) {
 			err = getpwnam_r(d->uname, &pwent, buffer,
 			    sizeof(buffer), &pw);
 			if (err != 0) {

@@ -1663,7 +1672,7 @@ pkg_add_fromdir(struct pkg *pkg, const char *src, struct pkgdb *db __unused)

 		} else {
 			d->uid = install_as_user ? st.st_uid : 0;
 		}
 		if (d->gname[0] != '\0') {
 		if (d->gname != NULL) {
 			err = getgrnam_r(d->gname, &grent, buffer,
 			    sizeof(buffer), &gr);
 			if (err != 0) {

@@ -1707,7 +1716,7 @@ pkg_add_fromdir(struct pkg *pkg, const char *src, struct pkgdb *db __unused)

 			close(fromfd);
 			pkg_fatal_errno("%s%s", src, f->path);
 		}
 		if (f->uname[0] != '\0') {
 		if (f->uname != NULL) {
 			err = getpwnam_r(f->uname, &pwent, buffer,
 			    sizeof(buffer), &pw);
 			if (err != 0) {

@@ -1720,7 +1729,7 @@ pkg_add_fromdir(struct pkg *pkg, const char *src, struct pkgdb *db __unused)

 			f->uid = install_as_user ? st.st_uid : 0;
 		}
 		if (f->gname[0] != '\0') {
 		if (f->gname != NULL) {
 			err = getgrnam_r(f->gname, &grent, buffer,
 			    sizeof(buffer), &gr);
 			if (err != 0) {

modified libpkg/pkg_attributes.c

@@ -63,10 +63,23 @@ pkg_dep_is_locked(struct pkg_dep const * const d)

 void
 pkg_file_free(struct pkg_file *file)
 {
 	free(file->path);
 	free(file->uname);
 	free(file->gname);
 	free(file->symlink_target);
 	free(file->sum);
 	free(file);
 }
 void
 pkg_dir_free(struct pkg_dir *dir)
 {
 	free(dir->path);
 	free(dir->uname);
 	free(dir->gname);
 	free(dir);
 }
 /*
  * Script
  */

@@ -120,6 +133,7 @@ pkg_config_file_free(struct pkg_config_file *c)

 	if (c == NULL)
 		return;
 	free(c->path);
 	free(c->content);
 	free(c);
 }

modified libpkg/pkg_create.c

@@ -128,13 +128,14 @@ pkg_create_from_dir(struct pkg *pkg, const char *root,

 			}
 			if (S_ISLNK(st.st_mode)) {
 				linklen = readlink(fpath, file->symlink_target, sizeof(file->symlink_target) - 1);
 				char link[MAXPATHLEN] = { 0 };
 				linklen = readlink(fpath, link, sizeof(link) -1);
 				if (linklen == -1) {
 					vec_free_and_free(&hardlinks, free);
 					pkg_emit_errno("pkg_create_from_dir", "readlink failed");
 					return (EPKG_FATAL);
 				}
 				file->symlink_target[linklen] = '\0';
 				file->symlink_target = xstrdup(link);
 			}
 			if (pc->timestamp > (time_t)-1) {

modified libpkg/pkg_manifest.c

@@ -1210,10 +1210,10 @@ pkg_emit_object(struct pkg *pkg, short flags)

 						      ucl_object_fromstring(file->sum),
 						      "sum", 0, false);
 				ucl_object_insert_key(file_attrs,
 						      ucl_object_fromstring(file->uname[0] != '\0' ? file->uname : "root"),
 						      ucl_object_fromstring(file->uname != NULL ? file->uname : "root"),
 						      "uname", 0, false);
 				ucl_object_insert_key(file_attrs,
 						      ucl_object_fromstring(file->gname[0] != '\0' ? file->gname : "wheel"),
 						      ucl_object_fromstring(file->gname != NULL ? file->gname : "wheel"),
 						      "gname", 0, false);
 				snprintf(perm_str, sizeof(perm_str), "%#4.4o", file->perm);
 				ucl_object_insert_key(file_attrs,

@@ -1222,7 +1222,7 @@ pkg_emit_object(struct pkg *pkg, short flags)

 				ucl_object_insert_key(file_attrs,
 						      ucl_object_fromint(file->fflags),
 						      "fflags", 0, false);
 				if (file->symlink_target[0] != '\0') {
 				if (file->symlink_target != NULL) {
 					ucl_object_insert_key(file_attrs,
 							      ucl_object_fromstring(file->symlink_target),
 							      "symlink_target", 0, false);

@@ -1260,10 +1260,10 @@ pkg_emit_object(struct pkg *pkg, short flags)

 				dir_attrs = ucl_object_typed_new(UCL_OBJECT);
 				ucl_object_insert_key(dir_attrs,
 						      ucl_object_fromstring(dir->uname),
 						      ucl_object_fromstring(dir->uname ? dir->uname : "root"),
 						      "uname", 0, false);
 				ucl_object_insert_key(dir_attrs,
 						      ucl_object_fromstring(dir->gname),
 						      ucl_object_fromstring(dir->gname ? dir->gname : "wheel"),
 						      "gname", 0, false);
 				snprintf(perm_str, sizeof(perm_str), "%#4.4o", dir->perm);
 				ucl_object_insert_key(dir_attrs,

modified libpkg/pkg_printf.c

@@ -1257,7 +1257,7 @@ xstring *

 format_file_symlink_target(xstring *buf, const void *data, struct percent_esc *p)
 {
 	const struct pkg_file *file = data;
 	return (string_val(buf, file->symlink_target, p));
 	return (string_val(buf, file->symlink_target ? file->symlink_target : "", p));
 }
 /*

modified libpkg/pkgdb.c

@@ -1781,11 +1781,11 @@ pkgdb_register_pkg(struct pkgdb *db, struct pkg *pkg, int forced,

 		sql_arg_t args[] = {
 			SQL_ARG(file->path),
 			SQL_ARG(file->sum),
 			SQL_ARG(_pkgdb_empty_str_null(file->uname)),
 			SQL_ARG(_pkgdb_empty_str_null(file->gname)),
 			SQL_ARG(file->uname),
 			SQL_ARG(file->gname),
 			SQL_ARG(file->perm),
 			SQL_ARG(file->fflags),
 			SQL_ARG(_pkgdb_empty_str_null(file->symlink_target)),
 			SQL_ARG(file->symlink_target),
 			SQL_ARG(file->time[1].tv_sec),
 			SQL_ARG(package_id),
 		};

@@ -1871,8 +1871,8 @@ pkgdb_register_pkg(struct pkgdb *db, struct pkg *pkg, int forced,

 	while (pkg_dirs(pkg, &dir) == EPKG_OK) {
 		sql_arg_t args[] = {
 			SQL_ARG(dir->path),
 			SQL_ARG(_pkgdb_empty_str_null(dir->uname)),
 			SQL_ARG(_pkgdb_empty_str_null(dir->gname)),
 			SQL_ARG(dir->uname),
 			SQL_ARG(dir->gname),
 			SQL_ARG(dir->perm),
 			SQL_ARG(dir->fflags),
 			SQL_ARG(dir->time[1].tv_sec),

modified libpkg/private/pkg.h

@@ -241,7 +241,7 @@ struct pkg {

 	kvlist_t		 annotations;
 	unsigned			flags;
 	int		rootfd;
 	char		rootpath[MAXPATHLEN];
 	char		*rootpath;
 	charv_t	dir_to_del;
 	pkg_t		 type;
 	struct pkg_repo		*repo;

@@ -366,7 +366,7 @@ typedef enum {

 } merge_status;
 struct pkg_config_file {
 	char path[MAXPATHLEN];
 	char *path;
 	char *content;
 	char *newcontent;
 	merge_status status;

@@ -374,17 +374,17 @@ struct pkg_config_file {

 };
 struct pkg_file {
 	char		 path[MAXPATHLEN];
 	char		*path;
 	int64_t		 size;
 	char		*sum;
 	char		 uname[MAXLOGNAME];
 	char		 gname[MAXLOGNAME];
 	char		*uname;
 	char		*gname;
 	mode_t		 perm;
 	uid_t		 uid;
 	gid_t		 gid;
 	char		 temppath[MAXPATHLEN];
 	char		*temppath;
 	u_long		 fflags;
 	char		 symlink_target[MAXPATHLEN];
 	char		*symlink_target;
 	struct pkg_config_file *config;
 	struct timespec	 time[2];
 	struct pkg_file	*next, *prev;

@@ -392,9 +392,9 @@ struct pkg_file {

 };
 struct pkg_dir {
 	char		 path[MAXPATHLEN];
 	char		 uname[MAXLOGNAME];
 	char		 gname[MAXLOGNAME];
 	char		*path;
 	char		*uname;
 	char		*gname;
 	mode_t		 perm;
 	u_long		 fflags;
 	uid_t		 uid;

@@ -733,6 +733,7 @@ DEFINE_VEC_INSERT_SORTED_PROTO(kvlist_t, pkg_kv, struct pkg_kv *);

 void pkg_dep_free(struct pkg_dep *);
 void pkg_file_free(struct pkg_file *);
 void pkg_dir_free(struct pkg_dir *);
 void pkg_option_free(struct pkg_option *);
 void pkg_conflict_free(struct pkg_conflict *);
 void pkg_config_file_free(struct pkg_config_file *);

modified libpkg/utils.c

@@ -1146,14 +1146,15 @@ get_uid_from_uname(const char *uname)

 	static struct passwd pwent;
 	struct passwd *result;
 	int err;
 	const char *testuname = uname ? uname : "";
 	if (pwent.pw_name != NULL && STREQ(uname, pwent.pw_name))
 	if (pwent.pw_name != NULL && STREQ(testuname, pwent.pw_name))
 		goto out;
 	pwent.pw_name = NULL;
 	err = getpwnam_r(uname, &pwent, user_buffer, sizeof(user_buffer),
 	err = getpwnam_r(testuname, &pwent, user_buffer, sizeof(user_buffer),
 	    &result);
 	if (err != 0) {
 		pkg_emit_errno("getpwnam_r", uname);
 		pkg_emit_errno("getpwnam_r", testuname);
 		return (0);
 	}
 	if (result == NULL)

@@ -1169,14 +1170,15 @@ get_gid_from_gname(const char *gname)

 	static struct group grent;
 	struct group *result;
 	int err;
 	const char *testgname = gname ? gname : "";
 	if (grent.gr_name != NULL && STREQ(gname, grent.gr_name))
 	if (grent.gr_name != NULL && STREQ(testgname, grent.gr_name))
 		goto out;
 	grent.gr_name = NULL;
 	err = getgrnam_r(gname, &grent, group_buffer, sizeof(group_buffer),
 	err = getgrnam_r(testgname, &grent, group_buffer, sizeof(group_buffer),
 	    &result);
 	if (err != 0) {
 		pkg_emit_errno("getgrnam_r",gname);
 		pkg_emit_errno("getgrnam_r", testgname);
 		return (0);
 	}
 	if (result == NULL)

modified tests/frontend/create.sh

@@ -543,6 +543,7 @@ EOF

 		-e empty \
 		-s exit:0 \
 		pkg create -M ./+MANIFEST -r ${TMPDIR}
 	cp test-1.pkg /tmp
 	cat << EOF > output.ucl
 name = "test";

@@ -867,12 +868,12 @@ create_from_plist_with_variables_body() {

 	genmanifest
 	genplist "
 @var key1
 @var key2
 @var key2
 @var key3 plop
 %%key1%%file1
 %%key2%%file2
 %%key3%%
 @var key3 @comment
 @var key3 @comment
 %%key3%% file3"
 	atf_check \