Radish alpha
H
HardenedBSD-pkg
rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9
HardenedBSD Package Manager
Radicle
Git
Commits › 36dfb48
Add a 'hash' mode to pkg repo
Allan Jude committed 5 years ago
commit 36dfb4895b0b307fe3321f6a4478521080a6e9b7
parent fd8e940
10 files changed +177 -13
modified docs/pkg.conf.5
@@ -237,6 +237,22 @@ Directory which
will load plugins from.

Default:

.Pa /usr/local/lib/pkg
+ 
.It Cm PKG_REPO_HASH: boolean
+ 
When set to a
+ 
.Sy true
+ 
value, make
+ 
.Xr pkg-repo 8
+ 
rename packages with the short hash of contents appended to the filename.
+ 
Default:
+ 
.Sy false
+ 
.It Cm PKG_REPO_SYMLINK: boolean
+ 
When set to a
+ 
.Sy true
+ 
value, make
+ 
.Xr pkg-repo 8
+ 
create a symlink between the short hash filename and the regular filename.
+ 
Default:
+ 
.Sy false

.It Cm PKG_SSH_ARGS: string

Extra arguments to pass to

.Xr ssh 1 .
modified libpkg/pkg.h.in
@@ -819,7 +819,7 @@ int pkg_is_installed(struct pkgdb *db, const char *name);
 */

typedef int(pkg_password_cb)(char *, int, int, void*);

int pkg_create_repo(char *path, const char *output_dir, bool filelist,
- 
	const char *metafile);
+ 
	const char *metafile, bool hash, bool hash_symlink);

int pkg_finish_repo(const char *output_dir, pkg_password_cb *cb, char **argv,

    int argc, bool filelist);
modified libpkg/pkg_config.c
@@ -382,6 +382,18 @@ static struct config_entry c[] = {
	},

	{

		PKG_BOOL,
+ 
		"PKG_REPO_HASH",
+ 
		"NO",
+ 
		"Rename packages with the short hash of their contents",
+ 
	},
+ 
	{
+ 
		PKG_BOOL,
+ 
		"PKG_REPO_SYMLINK",
+ 
		"NO",
+ 
		"Create symlinks from the hashed filename to the regular filename",
+ 
	},
+ 
	{
+ 
		PKG_BOOL,

		"AUTOCLEAN",

		"NO",

		"Always cleanup the cache directory after install/upgrade",
modified libpkg/pkg_repo_create.c
@@ -83,6 +83,87 @@ struct pkg_conflict_bulk {
};



static int
+ 
hash_file(struct pkg_repo_meta *meta, struct pkg *pkg, char *path)
+ 
{
+ 
	char tmp_repo[MAXPATHLEN] = { 0 };
+ 
	char tmp_name[MAXPATHLEN] = { 0 };
+ 
	char repo_name[MAXPATHLEN] = { 0 };
+ 
	char hash_name[MAXPATHLEN] = { 0 };
+ 
	char link_name[MAXPATHLEN] = { 0 };
+ 
	char *rel_repo = NULL;
+ 
	char *rel_dir = NULL;
+ 
	char *rel_link = NULL;
+ 
	char *ext = NULL;
+
+ 
	/* Don't rename symlinks */
+ 
	if (is_link(path))
+ 
		return (EPKG_OK);
+
+ 
	ext = strrchr(path, '.');
+
+ 
	strlcpy(tmp_name, path, sizeof(tmp_name));
+ 
	rel_dir = dirname(tmp_name);
+ 
	while (strstr(rel_dir, "/Hashed") != NULL) {
+ 
		rel_dir = dirname(rel_dir);
+ 
	}
+ 
	strlcpy(tmp_name, rel_dir, sizeof(tmp_name));
+ 
	rel_dir = (char *)&tmp_name;
+
+ 
	rel_repo = path;
+ 
	if (strncmp(rel_repo, meta->repopath, strlen(meta->repopath)) == 0) {
+ 
		rel_repo += strlen(meta->repopath);
+ 
		while (rel_repo[0] == '/')
+ 
			rel_repo++;
+ 
	}
+ 
	strlcpy(tmp_repo, rel_repo, sizeof(tmp_repo));
+ 
	rel_repo = dirname(tmp_repo);
+ 
	while (strstr(rel_repo, "/Hashed") != NULL) {
+ 
		rel_repo = dirname(rel_repo);
+ 
	}
+ 
	strlcpy(tmp_repo, rel_repo, sizeof(tmp_repo));
+ 
	rel_repo = (char *)&tmp_repo;
+
+ 
	pkg_snprintf(repo_name, sizeof(repo_name), "%S/%S/%n-%v%S%z%S",
+ 
	    rel_repo, PKG_HASH_DIR, pkg, pkg, PKG_HASH_SEPSTR, pkg, ext);
+ 
	pkg_snprintf(link_name, sizeof(repo_name), "%S/%n-%v%S",
+ 
	    rel_dir, pkg, pkg, ext);
+ 
	pkg_snprintf(hash_name, sizeof(hash_name), "%S/%S/%n-%v%S%z%S",
+ 
	    rel_dir, PKG_HASH_DIR, pkg, pkg, PKG_HASH_SEPSTR, pkg, ext);
+ 
	rel_link = (char *)&hash_name;
+ 
	rel_link += strlen(rel_dir);
+ 
	while (rel_link[0] == '/')
+ 
		rel_link++;
+
+ 
	snprintf(tmp_name, sizeof(tmp_name), "%s/%s", rel_dir, PKG_HASH_DIR);
+ 
	rel_dir = (char *)&tmp_name;
+ 
	if (!is_dir(rel_dir)) {
+ 
		pkg_debug(1, "Making directory: %s", rel_dir);
+ 
		(void)mkdirs(rel_dir);
+ 
	}
+
+ 
	if (strcmp(path, hash_name) != 0) {
+ 
		pkg_debug(1, "Rename the pkg from: %s to: %s", path, hash_name);
+ 
		if (rename(path, hash_name) == -1) {
+ 
			pkg_emit_errno("rename", hash_name);
+ 
			return (EPKG_FATAL);
+ 
		}
+ 
	}
+ 
	if (meta->hash_symlink) {
+ 
		pkg_debug(1, "Symlinking pkg file from: %s to: %s", rel_link,
+ 
		    link_name);
+ 
		(void)unlink(link_name);
+ 
		if (symlink(rel_link, link_name) == -1) {
+ 
			pkg_emit_errno("symlink", link_name);
+ 
			return (EPKG_FATAL);
+ 
		}
+ 
	}
+ 
	free(pkg->repopath);
+ 
	pkg->repopath = xstrdup(repo_name);
+
+ 
	return (EPKG_OK);
+ 
}
+
+ 
static int

pkg_digest_sort_compare_func(struct digest_list_entry *d1,

		struct digest_list_entry *d2)

{
@@ -136,6 +217,8 @@ pkg_create_repo_read_fts(struct pkg_fts_item **items, FTS *fts,
	FTSENT *fts_ent;

	struct pkg_fts_item *fts_cur;

	char *ext;
+ 
	int linklen = 0;
+ 
	char tmp_name[MAXPATHLEN] = { 0 };



	errno = 0;
@@ -164,6 +247,16 @@ pkg_create_repo_read_fts(struct pkg_fts_item **items, FTS *fts,
		}

		/* Follow symlinks. */

		if (fts_ent->fts_info == FTS_SL) {
+ 
			/* Skip symlinks to hashed packages */
+ 
			if (meta->hash) {
+ 
				linklen = readlink(fts_ent->fts_path,
+ 
				    (char *)&tmp_name, MAXPATHLEN);
+ 
				if (linklen < 0)
+ 
					continue;
+ 
				tmp_name[linklen] = '\0';
+ 
				if (strstr(tmp_name, PKG_HASH_DIR) != NULL)
+ 
					continue;
+ 
			}

			fts_set(fts, fts_ent, FTS_FOLLOW);

			/* Restart. Next entry will be the resolved file. */

			continue;
@@ -284,7 +377,13 @@ pkg_create_repo_worker(struct pkg_fts_item *start, size_t nelts,
			pkg->sum = pkg_checksum_file(cur->fts_accpath,

			    PKG_HASH_TYPE_SHA256_HEX);

			pkg->pkgsize = cur->fts_size;
- 
			pkg->repopath = xstrdup(cur->pkg_path);
+ 
			if (meta->hash) {
+ 
				ret = hash_file(meta, pkg, cur->fts_accpath);
+ 
				if (ret != EPKG_OK)
+ 
					goto cleanup;
+ 
			} else {
+ 
				pkg->repopath = xstrdup(cur->pkg_path);
+ 
			}



			/*

			 * TODO: use pkg_checksum for new manifests
@@ -452,7 +551,7 @@ fts_compare(const FTSENT *const *a, const FTSENT *const *b)


int

pkg_create_repo(char *path, const char *output_dir, bool filelist,
- 
	const char *metafile)
+ 
	const char *metafile, bool hash, bool hash_symlink)

{

	FTS *fts = NULL;

	struct pkg_fts_item *fts_items = NULL, *fts_cur, *fts_start;
@@ -512,6 +611,9 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,
	} else {

		meta = pkg_repo_meta_default();

	}
+ 
	meta->repopath = path;
+ 
	meta->hash = hash;
+ 
	meta->hash_symlink = hash_symlink;



	repopath[0] = path;

	repopath[1] = NULL;
@@ -584,9 +686,8 @@ pkg_create_repo(char *path, const char *output_dir, bool filelist,
				goto cleanup;

			}
- 
			if (pkg_create_repo_worker(fts_start, cur_jobs,
- 
					mfd, ffd, cur_pipe[1],
- 
					meta) == EPKG_FATAL) {
+ 
			if (pkg_create_repo_worker(fts_start, cur_jobs, mfd,
+ 
			    ffd, cur_pipe[1], meta) == EPKG_FATAL) {

				close(cur_pipe[0]);

				close(cur_pipe[1]);

				goto cleanup;
modified libpkg/private/pkg.h
@@ -51,6 +51,11 @@
#define PKG_NUM_SCRIPTS 9

#define PKG_NUM_LUA_SCRIPTS 5
+ 
#define PKG_HASH_SEP '~'
+ 
#define PKG_HASH_SEPSTR "~"
+
+ 
#define PKG_HASH_DIR "Hashed"
+ 


/*

 * Some compatibility checks

 */
@@ -496,6 +501,9 @@ struct pkg_repo_meta {
	time_t eol;



	int version;
+ 
	char *repopath;
+ 
	bool hash;
+ 
	bool hash_symlink;

};



struct pkg_repo_it_ops {
modified libpkg/private/utils.h
@@ -69,6 +69,7 @@ int file_to_bufferat(int, const char *, char **, off_t *);
int format_exec_cmd(char **, const char *, const char *, const char *, char *,

    int argc, char **argv);

int is_dir(const char *);
+ 
int is_link(const char *);



int rsa_new(struct rsa_key **, pkg_password_cb *, char *path);

void rsa_free(struct rsa_key *);
modified libpkg/repo/binary/fetch.c
@@ -75,15 +75,15 @@ pkg_repo_binary_get_cached_name(struct pkg_repo *repo, struct pkg *pkg,
		 * The real naming scheme:

		 * <cachedir>/<name>-<version>-<checksum>.txz

		 */
- 
		pkg_snprintf(dest, destlen, "%S/%n-%v-%z%S",
- 
				ctx.cachedir, pkg, pkg, pkg, ext);
+ 
		pkg_snprintf(dest, destlen, "%S/%n-%v%S%z%S",
+ 
		    ctx.cachedir, pkg, pkg, PKG_HASH_SEPSTR, pkg, ext);

		if (stat (dest, &st) == -1 || pkg->pkgsize != st.st_size)

			return (EPKG_FATAL);



	}

	else {
- 
		pkg_snprintf(dest, destlen, "%S/%n-%v-%z",
- 
				ctx.cachedir, pkg, pkg, pkg);
+ 
		pkg_snprintf(dest, destlen, "%S/%n-%v%S%z", ctx.cachedir, pkg,
+ 
		    pkg, PKG_HASH_SEPSTR, pkg);

	}



	return (EPKG_OK);
modified libpkg/utils.c
@@ -296,6 +296,14 @@ is_dir(const char *path)
	return (stat(path, &st) == 0 && S_ISDIR(st.st_mode));

}
+ 
int
+ 
is_link(const char *path)
+ 
{
+ 
	struct stat st;
+
+ 
	return (lstat(path, &st) == 0 && S_ISLNK(st.st_mode));
+ 
}
+ 


bool

string_end_with(const char *path, const char *str)

{
modified scripts/completion/_pkg.in
@@ -143,6 +143,8 @@ _pkg_config_opts() {
		'PKG_ENABLE_PLUGINS[activate plugin support]:boolean:(yes no)' \

		'PKG_ENV[key/value pair of environment variables]:key/value list' \

		'PKG_PLUGINS_DIR[specify directory for plugins]:directory:_files -/' \
+ 
		'PKG_REPO_HASH[make pkg_repo(8) rename files using a short hash]:boolean:(yes no)' \
+ 
		'PKG_REPO_SYMLINK[make pkg_repo(8) symlink the hashed filename to the regular filename]:boolean:(yes no)' \

		'PKG_SSH_ARGS[extra arguments for ssh(1)]:ssh(1) arguments' \

		'PLIST_KEYWORDS_DIR[directory containing definitions of plist keywords]:directory:_files -/' \

		'PLIST_ACCEPT_DIRECTORIES[accept directories listed like plain files in plist]:boolean:(yes no)' \
@@ -459,10 +461,12 @@ _pkg_args() {
			;;

		(repo)

			_arguments -A '-*' -s \
+ 
				'(-h --hash)'{-h,--hash}'[Rename repo files with a short hash appeneded]' \

				'(-l --list-files)'{-l,--list-files}'[generate list of all files in repo as filesite.txz archive]' \

				'(-q --quiet)'{-q,--quiet}'[force quiet output]' \

				'(-o --output-dir)'{-o+,--output-dir=}'[specify the location of the new repo]:repo location:_files -/' \

				'(-m --meta-file)'{-m+,--meta-file=}'[use specified file as repository meta file instead of the defaults]:meta file:_files' \
+ 
				'(-s --symlink)'{-s,--symlink}'[Create a symlink between the hashed and regular filename]' \

				':repository path:_files -/' \

				':RSA key: _alternative "files\:RSA key\:_files" "commands\:commands\:_cmdstring"'

			return
modified src/repo.c
@@ -96,17 +96,27 @@ exec_repo(int argc, char **argv)
	bool	 filelist = false;

	char	*output_dir = NULL;

	char	*meta_file = NULL;
+ 
	bool	 hash = false;
+ 
	bool	 hash_symlink = false;
+
+ 
	hash = pkg_object_bool(pkg_config_get("PKG_REPO_HASH"));
+ 
	hash_symlink = pkg_object_bool(pkg_config_get("PKG_REPO_SYMLINK"));



	struct option longopts[] = {
+ 
		{ "hash",	no_argument,		NULL,	'h' },

		{ "list-files", no_argument,		NULL,	'l' },
+ 
		{ "meta-file",	required_argument,	NULL,	'm' },

		{ "output-dir", required_argument,	NULL,	'o' },

		{ "quiet",	no_argument,		NULL,	'q' },
- 
		{ "meta-file",	required_argument,	NULL,	'm' },
+ 
		{ "symlink",	no_argument,		NULL,	's' },

		{ NULL,		0,			NULL,	0   },

	};
- 
	while ((ch = getopt_long(argc, argv, "+lo:qm:", longopts, NULL)) != -1) {
+ 
	while ((ch = getopt_long(argc, argv, "+hlo:qm:s", longopts, NULL)) != -1) {

		switch (ch) {
+ 
		case 'h':
+ 
			hash = true;
+ 
			break;

		case 'l':

			filelist = true;

			break;
@@ -119,6 +129,9 @@ exec_repo(int argc, char **argv)
		case 'm':

			meta_file = optarg;

			break;
+ 
		case 's':
+ 
			hash_symlink = true;
+ 
			break;

		default:

			usage_repo();

			return (EX_USAGE);
@@ -140,7 +153,8 @@ exec_repo(int argc, char **argv)
	if (output_dir == NULL)

		output_dir = argv[0];
- 
	ret = pkg_create_repo(argv[0], output_dir, filelist, meta_file);
+ 
	ret = pkg_create_repo(argv[0], output_dir, filelist, meta_file, hash,
+ 
	    hash_symlink);



	if (ret != EPKG_OK) {

		printf("Cannot create repository catalogue\n");