Radish alpha
H
HardenedBSD-pkg
rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9
HardenedBSD Package Manager
Radicle
Git
Commits › b297dbe
Merge pull request #817 from vstakhov/pkg-checksum
Vsevolod Stakhov committed 11 years ago
commit b297dbec4124ac7ad58341bf8c81e21eb453d183
parent 6088e07
12 files changed +523 -97
modified libpkg/Makefile.am
@@ -17,6 +17,7 @@ libpkg_la_SOURCES= pkg.c \
			pkg_add.c \

			pkg_attributes.c \

			pkg_audit.c \
+ 
			pkg_checksum.c \

			pkg_config.c \

			pkg_conflicts.c \

			pkg_cudf.c \
modified libpkg/pkg.c
@@ -306,9 +306,12 @@ pkg_vset(struct pkg *pkg, va_list ap)
				data = pkg_repo_url(r);

			}
- 
			ucl_object_replace_key(pkg->fields,
- 
			    ucl_object_fromstring_common(data, strlen(data), 0),
- 
			    pkg_keys[attr].name, strlen(pkg_keys[attr].name), false);
+ 
			if (data == NULL)
+ 
				ucl_object_delete_key(pkg->fields, pkg_keys[attr].name);
+ 
			else
+ 
				ucl_object_replace_key(pkg->fields,
+ 
				    ucl_object_fromstring_common(data, strlen(data), 0),
+ 
				    pkg_keys[attr].name, strlen(pkg_keys[attr].name), false);



			if (buf != NULL)

				free(buf);
modified libpkg/pkg.h.in
@@ -254,6 +254,7 @@ typedef enum {
	PKG_LICENSES,

	PKG_CATEGORIES,

	PKG_UNIQUEID,
+ 
	PKG_OLD_DIGEST,

	PKG_NUM_FIELDS, 	/* end of fields */

} pkg_attr;
added libpkg/pkg_checksum.c
@@ -0,0 +1,402 @@
+ 
/* Copyright (c) 2014, Vsevolod Stakhov
+ 
 * All rights reserved.
+ 
 *
+ 
 * Redistribution and use in source and binary forms, with or without
+ 
 * modification, are permitted provided that the following conditions are met:
+ 
 *       * Redistributions of source code must retain the above copyright
+ 
 *         notice, this list of conditions and the following disclaimer.
+ 
 *       * Redistributions in binary form must reproduce the above copyright
+ 
 *         notice, this list of conditions and the following disclaimer in the
+ 
 *         documentation and/or other materials provided with the distribution.
+ 
 *
+ 
 * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ 
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ 
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ 
 * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ 
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ 
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ 
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ 
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ 
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
 */
+
+ 
#include <sys/types.h>
+ 
#include <sys/sbuf.h>
+
+ 
#include <assert.h>
+ 
#include <ctype.h>
+ 
#include <errno.h>
+ 
#include <stdbool.h>
+ 
#include <stdlib.h>
+ 
#include <string.h>
+ 
#include <ucl.h>
+
+ 
#include "pkg.h"
+ 
#include "private/pkg.h"
+ 
#include "private/utils.h"
+ 
#include "private/event.h"
+
+ 
struct pkg_checksum_entry {
+ 
	const char *field;
+ 
	const char *value;
+ 
	struct pkg_checksum_entry *next, *prev;
+ 
};
+
+ 
typedef void (*pkg_checksum_hash_func)(struct pkg_checksum_entry *entries,
+ 
				unsigned char **out, size_t *outlen);
+ 
typedef void (*pkg_checksum_encode_func)(unsigned char *in, size_t inlen,
+ 
				char *out, size_t outlen);
+
+ 
static void pkg_checksum_hash_sha256(struct pkg_checksum_entry *entries,
+ 
				unsigned char **out, size_t *outlen);
+ 
static void pkg_checksum_encode_base32(unsigned char *in, size_t inlen,
+ 
				char *out, size_t outlen);
+ 
static void pkg_checksum_encode_hex(unsigned char *in, size_t inlen,
+ 
				char *out, size_t outlen);
+
+ 
static const struct _pkg_cksum_type {
+ 
	const char *name;
+ 
	size_t hlen;
+ 
	pkg_checksum_hash_func hfunc;
+ 
	pkg_checksum_encode_func encfunc;
+ 
} checksum_types[] = {
+ 
	[PKG_HASH_TYPE_SHA256_BASE32] = {
+ 
		"sha256_base32",
+ 
		PKG_CHECKSUM_SHA256_LEN,
+ 
		pkg_checksum_hash_sha256,
+ 
		pkg_checksum_encode_base32
+ 
	},
+ 
	[PKG_HASH_TYPE_SHA256_HEX] = {
+ 
		"sha256_hex",
+ 
		PKG_CHECKSUM_SHA256_LEN,
+ 
		pkg_checksum_hash_sha256,
+ 
		pkg_checksum_encode_hex
+ 
	},
+ 
	[PKG_HASH_TYPE_UNKNOWN] = {
+ 
		NULL,
+ 
		-1,
+ 
		NULL,
+ 
		NULL
+ 
	}
+ 
};
+
+ 
static void
+ 
pkg_checksum_add_object(const ucl_object_t *o, const char *key,
+ 
	struct pkg_checksum_entry **entries)
+ 
{
+ 
	struct pkg_checksum_entry *e;
+
+ 
	e = malloc(sizeof(*e));
+ 
	if (e == NULL) {
+ 
		pkg_emit_errno("malloc", "pkg_checksum_entry");
+ 
		return;
+ 
	}
+
+ 
	e->field = key;
+ 
	e->value = ucl_object_tostring(o);
+ 
	DL_APPEND(*entries, e);
+ 
}
+
+ 
static void
+ 
pkg_checksum_add_option(const struct pkg_option *o,
+ 
	struct pkg_checksum_entry **entries)
+ 
{
+ 
	struct pkg_checksum_entry *e;
+
+ 
	e = malloc(sizeof(*e));
+ 
	if (e == NULL) {
+ 
		pkg_emit_errno("malloc", "pkg_checksum_entry");
+ 
		return;
+ 
	}
+
+ 
	e->field = pkg_option_opt(o);
+ 
	e->value = pkg_option_value(o);
+ 
	DL_APPEND(*entries, e);
+ 
}
+
+ 
static int
+ 
pkg_checksum_entry_cmp(struct pkg_checksum_entry *e1,
+ 
	struct pkg_checksum_entry *e2)
+ 
{
+ 
	return (strcmp(e1->field, e2->field));
+ 
}
+
+ 
/*
+ 
 * At the moment we use the following fields to calculate the unique checksum
+ 
 * of the following fields:
+ 
 * - name
+ 
 * - origin
+ 
 * - version
+ 
 * - arch
+ 
 * - maintainer
+ 
 * - www
+ 
 * - message
+ 
 * - comment
+ 
 * - options
+ 
 */
+
+ 
int
+ 
pkg_checksum_generate(struct pkg *pkg, char *dest, size_t destlen,
+ 
	pkg_checksum_type_t type)
+ 
{
+ 
	const char *key;
+ 
	unsigned char *bdigest;
+ 
	size_t blen;
+ 
	struct pkg_checksum_entry *entries = NULL;
+ 
	const ucl_object_t *o;
+ 
	struct pkg_option *option = NULL;
+ 
	int i;
+ 
	int recopies[] = {
+ 
		PKG_NAME,
+ 
		PKG_ORIGIN,
+ 
		PKG_VERSION,
+ 
		PKG_ARCH,
+ 
		PKG_MAINTAINER,
+ 
		PKG_WWW,
+ 
		PKG_MESSAGE,
+ 
		PKG_COMMENT,
+ 
		-1
+ 
	};
+
+ 
	if (pkg == NULL || type >= PKG_HASH_TYPE_UNKNOWN ||
+ 
					destlen < checksum_types[type].hlen)
+ 
		return (EPKG_FATAL);
+
+ 
	for (i = 0; recopies[i] != -1; i++) {
+ 
		key = pkg_keys[recopies[i]].name;
+ 
		if ((o = ucl_object_find_key(pkg->fields, key)))
+ 
			pkg_checksum_add_object(o, key, &entries);
+ 
	}
+
+ 
	while (pkg_options(pkg, &option) == EPKG_OK) {
+ 
		pkg_checksum_add_option(option, &entries);
+ 
	}
+
+ 
	/* Sort before hashing */
+ 
	DL_SORT(entries, pkg_checksum_entry_cmp);
+
+ 
	checksum_types[type].hfunc(entries, &bdigest, &blen);
+ 
	if (blen == 0 || bdigest == NULL) {
+ 
		LL_FREE(entries, free);
+ 
		return (EPKG_FATAL);
+ 
	}
+
+ 
	i = snprintf(dest, destlen, "%d:%d:", PKG_CHECKSUM_CUR_VERSION, type);
+ 
	checksum_types[type].encfunc(bdigest, blen, dest + i, destlen - i);
+
+ 
	LL_FREE(entries, free);
+
+ 
	return (EPKG_OK);
+ 
}
+
+ 
bool
+ 
pkg_checksum_is_valid(const char *cksum, size_t clen)
+ 
{
+ 
	const char *semicolon;
+ 
	unsigned int value;
+
+ 
	if (clen < 4)
+ 
		return (false);
+
+ 
	semicolon = strchr(cksum, ':');
+ 
	if (semicolon == NULL || *semicolon == '\0')
+ 
		return (false);
+
+ 
	/* Test version */
+ 
	value = strtoul(cksum, NULL, 10);
+ 
	if (value != PKG_CHECKSUM_CUR_VERSION)
+ 
		return (false);
+
+ 
	cksum = semicolon + 1;
+ 
	semicolon = strchr(cksum, ':');
+ 
	if (semicolon == NULL || *semicolon == '\0')
+ 
		return (false);
+
+ 
	/* Test type */
+ 
	value = strtoul(cksum, NULL, 10);
+ 
	if (value >= PKG_HASH_TYPE_UNKNOWN)
+ 
		return (false);
+
+ 
	return (true);
+ 
}
+
+
+ 
pkg_checksum_type_t
+ 
pkg_checksum_get_type(const char *cksum, size_t clen)
+ 
{
+ 
	const char *semicolon;
+ 
	unsigned int value;
+
+ 
	semicolon = strchr(cksum, ':');
+ 
	if (semicolon != NULL && *semicolon != '\0') {
+ 
		value = strtoul(semicolon + 1, NULL, 10);
+ 
		if (value < PKG_HASH_TYPE_UNKNOWN)
+ 
			return (value);
+ 
	}
+
+ 
	return (PKG_HASH_TYPE_UNKNOWN);
+ 
}
+
+
+ 
static void
+ 
pkg_checksum_hash_sha256(struct pkg_checksum_entry *entries,
+ 
		unsigned char **out, size_t *outlen)
+ 
{
+ 
	SHA256_CTX sign_ctx;
+
+ 
	SHA256_Init(&sign_ctx);
+
+ 
	while(entries) {
+ 
		SHA256_Update(&sign_ctx, entries->value, strlen(entries->value));
+ 
		entries = entries->next;
+ 
	}
+ 
	*out = malloc(SHA256_DIGEST_LENGTH);
+ 
	if (*out != NULL) {
+ 
		SHA256_Final(*out, &sign_ctx);
+ 
		*outlen = SHA256_DIGEST_LENGTH;
+ 
	}
+ 
	else {
+ 
		pkg_emit_errno("malloc", "pkg_checksum_hash_sha256");
+ 
		*outlen = 0;
+ 
	}
+ 
}
+
+ 
/*
+ 
 * We use here z-base32 encoding described here:
+ 
 * http://philzimmermann.com/docs/human-oriented-base-32-encoding.txt
+ 
 */
+ 
static const char b32[]="ybndrfg8ejkmcpqxot1uwisza345h769";
+
+
+ 
static void
+ 
pkg_checksum_encode_base32(unsigned char *in, size_t inlen,
+ 
				char *out, size_t outlen)
+ 
{
+ 
	int i, remain = -1, r, x;
+
+ 
	if (outlen < inlen * 8 / 5) {
+ 
		pkg_emit_error("cannot encode base32 as outlen is not sufficient");
+ 
		return;
+ 
	}
+
+ 
	for (i = 0, r = 0; i < inlen; i++) {
+ 
		switch (i % 5) {
+ 
		case 0:
+ 
			/* 8 bits of input and 3 to remain */
+ 
			x = in[i];
+ 
			remain = in[i] >> 5;
+ 
			out[r++] = b32[x & 0x1F];
+ 
			break;
+ 
		case 1:
+ 
			/* 11 bits of input, 1 to remain */
+ 
			x = remain | in[i] << 3;
+ 
			out[r++] = b32[x & 0x1F];
+ 
			out[r++] = b32[x >> 5 & 0x1F];
+ 
			remain = x >> 10;
+ 
			break;
+ 
		case 2:
+ 
			/* 9 bits of input, 4 to remain */
+ 
			x = remain | in[i] << 1;
+ 
			out[r++] = b32[x & 0x1F];
+ 
			remain = x >> 5;
+ 
			break;
+ 
		case 3:
+ 
			/* 12 bits of input, 2 to remain */
+ 
			x = remain | in[i] << 4;
+ 
			out[r++] = b32[x & 0x1F];
+ 
			out[r++] = b32[x >> 5 & 0x1F];
+ 
			remain = x >> 10 & 0x3;
+ 
			break;
+ 
		case 4:
+ 
			/* 10 bits of output, nothing to remain */
+ 
			x = remain | in[i] << 2;
+ 
			out[r++] = b32[x & 0x1F];
+ 
			out[r++] = b32[x >> 5 & 0x1F];
+ 
			remain = -1;
+ 
			break;
+ 
		default:
+ 
			/* Not to be happen */
+ 
			break;
+ 
		}
+
+ 
	}
+ 
	if (remain >= 0)
+ 
		out[r++] = b32[remain];
+
+ 
	out[r] = 0;
+ 
}
+
+ 
static void
+ 
pkg_checksum_encode_hex(unsigned char *in, size_t inlen,
+ 
				char *out, size_t outlen)
+ 
{
+ 
	int i;
+
+ 
	if (outlen < inlen * 2) {
+ 
		pkg_emit_error("cannot encode hex as outlen is not sufficient");
+ 
		return;
+ 
	}
+
+ 
	for (i = 0; i < inlen; i++)
+ 
		sprintf(out + (i * 2), "%02x", in[i]);
+
+ 
	out[inlen * 2] = '\0';
+ 
}
+
+ 
pkg_checksum_type_t
+ 
pkg_checksum_type_from_string(const char *name)
+ 
{
+ 
	int i;
+ 
	for (i = 0; i < PKG_HASH_TYPE_UNKNOWN; i ++) {
+ 
		if (strcasecmp(name, checksum_types[i].name) == 0)
+ 
			return (i);
+ 
	}
+
+ 
	return (PKG_HASH_TYPE_UNKNOWN);
+ 
}
+
+ 
size_t
+ 
pkg_checksum_type_size(pkg_checksum_type_t type)
+ 
{
+ 
	return (checksum_types[type].hlen);
+ 
}
+
+ 
int
+ 
pkg_checksum_calculate(struct pkg *pkg, struct pkgdb *db)
+ 
{
+ 
	char *new_digest;
+ 
	struct pkg_repo *repo;
+ 
	const char *reponame;
+ 
	int rc = EPKG_OK;
+ 
	pkg_checksum_type_t type = 0;
+
+ 
	pkg_get(pkg, PKG_REPONAME, &reponame);
+ 
	if (reponame != NULL) {
+ 
		repo = pkg_repo_find_name(reponame);
+
+ 
		if (repo != NULL)
+ 
			type = repo->meta->digest_format;
+ 
	}
+
+ 
	new_digest = malloc(pkg_checksum_type_size(type));
+ 
	if (new_digest == NULL) {
+ 
		pkg_emit_errno("malloc", "pkg_checksum_type_t");
+ 
		return (EPKG_FATAL);
+ 
	}
+
+ 
	if (pkg_checksum_generate(pkg, new_digest, pkg_checksum_type_size(type), type)
+ 
			!= EPKG_OK) {
+ 
		free(new_digest);
+ 
		return (EPKG_FATAL);
+ 
	}
+
+ 
	pkg_set(pkg, PKG_DIGEST, new_digest);
+
+ 
	if (db != NULL)
+ 
		pkgdb_set_pkg_digest(db, pkg);
+
+ 
	free(new_digest);
+
+ 
	return (rc);
+ 
}
modified libpkg/pkg_jobs.c
@@ -498,69 +498,6 @@ iter_again:


#undef PRIORITY_CAN_UPDATE
- 
/*
- 
 * Calculate manifest for packages that lack it
- 
 */
- 
static int
- 
pkg_jobs_digest_manifest(struct pkg_jobs *j, struct pkg *pkg)
- 
{
- 
	char *new_digest;
- 
	int rc = EPKG_FATAL;
- 
	struct sbuf *sb;
- 
	char path[MAXPATHLEN], sha256[SHA256_DIGEST_LENGTH * 2 + 1];
- 
	struct archive *a;
- 
	struct archive_entry *ae;
-
- 
	/* Try to use the cached package */
- 
	pkg_snprintf(path, sizeof(path), "%R", pkg);
- 
	if (*path != '/')
- 
		pkg_repo_cached_name(pkg, path, sizeof(path));
- 
	a = archive_read_new();
- 
	archive_read_support_filter_all(a);
- 
	archive_read_support_format_tar(a);
- 
	if (archive_read_open_filename(a, path, 4096) == ARCHIVE_OK) {
- 
		const char *fpath;
-
- 
		while (archive_read_next_header(a, &ae) == ARCHIVE_OK) {
- 
			fpath = archive_entry_pathname(ae);
- 
			if (strcmp(fpath, "+COMPACT_MANIFEST") == 0) {
- 
				char *buffer;
- 
				size_t len = archive_entry_size(ae);
-
- 
				buffer = malloc(len);
- 
				archive_read_data(a, buffer, archive_entry_size(ae));
- 
				sha256_buf(buffer, len, sha256);
- 
				free(buffer);
- 
				pkg_debug(1, "manifest(%s): %.*s", buffer, len, buffer);
- 
				pkg_set(pkg, PKG_DIGEST, sha256);
- 
				pkgdb_set_pkg_digest(j->db, pkg);
- 
				rc = EPKG_OK;
- 
				break;
- 
			}
- 
		}
-
- 
		archive_read_close(a);
- 
	}
- 
	if (rc != EPKG_OK) {
- 
		/* XXX: broken with pkg 1.2 repos */
- 
		/* We need to calculate digest of this package */
- 
		sb = sbuf_new_auto();
- 
		rc = pkg_emit_manifest_sbuf(pkg, sb, PKG_MANIFEST_EMIT_COMPACT, &new_digest);
-
- 
		pkg_debug(1, "manifest(%s): %s", new_digest, sbuf_data(sb));
- 
		if (rc == EPKG_OK) {
- 
			pkg_set(pkg, PKG_DIGEST, new_digest);
- 
			pkgdb_set_pkg_digest(j->db, pkg);
- 
			free(new_digest);
- 
		}
-
- 
		sbuf_delete(sb);
- 
	}
-
- 
	archive_read_free(a);
-
- 
	return (rc);
- 
}



/**

 * Check whether a package is in the universe already or add it
@@ -579,7 +516,7 @@ pkg_jobs_handle_pkg_universe(struct pkg_jobs *j, struct pkg *pkg,
	if (digest == NULL) {

		pkg_debug(3, "no digest found for package %s (%s-%s)", uid,

				name, version);
- 
		if (pkg_jobs_digest_manifest(j, pkg) != EPKG_OK) {
+ 
		if (pkg_checksum_calculate(pkg, j->db) != EPKG_OK) {

			*found = NULL;

			return (EPKG_FATAL);

		}
@@ -963,7 +900,7 @@ pkg_jobs_process_remote_pkg(struct pkg_jobs *j, struct pkg *p,
	pkg_get(p, PKG_UNIQUEID, &uid, PKG_DIGEST, &digest);



	if (digest == NULL) {
- 
		if (pkg_jobs_digest_manifest(j, p) != EPKG_OK) {
+ 
		if (pkg_checksum_calculate(p, j->db) != EPKG_OK) {

			return (EPKG_FATAL);

		}

		pkg_get(p, PKG_DIGEST, &digest);
modified libpkg/pkg_repo_meta.c
@@ -35,7 +35,7 @@ static ucl_object_t *repo_meta_schema_v1 = NULL;
static void

pkg_repo_meta_set_default(struct pkg_repo_meta *meta)

{
- 
	meta->digest_format = strdup("sha256");
+ 
	meta->digest_format = PKG_HASH_TYPE_SHA256_BASE32;

	meta->packing_format = TXZ;



	/* Not use conflicts for now */
@@ -59,7 +59,6 @@ pkg_repo_meta_free(struct pkg_repo_meta *meta)
		free(meta->manifests);

		free(meta->digests);

		free(meta->fulldb);
- 
		free(meta->digest_format);

		free(meta->maintainer);

		free(meta->source);

		free(meta->source_identifier);
@@ -86,7 +85,7 @@ pkg_repo_meta_open_schema_v1()
			"maintainer = {type = string};\n"

			"source = {type = string};\n"

			"packing_format = {enum = [txz, tbz, tgz]};\n"
- 
			"digest_format = {enum = [sha256]};\n"
+ 
			"digest_format = {enum = [sha256_base32, sha256_hex]};\n"

			"digests = {type = string};\n"

			"manifests = {type = string};\n"

			"conflicts = {type = string};\n"
@@ -174,7 +173,6 @@ pkg_repo_meta_parse(ucl_object_t *top, struct pkg_repo_meta **target, int versio


	META_EXTRACT_STRING(maintainer);

	META_EXTRACT_STRING(source);
- 
	META_EXTRACT_STRING(digest_format);



	META_EXTRACT_STRING(conflicts);

	META_EXTRACT_STRING(digests);
@@ -198,6 +196,11 @@ pkg_repo_meta_parse(ucl_object_t *top, struct pkg_repo_meta **target, int versio
		meta->packing_format = packing_format_from_string(ucl_object_tostring(obj));

	}
+ 
	obj = ucl_object_find_key(top, "digest_format");
+ 
	if (obj != NULL && obj->type == UCL_STRING) {
+ 
		meta->digest_format = pkg_checksum_type_from_string(ucl_object_tostring(obj));
+ 
	}
+ 


	obj = ucl_object_find_key(top, "cert");

	while ((cur = ucl_iterate_object(obj, &iter, false)) != NULL) {

		cert = pkg_repo_meta_parse_cert(cur);
modified libpkg/pkg_repo_update.c
@@ -85,9 +85,10 @@ pkg_repo_register(struct pkg_repo *repo, sqlite3 *sqlite)
}



static int
- 
pkg_repo_add_from_manifest(char *buf, const char *origin, long offset,
- 
		const char *manifest_digest, sqlite3 *sqlite,
- 
		struct pkg_manifest_key **keys, struct pkg **p)
+ 
pkg_repo_add_from_manifest(char *buf, const char *origin, const char *digest,
+ 
		long offset, sqlite3 *sqlite,
+ 
		struct pkg_manifest_key **keys, struct pkg **p, bool is_legacy,
+ 
		struct pkg_repo *repo)

{

	int rc = EPKG_OK;

	struct pkg *pkg;
@@ -127,7 +128,16 @@ pkg_repo_add_from_manifest(char *buf, const char *origin, long offset,
		goto cleanup;

	}
- 
	rc = pkgdb_repo_add_package(pkg, NULL, sqlite, manifest_digest, true);
+ 
	pkg_set(pkg, PKG_REPONAME, repo->name);
+ 
	if (is_legacy) {
+ 
		pkg_set(pkg, PKG_OLD_DIGEST, digest);
+ 
		pkg_checksum_calculate(pkg, NULL);
+ 
	}
+ 
	else {
+ 
		pkg_set(pkg, PKG_DIGEST, digest);
+ 
	}
+
+ 
	rc = pkgdb_repo_add_package(pkg, NULL, sqlite, true);



cleanup:

	return (rc);
@@ -136,6 +146,7 @@ cleanup:
struct pkg_increment_task_item {

	char *origin;

	char *digest;
+ 
	char *olddigest;

	long offset;

	long length;

	UT_hash_handle hh;
@@ -219,7 +230,7 @@ pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mti
	size_t len = 0;

	int hash_it = 0;

	time_t now, last;
- 
	bool in_trans = false, new_repo = true;
+ 
	bool in_trans = false, new_repo = true, legacy_repo = false;



	if (access(name, R_OK) != -1)

		new_repo = false;
@@ -242,11 +253,6 @@ pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mti
		goto cleanup;

	}
- 
	while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
- 
		pkg_get(pkg, PKG_ORIGIN, &origin, PKG_DIGEST, &digest);
- 
		pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0);
- 
	}
- 


	if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL)

		pkg_emit_notice("repository %s has no meta file, using "

		    "default settings", repo->name);
@@ -281,6 +287,30 @@ pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mti
	fseek(fmanifest, 0, SEEK_END);

	len = ftell(fmanifest);
+ 
	/* Detect whether we have legacy repo */
+ 
	if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
+ 
		p = linebuf;
+ 
		origin = strsep(&p, ":");
+ 
		digest = strsep(&p, ":");
+ 
		if (digest == NULL) {
+ 
			pkg_emit_error("invalid digest file format");
+ 
			rc = EPKG_FATAL;
+ 
			goto cleanup;
+ 
		}
+ 
		if (!pkg_checksum_is_valid(digest, strlen(digest))) {
+ 
			legacy_repo = true;
+ 
			pkg_debug(1, "repository '%s' has a legacy digests format", repo->name);
+ 
		}
+ 
	}
+ 
	fseek(fdigests, 0, SEEK_SET);
+
+ 
	/* Load local repo data */
+ 
	while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
+ 
		pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST,
+ 
				&digest);
+ 
		pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0);
+ 
	}
+ 


	pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);

	/* load the while digests */

	while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
@@ -295,7 +325,6 @@ pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mti
		if (origin == NULL || digest == NULL ||

				offset == NULL) {

			pkg_emit_error("invalid digest file format");
- 
			assert(0);

			rc = EPKG_FATAL;

			goto cleanup;

		}
@@ -399,12 +428,13 @@ pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mti
		if (rc == EPKG_OK) {

			if (item->length != 0) {

				rc = pkg_repo_add_from_manifest(map + item->offset, item->origin,
- 
						item->length, item->digest,
- 
						sqlite, &keys, &pkg);
+ 
				    item->digest, item->length, sqlite, &keys, &pkg, legacy_repo,
+ 
				    repo);

			}

			else {

				rc = pkg_repo_add_from_manifest(map + item->offset, item->origin,
- 
						len - item->offset, item->digest, sqlite, &keys, &pkg);
+ 
				    item->digest, len - item->offset, sqlite, &keys, &pkg,
+ 
				    legacy_repo, repo);

			}

		}

		free(item->origin);
modified libpkg/pkgdb.c
@@ -1479,9 +1479,19 @@ pkgdb_it_next(struct pkgdb_it *it, struct pkg **pkg_p, unsigned flags)
	struct pkg	*pkg;

	int		 i;

	int		 ret;
+ 
	const char *digest;



	assert(it != NULL);
+ 
	/*
+ 
	 * XXX:
+ 
	 * Currently, we have a lot of issues related to pkg digests.
+ 
	 * So we want to ensure that we always have a valid package digest
+ 
	 * even if we work with pkg 1.2 repo. Therefore, we explicitly check
+ 
	 * manifest digests and set it to NULL if it is invalid.
+ 
	 *
+ 
	 */
+ 


	if (it->finished && (it->flags & PKGDB_IT_FLAG_ONCE))

		return (EPKG_END);
@@ -1497,6 +1507,10 @@ pkgdb_it_next(struct pkgdb_it *it, struct pkg **pkg_p, unsigned flags)


		populate_pkg(it->stmt, pkg);
+ 
		pkg_get(pkg, PKG_DIGEST, &digest);
+ 
		if (!pkg_checksum_is_valid(digest, strlen(digest)))
+ 
			pkg_set(pkg, PKG_DIGEST, NULL);
+ 


		for (i = 0; load_on_flag[i].load != NULL; i++) {

			if (flags & load_on_flag[i].flag) {

				if (it->db != NULL) {
modified libpkg/pkgdb_repo.c
@@ -59,7 +59,7 @@
/* The package repo schema minor revision.

   Minor schema changes don't prevent older pkgng

   versions accessing the repo. */
- 
#define REPO_SCHEMA_MINOR 9
+ 
#define REPO_SCHEMA_MINOR 10



/* REPO_SCHEMA_VERSION=2007 */

#define REPO_SCHEMA_VERSION (REPO_SCHEMA_MAJOR * 1000 + REPO_SCHEMA_MINOR)
@@ -90,10 +90,10 @@ static sql_prstmt sql_prepared_statements[PRSTMT_LAST] = {
		NULL,

		"INSERT INTO packages ("

		"origin, name, version, comment, desc, arch, maintainer, www, "
- 
		"prefix, pkgsize, flatsize, licenselogic, cksum, path, manifestdigest"
+ 
		"prefix, pkgsize, flatsize, licenselogic, cksum, path, manifestdigest, olddigest"

		")"
- 
		"VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
- 
		"TTTTTTTTTIIITTT",
+ 
		"VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
+ 
		"TTTTTTTTTIIITTTT",

	},

	[DEPS] = {

		NULL,
@@ -468,10 +468,11 @@ pkgdb_repo_cksum_exists(sqlite3 *sqlite, const char *cksum)


int

pkgdb_repo_add_package(struct pkg *pkg, const char *pkg_path,
- 
		sqlite3 *sqlite, const char *manifest_digest, bool forced)
+ 
		sqlite3 *sqlite, bool forced)

{

	const char *name, *version, *origin, *comment, *desc;

	const char *arch, *maintainer, *www, *prefix, *sum, *rpath;
+ 
	const char *olddigest, *manifestdigest;

	int64_t			 flatsize, pkgsize;

	int64_t			 licenselogic;

	int			 ret;
@@ -490,13 +491,14 @@ pkgdb_repo_add_package(struct pkg *pkg, const char *pkg_path,
			    PKG_LICENSE_LOGIC, &licenselogic, PKG_CKSUM, &sum,

			    PKG_PKGSIZE, &pkgsize, PKG_REPOPATH, &rpath,

			    PKG_LICENSES, &licenses, PKG_CATEGORIES, &categories,
- 
			    PKG_ANNOTATIONS, &annotations);
+ 
			    PKG_ANNOTATIONS, &annotations, PKG_OLD_DIGEST, &olddigest,
+ 
			    PKG_DIGEST, &manifestdigest);



try_again:

	if ((ret = run_prepared_statement(PKG, origin, name, version,

			comment, desc, arch, maintainer, www, prefix,

			pkgsize, flatsize, (int64_t)licenselogic, sum,
- 
			rpath, manifest_digest)) != SQLITE_DONE) {
+ 
			rpath, manifestdigest, olddigest)) != SQLITE_DONE) {

		if (ret == SQLITE_CONSTRAINT) {

			switch(maybe_delete_conflicting(origin,

					version, pkg_path, forced)) {
modified libpkg/private/pkg.h
@@ -274,13 +274,19 @@ struct pkg_repo_meta_key {
	UT_hash_handle hh;

};
+ 
typedef enum pkg_checksum_type_e {
+ 
	PKG_HASH_TYPE_SHA256_BASE32 = 0,
+ 
	PKG_HASH_TYPE_SHA256_HEX,
+ 
	PKG_HASH_TYPE_UNKNOWN
+ 
} pkg_checksum_type_t;
+ 


struct pkg_repo_meta {



	char *maintainer;

	char *source;



	pkg_formats packing_format;
- 
	char *digest_format; /* TODO: should be enumeration */
+ 
	pkg_checksum_type_t digest_format;



	char *digests;

	char *manifests;
@@ -393,6 +399,7 @@ static struct pkg_key {
	[PKG_LICENSES] = { "licenses", UCL_ARRAY },

	[PKG_CATEGORIES] = { "catagories", UCL_ARRAY },

	[PKG_UNIQUEID] = { "uniqueid", UCL_STRING },
+ 
	[PKG_OLD_DIGEST] = { "olddigest", UCL_STRING },

};



int pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url,
@@ -538,4 +545,17 @@ bool ucl_object_emit_sbuf(const ucl_object_t *obj, enum ucl_emitter emit_type,
bool ucl_object_emit_file(const ucl_object_t *obj, enum ucl_emitter emit_type,

    FILE *);
+ 
/* Hash is in format <version>:<typeid>:<hexhash> */
+ 
#define PKG_CHECKSUM_SHA256_LEN (SHA256_DIGEST_LENGTH * 2 + 10)
+ 
#define PKG_CHECKSUM_CUR_VERSION 1
+
+ 
int pkg_checksum_generate(struct pkg *pkg, char *dest, size_t destlen,
+ 
	pkg_checksum_type_t type);
+
+ 
bool pkg_checksum_is_valid(const char *cksum, size_t clen);
+ 
pkg_checksum_type_t pkg_checksum_get_type(const char *cksum, size_t clen);
+ 
pkg_checksum_type_t pkg_checksum_type_from_string(const char *name);
+ 
size_t pkg_checksum_type_size(pkg_checksum_type_t type);
+ 
int pkg_checksum_calculate(struct pkg *pkg, struct pkgdb *db);
+ 


#endif
modified libpkg/private/pkgdb.h
@@ -109,13 +109,12 @@ int pkgdb_repo_cksum_exists(sqlite3 *sqlite, const char *cksum);
 * @param pkg package structure

 * @param pkg_path path triggered package addition

 * @param sqlite sqlite pointer
- 
 * @param manifest_digest sha256 checksum of the manifest of the package

 * @param forced force adding of package even if it is outdated

 * @return EPKG_OK if package added, EPKG_END if package already exists and is newer than

 * inserted one, EPKG_FATAL if error occurred

 */

int pkgdb_repo_add_package(struct pkg *pkg, const char *pkg_path,
- 
		sqlite3 *sqlite, const char *manifest_digest, bool forced);
+ 
		sqlite3 *sqlite, bool forced);



/**

 * Remove specified pkg from repo
modified libpkg/private/repodb.h
@@ -60,7 +60,8 @@ static const char initsql[] = ""
	    /* relative path to the package in the repository */

	    "path TEXT NOT NULL,"

	    "pkg_format_version INTEGER,"
- 
	    "manifestdigest TEXT NULL"
+ 
	    "manifestdigest TEXT NULL,"
+ 
	    "olddigest TEXT NULL"

	");"

	"CREATE TABLE deps ("

	    "origin TEXT,"
@@ -354,6 +355,13 @@ static const struct repo_changes repo_upgrades[] = {
	 "CREATE INDEX IF NOT EXISTS %Q.packages_version ON packages(name, version);"

	 "CREATE UNIQUE INDEX IF NOT EXISTS %Q.packages_digest ON packages(manifestdigest);"

	},
+ 
	{2009,
+ 
	 2010,
+ 
	 "Add legacy digest field",
+
+ 
	 "ALTER TABLE %Q.packages ADD COLUMN olddigest TEXT NULL;"
+ 
	 "UPDATE %Q.packages SET olddigest=manifestdigest WHERE olddigest=NULL;"
+ 
	},

	/* Mark the end of the array */

	{ -1, -1, NULL, NULL, }
@@ -362,6 +370,12 @@ static const struct repo_changes repo_upgrades[] = {
/* How to downgrade a newer repo to match what the current system

   expects */

static const struct repo_changes repo_downgrades[] = {
+ 
	{2010,
+ 
	 2009,
+ 
	 "Drop olddigest field",
+
+ 
	 "ALTER TABLE %Q.packages REMOVE COLUMN olddigest;"
+ 
	},

	{2009,

	 2008,

	 "Drop indicies",