17804c4 Merge branch 'gsoc2014' — HardenedBSD-pkg

HardenedBSD-pkg

rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9

HardenedBSD Package Manager

Merge branch 'gsoc2014'

Vsevolod Stakhov committed 11 years ago

commit 17804c4335b919baf9a3f616b4823f2995973b12
parent 119d592

36 files changed +4306 -4148

modified configure.ac

@@ -229,13 +229,43 @@ AC_SUBST([TESTS])

 AC_SUBST([LDNS_LIBS])
 AC_SUBST([LDNS_CFLAGS])
 dnl We now support merely binary repos
 m4_define([repos], [binary])
 m4_define([repos_ldadd], [])
 m4_define([repos_makefiles], [])
 m4_define([repos_ldadd_static], [])
 dnl Add ldadd for repo
 m4_define([repos_add_ldadd], [
 	m4_append([$1], [m4_join([], [$(top_builddir)/libpkg/repo/],
 		[$3], [/librepo_],
 		[$3], [$2.la ])])
 ])
 dnl Create makefile definition for repo
 m4_define([repos_add_makefile], [
 	m4_append([$1], [m4_join([], [\], m4_newline(), [libpkg/repo/],
 		[$2], [/Makefile ])])
 ])
 dnl Apply macros to repos list
 m4_map([m4_curry([repos_add_ldadd], [repos_ldadd], [])], [repos])
 m4_map([m4_curry([repos_add_ldadd], [repos_ldadd_static], [_static])], [repos])
 m4_map([m4_curry([repos_add_makefile], [repos_makefiles])], [repos])
 REPOS="m4_expand([repos])"
 REPOS_LDADD='m4_normalize(m4_expand([repos_ldadd]))'
 REPOS_LDADD_STATIC='m4_normalize(m4_expand([repos_ldadd_static]))'
 AC_SUBST([REPOS])
 AC_SUBST([REPOS_LDADD])
 AC_SUBST([REPOS_LDADD_STATIC])
 AC_CONFIG_FILES(Makefile \
 		src/Makefile \
 		external/Makefile \
 		scripts/Makefile \
 		docs/Makefile \
 		tests/Makefile \
 		libpkg/Makefile
 		libpkg/Makefile \
 		libpkg/repo/Makefile \
 		libpkg/pkg.h \
 		libpkg/pkg.pc
 		scripts/periodic/400.status-pkg

@@ -246,6 +276,32 @@ AC_CONFIG_FILES(Makefile \

 		scripts/sbin/pkg2ng
 		scripts/completion/_pkg.bash
 		scripts/completion/_pkg.zsh)
 AC_CONFIG_FILES(m4_expand([repos_makefiles]))
 AC_CONFIG_HEADERS(pkg_config.h)
 AC_CONFIG_COMMANDS([pkg_repos.h], [[
 	NMODULES=pkg_repos.h.new
 	echo "#ifndef PKG_REPOS_H" > $NMODULES
 	echo "#define PKG_REPOS_H" >> $NMODULES
 	for rt in $REPOS ; do
 		echo "extern struct pkg_repo_ops pkg_repo_${rt}_ops;" >> $NMODULES;
 	done
 	echo "struct pkg_repo_ops* repos_ops[] = {" >> $NMODULES
 	for rt in $REPOS ; do
 		echo "&pkg_repo_${rt}_ops," >> $NMODULES
 	done
 	echo "NULL" >> $NMODULES
 	echo "};" >> $NMODULES
 	echo "#endif /* PKG_REPOS_H */" >> $NMODULES
 	if [ -f pkg_repos.h ] ; then
 		diff $NMODULES pkg_repos.h > /dev/null 2>&1
 		if [ $? -ne 0 ] ; then
 			cp $NMODULES pkg_repos.h
 		fi
 	else
 		cp $NMODULES pkg_repos.h
 	fi
 	rm $NMODULES
 ]], [REPOS=$REPOS])
 AC_OUTPUT

modified libpkg/Makefile.am

@@ -41,7 +41,8 @@ libpkg_la_SOURCES= pkg.c \

 			pkg_status.c \
 			pkg_version.c \
 			pkgdb.c \
 			pkgdb_repo.c \
 			pkgdb_iterator.c \
 			pkgdb_query.c \
 			rcscripts.c \
 			rsa.c \
 			ssh.c \

@@ -54,6 +55,7 @@ libpkg_la_LIBADD= $(top_builddir)/external/libucl.la \

 			$(top_builddir)/external/libsqlite.la \
 			$(top_builddir)/external/libyaml.la \
 			$(top_builddir)/external/libexpat.la \
 			@REPOS_LDADD@ \
 			@LIBELF_LIB@ \
 			@LIBSBUF_LIB@ \
 			@LIBEXECINFO_LIB@ \

@@ -73,7 +75,8 @@ libpkg_static_la_CFLAGS= $(pkg_common_cflags) -static

 libpkg_static_la_LIBADD=	$(top_builddir)/external/libucl_static.la \
 			$(top_builddir)/external/libsqlite_static.la \
 			$(top_builddir)/external/libexpat_static.la \
 			$(top_builddir)/external/libyaml_static.la
 			$(top_builddir)/external/libyaml_static.la \
 			@REPOS_LDADD_STATIC@
 libpkg_static_la_LDFLAGS=	-all-static

@@ -87,3 +90,5 @@ noinst_HEADERS= private/db_upgrades.h \

 			private/repodb.h \
 			private/thd_repo.h \
 			private/utils.h
 SUBDIRS = repo .
 \ No newline at end of file

modified libpkg/libpkg.ver

@@ -227,6 +227,10 @@ global:

 	pkgdb_stats;
 	pkgdb_upgrade_lock;
 	ports_parse_plist;
 	pkgdb_repo_search;
 	pkgdb_repo_query;
 	pkgdb_query_shlib_provide;
 	pkgdb_query_shlib_require;
 local:
 	*;
 };

modified libpkg/pkg.c

@@ -277,7 +277,6 @@ static int

 pkg_vset(struct pkg *pkg, va_list ap)
 {
 	int attr;
 	struct pkg_repo *r;
 	char *buf = NULL;
 	const char *data;
 	const char *str;

@@ -299,13 +298,6 @@ pkg_vset(struct pkg *pkg, va_list ap)

 				data = buf;
 			}
 			if (attr == PKG_REPOURL) {
 				r = pkg_repo_find_ident(str);
 				if (r == NULL)
 					break;
 				data = pkg_repo_url(r);
 			}
 			if (data == NULL)
 				ucl_object_delete_key(pkg->fields, pkg_keys[attr].name);
 			else

modified libpkg/pkg.h.in

@@ -309,10 +309,6 @@ typedef enum {

 } pkg_list;
 typedef enum {
 	REPO_BINARY_PKGS,
 } repo_t;
 typedef enum {
 	SRV,
 	HTTP,
 	NOMIRROR,

@@ -1051,9 +1047,9 @@ bool pkgdb_case_sensitive(void);

  */
 struct pkgdb_it * pkgdb_query(struct pkgdb *db, const char *pattern,
     match_t type);
 struct pkgdb_it * pkgdb_rquery(struct pkgdb *db, const char *pattern,
 struct pkgdb_it * pkgdb_repo_query(struct pkgdb *db, const char *pattern,
     match_t type, const char *reponame);
 struct pkgdb_it * pkgdb_search(struct pkgdb *db, const char *pattern,
 struct pkgdb_it * pkgdb_repo_search(struct pkgdb *db, const char *pattern,
     match_t type, pkgdb_field field, pkgdb_field sort, const char *reponame);
 /**

@@ -1061,8 +1057,8 @@ struct pkgdb_it * pkgdb_search(struct pkgdb *db, const char *pattern,

  */
 struct pkgdb_it * pkgdb_query_which(struct pkgdb *db, const char *path, bool glob);
 struct pkgdb_it * pkgdb_query_shlib_required(struct pkgdb *db, const char *shlib);
 struct pkgdb_it * pkgdb_query_shlib_provided(struct pkgdb *db, const char *shlib);
 struct pkgdb_it * pkgdb_query_shlib_require(struct pkgdb *db, const char *shlib);
 struct pkgdb_it * pkgdb_query_shlib_provide(struct pkgdb *db, const char *shlib);
 struct pkgdb_it * pkgdb_rquery_provide(struct pkgdb *db,
     const char *provide, const char *repo);

@@ -1575,16 +1571,13 @@ int pkg_repos_total_count(void);

 int pkg_repos_activated_count(void);
 int pkg_repos(struct pkg_repo **);
 const char *pkg_repo_url(struct pkg_repo *r);
 const char *pkg_repo_ident(struct pkg_repo *r);
 const char *pkg_repo_name(struct pkg_repo *r);
 const char * pkg_repo_ident_from_name(const char *repo_name);
 const char *pkg_repo_key(struct pkg_repo *r);
 const char *pkg_repo_fingerprints(struct pkg_repo *r);
 signature_t pkg_repo_signature_type(struct pkg_repo *r);
 bool pkg_repo_enabled(struct pkg_repo *r);
 mirror_t pkg_repo_mirror_type(struct pkg_repo *r);
 struct pkg_repo *pkg_repo_find_ident(const char *ident);
 struct pkg_repo *pkg_repo_find_name(const char *name);
 struct pkg_repo *pkg_repo_find(const char *name);
 /**
  * pkg_printf() and friends.  These parallel the similarly named libc

modified libpkg/pkg_checksum.c

@@ -373,7 +373,7 @@ pkg_checksum_calculate(struct pkg *pkg, struct pkgdb *db)

 	pkg_get(pkg, PKG_REPONAME, &reponame);
 	if (reponame != NULL) {
 		repo = pkg_repo_find_name(reponame);
 		repo = pkg_repo_find(reponame);
 		if (repo != NULL)
 			type = repo->meta->digest_format;

modified libpkg/pkg_config.c

@@ -47,8 +47,8 @@

 #include "pkg.h"
 #include "private/pkg.h"
 #include "private/event.h"
 #include "pkg_repos.h"
 #define REPO_NAME_PREFIX "repo-"
 #ifndef PORTSDIR
 #define PORTSDIR "/usr/ports"
 #endif

@@ -311,7 +311,8 @@ static struct config_entry c[] = {

 static bool parsed = false;
 static size_t c_size = NELEM(c);
 static struct pkg_repo	*pkg_repo_new(const char *name, const char *url);
 static struct pkg_repo* pkg_repo_new(const char *name,
 	const char *url, const char *type);
 static void
 connect_evpipe(const char *evpipe) {

@@ -397,15 +398,22 @@ disable_plugins_if_static(void)

 static void
 add_repo(const ucl_object_t *obj, struct pkg_repo *r, const char *rname)
 {
 	const ucl_object_t *cur;
 	ucl_object_t *tmp = NULL;
 	const ucl_object_t *cur, *enabled;
 	ucl_object_iter_t it = NULL;
 	bool enable = true;
 	const char *url = NULL, *pubkey = NULL, *mirror_type = NULL;
 	const char *signature_type = NULL, *fingerprints = NULL;
 	const char *key;
 	const char *type = NULL;
 	pkg_debug(1, "PkgConfig: parsing repository object %s", rname);
 	enabled = ucl_object_find_key(obj, "enabled");
 	if (enabled != NULL && !ucl_object_toboolean(enabled)) {
 		pkg_debug(1, "PkgConfig: skipping disabled repo %s", rname);
 		return;
 	}
 	while ((cur = ucl_iterate_object(obj, &it, true))) {
 		key = ucl_object_key(cur);
 		if (key == NULL)

@@ -427,24 +435,6 @@ add_repo(const ucl_object_t *obj, struct pkg_repo *r, const char *rname)

 				return;
 			}
 			pubkey = ucl_object_tostring(cur);
 		} else if (strcasecmp(key, "enabled") == 0) {
 			if (cur->type == UCL_STRING)
 				tmp = ucl_object_fromstring_common(ucl_object_tostring(cur),
 				    strlen(ucl_object_tostring(cur)), UCL_STRING_PARSE_BOOLEAN);
 			if (cur->type != UCL_BOOLEAN && (tmp != NULL && tmp->type != UCL_BOOLEAN)) {
 				pkg_emit_error("Expecting a boolean for the "
 				    "'%s' key of the '%s' repo",
 				    key, rname);
 				if (tmp != NULL)
 					ucl_object_unref(tmp);
 				return;
 			}
 			if (tmp != NULL)
 				pkg_emit_error("Warning: expecting a boolean for the '%s' key of the '%s' repo, "
 				    " the value has been correctly converted, please consider fixing", key, rname);
 			enable = ucl_object_toboolean(tmp != NULL ? tmp : cur);
 			if (tmp != NULL)
 				ucl_object_unref(tmp);
 		} else if (strcasecmp(key, "mirror_type") == 0) {
 			if (cur->type != UCL_STRING) {
 				pkg_emit_error("Expecting a string for the "

@@ -469,6 +459,14 @@ add_repo(const ucl_object_t *obj, struct pkg_repo *r, const char *rname)

 				return;
 			}
 			fingerprints = ucl_object_tostring(cur);
 		} else if (strcasecmp(key, "type") == 0) {
 			if (cur->type != UCL_STRING) {
 				pkg_emit_error("Expecting a string for the "
 					"'%s' key of the '%s' repo",
 					key, rname);
 				return;
 			}
 			type = ucl_object_tostring(cur);
 		}
 	}

@@ -478,7 +476,7 @@ add_repo(const ucl_object_t *obj, struct pkg_repo *r, const char *rname)

 	}
 	if (r == NULL)
 		r = pkg_repo_new(rname, url);
 		r = pkg_repo_new(rname, url, type);
 	if (signature_type != NULL) {
 		if (strcasecmp(signature_type, "pubkey") == 0)

@@ -523,7 +521,7 @@ walk_repo_obj(const ucl_object_t *obj, const char *file)

 	while ((cur = ucl_iterate_object(obj, &it, true))) {
 		key = ucl_object_key(cur);
 		pkg_debug(1, "PkgConfig: parsing key '%s'", key);
 		r = pkg_repo_find_ident(key);
 		r = pkg_repo_find(key);
 		if (r != NULL)
 			pkg_debug(1, "PkgConfig: overwriting repository %s", key);
 		if (cur->type == UCL_OBJECT)

@@ -917,20 +915,42 @@ pkg_init(const char *path, const char *reposdir)

 	return (EPKG_OK);
 }
 static struct pkg_repo_ops*
 pkg_repo_find_type(const char *type)
 {
 	struct pkg_repo_ops *found = NULL, **cur;
 	/* Default repo type */
 	if (type == NULL)
 		return (pkg_repo_find_type("binary"));
 	cur = &repos_ops[0];
 	while (*cur != NULL) {
 		if (strcasecmp(type, (*cur)->type) == 0) {
 			found = *cur;
 		}
 		cur ++;
 	}
 	if (found == NULL)
 		return (pkg_repo_find_type("binary"));
 	return (found);
 }
 static struct pkg_repo *
 pkg_repo_new(const char *name, const char *url)
 pkg_repo_new(const char *name, const char *url, const char *type)
 {
 	struct pkg_repo *r;
 	r = calloc(1, sizeof(struct pkg_repo));
 	r->type = REPO_BINARY_PKGS;
 	r->update = pkg_repo_update_binary_pkgs;
 	r->ops = pkg_repo_find_type(type);
 	r->url = strdup(url);
 	r->signature_type = SIG_NONE;
 	r->mirror_type = NOMIRROR;
 	r->enable = true;
 	r->meta = pkg_repo_meta_default();
 	asprintf(&r->name, REPO_NAME_PREFIX"%s", name);
 	r->name = strdup(name);
 	HASH_ADD_KEYPTR(hh, repos, r->name, strlen(r->name), r);
 	return (r);

@@ -1000,23 +1020,6 @@ pkg_repo_url(struct pkg_repo *r)

 	return (r->url);
 }
 /* The repo identifier from pkg.conf(5): without the 'repo-' prefix */
 const char *
 pkg_repo_ident(struct pkg_repo *r)
 {
 	return (r->name + strlen(REPO_NAME_PREFIX));
 }
 /* Ditto: The repo identifier from pkg.conf(5): without the 'repo-' prefix */
 const char *
 pkg_repo_ident_from_name(const char *repo_name)
 {
 	if (repo_name == NULL)
 		return "local";
 	return (repo_name + strlen(REPO_NAME_PREFIX));
 }
 /* The basename of the sqlite DB file and the database name */
 const char *
 pkg_repo_name(struct pkg_repo *r)

@@ -1054,27 +1057,10 @@ pkg_repo_mirror_type(struct pkg_repo *r)

 	return (r->mirror_type);
 }
 /* Locate the repo by the identifying tag from pkg.conf(5) */
 struct pkg_repo *
 pkg_repo_find_ident(const char *repoident)
 {
 	struct pkg_repo *r;
 	char *name;
 	asprintf(&name, REPO_NAME_PREFIX"%s", repoident);
 	if (name == NULL)
 		return (NULL);	/* Out of memory */
 	r = pkg_repo_find_name(name);
 	free(name);
 	return (r);
 }
 /* Locate the repo by the file basename / database name */
 struct pkg_repo *
 pkg_repo_find_name(const char *reponame)
 pkg_repo_find(const char *reponame)
 {
 	struct pkg_repo *r;

modified libpkg/pkg_delete.c

@@ -50,6 +50,7 @@ pkg_delete(struct pkg *pkg, struct pkgdb *db, unsigned flags)

 	assert(pkg != NULL);
 	assert(db != NULL);
 #if 0
 	/*
 	 * Do not trust the existing entries as it may have changed if we
 	 * delete packages in batch.

@@ -70,6 +71,7 @@ pkg_delete(struct pkg *pkg, struct pkgdb *db, unsigned flags)

 		return (ret);
 	if ((ret = pkgdb_load_annotations(db, pkg)) != EPKG_OK)
 		return (ret);
 #endif
 	if ((flags & PKG_DELETE_UPGRADE) == 0)
 		pkg_emit_deinstall_begin(pkg);

modified libpkg/pkg_jobs.c

@@ -87,7 +87,7 @@ pkg_jobs_set_flags(struct pkg_jobs *j, pkg_flags flags)

 int
 pkg_jobs_set_repository(struct pkg_jobs *j, const char *ident)
 {
 	if ((pkg_repo_find_ident(ident)) == NULL) {
 	if ((pkg_repo_find(ident)) == NULL) {
 		pkg_emit_error("Unknown repository: %s", ident);
 		return (EPKG_FATAL);
 	}

@@ -728,7 +728,7 @@ pkg_jobs_add_universe(struct pkg_jobs *j, struct pkg *pkg,

 				continue;
 			/* Not found, search in the repos */
 			it = pkgdb_find_shlib_provide(j->db, pkg_shlib_name(shlib), j->reponame);
 			it = pkgdb_repo_shlib_provide(j->db, pkg_shlib_name(shlib), j->reponame);
 			if (it != NULL) {
 				rpkg = NULL;
 				prhead = NULL;

@@ -998,7 +998,7 @@ pkg_jobs_try_remote_candidate(struct pkg_jobs *j, const char *pattern,

 	const char *fuid;
 	struct pkg_job_universe_item *unit;
 	if ((it = pkgdb_rquery(j->db, pattern, m, j->reponame)) == NULL)
 	if ((it = pkgdb_repo_query(j->db, pattern, m, j->reponame)) == NULL)
 		return (EPKG_FATAL);
 	qmsg = sbuf_new_auto();

@@ -1103,7 +1103,7 @@ pkg_jobs_find_remote_pkg(struct pkg_jobs *j, const char *pattern,

 	if (j->type == PKG_JOBS_UPGRADE && (j->flags & PKG_FLAG_FORCE) == PKG_FLAG_FORCE)
 		force = true;
 	if ((it = pkgdb_rquery(j->db, pattern, m, j->reponame)) == NULL)
 	if ((it = pkgdb_repo_query(j->db, pattern, m, j->reponame)) == NULL)
 		rc = EPKG_FATAL;
 	while (it != NULL && pkgdb_it_next(it, &p, flags) == EPKG_OK) {

@@ -1264,7 +1264,7 @@ get_remote_pkg(struct pkg_jobs *j, const char *uid, unsigned flag)

 				PKG_LOAD_ANNOTATIONS|PKG_LOAD_CONFLICTS;
 	}
 	if ((it = pkgdb_rquery(j->db, uid, MATCH_EXACT, j->reponame)) == NULL)
 	if ((it = pkgdb_repo_query(j->db, uid, MATCH_EXACT, j->reponame)) == NULL)
 		return (NULL);
 	if (pkgdb_it_next(it, &pkg, flag) != EPKG_OK)

@@ -1898,7 +1898,7 @@ pkg_jobs_check_remote_candidate(struct pkg_jobs *j, struct pkg *pkg)

 	sqlite3_snprintf(sizeof(sqlbuf), sqlbuf, " WHERE manifestdigest=%Q", digest);
 	it = pkgdb_rquery(j->db, sqlbuf, MATCH_CONDITION, j->reponame);
 	it = pkgdb_repo_query(j->db, sqlbuf, MATCH_CONDITION, j->reponame);
 	if (it != NULL) {
 		/*
 		 * If we have the same package in a remote repo, it is not an

modified libpkg/pkg_printf.c

@@ -1313,7 +1313,7 @@ format_repo_ident(struct sbuf *sbuf, const void *data, struct percent_esc *p)

 	const char		*reponame;
 	pkg_get(pkg, PKG_REPONAME, &reponame);
 	return (string_val(sbuf, pkg_repo_ident_from_name(reponame), p));
 	return (string_val(sbuf, reponame, p));
 }
 /*

modified libpkg/pkg_repo.c

@@ -153,7 +153,7 @@ pkg_repo_fetch_package(struct pkg *pkg)

 	 * For a single attached database the repository URL should be
 	 * defined by PACKAGESITE.
 	 */
 	repo = pkg_repo_find_name(reponame);
 	repo = pkg_repo_find(reponame);
 	packagesite = pkg_repo_url(repo);
 	if (packagesite == NULL || packagesite[0] == '\0') {

modified libpkg/pkg_repo_update.c

@@ -48,558 +48,8 @@

 #include "private/repodb.h"
 #include "private/pkg.h"
 static int
 pkg_repo_register(struct pkg_repo *repo, sqlite3 *sqlite)
 {
 	sqlite3_stmt *stmt;
 	const char sql[] = ""
 	    "INSERT OR REPLACE INTO repodata (key, value) "
 	    "VALUES (\"packagesite\", ?1);";
 	/* register the packagesite */
 	if (sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repodata ("
 			"   key TEXT UNIQUE NOT NULL,"
 			"   value TEXT NOT NULL"
 			");") != EPKG_OK) {
 		pkg_emit_error("Unable to register the packagesite in the "
 				"database");
 		return (EPKG_FATAL);
 	}
 	if (sqlite3_prepare_v2(sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	sqlite3_bind_text(stmt, 1, pkg_repo_url(repo), -1, SQLITE_STATIC);
 	if (sqlite3_step(stmt) != SQLITE_DONE) {
 		ERROR_SQLITE(sqlite, sql);
 		sqlite3_finalize(stmt);
 		return (EPKG_FATAL);
 	}
 	sqlite3_finalize(stmt);
 	return (EPKG_OK);
 }
 static int
 pkg_repo_add_from_manifest(char *buf, const char *origin, const char *digest,
 		long offset, sqlite3 *sqlite,
 		struct pkg_manifest_key **keys, struct pkg **p, bool is_legacy,
 		struct pkg_repo *repo)
 {
 	int rc = EPKG_OK;
 	struct pkg *pkg;
 	const char *local_origin, *pkg_arch;
 	if (*p == NULL) {
 		rc = pkg_new(p, PKG_REMOTE);
 		if (rc != EPKG_OK)
 			return (EPKG_FATAL);
 	} else {
 		pkg_reset(*p, PKG_REMOTE);
 	}
 	pkg = *p;
 	pkg_manifest_keys_new(keys);
 	rc = pkg_parse_manifest(pkg, buf, offset, *keys);
 	if (rc != EPKG_OK) {
 		goto cleanup;
 	}
 	rc = pkg_is_valid(pkg);
 	if (rc != EPKG_OK) {
 		goto cleanup;
 	}
 	/* Ensure that we have a proper origin and arch*/
 	pkg_get(pkg, PKG_ORIGIN, &local_origin, PKG_ARCH, &pkg_arch);
 	if (local_origin == NULL || strcmp(local_origin, origin) != 0) {
 		pkg_emit_error("manifest contains origin %s while we wanted to add origin %s",
 				local_origin ? local_origin : "NULL", origin);
 		rc = EPKG_FATAL;
 		goto cleanup;
 	}
 	if (pkg_arch == NULL || !is_valid_abi(pkg_arch, true)) {
 		rc = EPKG_FATAL;
 		goto cleanup;
 	}
 	pkg_set(pkg, PKG_REPONAME, repo->name);
 	if (is_legacy) {
 		pkg_set(pkg, PKG_OLD_DIGEST, digest);
 		pkg_checksum_calculate(pkg, NULL);
 	}
 	else {
 		pkg_set(pkg, PKG_DIGEST, digest);
 	}
 	rc = pkgdb_repo_add_package(pkg, NULL, sqlite, true);
 cleanup:
 	return (rc);
 }
 struct pkg_increment_task_item {
 	char *origin;
 	char *digest;
 	char *olddigest;
 	long offset;
 	long length;
 	UT_hash_handle hh;
 };
 static void
 pkg_repo_update_increment_item_new(struct pkg_increment_task_item **head, const char *origin,
 		const char *digest, long offset, long length)
 {
 	struct pkg_increment_task_item *item;
 	item = calloc(1, sizeof(struct pkg_increment_task_item));
 	item->origin = strdup(origin);
 	if (digest == NULL)
 		digest = "";
 	item->digest = strdup(digest);
 	item->offset = offset;
 	item->length = length;
 	HASH_ADD_KEYPTR(hh, *head, item->origin, strlen(item->origin), item);
 }
 static void __unused
 pkg_repo_parse_conflicts_file(FILE *f, sqlite3 *sqlite)
 {
 	size_t linecap = 0;
 	ssize_t linelen;
 	char *linebuf = NULL, *p, **deps;
 	const char *origin, *pdep;
 	int ndep, i;
 	const char conflicts_clean_sql[] = ""
 			"DELETE FROM pkg_conflicts;";
 	pkg_debug(4, "pkg_parse_conflicts_file: running '%s'", conflicts_clean_sql);
 	(void)sql_exec(sqlite, conflicts_clean_sql);
 	while ((linelen = getline(&linebuf, &linecap, f)) > 0) {
 		p = linebuf;
 		origin = strsep(&p, ":");
 		/* Check dependencies number */
 		pdep = p;
 		ndep = 1;
 		while (*pdep != '\0') {
 			if (*pdep == ',')
 				ndep ++;
 			pdep ++;
 		}
 		deps = malloc(sizeof(char *) * ndep);
 		for (i = 0; i < ndep; i ++) {
 			deps[i] = strsep(&p, ",\n");
 		}
 		pkgdb_repo_register_conflicts(origin, deps, ndep, sqlite);
 		free(deps);
 	}
 	if (linebuf != NULL)
 		free(linebuf);
 }
 static int
 pkg_repo_update_incremental(const char *name, struct pkg_repo *repo, time_t *mtime)
 {
 	FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/;
 	sqlite3 *sqlite = NULL;
 	struct pkg *pkg = NULL;
 	int rc = EPKG_FATAL;
 	const char *origin, *digest, *offset, *length;
 	struct pkgdb_it *it = NULL;
 	char *linebuf = NULL, *p;
 	int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0;
 	long num_offset, num_length;
 	time_t local_t = *mtime;
 	time_t digest_t;
 	time_t packagesite_t;
 	struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
 			*item, *tmp_item;
 	struct pkg_manifest_key *keys = NULL;
 	size_t linecap = 0;
 	ssize_t linelen;
 	char *map = MAP_FAILED;
 	size_t len = 0;
 	int hash_it = 0;
 	bool in_trans = false, new_repo = true, legacy_repo = false, reuse_repo;
 	if (access(name, R_OK) != -1)
 		new_repo = false;
 	pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
 	if ((rc = pkgdb_repo_open(name, false, &sqlite, &reuse_repo)) != EPKG_OK) {
 		return (EPKG_FATAL);
 	}
 	if (!reuse_repo) {
 		pkg_debug(1, "Pkgrepo, need to re-create database '%s'", name);
 		local_t = 0;
 		*mtime = 0;
 	}
 	if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK) {
 		goto cleanup;
 	}
 	if ((rc = pkg_repo_register(repo, sqlite)) != EPKG_OK)
 		goto cleanup;
 	it = pkgdb_repo_origins(sqlite);
 	if (it == NULL) {
 		rc = EPKG_FATAL;
 		goto cleanup;
 	}
 	if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL)
 		pkg_emit_notice("repository %s has no meta file, using "
 		    "default settings", repo->name);
 	fdigests = pkg_repo_fetch_remote_extract_tmp(repo,
 			repo->meta->digests, &local_t, &rc);
 	if (fdigests == NULL) {
 		if (rc == EPKG_FATAL)
 			/* Destroy repo completely */
 			if (new_repo)
 				unlink(name);
 		goto cleanup;
 	}
 	digest_t = local_t;
 	local_t = *mtime;
 	fmanifest = pkg_repo_fetch_remote_extract_tmp(repo,
 			repo->meta->manifests, &local_t, &rc);
 	if (fmanifest == NULL) {
 		if (rc == EPKG_FATAL)
 			/* Destroy repo completely */
 			if (new_repo)
 				unlink(name);
 		goto cleanup;
 	}
 	packagesite_t = digest_t;
 	*mtime = packagesite_t > digest_t ? packagesite_t : digest_t;
 	/*fconflicts = repo_fetch_remote_extract_tmp(repo,
 			repo_conflicts_archive, "txz", &local_t,
 			&rc, repo_conflicts_file);*/
 	fseek(fmanifest, 0, SEEK_END);
 	len = ftell(fmanifest);
 	/* Detect whether we have legacy repo */
 	if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
 		p = linebuf;
 		origin = strsep(&p, ":");
 		digest = strsep(&p, ":");
 		if (digest == NULL) {
 			pkg_emit_error("invalid digest file format");
 			rc = EPKG_FATAL;
 			goto cleanup;
 		}
 		if (!pkg_checksum_is_valid(digest, strlen(digest))) {
 			legacy_repo = true;
 			pkg_debug(1, "repository '%s' has a legacy digests format", repo->name);
 		}
 	}
 	fseek(fdigests, 0, SEEK_SET);
 	/* Load local repo data */
 	while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
 		pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST,
 				&digest);
 		pkg_repo_update_increment_item_new(&ldel, origin, digest, 4, 0);
 	}
 	pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
 	/* load the while digests */
 	while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
 		p = linebuf;
 		origin = strsep(&p, ":");
 		digest = strsep(&p, ":");
 		offset = strsep(&p, ":");
 		/* files offset */
 		strsep(&p, ":");
 		length = strsep(&p, ":");
 		if (origin == NULL || digest == NULL ||
 				offset == NULL) {
 			pkg_emit_error("invalid digest file format");
 			rc = EPKG_FATAL;
 			goto cleanup;
 		}
 		errno = 0;
 		num_offset = (long)strtoul(offset, NULL, 10);
 		if (errno != 0) {
 			pkg_emit_errno("strtoul", "digest format error");
 			rc = EPKG_FATAL;
 			goto cleanup;
 		}
 		if (length != NULL) {
 			errno = 0;
 			num_length = (long)strtoul(length, NULL, 10);
 			if (errno != 0) {
 				pkg_emit_errno("strtoul", "digest format error");
 				rc = EPKG_FATAL;
 				goto cleanup;
 			}
 		}
 		else {
 			num_length = 0;
 		}
 		processed++;
 		HASH_FIND_STR(ldel, origin, item);
 		if (item == NULL) {
 			added++;
 			pkg_repo_update_increment_item_new(&ladd, origin, digest, num_offset,
 					num_length);
 		} else {
 			if (strcmp(digest, item->digest) == 0) {
 				free(item->origin);
 				free(item->digest);
 				HASH_DEL(ldel, item);
 				free(item);
 				item = NULL;
 			} else {
 				free(item->origin);
 				free(item->digest);
 				HASH_DEL(ldel, item);
 				free(item);
 				item = NULL;
 				pkg_repo_update_increment_item_new(&ladd, origin, digest,
 						num_offset, num_length);
 				updated++;
 			}
 		}
 	}
 	rc = EPKG_OK;
 	pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
 	sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repo_update (x INTEGER);");
 	in_trans = true;
 	rc = pkgdb_transaction_begin(sqlite, "REPO");
 	if (rc != EPKG_OK)
 		goto cleanup;
 	removed = HASH_COUNT(ldel);
 	hash_it = 0;
 	pkg_emit_progress_start("Removing expired entries");
 	HASH_ITER(hh, ldel, item, tmp_item) {
 		pkg_emit_progress_tick(++hash_it, removed);
 		if (rc == EPKG_OK) {
 			rc = pkgdb_repo_remove_package(item->origin);
 		}
 		free(item->origin);
 		free(item->digest);
 		HASH_DEL(ldel, item);
 		free(item);
 	}
 	pkg_debug(1, "Pkgrepo, pushing new entries for '%s'", name);
 	pkg = NULL;
 	if (len > 0 && len < SSIZE_MAX) {
 		map = mmap(NULL, len, PROT_READ, MAP_SHARED, fileno(fmanifest), 0);
 		fclose(fmanifest);
 	} else {
 		if (len == 0)
 			pkg_emit_error("Empty catalog");
 		else
 			pkg_emit_error("Catalog too large");
 		goto cleanup;
 	}
 	hash_it = 0;
 	pushed = HASH_COUNT(ladd);
 	pkg_emit_progress_start("Adding new entries");
 	HASH_ITER(hh, ladd, item, tmp_item) {
 		pkg_emit_progress_tick(++hash_it, pushed);
 		if (rc == EPKG_OK) {
 			if (item->length != 0) {
 				rc = pkg_repo_add_from_manifest(map + item->offset, item->origin,
 				    item->digest, item->length, sqlite, &keys, &pkg, legacy_repo,
 				    repo);
 			}
 			else {
 				rc = pkg_repo_add_from_manifest(map + item->offset, item->origin,
 				    item->digest, len - item->offset, sqlite, &keys, &pkg,
 				    legacy_repo, repo);
 			}
 		}
 		free(item->origin);
 		free(item->digest);
 		HASH_DEL(ladd, item);
 		free(item);
 	}
 	pkg_manifest_keys_free(keys);
 	pkg_emit_incremental_update(updated, removed, added, processed);
 cleanup:
 	if (in_trans) {
 		if (rc != EPKG_OK)
 			pkgdb_transaction_rollback(sqlite, "REPO");
 		if (pkgdb_transaction_commit(sqlite, "REPO") != EPKG_OK)
 			rc = EPKG_FATAL;
 	}
 	pkgdb_repo_finalize_statements();
 	if (rc == EPKG_OK)
 		sql_exec(sqlite, "DROP TABLE repo_update;");
 	if (pkg != NULL)
 		pkg_free(pkg);
 	if (it != NULL)
 		pkgdb_it_free(it);
 	if (map == MAP_FAILED && fmanifest)
 		fclose(fmanifest);
 	if (fdigests)
 		fclose(fdigests);
 	/* if (fconflicts)
 		fclose(fconflicts);*/
 	if (map != MAP_FAILED)
 		munmap(map, len);
 	if (linebuf != NULL)
 		free(linebuf);
 	sqlite3_close(sqlite);
 	return (rc);
 }
 int
 pkg_repo_update_binary_pkgs(struct pkg_repo *repo, bool force)
 {
 	char filepath[MAXPATHLEN];
 	const char *dbdir = NULL;
 	struct stat st;
 	time_t t = 0;
 	sqlite3 *sqlite = NULL;
 	char *req = NULL;
 	int64_t res;
 	bool got_meta = false;
 	sqlite3_initialize();
 	if (!pkg_repo_enabled(repo))
 		return (EPKG_OK);
 	dbdir = pkg_object_string(pkg_config_get("PKG_DBDIR"));
 	pkg_debug(1, "PkgRepo: verifying update for %s", pkg_repo_name(repo));
 	snprintf(filepath, sizeof(filepath), "%s/%s.meta", dbdir, pkg_repo_name(repo));
 	if (stat(filepath, &st) != -1) {
 		t = force ? 0 : st.st_mtime;
 		got_meta = true;
 	}
 	snprintf(filepath, sizeof(filepath), "%s/%s.sqlite", dbdir, pkg_repo_name(repo));
 	if (stat(filepath, &st) != -1) {
 		if (!got_meta && !force)
 			t = st.st_mtime;
 	}
 	if (t != 0) {
 		if (sqlite3_open(filepath, &sqlite) != SQLITE_OK) {
 			pkg_emit_error("Unable to open local database");
 			return (EPKG_FATAL);
 		}
 		if (get_pragma(sqlite, "SELECT count(name) FROM sqlite_master "
 		    "WHERE type='table' AND name='repodata';", &res, false) != EPKG_OK) {
 			pkg_emit_error("Unable to query repository");
 			sqlite3_close(sqlite);
 			return (EPKG_FATAL);
 		}
 		if (res != 1) {
 			t = 0;
 			pkg_emit_notice("Repository %s contains no repodata table, "
 					"need to re-create database", repo->name);
 			if (sqlite != NULL) {
 				sqlite3_close(sqlite);
 				sqlite = NULL;
 			}
 		}
 	}
 	if (t != 0) {
 		req = sqlite3_mprintf("select count(key) from repodata "
 		    "WHERE key = \"packagesite\" and value = '%q'", pkg_repo_url(repo));
 		res = 0;
 		/*
 		 * Ignore error here:
 		 * if an error occure it means the database is unusable
 		 * therefor it is better to rebuild it from scratch
 		 */
 		get_pragma(sqlite, req, &res, true);
 		sqlite3_free(req);
 		if (res == 1) {
 			/* Test for incomplete upgrade */
 			if (sqlite3_exec(sqlite, "INSERT INTO repo_update VALUES(1);",
 					NULL, NULL, NULL) == SQLITE_OK) {
 				res = -1;
 				pkg_emit_notice("The previous update of %s was not completed "
 						"successfully, re-create repo database", repo->name);
 			}
 		}
 		else {
 			pkg_emit_notice("Repository %s has a wrong packagesite, need to "
 					"re-create database", repo->name);
 		}
 		if (res != 1) {
 			t = 0;
 			if (sqlite != NULL) {
 				sqlite3_close(sqlite);
 				sqlite = NULL;
 			}
 			unlink(filepath);
 		}
 	}
 	res = pkg_repo_update_incremental(filepath, repo, &t);
 	if (res != EPKG_OK && res != EPKG_UPTODATE) {
 		pkg_emit_notice("Unable to find catalogs");
 		goto cleanup;
 	}
 cleanup:
 	/* Set mtime from http request if possible */
 	if (t != 0) {
 		struct timeval ftimes[2] = {
 			{
 			.tv_sec = t,
 			.tv_usec = 0
 			},
 			{
 			.tv_sec = t,
 			.tv_usec = 0
 			}
 		};
 		if (got_meta)
 			snprintf(filepath, sizeof(filepath), "%s/%s.meta", dbdir, pkg_repo_name(repo));
 		utimes(filepath, ftimes);
 	}
 	return (res);
 }
 int
 pkg_update(struct pkg_repo *repo, bool force)
 {
 	return (repo->update(repo, force));
 	return (repo->ops->update(repo, force));
 }

modified libpkg/pkgdb.c

@@ -77,15 +77,7 @@

 #define DBVERSION (DB_SCHEMA_MAJOR * 1000 + DB_SCHEMA_MINOR)
 static void pkgdb_regex(sqlite3_context *, int, sqlite3_value **);
 static void pkgdb_split_uid(sqlite3_context *, int, sqlite3_value **);
 static void pkgdb_split_version(sqlite3_context *, int, sqlite3_value **);
 static void pkgdb_regex_delete(void *);
 static int pkgdb_upgrade(struct pkgdb *);
 static void populate_pkg(sqlite3_stmt *stmt, struct pkg *pkg);
 static void pkgdb_detach_remotes(sqlite3 *);
 static int sqlcmd_init(sqlite3 *db, __unused const char **err,
     __unused const void *noused);
 static int prstmt_initialize(struct pkgdb *db);
 /* static int run_prstmt(sql_prstmt_index s, ...); */
 static void prstmt_finalize(struct pkgdb *db);

@@ -94,193 +86,7 @@ static int pkgdb_insert_scripts(struct pkg *pkg, int64_t package_id, sqlite3 *s)

 extern int sqlite3_shell(int, char**);
 /*
  * Keep entries sorted by name!
  */
 static struct column_mapping {
 	const char * const name;
 	pkg_attr type;
 	enum {
 		PKG_SQLITE_STRING,
 		PKG_SQLITE_INT64,
 		PKG_SQLITE_BOOL
 	} pkg_type;
 } columns[] = {
 	{ "arch",	PKG_ARCH, PKG_SQLITE_STRING },
 	{ "automatic",	PKG_AUTOMATIC, PKG_SQLITE_BOOL },
 	{ "cksum",	PKG_CKSUM, PKG_SQLITE_STRING },
 	{ "comment",	PKG_COMMENT, PKG_SQLITE_STRING },
 	{ "dbname",	PKG_REPONAME, PKG_SQLITE_STRING },
 	{ "desc",	PKG_DESC, PKG_SQLITE_STRING },
 	{ "flatsize",	PKG_FLATSIZE, PKG_SQLITE_INT64 },
 	{ "id",		PKG_ROWID, PKG_SQLITE_INT64 },
 	{ "licenselogic", PKG_LICENSE_LOGIC, PKG_SQLITE_INT64 },
 	{ "locked",	PKG_LOCKED, PKG_SQLITE_BOOL },
 	{ "maintainer",	PKG_MAINTAINER, PKG_SQLITE_STRING },
 	{ "manifestdigest",	PKG_DIGEST, PKG_SQLITE_STRING },
 	{ "message",	PKG_MESSAGE, PKG_SQLITE_STRING },
 	{ "name",	PKG_NAME, PKG_SQLITE_STRING },
 	{ "oldflatsize", PKG_OLD_FLATSIZE, PKG_SQLITE_INT64 },
 	{ "oldversion",	PKG_OLD_VERSION, PKG_SQLITE_STRING },
 	{ "origin",	PKG_ORIGIN, PKG_SQLITE_STRING },
 	{ "pkgsize",	PKG_PKGSIZE, PKG_SQLITE_INT64 },
 	{ "prefix",	PKG_PREFIX, PKG_SQLITE_STRING },
 	{ "repopath",	PKG_REPOPATH, PKG_SQLITE_STRING },
 	{ "rowid",	PKG_ROWID, PKG_SQLITE_INT64 },
 	{ "time",	PKG_TIME, PKG_SQLITE_INT64 },
 	{ "uniqueid",	PKG_UNIQUEID, PKG_SQLITE_STRING },
 	{ "version",	PKG_VERSION, PKG_SQLITE_STRING },
 	{ "weight",	-1, PKG_SQLITE_INT64 },
 	{ "www",	PKG_WWW, PKG_SQLITE_STRING },
 	{ NULL,		-1, PKG_SQLITE_STRING }
 };
 static int
 load_val(sqlite3 *db, struct pkg *pkg, const char *sql, unsigned flags,
     int (*pkg_adddata)(struct pkg *pkg, const char *data), int list)
 {
 	sqlite3_stmt	*stmt;
 	int		 ret;
 	int64_t		 rowid;
 	assert(db != NULL && pkg != NULL);
 	if (pkg->flags & flags)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_adddata(pkg, sqlite3_column_text(stmt, 0));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		if (list != -1)
 			pkg_list_free(pkg, list);
 		ERROR_SQLITE(db, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= flags;
 	return (EPKG_OK);
 }
 static int
 load_tag_val(sqlite3 *db, struct pkg *pkg, const char *sql, unsigned flags,
 	     int (*pkg_addtagval)(struct pkg *pkg, const char *tag, const char *val),
 	     int list)
 {
 	sqlite3_stmt	*stmt;
 	int		 ret;
 	int64_t		 rowid;
 	assert(db != NULL && pkg != NULL);
 	if (pkg->flags & flags)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_addtagval(pkg, sqlite3_column_text(stmt, 0),
 			      sqlite3_column_text(stmt, 1));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		if (list != -1)
 			pkg_list_free(pkg, list);
 		ERROR_SQLITE(db, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= flags;
 	return (EPKG_OK);
 }
 static int
 compare_column_func(const void *pkey, const void *pcolumn)
 {
 	const char *key = (const char*)pkey;
 	const struct column_mapping *column =
 			(const struct column_mapping*)pcolumn;
 	return strcmp(key, column->name);
 }
 static void
 populate_pkg(sqlite3_stmt *stmt, struct pkg *pkg) {
 	int		 icol = 0;
 	const char	*colname;
 	assert(stmt != NULL);
 	for (icol = 0; icol < sqlite3_column_count(stmt); icol++) {
 		colname = sqlite3_column_name(stmt, icol);
 		struct column_mapping *column;
 		switch (sqlite3_column_type(stmt, icol)) {
 		case SQLITE_TEXT:
 			column = bsearch(colname, columns, NELEM(columns) - 1,
 					sizeof(columns[0]), compare_column_func);
 			if (column == NULL) {
 				pkg_emit_error("unknown column %s", colname);
 			}
 			else {
 				if (column->pkg_type == PKG_SQLITE_STRING)
 					pkg_set(pkg, column->type,
 						sqlite3_column_text(stmt, icol));
 				else
 					pkg_emit_error("want string for column %s and got number",
 							colname);
 			}
 			break;
 		case SQLITE_INTEGER:
 			column = bsearch(colname, columns, NELEM(columns) - 1,
 					sizeof(columns[0]), compare_column_func);
 			if (column == NULL) {
 				pkg_emit_error("Unknown column %s", colname);
 			}
 			else {
 				if (column->pkg_type == PKG_SQLITE_INT64)
 					pkg_set(pkg, column->type,
 						sqlite3_column_int64(stmt, icol));
 				else if (column->pkg_type == PKG_SQLITE_BOOL)
 					pkg_set(pkg, column->type,
 							(bool)sqlite3_column_int(stmt, icol));
 				else
 					pkg_emit_error("want number for column %s and got string",
 							colname);
 			}
 			break;
 		case SQLITE_BLOB:
 		case SQLITE_FLOAT:
 			pkg_emit_error("wrong type for column: %s",
 			    colname);
 			/* just ignore currently */
 			break;
 		case SQLITE_NULL:
 			break;
 		}
 	}
 }
 static void
 void
 pkgdb_regex(sqlite3_context *ctx, int argc, sqlite3_value **argv)
 {
 	const unsigned char	*regex = NULL;

@@ -353,19 +159,19 @@ pkgdb_split_common(sqlite3_context *ctx, int argc, sqlite3_value **argv,

 	}
 }
 static void
 void
 pkgdb_split_uid(sqlite3_context *ctx, int argc, sqlite3_value **argv)
 {
 	pkgdb_split_common(ctx, argc, argv, '~', "name", "origin");
 }
 static void
 void
 pkgdb_split_version(sqlite3_context *ctx, int argc, sqlite3_value **argv)
 {
 	pkgdb_split_common(ctx, argc, argv, '-', "name", "version");
 }
 static void
 void
 pkgdb_regex_delete(void *p)
 {
 	regex_t	*re = (regex_t *)p;

@@ -374,7 +180,7 @@ pkgdb_regex_delete(void *p)

 	free(re);
 }
 static void
 void
 pkgdb_now(sqlite3_context *ctx, int argc, __unused sqlite3_value **argv)
 {
 	if (argc != 0) {

@@ -386,7 +192,7 @@ pkgdb_now(sqlite3_context *ctx, int argc, __unused sqlite3_value **argv)

 	sqlite3_result_int64(ctx, (int64_t)time(NULL));
 }
 static void
 void
 pkgdb_myarch(sqlite3_context *ctx, int argc, sqlite3_value **argv)
 {
 	const unsigned char	*arch = NULL;

@@ -834,101 +640,8 @@ pkgdb_init(sqlite3 *sdb)

 	return (sql_exec(sdb, sql, DBVERSION));
 }
 /**
  * Initialize the local cache of the remote database with indicies
  */
 int
 pkgdb_remote_init(struct pkgdb *db, const char *repo)
 {
 	struct sbuf	*sql = NULL;
 	const char	*reponame = NULL;
 	int		 ret;
 	const char	 init_sql[] = ""
 	"BEGIN;"
 	"CREATE INDEX IF NOT EXISTS '%s'.deps_origin ON deps(origin);"
 	"CREATE INDEX IF NOT EXISTS '%s'.pkg_digest_id ON packages(origin, manifestdigest);"
 	"CREATE INDEX IF NOT EXISTS '%s'.pkg_conflicts_pid ON pkg_conflicts(package_id);"
 	"CREATE INDEX IF NOT EXISTS '%s'.pkg_conflicts_cid ON pkg_conflicts(conflict_id);"
 	"CREATE INDEX IF NOT EXISTS '%s'.pkg_provides_id ON pkg_provides(package_id);"
 	"COMMIT;"
 	;
 	if ((reponame = pkgdb_get_reponame(db, repo)) == NULL) {
 		return (EPKG_FATAL);
 	}
 	sql = sbuf_new_auto();
 	sbuf_printf(sql, init_sql, reponame, reponame, reponame, reponame, reponame);
 	ret = sql_exec(db->sqlite, sbuf_data(sql));
 	sbuf_delete(sql);
 	return (ret);
 }
 static int
 pkgdb_open_multirepos(const char *dbdir, struct pkgdb *db,
 		      const char *reponame)
 {
 	int		  ret;
 	char		  remotepath[MAXPATHLEN];
 	struct pkg_repo	 *r = NULL;
 	int		  repocount = 0;
 	while (pkg_repos(&r) == EPKG_OK) {
 		if (reponame != NULL) {
 			if (strcmp(pkg_repo_ident(r), reponame) != 0)
 				continue;
 		} else {
 			if (!pkg_repo_enabled(r))
 				continue;
 		}
 		/* is it already attached? */
 		if (pkgdb_is_attached(db->sqlite, pkg_repo_name(r))) {
 			pkg_emit_error("repository '%s' is already "
 			    "listed, ignoring", pkg_repo_ident(r));
 			continue;
 		}
 		snprintf(remotepath, sizeof(remotepath), "%s/%s.sqlite",
 			 dbdir, pkg_repo_name(r));
 		if (access(remotepath, R_OK) != 0) {
 			pkg_emit_noremotedb(pkg_repo_ident(r));
 			pkgdb_close(db);
 			return (EPKG_ENODB);
 		}
 		ret = sql_exec(db->sqlite, "ATTACH '%s' AS '%s';",
 		          remotepath, pkg_repo_name(r));
 		if (ret != EPKG_OK) {
 			pkgdb_close(db);
 			return (EPKG_FATAL);
 		}
 		switch (pkgdb_repo_check_version(db, pkg_repo_name(r))) {
 		case EPKG_FATAL:
 			pkgdb_close(db);
 			return (EPKG_FATAL);
 			break;
 		case EPKG_REPOSCHEMA:
 			ret = sql_exec(db->sqlite, "DETACH DATABASE '%s'",
 				  pkg_repo_name(r));
 			if (ret != EPKG_OK) {
 				pkgdb_close(db);
 				return (EPKG_FATAL);
 			}
 			break;
 		default:
 			repocount++;
 			break;
 		}
 	}
 	return (repocount > 0 ? EPKG_OK : EPKG_FATAL);
 }
 static int
 file_mode_insecure(const char *path, bool install_as_user)
 pkgdb_is_insecure_mode(const char *path, bool install_as_user)
 {
 	uid_t		fileowner;
 	gid_t		filegroup;

@@ -987,8 +700,8 @@ file_mode_insecure(const char *path, bool install_as_user)

 	return (EPKG_OK);
 }
 static int
 database_access(unsigned mode, const char* dbdir, const char *dbname)
 int
 pkgdb_check_access(unsigned mode, const char* dbdir, const char *dbname)
 {
 	char		 dbpath[MAXPATHLEN];
 	int		 retval;

@@ -996,13 +709,13 @@ database_access(unsigned mode, const char* dbdir, const char *dbname)

 	bool		 install_as_user;
 	if (dbname != NULL)
 		snprintf(dbpath, sizeof(dbpath), "%s/%s.sqlite", dbdir, dbname);
 		snprintf(dbpath, sizeof(dbpath), "%s/%s", dbdir, dbname);
 	else
 		strlcpy(dbpath, dbdir, sizeof(dbpath));
 	install_as_user = (getenv("INSTALL_AS_USER") != NULL);
 	retval = file_mode_insecure(dbpath, install_as_user);
 	retval = pkgdb_is_insecure_mode(dbpath, install_as_user);
 	database_exists = (retval != EPKG_ENODB);

@@ -1089,17 +802,17 @@ pkgdb_access(unsigned mode, unsigned database)

 	   Otherwise, just test for read access */
 	if ((mode & PKGDB_MODE_CREATE) != 0) {
 		retval = database_access(PKGDB_MODE_READ|PKGDB_MODE_WRITE,
 		retval = pkgdb_check_access(PKGDB_MODE_READ|PKGDB_MODE_WRITE,
 					 dbdir, NULL);
 	} else
 		retval = database_access(PKGDB_MODE_READ, dbdir, NULL);
 		retval = pkgdb_check_access(PKGDB_MODE_READ, dbdir, NULL);
 	if (retval != EPKG_OK)
 		return (retval);
 	/* Test local.sqlite, if required */
 	if ((database & PKGDB_DB_LOCAL) != 0) {
 		retval = database_access(mode, dbdir, "local");
 		retval = pkgdb_check_access(mode, dbdir, "local.sqlite");
 		if (retval != EPKG_OK)
 			return (retval);
 	}

@@ -1112,7 +825,7 @@ pkgdb_access(unsigned mode, unsigned database)

 			if (!pkg_repo_enabled(r))
 				continue;
 			retval = database_access(mode, dbdir, pkg_repo_name(r));
 			retval = r->ops->access(r, mode);
 			if (retval != EPKG_OK)
 				return (retval);
 		}

@@ -1136,6 +849,32 @@ pkgdb_open(struct pkgdb **db_p, pkgdb_t type)

 	return (pkgdb_open_all(db_p, type, NULL));
 }
 static int
 pkgdb_open_repos(struct pkgdb *db, const char *reponame)
 {
 	struct pkg_repo *r = NULL;
 	struct _pkg_repo_list_item *item;
 	while (pkg_repos(&r) == EPKG_OK) {
 		if (reponame == NULL || strcasecmp(r->name, reponame) == 0) {
 			/* We need read only access here */
 			if (r->ops->open(r, R_OK) == EPKG_OK) {
 				item = malloc(sizeof(*item));
 				if (item == NULL) {
 					pkg_emit_errno("malloc", "_pkg_repo_list_item");
 					return (EPKG_FATAL);
 				}
 				r->ops->init(r);
 				item->repo = r;
 				LL_PREPEND(db->repos, item);
 			}
 		}
 	}
 	return (EPKG_OK);
 }
 int
 pkgdb_open_all(struct pkgdb **db_p, pkgdb_t type, const char *reponame)
 {

@@ -1150,21 +889,16 @@ pkgdb_open_all(struct pkgdb **db_p, pkgdb_t type, const char *reponame)

 	int		 ret;
 	if (*db_p != NULL) {
 		assert((*db_p)->lock_count == 0);
 		reopen = true;
 		db = *db_p;
 		if (db->type == type)
 			return (EPKG_OK);
 	}
 	dbdir = pkg_object_string(pkg_config_get("PKG_DBDIR"));
 	if (!reopen && (db = calloc(1, sizeof(struct pkgdb))) == NULL) {
 		pkg_emit_errno("malloc", "pkgdb");
 		return EPKG_FATAL;
 		return (EPKG_FATAL);
 	}
 	db->type = type;
 	db->lock_count = 0;
 	db->prstmt_initialized = false;
 	if (!reopen) {

@@ -1225,7 +959,7 @@ pkgdb_open_all(struct pkgdb **db_p, pkgdb_t type, const char *reponame)

 		}
 		/* Create our functions */
 		sqlcmd_init(db->sqlite, NULL, NULL);
 		pkgdb_sqlcmd_init(db->sqlite, NULL, NULL);
 		if (pkgdb_upgrade(db) != EPKG_OK) {
 			pkgdb_close(db);

@@ -1245,8 +979,7 @@ pkgdb_open_all(struct pkgdb **db_p, pkgdb_t type, const char *reponame)

 	if (type == PKGDB_REMOTE || type == PKGDB_MAYBE_REMOTE) {
 		if (reponame != NULL || pkg_repos_activated_count() > 0) {
 			db->type = PKGDB_REMOTE;
 			ret = pkgdb_open_multirepos(dbdir, db, reponame);
 			ret = pkgdb_open_repos(db, reponame);
 			if (ret != EPKG_OK)
 				return (ret);
 		} else if (type == PKGDB_REMOTE) {

@@ -1276,6 +1009,8 @@ pkgdb_open_all(struct pkgdb **db_p, pkgdb_t type, const char *reponame)

 void
 pkgdb_close(struct pkgdb *db)
 {
 	struct _pkg_repo_list_item *cur, *tmp;
 	if (db == NULL)
 		return;

@@ -1283,9 +1018,10 @@ pkgdb_close(struct pkgdb *db)

 		prstmt_finalize(db);
 	if (db->sqlite != NULL) {
 		assert(db->lock_count == 0);
 		if (db->type == PKGDB_REMOTE) {
 			pkgdb_detach_remotes(db->sqlite);
 		LL_FOREACH_SAFE(db->repos, cur, tmp) {
 			cur->repo->ops->close(cur->repo, false);
 			free(cur);
 		}
 		if (!sqlite3_db_readonly(db->sqlite, "main"))

@@ -1328,1123 +1064,130 @@ pkgdb_transaction_begin(sqlite3 *sqlite, const char *savepoint)

 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	} else {
 		char sql[128] = "SAVEPOINT ";
 		strlcat(sql, savepoint, sizeof(sql));
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	}
 	if (ret == SQLITE_OK)
 		PKGDB_SQLITE_RETRY_ON_BUSY(ret)
 			ret = sqlite3_step(stmt);
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_OK && ret != SQLITE_DONE)
 		ERROR_SQLITE(sqlite, psql);
 	return (ret == SQLITE_OK || ret == SQLITE_DONE ? EPKG_OK : EPKG_FATAL);
 }
 int
 pkgdb_transaction_commit(sqlite3 *sqlite, const char *savepoint)
 {
 	int		 ret;
 	sqlite3_stmt	*stmt;
 	const char *psql;
 	assert(sqlite != NULL);
 	if (savepoint == NULL || savepoint[0] == '\0') {
 		const char sql[] = "COMMIT TRANSACTION";
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	} else {
 		char sql[128] = "RELEASE SAVEPOINT ";
 		strlcat(sql, savepoint, sizeof(sql));
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	}
 	if (ret == SQLITE_OK)
 		PKGDB_SQLITE_RETRY_ON_BUSY(ret)
 			ret = sqlite3_step(stmt);
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_OK && ret != SQLITE_DONE)
 		ERROR_SQLITE(sqlite, psql);
 	return (ret == SQLITE_OK || ret == SQLITE_DONE ? EPKG_OK : EPKG_FATAL);
 }
 int
 pkgdb_transaction_rollback(sqlite3 *sqlite, const char *savepoint)
 {
 	int		 ret;
 	sqlite3_stmt	*stmt;
 	const char *psql;
 	assert(sqlite != NULL);
 	if (savepoint == NULL || savepoint[0] == '\0') {
 		const char sql[] = "ROLLBACK TRANSACTION";
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	} else {
 		char sql[128] = "ROLLBACK TO SAVEPOINT ";
 		strlcat(sql, savepoint, sizeof(sql));
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	}
 	if (ret == SQLITE_OK)
 		PKGDB_SQLITE_RETRY_ON_BUSY(ret)
 			ret = sqlite3_step(stmt);
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_OK && ret != SQLITE_DONE)
 		ERROR_SQLITE(sqlite, psql);
 	return (ret == SQLITE_OK || ret == SQLITE_DONE ? EPKG_OK : EPKG_FATAL);
 }
 struct pkgdb_it *
 pkgdb_it_new(struct pkgdb *db, sqlite3_stmt *s, int type, short flags)
 {
 	struct pkgdb_it	*it;
 	assert(db != NULL && s != NULL);
 	assert(!(flags & (PKGDB_IT_FLAG_CYCLED & PKGDB_IT_FLAG_ONCE)));
 	assert(!(flags & (PKGDB_IT_FLAG_AUTO & (PKGDB_IT_FLAG_CYCLED | PKGDB_IT_FLAG_ONCE))));
 	if ((it = malloc(sizeof(struct pkgdb_it))) == NULL) {
 		pkg_emit_errno("malloc", "pkgdb_it");
 		sqlite3_finalize(s);
 		return (NULL);
 	}
 	it->db = db;
 	it->sqlite = db->sqlite;
 	it->stmt = s;
 	it->type = type;
 	it->flags = flags;
 	it->finished = 0;
 	return (it);
 }
 static struct load_on_flag {
 	int	flag;
 	int	(*load)(struct pkgdb *db, struct pkg *p);
 } load_on_flag[] = {
 	{ PKG_LOAD_DEPS,		pkgdb_load_deps },
 	{ PKG_LOAD_RDEPS,		pkgdb_load_rdeps },
 	{ PKG_LOAD_FILES,		pkgdb_load_files },
 	{ PKG_LOAD_DIRS,		pkgdb_load_dirs },
 	{ PKG_LOAD_SCRIPTS,		pkgdb_load_scripts },
 	{ PKG_LOAD_OPTIONS,		pkgdb_load_options },
 	{ PKG_LOAD_MTREE,		pkgdb_load_mtree },
 	{ PKG_LOAD_CATEGORIES,		pkgdb_load_category },
 	{ PKG_LOAD_LICENSES,		pkgdb_load_license },
 	{ PKG_LOAD_USERS,		pkgdb_load_user },
 	{ PKG_LOAD_GROUPS,		pkgdb_load_group },
 	{ PKG_LOAD_SHLIBS_REQUIRED,	pkgdb_load_shlib_required },
 	{ PKG_LOAD_SHLIBS_PROVIDED,	pkgdb_load_shlib_provided },
 	{ PKG_LOAD_ANNOTATIONS,		pkgdb_load_annotations },
 	{ PKG_LOAD_CONFLICTS,		pkgdb_load_conflicts },
 	{ PKG_LOAD_PROVIDES,		pkgdb_load_provides },
 	{ -1,			        NULL }
 };
 int
 pkgdb_it_next(struct pkgdb_it *it, struct pkg **pkg_p, unsigned flags)
 {
 	struct pkg	*pkg;
 	int		 i;
 	int		 ret;
 	const char *digest;
 	assert(it != NULL);
 	/*
 	 * XXX:
 	 * Currently, we have a lot of issues related to pkg digests.
 	 * So we want to ensure that we always have a valid package digest
 	 * even if we work with pkg 1.2 repo. Therefore, we explicitly check
 	 * manifest digests and set it to NULL if it is invalid.
 	 *
 	 */
 	if (it->finished && (it->flags & PKGDB_IT_FLAG_ONCE))
 		return (EPKG_END);
 	switch (sqlite3_step(it->stmt)) {
 	case SQLITE_ROW:
 		if (*pkg_p == NULL) {
 			ret = pkg_new(pkg_p, it->type);
 			if (ret != EPKG_OK)
 				return (ret);
 		} else
 			pkg_reset(*pkg_p, it->type);
 		pkg = *pkg_p;
 		populate_pkg(it->stmt, pkg);
 		pkg_get(pkg, PKG_DIGEST, &digest);
 		if (digest != NULL && !pkg_checksum_is_valid(digest, strlen(digest)))
 			pkg_set(pkg, PKG_DIGEST, NULL);
 		for (i = 0; load_on_flag[i].load != NULL; i++) {
 			if (flags & load_on_flag[i].flag) {
 				if (it->db != NULL) {
 					ret = load_on_flag[i].load(it->db, pkg);
 					if (ret != EPKG_OK)
 						return (ret);
 				}
 				else {
 					pkg_emit_error("invalid iterator passed to pkgdb_it_next");
 					return (EPKG_FATAL);
 				}
 			}
 		}
 		return (EPKG_OK);
 	case SQLITE_DONE:
 		it->finished ++;
 		if (it->flags & PKGDB_IT_FLAG_CYCLED) {
 			sqlite3_reset(it->stmt);
 			return (EPKG_OK);
 		}
 		else {
 			if (it->flags & PKGDB_IT_FLAG_AUTO)
 				pkgdb_it_free(it);
 			return (EPKG_END);
 		}
 		break;
 	default:
 		ERROR_SQLITE(it->sqlite, "iterator");
 		return (EPKG_FATAL);
 	}
 }
 void
 pkgdb_it_reset(struct pkgdb_it *it)
 {
 	if (it == NULL)
 		return;
 	it->finished = 0;
 	sqlite3_reset(it->stmt);
 }
 void
 pkgdb_it_free(struct pkgdb_it *it)
 {
 	if (it == NULL)
 		return;
 	sqlite3_finalize(it->stmt);
 	free(it);
 }
 /* By default, MATCH_EXACT and MATCH_REGEX are case sensitive.  This
  * is modified in many actions according to the value of
  * CASE_SENSITIVE_MATCH in pkg.conf and then possbily reset again in
  * pkg search et al according to command line flags */
 static bool _case_sensitive_flag = false;
 void
 pkgdb_set_case_sensitivity(bool case_sensitive)
 {
 	_case_sensitive_flag = case_sensitive;
 	return;
 }
 bool
 pkgdb_case_sensitive(void)
 {
 	return (_case_sensitive_flag);
 }
 const char *
 pkgdb_get_pattern_query(const char *pattern, match_t match)
 {
 	char		*checkorigin = NULL;
 	char		*checkuid = NULL;
 	const char	*comp = NULL;
 	if (pattern != NULL) {
 		checkuid = strchr(pattern, '~');
 		if (checkuid == NULL)
 			checkorigin = strchr(pattern, '/');
 	}
 	switch (match) {
 	case MATCH_ALL:
 		comp = "";
 		break;
 	case MATCH_EXACT:
 		if (pkgdb_case_sensitive()) {
 			if (checkuid == NULL) {
 				if (checkorigin == NULL)
 					comp = " WHERE name = ?1 "
 					    "OR (name = SPLIT_VERSION('name', ?1) AND "
 					    " version = SPLIT_VERSION('version', ?1))";
 				else
 					comp = " WHERE origin = ?1";
 			} else {
 				comp = " WHERE name = SPLIT_UID('name', ?1) AND "
 						"origin = SPLIT_UID('origin', ?1)";
 			}
 		} else {
 			if (checkuid == NULL) {
 				if (checkorigin == NULL)
 					comp = " WHERE name = ?1 COLLATE NOCASE "
 							"OR (name = SPLIT_VERSION('name', ?1) COLLATE NOCASE AND "
 							" version = SPLIT_VERSION('version', ?1))";
 				else
 					comp = " WHERE origin = ?1 COLLATE NOCASE";
 			} else {
 				comp = " WHERE name = SPLIT_UID('name', ?1) COLLATE NOCASE AND "
 						"origin = SPLIT_UID('origin', ?1) COLLATE NOCASE";
 			}
 		}
 		break;
 	case MATCH_GLOB:
 		if (checkuid == NULL) {
 			if (checkorigin == NULL)
 				comp = " WHERE name GLOB ?1 "
 					"OR (name GLOB SPLIT_VERSION('name', ?1) AND "
 					" version GLOB SPLIT_VERSION('version', ?1))";
 			else
 				comp = " WHERE origin GLOB ?1";
 		} else {
 			comp = " WHERE name = SPLIT_UID('name', ?1) AND "
 					"origin = SPLIT_UID('origin', ?1)";
 		}
 		break;
 	case MATCH_REGEX:
 		if (checkuid == NULL) {
 			if (checkorigin == NULL)
 				comp = " WHERE name REGEXP ?1 "
 				    "OR name || '-' || version REGEXP ?1";
 			else
 				comp = " WHERE origin REGEXP ?1";
 		} else {
 			comp = " WHERE name = SPLIT_UID('name', ?1) AND "
 					"origin = SPLIT_UID('origin', ?1)";
 		}
 		break;
 	case MATCH_CONDITION:
 		comp = pattern;
 		break;
 	case MATCH_FTS:
 		if (checkorigin == NULL)
 			comp = " WHERE id IN (SELECT id FROM pkg_search WHERE name MATCH ?1)";
 		else
 			comp = " WHERE id IN (SELECT id FROM pkg_search WHERE origin MATCH ?1)";
 		break;
 	}
 	return (comp);
 }
 static const char *
 pkgdb_get_match_how(match_t match)
 {
 	const char	*how = NULL;
 	switch (match) {
 	case MATCH_ALL:
 		how = NULL;
 		break;
 	case MATCH_EXACT:
 		if (pkgdb_case_sensitive())
 			how = "%s = ?1";
 		else
 			how = "%s = ?1 COLLATE NOCASE";
 		break;
 	case MATCH_GLOB:
 		how = "%s GLOB ?1";
 		break;
 	case MATCH_REGEX:
 		how = "%s REGEXP ?1";
 		break;
 	case MATCH_CONDITION:
 		/* Should not be called by pkgdb_get_match_how(). */
 		assert(0);
 		break;
 	case MATCH_FTS:
 		how = "id IN (SELECT id FROM pkg_search WHERE %s MATCH ?1)";
 		break;
 	}
 	return (how);
 }
 struct pkgdb_it *
 pkgdb_query(struct pkgdb *db, const char *pattern, match_t match)
 {
 	char		 sql[BUFSIZ];
 	sqlite3_stmt	*stmt;
 	const char	*comp = NULL;
 	assert(db != NULL);
 	assert(match == MATCH_ALL || (pattern != NULL && pattern[0] != '\0'));
 	comp = pkgdb_get_pattern_query(pattern, match);
 	sqlite3_snprintf(sizeof(sql), sql,
 			"SELECT id, origin, name, name || '~' || origin as uniqueid, "
 				"version, comment, desc, "
 				"message, arch, maintainer, www, "
 				"prefix, flatsize, licenselogic, automatic, "
 				"locked, time, manifestdigest "
 			"FROM packages AS p%s "
 			"ORDER BY p.name;", comp);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (NULL);
 	}
 	if (match != MATCH_ALL && match != MATCH_CONDITION)
 		sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_query_which(struct pkgdb *db, const char *path, bool glob)
 {
 	sqlite3_stmt	*stmt;
 	char	sql[BUFSIZ];
 	assert(db != NULL);
 	sqlite3_snprintf(sizeof(sql), sql,
 			"SELECT p.id, p.origin, p.name, p.name || '~' || p.origin as uniqueid, "
 			"p.version, p.comment, p.desc, "
 			"p.message, p.arch, p.maintainer, p.www, "
 			"p.prefix, p.flatsize, p.time "
 			"FROM packages AS p "
 			"LEFT JOIN files AS f ON p.id = f.package_id "
 			"WHERE f.path %s ?1 GROUP BY p.id;", glob ? "GLOB" : "=");
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (NULL);
 	}
 	sqlite3_bind_text(stmt, 1, path, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_query_shlib_required(struct pkgdb *db, const char *shlib)
 {
 	sqlite3_stmt	*stmt;
 	const char	 sql[] = ""
 		"SELECT p.id, p.origin, p.name, p.name || '~' || p.origin as uniqueid, "
 			"p.version, p.comment, p.desc, "
 			"p.message, p.arch, p.maintainer, p.www, "
 			"p.prefix, p.flatsize, p.time "
 			"FROM packages AS p, pkg_shlibs_required AS ps, shlibs AS s "
 			"WHERE p.id = ps.package_id "
 				"AND ps.shlib_id = s.id "
 				"AND s.name = ?1;";
 	assert(db != NULL);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (NULL);
 	}
 	sqlite3_bind_text(stmt, 1, shlib, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_query_shlib_provided(struct pkgdb *db, const char *shlib)
 {
 	sqlite3_stmt	*stmt;
 	const char	 sql[] = ""
 		"SELECT p.id, p.origin, p.name, p.name || '~' || p.origin as uniqueid, "
 			"p.version, p.comment, p.desc, "
 			"p.message, p.arch, p.maintainer, p.www, "
 			"p.prefix, p.flatsize, p.time "
 			"FROM packages AS p, pkg_shlibs_provided AS ps, shlibs AS s "
 			"WHERE p.id = ps.package_id "
 				"AND ps.shlib_id = s.id "
 				"AND s.name = ?1;";
 	assert(db != NULL);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (NULL);
 	}
 	sqlite3_bind_text(stmt, 1, shlib, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 int
 pkgdb_is_dir_used(struct pkgdb *db, const char *dir, int64_t *res)
 {
 	sqlite3_stmt	*stmt;
 	int		 ret;
 	const char	 sql[] = ""
 		"SELECT count(package_id) FROM pkg_directories, directories "
 		"WHERE directory_id = directories.id AND directories.path = ?1;";
 	assert(db != NULL);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	sqlite3_bind_text(stmt, 1, dir, -1, SQLITE_TRANSIENT);
 	ret = sqlite3_step(stmt);
 	if (ret == SQLITE_ROW)
 		*res = sqlite3_column_int64(stmt, 0);
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_ROW) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	return (EPKG_OK);
 }
 int
 pkgdb_load_deps(struct pkgdb *db, struct pkg *pkg)
 {
 	sqlite3_stmt	*stmt = NULL;
 	int		 ret = EPKG_OK;
 	int64_t		 rowid;
 	char		 sql[BUFSIZ];
 	const char	*reponame = NULL;
 	const char	*mainsql = ""
 		"SELECT d.name, d.origin, d.version, p.locked "
 		"FROM main.deps AS d "
 		"LEFT JOIN main.packages AS p ON p.origin = d.origin "
 		"AND p.name = d.name "
 		"WHERE d.package_id = ?1 ORDER BY d.origin DESC;";
 	const char	*reposql = ""
 		"SELECT d.name, d.origin, d.version, 0 "
 		"FROM %Q.deps AS d "
 		"WHERE d.package_id = ?1 ORDER BY d.origin DESC;";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->flags & PKG_LOAD_DEPS)
 		return (EPKG_OK);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, reposql, reponame);
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL);
 	} else {
 		pkg_debug(4, "Pkgdb: running '%s'", mainsql);
 		ret = sqlite3_prepare_v2(db->sqlite, mainsql, -1, &stmt, NULL);
 	}
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_adddep(pkg, sqlite3_column_text(stmt, 0),
 			   sqlite3_column_text(stmt, 1),
 			   sqlite3_column_text(stmt, 2),
 			   sqlite3_column_int(stmt, 3));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		pkg_list_free(pkg, PKG_DEPS);
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_DEPS;
 	return (EPKG_OK);
 }
 int
 pkgdb_load_rdeps(struct pkgdb *db, struct pkg *pkg)
 {
 	sqlite3_stmt	*stmt = NULL;
 	int		 ret;
 	const char	*uniqueid;
 	const char	*reponame = NULL;
 	char		 sql[BUFSIZ];
 	const char	*mainsql = ""
 		"SELECT p.name, p.origin, p.version, p.locked "
 		"FROM main.packages AS p "
 		"INNER JOIN main.deps AS d ON p.id = d.package_id "
 		"WHERE d.name = SPLIT_UID('name', ?1) AND "
 		"d.origin = SPLIT_UID('origin', ?1);";
 	const char	*reposql = ""
 		"SELECT p.name, p.origin, p.version, 0 "
 		"FROM %Q.packages AS p "
 		"INNER JOIN %Q.deps AS d ON p.id = d.package_id "
 		"WHERE d.name = SPLIT_UID('name', ?1) AND "
 		"d.origin = SPLIT_UID('origin', ?1);";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->flags & PKG_LOAD_RDEPS)
 		return (EPKG_OK);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, reposql, reponame, reponame);
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL);
 	} else {
 		pkg_debug(4, "Pkgdb: running '%s'", mainsql);
 		ret = sqlite3_prepare_v2(db->sqlite, mainsql, -1, &stmt, NULL);
 	}
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, mainsql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_UNIQUEID, &uniqueid);
 	sqlite3_bind_text(stmt, 1, uniqueid, -1, SQLITE_STATIC);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_addrdep(pkg, sqlite3_column_text(stmt, 0),
 			    sqlite3_column_text(stmt, 1),
 			    sqlite3_column_text(stmt, 2),
 			    sqlite3_column_int(stmt, 3));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		pkg_list_free(pkg, PKG_RDEPS);
 		ERROR_SQLITE(db->sqlite, mainsql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_RDEPS;
 	return (EPKG_OK);
 }
 int
 pkgdb_load_files(struct pkgdb *db, struct pkg *pkg)
 {
 	sqlite3_stmt	*stmt = NULL;
 	int		 ret;
 	int64_t		 rowid;
 	const char	 sql[] = ""
 		"SELECT path, sha256 "
 		"FROM files "
 		"WHERE package_id = ?1 "
 		"ORDER BY PATH ASC";
 	assert(db != NULL && pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	if (pkg->flags & PKG_LOAD_FILES)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_addfile(pkg, sqlite3_column_text(stmt, 0),
 		    sqlite3_column_text(stmt, 1), false);
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		pkg_list_free(pkg, PKG_FILES);
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_FILES;
 	return (EPKG_OK);
 }
 int
 pkgdb_load_dirs(struct pkgdb *db, struct pkg *pkg)
 {
 	const char	 sql[] = ""
 		"SELECT path, try "
 		"FROM pkg_directories, directories "
 		"WHERE package_id = ?1 "
 		"AND directory_id = directories.id "
 		"ORDER by path DESC";
 	sqlite3_stmt	*stmt;
 	int		 ret;
 	int64_t		 rowid;
 	assert(db != NULL && pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	if (pkg->flags & PKG_LOAD_DIRS)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_adddir(pkg, sqlite3_column_text(stmt, 0),
 		    sqlite3_column_int(stmt, 1), false);
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		pkg_list_free(pkg, PKG_DIRS);
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_DIRS;
 	return (EPKG_OK);
 }
 int
 pkgdb_load_license(struct pkgdb *db, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*reponame = NULL;
 	const char	*basesql = ""
 		"SELECT name "
 		"FROM %Q.pkg_licenses, %Q.licenses AS l "
 		"WHERE package_id = ?1 "
 			"AND license_id = l.id "
 		"ORDER by name DESC";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, basesql, reponame, reponame);
 	} else
 		sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	return (load_val(db->sqlite, pkg, sql, PKG_LOAD_LICENSES,
 	    pkg_addlicense, PKG_LICENSES));
 }
 int
 pkgdb_load_category(struct pkgdb *db, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*reponame = NULL;
 	const char	*basesql = ""
 		"SELECT name "
 		"FROM %Q.pkg_categories, %Q.categories AS c "
 		"WHERE package_id = ?1 "
 			"AND category_id = c.id "
 		"ORDER by name DESC";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, basesql, reponame, reponame);
 	} else
 		sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	return (load_val(db->sqlite, pkg, sql, PKG_LOAD_CATEGORIES,
 	    pkg_addcategory, PKG_CATEGORIES));
 }
 int
 pkgdb_load_user(struct pkgdb *db, struct pkg *pkg)
 {
 	/*struct pkg_user *u = NULL;
 	struct passwd *pwd = NULL;*/
 	int		ret;
 	const char	sql[] = ""
 		"SELECT users.name "
 		"FROM pkg_users, users "
 		"WHERE package_id = ?1 "
 			"AND user_id = users.id "
 		"ORDER by name DESC";
 	assert(db != NULL && pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	ret = load_val(db->sqlite, pkg, sql, PKG_LOAD_USERS,
 	    pkg_adduser, PKG_USERS);
 	/* TODO get user uidstr from local database */
 /*	while (pkg_users(pkg, &u) == EPKG_OK) {
 		pwd = getpwnam(pkg_user_name(u));
 		if (pwd == NULL)
 			continue;
 		strlcpy(u->uidstr, pw_make(pwd), sizeof(u->uidstr));
 	}*/
 	return (ret);
 }
 int
 pkgdb_load_group(struct pkgdb *db, struct pkg *pkg)
 {
 	struct pkg_group	*g = NULL;
 	struct group		*grp = NULL;
 	int			 ret;
 	const char		 sql[] = ""
 		"SELECT groups.name "
 		"FROM pkg_groups, groups "
 		"WHERE package_id = ?1 "
 			"AND group_id = groups.id "
 		"ORDER by name DESC";
 	assert(db != NULL && pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	ret = load_val(db->sqlite, pkg, sql, PKG_LOAD_GROUPS,
 	    pkg_addgroup, PKG_GROUPS);
 	while (pkg_groups(pkg, &g) == EPKG_OK) {
 		grp = getgrnam(pkg_group_name(g));
 		if (grp == NULL)
 			continue;
 		strlcpy(g->gidstr, gr_make(grp), sizeof(g->gidstr));
 	}
 					 &stmt, NULL);
 	} else {
 		char sql[128] = "SAVEPOINT ";
 	return (ret);
 }
 		strlcat(sql, savepoint, sizeof(sql));
 int
 pkgdb_load_shlib_required(struct pkgdb *db, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*reponame = NULL;
 	const char	*basesql = ""
 		"SELECT name "
 		"FROM %Q.pkg_shlibs_required, %Q.shlibs AS s "
 		"WHERE package_id = ?1 "
 			"AND shlib_id = s.id "
 		"ORDER by name DESC";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, basesql, reponame, reponame);
 	} else
 		sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	}
 	return (load_val(db->sqlite, pkg, sql, PKG_LOAD_SHLIBS_REQUIRED,
 	    pkg_addshlib_required, PKG_SHLIBS_REQUIRED));
 }
 	if (ret == SQLITE_OK)
 		PKGDB_SQLITE_RETRY_ON_BUSY(ret)
 			ret = sqlite3_step(stmt);
 	sqlite3_finalize(stmt);
 int
 pkgdb_load_shlib_provided(struct pkgdb *db, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*reponame = NULL;
 	const char	*basesql = ""
 		"SELECT name "
 		"FROM %Q.pkg_shlibs_provided, %Q.shlibs AS s "
 		"WHERE package_id = ?1 "
 			"AND shlib_id = s.id "
 		"ORDER by name DESC";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, basesql, reponame, reponame);
 	} else
 		sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	if (ret != SQLITE_OK && ret != SQLITE_DONE)
 		ERROR_SQLITE(sqlite, psql);
 	return (load_val(db->sqlite, pkg, sql, PKG_LOAD_SHLIBS_PROVIDED,
 	    pkg_addshlib_provided, PKG_SHLIBS_PROVIDED));
 	return (ret == SQLITE_OK || ret == SQLITE_DONE ? EPKG_OK : EPKG_FATAL);
 }
 int
 pkgdb_load_annotations(struct pkgdb *db, struct pkg *pkg)
 pkgdb_transaction_commit(sqlite3 *sqlite, const char *savepoint)
 {
 	char		 sql[BUFSIZ];
 	const char	*reponame = NULL;
 	const char	*basesql = ""
 		"SELECT k.annotation AS tag, v.annotation AS value"
 		"  FROM %Q.pkg_annotation p"
 		"    JOIN %Q.annotation k ON (p.tag_id = k.annotation_id)"
 		"    JOIN %Q.annotation v ON (p.value_id = v.annotation_id)"
 		"  WHERE p.package_id = ?1"
 		"  ORDER BY tag, value";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, basesql, reponame,
 		    reponame, reponame);
 	} else
 		sqlite3_snprintf(sizeof(sql), sql, basesql, "main",
                     "main", "main");
 	int		 ret;
 	sqlite3_stmt	*stmt;
 	const char *psql;
 	return (load_tag_val(db->sqlite, pkg, sql, PKG_LOAD_ANNOTATIONS,
 		   pkg_addannotation, PKG_ANNOTATIONS));
 }
 	assert(sqlite != NULL);
 int
 pkgdb_load_scripts(struct pkgdb *db, struct pkg *pkg)
 {
 	sqlite3_stmt	*stmt = NULL;
 	int		 ret;
 	int64_t		 rowid;
 	const char	 sql[] = ""
 		"SELECT script, type "
 		"FROM pkg_script JOIN script USING(script_id) "
 		"WHERE package_id = ?1";
 	if (savepoint == NULL || savepoint[0] == '\0') {
 		const char sql[] = "COMMIT TRANSACTION";
 		psql = sql;
 	assert(db != NULL && pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	} else {
 		char sql[128] = "RELEASE SAVEPOINT ";
 	if (pkg->flags & PKG_LOAD_SCRIPTS)
 		return (EPKG_OK);
 		strlcat(sql, savepoint, sizeof(sql));
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	if (ret == SQLITE_OK)
 		PKGDB_SQLITE_RETRY_ON_BUSY(ret)
 			ret = sqlite3_step(stmt);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_addscript(pkg, sqlite3_column_text(stmt, 0),
 		    sqlite3_column_int(stmt, 1));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	if (ret != SQLITE_OK && ret != SQLITE_DONE)
 		ERROR_SQLITE(sqlite, psql);
 	pkg->flags |= PKG_LOAD_SCRIPTS;
 	return (EPKG_OK);
 	return (ret == SQLITE_OK || ret == SQLITE_DONE ? EPKG_OK : EPKG_FATAL);
 }
 int
 pkgdb_load_options(struct pkgdb *db, struct pkg *pkg)
 pkgdb_transaction_rollback(sqlite3 *sqlite, const char *savepoint)
 {
 	const char	*reponame;
 	char		 sql[BUFSIZ];
 	unsigned int	 i;
 	struct optionsql {
 		const char	 *sql;
 		int		(*pkg_addtagval)(struct pkg *pkg,
 						  const char *tag,
 						  const char *val);
 		int		  nargs;
 	}			  optionsql[] = {
 		{
 			"SELECT option, value "
 			"FROM %Q.option JOIN %Q.pkg_option USING(option_id) "
 			"WHERE package_id = ?1 ORDER BY option",
 			pkg_addoption,
 ,
 		},
 		{
 			"SELECT option, default_value "
 			"FROM %Q.option JOIN %Q.pkg_option_default USING(option_id) "
 			"WHERE package_id = ?1 ORDER BY option",
 			pkg_addoption_default,
 ,
 		},
 		{
 			"SELECT option, description "
 			"FROM %Q.option JOIN %Q.pkg_option_desc USING(option_id) "
 			"JOIN %Q.option_desc USING(option_desc_id) ORDER BY option",
 			pkg_addoption_description,
 ,
 		}
 	};
 	const char		 *opt_sql;
 	int			(*pkg_addtagval)(struct pkg *pkg,
 						 const char *tag,
 						 const char *val);
 	int			  nargs, ret;
 	int		 ret;
 	sqlite3_stmt	*stmt;
 	const char *psql;
 	assert(db != NULL && pkg != NULL);
 	assert(sqlite != NULL);
 	if (pkg->flags & PKG_LOAD_OPTIONS)
 		return (EPKG_OK);
 	if (savepoint == NULL || savepoint[0] == '\0') {
 		const char sql[] = "ROLLBACK TRANSACTION";
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	} else {
 		reponame = "main";
 		char sql[128] = "ROLLBACK TO SAVEPOINT ";
 		strlcat(sql, savepoint, sizeof(sql));
 		psql = sql;
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_prepare_v2(sqlite, sql, strlen(sql) + 1,
 					 &stmt, NULL);
 	}
 	for (i = 0; i < NELEM(optionsql); i++) {
 		opt_sql       = optionsql[i].sql;
 		pkg_addtagval = optionsql[i].pkg_addtagval;
 		nargs         = optionsql[i].nargs;
 	if (ret == SQLITE_OK)
 		PKGDB_SQLITE_RETRY_ON_BUSY(ret)
 			ret = sqlite3_step(stmt);
 	sqlite3_finalize(stmt);
 		switch(nargs) {
 		case 1:
 			sqlite3_snprintf(sizeof(sql), sql, opt_sql, reponame);
 			break;
 		case 2:
 			sqlite3_snprintf(sizeof(sql), sql, opt_sql, reponame,
 					 reponame);
 			break;
 		case 3:
 			sqlite3_snprintf(sizeof(sql), sql, opt_sql, reponame,
 					 reponame, reponame);
 			break;
 		default:
 			/* Nothing needs 4 or more, yet... */
 			return (EPKG_FATAL);
 			break;
 		}
 	if (ret != SQLITE_OK && ret != SQLITE_DONE)
 		ERROR_SQLITE(sqlite, psql);
 		pkg_debug(4, "Pkgdb> adding option");
 		ret = load_tag_val(db->sqlite, pkg, sql, PKG_LOAD_OPTIONS,
 				   pkg_addtagval, PKG_OPTIONS);
 		if (ret != EPKG_OK)
 			break;
 	}
 	return (ret);
 	return (ret == SQLITE_OK || ret == SQLITE_DONE ? EPKG_OK : EPKG_FATAL);
 }
 int
 pkgdb_load_mtree(struct pkgdb *db, struct pkg *pkg)
 {
 	const char	sql[] = ""
 		"SELECT m.content "
 		"FROM mtree AS m, packages AS p "
 		"WHERE m.id = p.mtree_id "
 			"AND p.id = ?1;";
 	assert(db != NULL && pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	return (load_val(db->sqlite, pkg, sql, PKG_LOAD_MTREE, pkg_set_mtree, -1));
 }
 int
 pkgdb_load_conflicts(struct pkgdb *db, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*reponame = NULL;
 	const char	*basesql = ""
 			"SELECT packages.origin "
 			"FROM %Q.pkg_conflicts "
 			"LEFT JOIN %Q.packages ON "
 			"packages.id = pkg_conflicts.conflict_id "
 			"WHERE package_id = ?1";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, basesql, reponame, reponame);
 	} else
 		sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 /* By default, MATCH_EXACT and MATCH_REGEX are case sensitive.  This
  * is modified in many actions according to the value of
  * CASE_SENSITIVE_MATCH in pkg.conf and then possbily reset again in
  * pkg search et al according to command line flags */
 	return (load_val(db->sqlite, pkg, sql, PKG_LOAD_CONFLICTS,
 			pkg_addconflict, PKG_CONFLICTS));
 }
 static bool _case_sensitive_flag = false;
 int
 pkgdb_load_provides(struct pkgdb *db, struct pkg *pkg)
 void
 pkgdb_set_case_sensitivity(bool case_sensitive)
 {
 	char		 sql[BUFSIZ];
 	const char	*reponame = NULL;
 	const char	*basesql = ""
 		"SELECT provide "
 		"FROM %Q.provides "
 		"WHERE package_id = ?1";
 	assert(db != NULL && pkg != NULL);
 	if (pkg->type == PKG_REMOTE) {
 		assert(db->type == PKGDB_REMOTE);
 		pkg_get(pkg, PKG_REPONAME, &reponame);
 		sqlite3_snprintf(sizeof(sql), sql, basesql, reponame, reponame);
 	} else
 		sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	_case_sensitive_flag = case_sensitive;
 	return;
 }
 	return (load_val(db->sqlite, pkg, sql, PKG_LOAD_PROVIDES,
 			pkg_addconflict, PKG_PROVIDES));
 bool
 pkgdb_case_sensitive(void)
 {
 	return (_case_sensitive_flag);
 }
 typedef enum _sql_prstmt_index {

@@ -3513,111 +2256,6 @@ sql_exec(sqlite3 *s, const char *sql, ...)

 	return (ret);
 }
 bool
 pkgdb_is_attached(sqlite3 *s, const char *name)
 {
 	sqlite3_stmt	*stmt;
 	const char	*dbname;
 	int		 ret;
 	assert(s != NULL);
 	ret = sqlite3_prepare_v2(s, "PRAGMA database_list;", -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(s, "PRAGMA database_list;");
 		return false;
 	}
 	while (sqlite3_step(stmt) != SQLITE_DONE) {
 		dbname = sqlite3_column_text(stmt, 1);
 		if (!strcmp(dbname, name)) {
 			sqlite3_finalize(stmt);
 			return (true);
 		}
 	}
 	sqlite3_finalize(stmt);
 	return (false);
 }
 int
 pkgdb_sql_all_attached(sqlite3 *s, struct sbuf *sql, const char *multireposql,
     const char *compound)
 {
 	sqlite3_stmt	*stmt;
 	const char	*dbname;
 	int		 dbcount = 0;
 	int		 ret;
 	assert(s != NULL);
 	assert(compound != NULL);
 	ret = sqlite3_prepare_v2(s, "PRAGMA database_list;", -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(s, "PRAGMA database_list;");
 		return (EPKG_FATAL);
 	}
 	while (sqlite3_step(stmt) != SQLITE_DONE) {
 		dbname = sqlite3_column_text(stmt, 1);
 		if ((strcmp(dbname, "main") == 0) ||
 		    (strcmp(dbname, "temp") == 0))
 			continue;
 		dbcount++;
 		if (dbcount > 1)
 			sbuf_cat(sql, compound);
 		/* replace any occurences of the dbname in the resulting SQL */
 		sbuf_printf(sql, multireposql, dbname);
 	}
 	sqlite3_finalize(stmt);
 	/* The generated SQL statement will not be syntactically
 	 * correct unless there is at least one other attached DB than
 	 * main or temp */
 	return (dbcount > 0 ? EPKG_OK : EPKG_FATAL);
 }
 static void
 pkgdb_detach_remotes(sqlite3 *s)
 {
 	sqlite3_stmt	*stmt;
 	struct sbuf	*sql = NULL;
 	const char	*dbname;
 	int		 ret;
 	assert(s != NULL);
 	ret = sqlite3_prepare_v2(s, "PRAGMA database_list;", -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(s, "PRAGMA database_list;");
 		return;
 	}
 	sql = sbuf_new_auto();
 	while (sqlite3_step(stmt) != SQLITE_DONE) {
 		dbname = sqlite3_column_text(stmt, 1);
 		if ((strcmp(dbname, "main") == 0) ||
 		    (strcmp(dbname, "temp") == 0))
 			continue;
 		sbuf_clear(sql);
 		sbuf_printf(sql, "DETACH '%s';", dbname);
 		sbuf_finish(sql);
 		sql_exec(s, sbuf_get(sql));
 	}
 	sqlite3_finalize(stmt);
 	sbuf_delete(sql);
 }
 int
 get_pragma(sqlite3 *s, const char *sql, int64_t *res, bool silence)
 {

@@ -3714,142 +2352,6 @@ pkgdb_compact(struct pkgdb *db)

 	return (sql_exec(db->sqlite, "VACUUM;"));
 }
 static int
 pkgdb_search_build_search_query(struct sbuf *sql, match_t match,
     pkgdb_field field, pkgdb_field sort)
 {
 	const char	*how = NULL;
 	const char	*what = NULL;
 	const char	*orderby = NULL;
 	how = pkgdb_get_match_how(match);
 	switch (field) {
 	case FIELD_NONE:
 		what = NULL;
 		break;
 	case FIELD_ORIGIN:
 		what = "origin";
 		break;
 	case FIELD_NAME:
 		what = "name";
 		break;
 	case FIELD_NAMEVER:
 		what = "name || '-' || version";
 		break;
 	case FIELD_COMMENT:
 		what = "comment";
 		break;
 	case FIELD_DESC:
 		what = "desc";
 		break;
 	}
 	if (what != NULL && how != NULL)
 		sbuf_printf(sql, how, what);
 	switch (sort) {
 	case FIELD_NONE:
 		orderby = NULL;
 		break;
 	case FIELD_ORIGIN:
 		orderby = " ORDER BY origin";
 		break;
 	case FIELD_NAME:
 		orderby = " ORDER BY name";
 		break;
 	case FIELD_NAMEVER:
 		orderby = " ORDER BY name, version";
 		break;
 	case FIELD_COMMENT:
 		orderby = " ORDER BY comment";
 		break;
 	case FIELD_DESC:
 		orderby = " ORDER BY desc";
 		break;
 	}
 	if (orderby != NULL)
 		sbuf_cat(sql, orderby);
 	return (EPKG_OK);
 }
 struct pkgdb_it *
 pkgdb_search(struct pkgdb *db, const char *pattern, match_t match,
     pkgdb_field field, pkgdb_field sort, const char *reponame)
 {
 	sqlite3_stmt	*stmt = NULL;
 	struct sbuf	*sql = NULL;
 	int		 ret;
 	const char	*rname;
 	const char	*basesql = ""
 		"SELECT id, origin, name, version, comment, "
 		"prefix, desc, arch, maintainer, www, "
 		"licenselogic, flatsize, pkgsize, "
 		"cksum, path AS repopath ";
 	const char	*multireposql = ""
 		"SELECT id, origin, name, version, comment, "
 		"prefix, desc, arch, maintainer, www, "
 		"licenselogic, flatsize, pkgsize, "
 		"cksum, path, '%1$s' AS dbname "
 		"FROM '%1$s'.packages ";
 	assert(db != NULL);
 	assert(pattern != NULL && pattern[0] != '\0');
 	assert(db->type == PKGDB_REMOTE);
 	sql = sbuf_new_auto();
 	sbuf_cat(sql, basesql);
 	/* add the dbname column to the SELECT */
 	sbuf_cat(sql, ", dbname FROM (");
 	if (reponame != NULL) {
 		if ((rname = pkgdb_get_reponame(db, reponame)) != NULL)
 			sbuf_printf(sql, multireposql, rname, rname);
 		else {
 			pkg_emit_error("Repository %s can't be loaded",
 					reponame);
 			sbuf_delete(sql);
 			return (NULL);
 		}
 	} else {
 		if (pkg_repos_activated_count() == 0) {
 			pkg_emit_error("No active repositories configured");
 			sbuf_delete(sql);
 			return (NULL);
 		}
 		/* test on all the attached databases */
 		if (pkgdb_sql_all_attached(db->sqlite, sql,
 		    multireposql, " UNION ALL ") != EPKG_OK) {
 			sbuf_delete(sql);
 			return (NULL);
 		}
 	}
 	/* close the UNIONs and build the search query */
 	sbuf_cat(sql, ") WHERE ");
 	pkgdb_search_build_search_query(sql, match, field, sort);
 	sbuf_cat(sql, ";");
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s'", sbuf_get(sql));
 	ret = sqlite3_prepare_v2(db->sqlite, sbuf_get(sql), -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE));
 }
 int
 pkgdb_integrity_append(struct pkgdb *db, struct pkg *p,
 		conflict_func_cb cb, void *cbdata)

@@ -4073,7 +2575,7 @@ pkgdb_integrity_conflict_local(struct pkgdb *db, const char *uniqueid)

 	sqlite3_bind_text(stmt, 1, uniqueid, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 	return (pkgdb_it_new_sqlite(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 static int

@@ -4209,8 +2711,8 @@ pkgdb_file_set_cksum(struct pkgdb *db, struct pkg_file *file,

  * create our custom functions in the sqlite3 connection.
  * Used both in the shell and pkgdb_open
  */
 static int
 sqlcmd_init(sqlite3 *db, __unused const char **err,
 int
 pkgdb_sqlcmd_init(sqlite3 *db, __unused const char **err,
 	    __unused const void *noused)
 {
 	sqlite3_create_function(db, "now", 0, SQLITE_ANY, NULL,

@@ -4242,7 +2744,7 @@ pkgshell_open(const char **reponame)

 	char		 localpath[MAXPATHLEN];
 	const char	*dbdir;
 	sqlite3_auto_extension((void(*)(void))sqlcmd_init);
 	sqlite3_auto_extension((void(*)(void))pkgdb_sqlcmd_init);
 	dbdir = pkg_object_string(pkg_config_get("PKG_DBDIR"));

@@ -4560,6 +3062,8 @@ pkgdb_stats(struct pkgdb *db, pkg_stats_t type)

 	case PKG_STATS_LOCAL_SIZE:
 		sbuf_printf(sql, "SELECT SUM(flatsize) FROM main.packages;");
 		break;
 	/* TODO: broken now */
 #if 0
 	case PKG_STATS_REMOTE_UNIQUE:
 		sbuf_printf(sql, "SELECT COUNT(c) FROM ");

@@ -4612,6 +3116,7 @@ pkgdb_stats(struct pkgdb *db, pkg_stats_t type)

 		/* close parentheses for the compound statement */
 		sbuf_printf(sql, ");");
 		break;
 #endif
 	}
 	sbuf_finish(sql);

added libpkg/pkgdb_iterator.c

@@ -0,0 +1,1012 @@

 /*-
  * Copyright (c) 2011-2012 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org>
  * Copyright (c) 2011 Will Andrews <will@FreeBSD.org>
  * Copyright (c) 2011 Philippe Pepiot <phil@philpep.org>
  * Copyright (c) 2011-2012 Marin Atanasov Nikolov <dnaeon@gmail.com>
  * Copyright (c) 2012-2013 Matthew Seaman <matthew@FreeBSD.org>
  * Copyright (c) 2012 Bryan Drewery <bryan@shatow.net>
  * Copyright (c) 2013 Gerald Pfeifer <gerald@pfeifer.com>
  * Copyright (c) 2013-2014 Vsevolod Stakhov <vsevolod@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <assert.h>
 #include <errno.h>
 #include <regex.h>
 #include <grp.h>
 #include <libutil.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdbool.h>
 #include <string.h>
 #include <unistd.h>
 #include <signal.h>
 #include <sqlite3.h>
 #include "pkg.h"
 #include "private/event.h"
 #include "private/pkg.h"
 #include "private/pkgdb.h"
 #include "private/utils.h"
 /*
  * Keep entries sorted by name!
  */
 static struct column_mapping {
 	const char * const name;
 	pkg_attr type;
 	enum {
 		PKG_SQLITE_STRING,
 		PKG_SQLITE_INT64,
 		PKG_SQLITE_BOOL
 	} pkg_type;
 } columns[] = {
 	{ "arch",	PKG_ARCH, PKG_SQLITE_STRING },
 	{ "automatic",	PKG_AUTOMATIC, PKG_SQLITE_BOOL },
 	{ "cksum",	PKG_CKSUM, PKG_SQLITE_STRING },
 	{ "comment",	PKG_COMMENT, PKG_SQLITE_STRING },
 	{ "dbname",	PKG_REPONAME, PKG_SQLITE_STRING },
 	{ "desc",	PKG_DESC, PKG_SQLITE_STRING },
 	{ "flatsize",	PKG_FLATSIZE, PKG_SQLITE_INT64 },
 	{ "id",		PKG_ROWID, PKG_SQLITE_INT64 },
 	{ "licenselogic", PKG_LICENSE_LOGIC, PKG_SQLITE_INT64 },
 	{ "locked",	PKG_LOCKED, PKG_SQLITE_BOOL },
 	{ "maintainer",	PKG_MAINTAINER, PKG_SQLITE_STRING },
 	{ "manifestdigest",	PKG_DIGEST, PKG_SQLITE_STRING },
 	{ "message",	PKG_MESSAGE, PKG_SQLITE_STRING },
 	{ "name",	PKG_NAME, PKG_SQLITE_STRING },
 	{ "oldflatsize", PKG_OLD_FLATSIZE, PKG_SQLITE_INT64 },
 	{ "oldversion",	PKG_OLD_VERSION, PKG_SQLITE_STRING },
 	{ "origin",	PKG_ORIGIN, PKG_SQLITE_STRING },
 	{ "pkgsize",	PKG_PKGSIZE, PKG_SQLITE_INT64 },
 	{ "prefix",	PKG_PREFIX, PKG_SQLITE_STRING },
 	{ "repopath",	PKG_REPOPATH, PKG_SQLITE_STRING },
 	{ "rowid",	PKG_ROWID, PKG_SQLITE_INT64 },
 	{ "time",	PKG_TIME, PKG_SQLITE_INT64 },
 	{ "uniqueid",	PKG_UNIQUEID, PKG_SQLITE_STRING },
 	{ "version",	PKG_VERSION, PKG_SQLITE_STRING },
 	{ "weight",	-1, PKG_SQLITE_INT64 },
 	{ "www",	PKG_WWW, PKG_SQLITE_STRING },
 	{ NULL,		-1, PKG_SQLITE_STRING }
 };
 static int
 load_val(sqlite3 *db, struct pkg *pkg, const char *sql, unsigned flags,
     int (*pkg_adddata)(struct pkg *pkg, const char *data), int list)
 {
 	sqlite3_stmt	*stmt;
 	int		 ret;
 	int64_t		 rowid;
 	assert(db != NULL && pkg != NULL);
 	if (pkg->flags & flags)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_adddata(pkg, sqlite3_column_text(stmt, 0));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		if (list != -1)
 			pkg_list_free(pkg, list);
 		ERROR_SQLITE(db, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= flags;
 	return (EPKG_OK);
 }
 static int
 load_tag_val(sqlite3 *db, struct pkg *pkg, const char *sql, unsigned flags,
 	     int (*pkg_addtagval)(struct pkg *pkg, const char *tag, const char *val),
 	     int list)
 {
 	sqlite3_stmt	*stmt;
 	int		 ret;
 	int64_t		 rowid;
 	assert(db != NULL && pkg != NULL);
 	if (pkg->flags & flags)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_addtagval(pkg, sqlite3_column_text(stmt, 0),
 			      sqlite3_column_text(stmt, 1));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		if (list != -1)
 			pkg_list_free(pkg, list);
 		ERROR_SQLITE(db, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= flags;
 	return (EPKG_OK);
 }
 static int
 compare_column_func(const void *pkey, const void *pcolumn)
 {
 	const char *key = (const char*)pkey;
 	const struct column_mapping *column =
 			(const struct column_mapping*)pcolumn;
 	return strcmp(key, column->name);
 }
 static int
 pkgdb_load_deps(sqlite3 *sqlite, struct pkg *pkg)
 {
 	sqlite3_stmt	*stmt = NULL;
 	int		 ret = EPKG_OK;
 	int64_t		 rowid;
 	char		 sql[BUFSIZ];
 	const char	*mainsql = ""
 		"SELECT d.name, d.origin, d.version, 0 "
 		"FROM main.deps AS d "
 		"LEFT JOIN main.packages AS p ON p.origin = d.origin "
 		"AND p.name = d.name "
 		"WHERE d.package_id = ?1 ORDER BY d.origin DESC;";
 	assert(pkg != NULL);
 	if (pkg->flags & PKG_LOAD_DEPS)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", mainsql);
 	ret = sqlite3_prepare_v2(sqlite, mainsql, -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	/* XXX: why we used locked here ? */
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_adddep(pkg, sqlite3_column_text(stmt, 0),
 			   sqlite3_column_text(stmt, 1),
 			   sqlite3_column_text(stmt, 2),
 			   sqlite3_column_int(stmt, 3));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		pkg_list_free(pkg, PKG_DEPS);
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_DEPS;
 	return (EPKG_OK);
 }
 static int
 pkgdb_load_rdeps(sqlite3 *sqlite, struct pkg *pkg)
 {
 	sqlite3_stmt	*stmt = NULL;
 	int		 ret;
 	const char	*uniqueid;
 	const char	*mainsql = ""
 		"SELECT p.name, p.origin, p.version, 0 "
 		"FROM main.packages AS p "
 		"INNER JOIN main.deps AS d ON p.id = d.package_id "
 		"WHERE d.name = SPLIT_UID('name', ?1) AND "
 		"d.origin = SPLIT_UID('origin', ?1);";
 	assert(pkg != NULL);
 	if (pkg->flags & PKG_LOAD_RDEPS)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", mainsql);
 	ret = sqlite3_prepare_v2(sqlite, mainsql, -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, mainsql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_UNIQUEID, &uniqueid);
 	sqlite3_bind_text(stmt, 1, uniqueid, -1, SQLITE_STATIC);
 	/* XXX: why we used locked here ? */
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_addrdep(pkg, sqlite3_column_text(stmt, 0),
 			    sqlite3_column_text(stmt, 1),
 			    sqlite3_column_text(stmt, 2),
 			    sqlite3_column_int(stmt, 3));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		pkg_list_free(pkg, PKG_RDEPS);
 		ERROR_SQLITE(sqlite, mainsql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_RDEPS;
 	return (EPKG_OK);
 }
 static int
 pkgdb_load_files(sqlite3 *sqlite, struct pkg *pkg)
 {
 	sqlite3_stmt	*stmt = NULL;
 	int		 ret;
 	int64_t		 rowid;
 	const char	 sql[] = ""
 		"SELECT path, sha256 "
 		"FROM files "
 		"WHERE package_id = ?1 "
 		"ORDER BY PATH ASC";
 	assert( pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	if (pkg->flags & PKG_LOAD_FILES)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_addfile(pkg, sqlite3_column_text(stmt, 0),
 		    sqlite3_column_text(stmt, 1), false);
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		pkg_list_free(pkg, PKG_FILES);
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_FILES;
 	return (EPKG_OK);
 }
 static int
 pkgdb_load_dirs(sqlite3 *sqlite, struct pkg *pkg)
 {
 	const char	 sql[] = ""
 		"SELECT path, try "
 		"FROM pkg_directories, directories "
 		"WHERE package_id = ?1 "
 		"AND directory_id = directories.id "
 		"ORDER by path DESC";
 	sqlite3_stmt	*stmt;
 	int		 ret;
 	int64_t		 rowid;
 	assert(pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	if (pkg->flags & PKG_LOAD_DIRS)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_adddir(pkg, sqlite3_column_text(stmt, 0),
 		    sqlite3_column_int(stmt, 1), false);
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		pkg_list_free(pkg, PKG_DIRS);
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_DIRS;
 	return (EPKG_OK);
 }
 static int
 pkgdb_load_license(sqlite3 *sqlite, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*basesql = ""
 		"SELECT name "
 		"FROM %Q.pkg_licenses, %Q.licenses AS l "
 		"WHERE package_id = ?1 "
 			"AND license_id = l.id "
 		"ORDER by name DESC";
 	assert(pkg != NULL);
 	sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	return (load_val(sqlite, pkg, sql, PKG_LOAD_LICENSES,
 	    pkg_addlicense, PKG_LICENSES));
 }
 static int
 pkgdb_load_category(sqlite3 *sqlite, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*basesql = ""
 		"SELECT name "
 		"FROM %Q.pkg_categories, %Q.categories AS c "
 		"WHERE package_id = ?1 "
 			"AND category_id = c.id "
 		"ORDER by name DESC";
 	assert(pkg != NULL);
 	sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	return (load_val(sqlite, pkg, sql, PKG_LOAD_CATEGORIES,
 	    pkg_addcategory, PKG_CATEGORIES));
 }
 static int
 pkgdb_load_user(sqlite3 *sqlite, struct pkg *pkg)
 {
 	/*struct pkg_user *u = NULL;
 	struct passwd *pwd = NULL;*/
 	int		ret;
 	const char	sql[] = ""
 		"SELECT users.name "
 		"FROM pkg_users, users "
 		"WHERE package_id = ?1 "
 			"AND user_id = users.id "
 		"ORDER by name DESC";
 	assert(pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	ret = load_val(sqlite, pkg, sql, PKG_LOAD_USERS,
 	    pkg_adduser, PKG_USERS);
 	/* TODO get user uidstr from local database */
 /*	while (pkg_users(pkg, &u) == EPKG_OK) {
 		pwd = getpwnam(pkg_user_name(u));
 		if (pwd == NULL)
 			continue;
 		strlcpy(u->uidstr, pw_make(pwd), sizeof(u->uidstr));
 	}*/
 	return (ret);
 }
 static int
 pkgdb_load_group(sqlite3 *sqlite, struct pkg *pkg)
 {
 	struct pkg_group	*g = NULL;
 	struct group		*grp = NULL;
 	int			 ret;
 	const char		 sql[] = ""
 		"SELECT groups.name "
 		"FROM pkg_groups, groups "
 		"WHERE package_id = ?1 "
 			"AND group_id = groups.id "
 		"ORDER by name DESC";
 	assert(pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	ret = load_val(sqlite, pkg, sql, PKG_LOAD_GROUPS,
 	    pkg_addgroup, PKG_GROUPS);
 	while (pkg_groups(pkg, &g) == EPKG_OK) {
 		grp = getgrnam(pkg_group_name(g));
 		if (grp == NULL)
 			continue;
 		strlcpy(g->gidstr, gr_make(grp), sizeof(g->gidstr));
 	}
 	return (ret);
 }
 static int
 pkgdb_load_shlib_required(sqlite3 *sqlite, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*basesql = ""
 		"SELECT name "
 		"FROM %Q.pkg_shlibs_required, %Q.shlibs AS s "
 		"WHERE package_id = ?1 "
 			"AND shlib_id = s.id "
 		"ORDER by name DESC";
 	assert(pkg != NULL);
 	sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	return (load_val(sqlite, pkg, sql, PKG_LOAD_SHLIBS_REQUIRED,
 	    pkg_addshlib_required, PKG_SHLIBS_REQUIRED));
 }
 static int
 pkgdb_load_shlib_provided(sqlite3 *sqlite, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*basesql = ""
 		"SELECT name "
 		"FROM %Q.pkg_shlibs_provided, %Q.shlibs AS s "
 		"WHERE package_id = ?1 "
 			"AND shlib_id = s.id "
 		"ORDER by name DESC";
 	assert(pkg != NULL);
 	sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	return (load_val(sqlite, pkg, sql, PKG_LOAD_SHLIBS_PROVIDED,
 	    pkg_addshlib_provided, PKG_SHLIBS_PROVIDED));
 }
 static int
 pkgdb_load_annotations(sqlite3 *sqlite, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*basesql = ""
 		"SELECT k.annotation AS tag, v.annotation AS value"
 		"  FROM %Q.pkg_annotation p"
 		"    JOIN %Q.annotation k ON (p.tag_id = k.annotation_id)"
 		"    JOIN %Q.annotation v ON (p.value_id = v.annotation_id)"
 		"  WHERE p.package_id = ?1"
 		"  ORDER BY tag, value";
 	sqlite3_snprintf(sizeof(sql), sql, basesql, "main",
                     "main", "main");
 	return (load_tag_val(sqlite, pkg, sql, PKG_LOAD_ANNOTATIONS,
 		   pkg_addannotation, PKG_ANNOTATIONS));
 }
 static int
 pkgdb_load_scripts(sqlite3 *sqlite, struct pkg *pkg)
 {
 	sqlite3_stmt	*stmt = NULL;
 	int		 ret;
 	int64_t		 rowid;
 	const char	 sql[] = ""
 		"SELECT script, type "
 		"FROM pkg_script JOIN script USING(script_id) "
 		"WHERE package_id = ?1";
 	assert(pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	if (pkg->flags & PKG_LOAD_SCRIPTS)
 		return (EPKG_OK);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg_get(pkg, PKG_ROWID, &rowid);
 	sqlite3_bind_int64(stmt, 1, rowid);
 	while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
 		pkg_addscript(pkg, sqlite3_column_text(stmt, 0),
 		    sqlite3_column_int(stmt, 1));
 	}
 	sqlite3_finalize(stmt);
 	if (ret != SQLITE_DONE) {
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	pkg->flags |= PKG_LOAD_SCRIPTS;
 	return (EPKG_OK);
 }
 static int
 pkgdb_load_options(sqlite3 *sqlite, struct pkg *pkg)
 {
 	const char	*reponame;
 	char		 sql[BUFSIZ];
 	unsigned int	 i;
 	struct optionsql {
 		const char	 *sql;
 		int		(*pkg_addtagval)(struct pkg *pkg,
 						  const char *tag,
 						  const char *val);
 		int		  nargs;
 	}			  optionsql[] = {
 		{
 			"SELECT option, value "
 			"FROM %Q.option JOIN %Q.pkg_option USING(option_id) "
 			"WHERE package_id = ?1 ORDER BY option",
 			pkg_addoption,
 ,
 		},
 		{
 			"SELECT option, default_value "
 			"FROM %Q.option JOIN %Q.pkg_option_default USING(option_id) "
 			"WHERE package_id = ?1 ORDER BY option",
 			pkg_addoption_default,
 ,
 		},
 		{
 			"SELECT option, description "
 			"FROM %Q.option JOIN %Q.pkg_option_desc USING(option_id) "
 			"JOIN %Q.option_desc USING(option_desc_id) ORDER BY option",
 			pkg_addoption_description,
 ,
 		}
 	};
 	const char		 *opt_sql;
 	int			(*pkg_addtagval)(struct pkg *pkg,
 						 const char *tag,
 						 const char *val);
 	int			  nargs, ret;
 	assert(pkg != NULL);
 	if (pkg->flags & PKG_LOAD_OPTIONS)
 		return (EPKG_OK);
 	reponame = "main";
 	for (i = 0; i < NELEM(optionsql); i++) {
 		opt_sql       = optionsql[i].sql;
 		pkg_addtagval = optionsql[i].pkg_addtagval;
 		nargs         = optionsql[i].nargs;
 		switch(nargs) {
 		case 1:
 			sqlite3_snprintf(sizeof(sql), sql, opt_sql, reponame);
 			break;
 		case 2:
 			sqlite3_snprintf(sizeof(sql), sql, opt_sql, reponame,
 					 reponame);
 			break;
 		case 3:
 			sqlite3_snprintf(sizeof(sql), sql, opt_sql, reponame,
 					 reponame, reponame);
 			break;
 		default:
 			/* Nothing needs 4 or more, yet... */
 			return (EPKG_FATAL);
 			break;
 		}
 		pkg_debug(4, "Pkgdb> adding option");
 		ret = load_tag_val(sqlite, pkg, sql, PKG_LOAD_OPTIONS,
 				   pkg_addtagval, PKG_OPTIONS);
 		if (ret != EPKG_OK)
 			break;
 	}
 	return (ret);
 }
 static int
 pkgdb_load_mtree(sqlite3 *sqlite, struct pkg *pkg)
 {
 	const char	sql[] = ""
 		"SELECT m.content "
 		"FROM mtree AS m, packages AS p "
 		"WHERE m.id = p.mtree_id "
 			"AND p.id = ?1;";
 	assert(pkg != NULL);
 	assert(pkg->type == PKG_INSTALLED);
 	return (load_val(sqlite, pkg, sql, PKG_LOAD_MTREE, pkg_set_mtree, -1));
 }
 static int
 pkgdb_load_conflicts(sqlite3 *sqlite, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*basesql = ""
 			"SELECT packages.origin "
 			"FROM %Q.pkg_conflicts "
 			"LEFT JOIN %Q.packages ON "
 			"packages.id = pkg_conflicts.conflict_id "
 			"WHERE package_id = ?1";
 	assert(pkg != NULL);
 	sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	return (load_val(sqlite, pkg, sql, PKG_LOAD_CONFLICTS,
 			pkg_addconflict, PKG_CONFLICTS));
 }
 static int
 pkgdb_load_provides(sqlite3 *sqlite, struct pkg *pkg)
 {
 	char		 sql[BUFSIZ];
 	const char	*basesql = ""
 		"SELECT provide "
 		"FROM %Q.provides "
 		"WHERE package_id = ?1";
 	assert(pkg != NULL);
 	sqlite3_snprintf(sizeof(sql), sql, basesql, "main", "main");
 	return (load_val(sqlite, pkg, sql, PKG_LOAD_PROVIDES,
 			pkg_addconflict, PKG_PROVIDES));
 }
 static void
 populate_pkg(sqlite3_stmt *stmt, struct pkg *pkg) {
 	int		 icol = 0;
 	const char	*colname;
 	assert(stmt != NULL);
 	for (icol = 0; icol < sqlite3_column_count(stmt); icol++) {
 		colname = sqlite3_column_name(stmt, icol);
 		struct column_mapping *column;
 		switch (sqlite3_column_type(stmt, icol)) {
 		case SQLITE_TEXT:
 			column = bsearch(colname, columns, NELEM(columns) - 1,
 					sizeof(columns[0]), compare_column_func);
 			if (column == NULL) {
 				pkg_emit_error("unknown column %s", colname);
 			}
 			else {
 				if (column->pkg_type == PKG_SQLITE_STRING)
 					pkg_set(pkg, column->type,
 						sqlite3_column_text(stmt, icol));
 				else
 					pkg_emit_error("want string for column %s and got number",
 							colname);
 			}
 			break;
 		case SQLITE_INTEGER:
 			column = bsearch(colname, columns, NELEM(columns) - 1,
 					sizeof(columns[0]), compare_column_func);
 			if (column == NULL) {
 				pkg_emit_error("Unknown column %s", colname);
 			}
 			else {
 				if (column->pkg_type == PKG_SQLITE_INT64)
 					pkg_set(pkg, column->type,
 						sqlite3_column_int64(stmt, icol));
 				else if (column->pkg_type == PKG_SQLITE_BOOL)
 					pkg_set(pkg, column->type,
 							(bool)sqlite3_column_int(stmt, icol));
 				else
 					pkg_emit_error("want number for column %s and got string",
 							colname);
 			}
 			break;
 		case SQLITE_BLOB:
 		case SQLITE_FLOAT:
 			pkg_emit_error("wrong type for column: %s",
 			    colname);
 			/* just ignore currently */
 			break;
 		case SQLITE_NULL:
 			break;
 		}
 	}
 }
 static struct load_on_flag {
 	int	flag;
 	int	(*load)(sqlite3 *sqlite, struct pkg *p);
 } load_on_flag[] = {
 	{ PKG_LOAD_DEPS,		pkgdb_load_deps },
 	{ PKG_LOAD_RDEPS,		pkgdb_load_rdeps },
 	{ PKG_LOAD_FILES,		pkgdb_load_files },
 	{ PKG_LOAD_DIRS,		pkgdb_load_dirs },
 	{ PKG_LOAD_SCRIPTS,		pkgdb_load_scripts },
 	{ PKG_LOAD_OPTIONS,		pkgdb_load_options },
 	{ PKG_LOAD_MTREE,		pkgdb_load_mtree },
 	{ PKG_LOAD_CATEGORIES,		pkgdb_load_category },
 	{ PKG_LOAD_LICENSES,		pkgdb_load_license },
 	{ PKG_LOAD_USERS,		pkgdb_load_user },
 	{ PKG_LOAD_GROUPS,		pkgdb_load_group },
 	{ PKG_LOAD_SHLIBS_REQUIRED,	pkgdb_load_shlib_required },
 	{ PKG_LOAD_SHLIBS_PROVIDED,	pkgdb_load_shlib_provided },
 	{ PKG_LOAD_ANNOTATIONS,		pkgdb_load_annotations },
 	{ PKG_LOAD_CONFLICTS,		pkgdb_load_conflicts },
 	{ PKG_LOAD_PROVIDES,		pkgdb_load_provides },
 	{ -1,			        NULL }
 };
 static void
 pkgdb_sqlite_it_reset(struct pkgdb_sqlite_it *it)
 {
 	if (it == NULL)
 		return;
 	it->finished = 0;
 	sqlite3_reset(it->stmt);
 }
 static void
 pkgdb_sqlite_it_free(struct pkgdb_sqlite_it *it)
 {
 	if (it == NULL)
 		return;
 	sqlite3_finalize(it->stmt);
 }
 static int
 pkgdb_sqlite_it_next(struct pkgdb_sqlite_it *it,
 	struct pkg **pkg_p, unsigned flags)
 {
 	struct pkg	*pkg;
 	int		 i;
 	int		 ret;
 	const char *digest;
 	assert(it != NULL);
 	/*
 	 * XXX:
 	 * Currently, we have a lot of issues related to pkg digests.
 	 * So we want to ensure that we always have a valid package digest
 	 * even if we work with pkg 1.2 repo. Therefore, we explicitly check
 	 * manifest digests and set it to NULL if it is invalid.
 	 *
 	 */
 	if (it->finished && (it->flags & PKGDB_IT_FLAG_ONCE))
 		return (EPKG_END);
 	switch (sqlite3_step(it->stmt)) {
 	case SQLITE_ROW:
 		if (*pkg_p == NULL) {
 			ret = pkg_new(pkg_p, it->pkg_type);
 			if (ret != EPKG_OK)
 				return (ret);
 		} else
 			pkg_reset(*pkg_p, it->pkg_type);
 		pkg = *pkg_p;
 		populate_pkg(it->stmt, pkg);
 		pkg_get(pkg, PKG_DIGEST, &digest);
 		if (digest != NULL && !pkg_checksum_is_valid(digest, strlen(digest)))
 			pkg_set(pkg, PKG_DIGEST, NULL);
 		for (i = 0; load_on_flag[i].load != NULL; i++) {
 			if (flags & load_on_flag[i].flag) {
 				if (it->sqlite != NULL) {
 					ret = load_on_flag[i].load(it->sqlite, pkg);
 					if (ret != EPKG_OK)
 						return (ret);
 				}
 				else {
 					pkg_emit_error("invalid iterator passed to pkgdb_it_next");
 					return (EPKG_FATAL);
 				}
 			}
 		}
 		return (EPKG_OK);
 	case SQLITE_DONE:
 		it->finished ++;
 		if (it->flags & PKGDB_IT_FLAG_CYCLED) {
 			sqlite3_reset(it->stmt);
 			return (EPKG_OK);
 		}
 		else {
 			if (it->flags & PKGDB_IT_FLAG_AUTO)
 				pkgdb_sqlite_it_free(it);
 			return (EPKG_END);
 		}
 		break;
 	default:
 		ERROR_SQLITE(it->sqlite, "iterator");
 		return (EPKG_FATAL);
 	}
 }
 int
 pkgdb_it_next(struct pkgdb_it *it, struct pkg **pkg_p, unsigned flags)
 {
 	struct pkg_repo_it *rit;
 	assert(it != NULL);
 	switch (it->type) {
 	case PKGDB_IT_LOCAL:
 		return (pkgdb_sqlite_it_next(&it->un.local, pkg_p, flags));
 		break;
 	case PKGDB_IT_REPO:
 		if (it->un.remote != NULL) {
 			rit = it->un.remote->it;
 			if (rit->ops->next(rit, pkg_p, flags) != EPKG_OK) {
 				/*
 				 * Detach this iterator from list and switch to another
 				 */
 				struct _pkg_repo_it_set *tmp;
 				rit->ops->free(rit);
 				tmp = it->un.remote;
 				it->un.remote = tmp->next;
 				free(tmp);
 				return (pkgdb_it_next(it, pkg_p, flags));
 			}
 			return (EPKG_OK);
 		}
 		/*
 		 * All done
 		 */
 		return (EPKG_END);
 		break;
 	}
 	return (EPKG_FATAL);
 }
 void
 pkgdb_it_reset(struct pkgdb_it *it)
 {
 	struct _pkg_repo_it_set *cur;
 	assert(it != NULL);
 	switch (it->type) {
 		case PKGDB_IT_LOCAL:
 			pkgdb_sqlite_it_reset(&it->un.local);
 			break;
 		case PKGDB_IT_REPO:
 			LL_FOREACH(it->un.remote, cur) {
 				cur->it->ops->reset(cur->it);
 			}
 			break;
 	}
 }
 void
 pkgdb_it_free(struct pkgdb_it *it)
 {
 	struct _pkg_repo_it_set *cur, *tmp;
 	if (it == NULL)
 		return;
 	switch (it->type) {
 		case PKGDB_IT_LOCAL:
 			pkgdb_sqlite_it_free(&it->un.local);
 			break;
 		case PKGDB_IT_REPO:
 			LL_FOREACH_SAFE(it->un.remote, cur, tmp) {
 				cur->it->ops->free(cur->it);
 				free(cur);
 			}
 			break;
 	}
 	free(it);
 }
 struct pkgdb_it *
 pkgdb_it_new_sqlite(struct pkgdb *db, sqlite3_stmt *s, int type, short flags)
 {
 	struct pkgdb_it	*it;
 	assert(db != NULL && s != NULL);
 	assert(!(flags & (PKGDB_IT_FLAG_CYCLED & PKGDB_IT_FLAG_ONCE)));
 	assert(!(flags & (PKGDB_IT_FLAG_AUTO & (PKGDB_IT_FLAG_CYCLED | PKGDB_IT_FLAG_ONCE))));
 	if ((it = malloc(sizeof(struct pkgdb_it))) == NULL) {
 		pkg_emit_errno("malloc", "pkgdb_it");
 		sqlite3_finalize(s);
 		return (NULL);
 	}
 	it->type = PKGDB_IT_LOCAL;
 	it->db = db;
 	it->un.local.sqlite = db->sqlite;
 	it->un.local.stmt = s;
 	it->un.local.pkg_type = type;
 	it->un.local.flags = flags;
 	it->un.local.finished = 0;
 	return (it);
 }
 struct pkgdb_it *
 pkgdb_it_new_repo(struct pkgdb *db)
 {
 	struct pkgdb_it	*it;
 	if ((it = malloc(sizeof(struct pkgdb_it))) == NULL) {
 		pkg_emit_errno("malloc", "pkgdb_it");
 		return (NULL);
 	}
 	it->type = PKGDB_IT_REPO;
 	it->db = db;
 	it->un.remote = NULL;
 	return (it);
 }
 void
 pkgdb_it_repo_attach(struct pkgdb_it *it, struct pkg_repo_it *rit)
 {
 	struct _pkg_repo_it_set *item;
 	if ((item = malloc(sizeof(struct _pkg_repo_it_set))) == NULL) {
 		pkg_emit_errno("malloc", "_pkg_repo_it_set");
 	}
 	else {
 		item->it = rit;
 		LL_PREPEND(it->un.remote, item);
 	}
 }

added libpkg/pkgdb_query.c

@@ -0,0 +1,349 @@

 /*-
  * Copyright (c) 2011-2012 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org>
  * Copyright (c) 2011 Will Andrews <will@FreeBSD.org>
  * Copyright (c) 2011 Philippe Pepiot <phil@philpep.org>
  * Copyright (c) 2011-2012 Marin Atanasov Nikolov <dnaeon@gmail.com>
  * Copyright (c) 2012-2013 Matthew Seaman <matthew@FreeBSD.org>
  * Copyright (c) 2012 Bryan Drewery <bryan@shatow.net>
  * Copyright (c) 2013 Gerald Pfeifer <gerald@pfeifer.com>
  * Copyright (c) 2013-2014 Vsevolod Stakhov <vsevolod@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <assert.h>
 #include <errno.h>
 #include <regex.h>
 #include <grp.h>
 #include <libutil.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdbool.h>
 #include <string.h>
 #include <unistd.h>
 #include <signal.h>
 #include <sqlite3.h>
 #include "pkg.h"
 #include "private/event.h"
 #include "private/pkg.h"
 #include "private/pkgdb.h"
 #include "private/utils.h"
 const char *
 pkgdb_get_pattern_query(const char *pattern, match_t match)
 {
 	char		*checkorigin = NULL;
 	char		*checkuid = NULL;
 	const char	*comp = NULL;
 	if (pattern != NULL) {
 		checkuid = strchr(pattern, '~');
 		if (checkuid == NULL)
 			checkorigin = strchr(pattern, '/');
 	}
 	switch (match) {
 	case MATCH_ALL:
 		comp = "";
 		break;
 	case MATCH_EXACT:
 		if (pkgdb_case_sensitive()) {
 			if (checkuid == NULL) {
 				if (checkorigin == NULL)
 					comp = " WHERE name = ?1 "
 					    "OR (name = SPLIT_VERSION('name', ?1) AND "
 					    " version = SPLIT_VERSION('version', ?1))";
 				else
 					comp = " WHERE origin = ?1";
 			} else {
 				comp = " WHERE name = SPLIT_UID('name', ?1) AND "
 						"origin = SPLIT_UID('origin', ?1)";
 			}
 		} else {
 			if (checkuid == NULL) {
 				if (checkorigin == NULL)
 					comp = " WHERE name = ?1 COLLATE NOCASE "
 							"OR (name = SPLIT_VERSION('name', ?1) COLLATE NOCASE AND "
 							" version = SPLIT_VERSION('version', ?1))";
 				else
 					comp = " WHERE origin = ?1 COLLATE NOCASE";
 			} else {
 				comp = " WHERE name = SPLIT_UID('name', ?1) COLLATE NOCASE AND "
 						"origin = SPLIT_UID('origin', ?1) COLLATE NOCASE";
 			}
 		}
 		break;
 	case MATCH_GLOB:
 		if (checkuid == NULL) {
 			if (checkorigin == NULL)
 				comp = " WHERE name GLOB ?1 "
 					"OR (name GLOB SPLIT_VERSION('name', ?1) AND "
 					" version GLOB SPLIT_VERSION('version', ?1))";
 			else
 				comp = " WHERE origin GLOB ?1";
 		} else {
 			comp = " WHERE name = SPLIT_UID('name', ?1) AND "
 					"origin = SPLIT_UID('origin', ?1)";
 		}
 		break;
 	case MATCH_REGEX:
 		if (checkuid == NULL) {
 			if (checkorigin == NULL)
 				comp = " WHERE name REGEXP ?1 "
 				    "OR name || '-' || version REGEXP ?1";
 			else
 				comp = " WHERE origin REGEXP ?1";
 		} else {
 			comp = " WHERE name = SPLIT_UID('name', ?1) AND "
 					"origin = SPLIT_UID('origin', ?1)";
 		}
 		break;
 	case MATCH_CONDITION:
 		comp = pattern;
 		break;
 	case MATCH_FTS:
 		if (checkorigin == NULL)
 			comp = " WHERE id IN (SELECT id FROM pkg_search WHERE name MATCH ?1)";
 		else
 			comp = " WHERE id IN (SELECT id FROM pkg_search WHERE origin MATCH ?1)";
 		break;
 	}
 	return (comp);
 }
 struct pkgdb_it *
 pkgdb_query(struct pkgdb *db, const char *pattern, match_t match)
 {
 	char		 sql[BUFSIZ];
 	sqlite3_stmt	*stmt;
 	const char	*comp = NULL;
 	assert(db != NULL);
 	assert(match == MATCH_ALL || (pattern != NULL && pattern[0] != '\0'));
 	comp = pkgdb_get_pattern_query(pattern, match);
 	sqlite3_snprintf(sizeof(sql), sql,
 			"SELECT id, origin, name, name || '~' || origin as uniqueid, "
 				"version, comment, desc, "
 				"message, arch, maintainer, www, "
 				"prefix, flatsize, licenselogic, automatic, "
 				"locked, time, manifestdigest "
 			"FROM packages AS p%s "
 			"ORDER BY p.name;", comp);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (NULL);
 	}
 	if (match != MATCH_ALL && match != MATCH_CONDITION)
 		sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new_sqlite(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_query_which(struct pkgdb *db, const char *path, bool glob)
 {
 	sqlite3_stmt	*stmt;
 	char	sql[BUFSIZ];
 	assert(db != NULL);
 	sqlite3_snprintf(sizeof(sql), sql,
 			"SELECT p.id, p.origin, p.name, p.name || '~' || p.origin as uniqueid, "
 			"p.version, p.comment, p.desc, "
 			"p.message, p.arch, p.maintainer, p.www, "
 			"p.prefix, p.flatsize, p.time "
 			"FROM packages AS p "
 			"LEFT JOIN files AS f ON p.id = f.package_id "
 			"WHERE f.path %s ?1 GROUP BY p.id;", glob ? "GLOB" : "=");
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (NULL);
 	}
 	sqlite3_bind_text(stmt, 1, path, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new_sqlite(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_query_shlib_require(struct pkgdb *db, const char *shlib)
 {
 	sqlite3_stmt	*stmt;
 	const char	 sql[] = ""
 		"SELECT p.id, p.origin, p.name, p.name || '~' || p.origin as uniqueid, "
 			"p.version, p.comment, p.desc, "
 			"p.message, p.arch, p.maintainer, p.www, "
 			"p.prefix, p.flatsize, p.time "
 			"FROM packages AS p, pkg_shlibs_required AS ps, shlibs AS s "
 			"WHERE p.id = ps.package_id "
 				"AND ps.shlib_id = s.id "
 				"AND s.name = ?1;";
 	assert(db != NULL);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (NULL);
 	}
 	sqlite3_bind_text(stmt, 1, shlib, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new_sqlite(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_query_shlib_provide(struct pkgdb *db, const char *shlib)
 {
 	sqlite3_stmt	*stmt;
 	const char	 sql[] = ""
 		"SELECT p.id, p.origin, p.name, p.name || '~' || p.origin as uniqueid, "
 			"p.version, p.comment, p.desc, "
 			"p.message, p.arch, p.maintainer, p.www, "
 			"p.prefix, p.flatsize, p.time "
 			"FROM packages AS p, pkg_shlibs_provided AS ps, shlibs AS s "
 			"WHERE p.id = ps.package_id "
 				"AND ps.shlib_id = s.id "
 				"AND s.name = ?1;";
 	assert(db != NULL);
 	pkg_debug(4, "Pkgdb: running '%s'", sql);
 	if (sqlite3_prepare_v2(db->sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sql);
 		return (NULL);
 	}
 	sqlite3_bind_text(stmt, 1, shlib, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new_sqlite(db, stmt, PKG_INSTALLED, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_repo_query(struct pkgdb *db, const char *pattern, match_t match,
     const char *repo)
 {
 	struct pkgdb_it *it;
 	struct pkg_repo_it *rit;
 	struct _pkg_repo_list_item *cur;
 	it = pkgdb_it_new_repo(db);
 	if (it == NULL)
 		return (NULL);
 	LL_FOREACH(db->repos, cur) {
 		if (repo == NULL || strcasecmp(cur->repo->name, repo) == 0) {
 			rit = cur->repo->ops->query(cur->repo, pattern, match);
 			if (rit != NULL)
 				pkgdb_it_repo_attach(it, rit);
 		}
 	}
 	return (it);
 }
 struct pkgdb_it *
 pkgdb_repo_shlib_require(struct pkgdb *db, const char *require, const char *repo)
 {
 	struct pkgdb_it *it;
 	struct pkg_repo_it *rit;
 	struct _pkg_repo_list_item *cur;
 	it = pkgdb_it_new_repo(db);
 	if (it == NULL)
 		return (NULL);
 	LL_FOREACH(db->repos, cur) {
 		if (repo == NULL || strcasecmp(cur->repo->name, repo) == 0) {
 			if (cur->repo->ops->shlib_required != NULL) {
 				rit = cur->repo->ops->shlib_required(cur->repo, require);
 				if (rit != NULL)
 					pkgdb_it_repo_attach(it, rit);
 			}
 		}
 	}
 	return (it);
 }
 struct pkgdb_it *
 pkgdb_repo_shlib_provide(struct pkgdb *db, const char *require, const char *repo)
 {
 	struct pkgdb_it *it;
 	struct pkg_repo_it *rit;
 	struct _pkg_repo_list_item *cur;
 	it = pkgdb_it_new_repo(db);
 	if (it == NULL)
 		return (NULL);
 	LL_FOREACH(db->repos, cur) {
 		if (repo == NULL || strcasecmp(cur->repo->name, repo) == 0) {
 			if (cur->repo->ops->shlib_required != NULL) {
 				rit = cur->repo->ops->shlib_provided(cur->repo, require);
 				if (rit != NULL)
 					pkgdb_it_repo_attach(it, rit);
 			}
 		}
 	}
 	return (it);
 }
 struct pkgdb_it *
 pkgdb_repo_search(struct pkgdb *db, const char *pattern, match_t match,
     pkgdb_field field, pkgdb_field sort, const char *repo)
 {
 	struct pkgdb_it *it;
 	struct pkg_repo_it *rit;
 	struct _pkg_repo_list_item *cur;
 	it = pkgdb_it_new_repo(db);
 	if (it == NULL)
 		return (NULL);
 	LL_FOREACH(db->repos, cur) {
 		if (repo == NULL || strcasecmp(cur->repo->name, repo) == 0) {
 			if (cur->repo->ops->search != NULL) {
 				rit = cur->repo->ops->search(cur->repo, pattern, match,
 					field, sort);
 				if (rit != NULL)
 					pkgdb_it_repo_attach(it, rit);
 			}
 		}
 	}
 	return (it);
 }

deleted libpkg/pkgdb_repo.c

@@ -1,1277 +0,0 @@

 /*-
  * Copyright (c) 2013 Vsevolod Stakhov <vsevolod@FreeBSD.org>
  * Copyright (c) 2011-2012 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2011-2012 Julien Laffaye <jlaffaye@FreeBSD.org>
  * Copyright (c) 2011 Will Andrews <will@FreeBSD.org>
  * Copyright (c) 2011 Philippe Pepiot <phil@philpep.org>
  * Copyright (c) 2011-2012 Marin Atanasov Nikolov <dnaeon@gmail.com>
  * Copyright (c) 2012-2013 Matthew Seaman <matthew@FreeBSD.org>
  * Copyright (c) 2012 Bryan Drewery <bryan@shatow.net>
  * Copyright (c) 2013 Gerald Pfeifer <gerald@pfeifer.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer
  *    in this position and unchanged.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/param.h>
 #include <assert.h>
 #include <errno.h>
 #include <regex.h>
 #include <grp.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdbool.h>
 #include <string.h>
 #include <unistd.h>
 #include <libgen.h>
 #include <sqlite3.h>
 #include "pkg.h"
 #include "private/event.h"
 #include "private/pkg.h"
 #include "private/pkgdb.h"
 #include "private/repodb.h"
 /* The package repo schema major revision */
 #define REPO_SCHEMA_MAJOR 2
 /* The package repo schema minor revision.
    Minor schema changes don't prevent older pkgng
    versions accessing the repo. */
 #define REPO_SCHEMA_MINOR 10
 /* REPO_SCHEMA_VERSION=2007 */
 #define REPO_SCHEMA_VERSION (REPO_SCHEMA_MAJOR * 1000 + REPO_SCHEMA_MINOR)
 typedef enum _sql_prstmt_index {
 	PKG = 0,
 	DEPS,
 	CAT1,
 	CAT2,
 	LIC1,
 	LIC2,
 	OPT1,
 	OPT2,
 	SHLIB1,
 	SHLIB_REQD,
 	SHLIB_PROV,
 	ANNOTATE1,
 	ANNOTATE2,
 	EXISTS,
 	VERSION,
 	DELETE,
 	FTS_APPEND,
 	PRSTMT_LAST,
 } sql_prstmt_index;
 static sql_prstmt sql_prepared_statements[PRSTMT_LAST] = {
 	[PKG] = {
 		NULL,
 		"INSERT INTO packages ("
 		"origin, name, version, comment, desc, arch, maintainer, www, "
 		"prefix, pkgsize, flatsize, licenselogic, cksum, path, manifestdigest, olddigest"
 		")"
 		"VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
 		"TTTTTTTTTIIITTTT",
 	},
 	[DEPS] = {
 		NULL,
 		"INSERT INTO deps (origin, name, version, package_id) "
 		"VALUES (?1, ?2, ?3, ?4)",
 		"TTTI",
 	},
 	[CAT1] = {
 		NULL,
 		"INSERT OR IGNORE INTO categories(name) VALUES(?1)",
 		"T",
 	},
 	[CAT2] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_categories(package_id, category_id) "
 		"VALUES (?1, (SELECT id FROM categories WHERE name = ?2))",
 		"IT",
 	},
 	[LIC1] = {
 		NULL,
 		"INSERT OR IGNORE INTO licenses(name) VALUES(?1)",
 		"T",
 	},
 	[LIC2] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_licenses(package_id, license_id) "
 		"VALUES (?1, (SELECT id FROM licenses WHERE name = ?2))",
 		"IT",
 	},
 	[OPT1] = {
 		NULL,
 		"INSERT OR IGNORE INTO option(option) "
 		"VALUES (?1)",
 		"T",
 	},
 	[OPT2] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_option (option_id, value, package_id) "
 		"VALUES (( SELECT option_id FROM option WHERE option = ?1), ?2, ?3)",
 		"TTI",
 	},
 	[SHLIB1] = {
 		NULL,
 		"INSERT OR IGNORE INTO shlibs(name) VALUES(?1)",
 		"T",
 	},
 	[SHLIB_REQD] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_shlibs_required(package_id, shlib_id) "
 		"VALUES (?1, (SELECT id FROM shlibs WHERE name = ?2))",
 		"IT",
 	},
 	[SHLIB_PROV] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_shlibs_provided(package_id, shlib_id) "
 		"VALUES (?1, (SELECT id FROM shlibs WHERE name = ?2))",
 		"IT",
 	},
 	[EXISTS] = {
 		NULL,
 		"SELECT count(*) FROM packages WHERE cksum=?1",
 		"T",
 	},
 	[ANNOTATE1] = {
 		NULL,
 		"INSERT OR IGNORE INTO annotation(annotation) "
 		"VALUES (?1)",
 		"T",
 	},
 	[ANNOTATE2] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_annotation(package_id, tag_id, value_id) "
 		"VALUES (?1,"
 		" (SELECT annotation_id FROM annotation WHERE annotation=?2),"
 		" (SELECT annotation_id FROM annotation WHERE annotation=?3))",
 		"ITT",
 	},
 	[VERSION] = {
 		NULL,
 		"SELECT version FROM packages WHERE origin=?1",
 		"T",
 	},
 	[DELETE] = {
 		NULL,
 		"DELETE FROM packages WHERE origin=?1;"
 		"DELETE FROM pkg_search WHERE origin=?1;",
 		"TT",
 	},
 	[FTS_APPEND] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_search(id, name, origin) "
 		"VALUES (?1, ?2 || '-' || ?3, ?4);",
 		"ITTT"
 	}
 	/* PRSTMT_LAST */
 };
 static void
 file_exists(sqlite3_context *ctx, int argc, sqlite3_value **argv)
 {
 	char	 fpath[MAXPATHLEN];
 	sqlite3	*db = sqlite3_context_db_handle(ctx);
 	char	*path = dirname(sqlite3_db_filename(db, "main"));
 	char	 cksum[SHA256_DIGEST_LENGTH * 2 +1];
 	if (argc != 2) {
 		sqlite3_result_error(ctx, "file_exists needs two argument", -1);
 		return;
 	}
 	snprintf(fpath, sizeof(fpath), "%s/%s", path, sqlite3_value_text(argv[0]));
 	if (access(fpath, R_OK) == 0) {
 		sha256_file(fpath, cksum);
 		if (strcmp(cksum, sqlite3_value_text(argv[1])) == 0)
 			sqlite3_result_int(ctx, 1);
 		else
 			sqlite3_result_int(ctx, 0);
 	} else {
 		sqlite3_result_int(ctx, 0);
 	}
 }
 static int
 get_repo_user_version(sqlite3 *sqlite, const char *database, int *reposcver)
 {
 	sqlite3_stmt *stmt;
 	int retcode;
 	char sql[BUFSIZ];
 	const char *fmt = "PRAGMA %Q.user_version";
 	assert(database != NULL);
 	sqlite3_snprintf(sizeof(sql), sql, fmt, database);
 	if (sqlite3_prepare_v2(sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	if (sqlite3_step(stmt) == SQLITE_ROW) {
 		*reposcver = sqlite3_column_int(stmt, 0);
 		retcode = EPKG_OK;
 	} else {
 		*reposcver = -1;
 		retcode = EPKG_FATAL;
 	}
 	sqlite3_finalize(stmt);
 	return (retcode);
 }
 static int
 initialize_prepared_statements(sqlite3 *sqlite)
 {
 	sql_prstmt_index i, last;
 	int ret;
 	last = PRSTMT_LAST;
 	for (i = 0; i < last; i++) {
 		ret = sqlite3_prepare_v2(sqlite, SQL(i), -1, &STMT(i), NULL);
 		if (ret != SQLITE_OK) {
 			ERROR_SQLITE(sqlite, SQL(i));
 			return (EPKG_FATAL);
 		}
 	}
 	return (EPKG_OK);
 }
 static int
 run_prepared_statement(sql_prstmt_index s, ...)
 {
 	int retcode;	/* Returns SQLITE error code */
 	va_list ap;
 	sqlite3_stmt *stmt;
 	int i;
 	const char *argtypes;
 	stmt = STMT(s);
 	argtypes = sql_prepared_statements[s].argtypes;
 	sqlite3_reset(stmt);
 	va_start(ap, s);
 	for (i = 0; argtypes[i] != '\0'; i++)
 	{
 		switch (argtypes[i]) {
 		case 'T':
 			sqlite3_bind_text(stmt, i + 1, va_arg(ap, const char*),
 			    -1, SQLITE_STATIC);
 			break;
 		case 'I':
 			sqlite3_bind_int64(stmt, i + 1, va_arg(ap, int64_t));
 			break;
 		}
 	}
 	va_end(ap);
 	retcode = sqlite3_step(stmt);
 	return (retcode);
 }
 void
 pkgdb_repo_finalize_statements(void)
 {
 	sql_prstmt_index i, last;
 	last = PRSTMT_LAST;
 	for (i = 0; i < last; i++)
 	{
 		if (STMT(i) != NULL) {
 			sqlite3_finalize(STMT(i));
 			STMT(i) = NULL;
 		}
 	}
 	return;
 }
 int
 pkgdb_repo_open(const char *repodb, bool force, sqlite3 **sqlite,
 	bool *incremental)
 {
 	bool db_not_open;
 	int reposcver;
 	int retcode = EPKG_OK;
 	if (access(repodb, R_OK) == 0)
 		*incremental = true;
 	else
 		*incremental = false;
 	sqlite3_initialize();
 	db_not_open = true;
 	while (db_not_open) {
 		if (sqlite3_open(repodb, sqlite) != SQLITE_OK) {
 			sqlite3_shutdown();
 			return (EPKG_FATAL);
 		}
 		db_not_open = false;
 		/* If the schema is too old, or we're forcing a full
 			   update, then we cannot do an incremental update.
 			   Delete the existing repo, and promote this to a
 			   full update */
 		if (!*incremental)
 			continue;
 		retcode = get_repo_user_version(*sqlite, "main", &reposcver);
 		if (retcode != EPKG_OK)
 			return (EPKG_FATAL);
 		if (force || reposcver != REPO_SCHEMA_VERSION) {
 			if (reposcver != REPO_SCHEMA_VERSION)
 				pkg_emit_error("re-creating repo to upgrade schema version "
 						"from %d to %d", reposcver,
 						REPO_SCHEMA_VERSION);
 			sqlite3_close(*sqlite);
 			unlink(repodb);
 			*incremental = false;
 			db_not_open = true;
 		}
 	}
 	sqlite3_create_function(*sqlite, "file_exists", 2, SQLITE_ANY, NULL,
 	    file_exists, NULL, NULL);
 	if (!*incremental) {
 		retcode = sql_exec(*sqlite, initsql, REPO_SCHEMA_VERSION);
 		if (retcode != EPKG_OK)
 			return (retcode);
 	}
 	return (EPKG_OK);
 }
 int
 pkgdb_repo_init(sqlite3 *sqlite)
 {
 	int retcode = EPKG_OK;
 	retcode = sql_exec(sqlite, "PRAGMA synchronous=default");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = sql_exec(sqlite, "PRAGMA foreign_keys=on");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = initialize_prepared_statements(sqlite);
 	if (retcode != EPKG_OK)
 		return (retcode);
 	return (EPKG_OK);
 }
 int
 pkgdb_repo_close(sqlite3 *sqlite, bool commit)
 {
 	int retcode = EPKG_OK;
 	if (sqlite == NULL)
 		return (retcode);
 	if (commit) {
 		if (pkgdb_transaction_commit(sqlite, NULL) != SQLITE_OK)
 			retcode = EPKG_FATAL;
 	}
 	else {
 		if (pkgdb_transaction_rollback(sqlite, NULL) != SQLITE_OK)
 				retcode = EPKG_FATAL;
 	}
 	pkgdb_repo_finalize_statements();
 	return (retcode);
 }
 static int
 maybe_delete_conflicting(const char *origin, const char *version,
 			 const char *pkg_path, bool forced)
 {
 	int ret = EPKG_FATAL;
 	const char *oversion;
 	if (run_prepared_statement(VERSION, origin) != SQLITE_ROW)
 		return (EPKG_FATAL); /* sqlite error */
 	oversion = sqlite3_column_text(STMT(VERSION), 0);
 	if (!forced) {
 		switch(pkg_version_cmp(oversion, version)) {
 		case -1:
 			pkg_emit_error("duplicate package origin: replacing older "
 					"version %s in repo with package %s for "
 					"origin %s", oversion, pkg_path, origin);
 			if (run_prepared_statement(DELETE, origin, origin) != SQLITE_DONE)
 				return (EPKG_FATAL); /* sqlite error */
 			ret = EPKG_OK;	/* conflict cleared */
 			break;
 		case 0:
 		case 1:
 			pkg_emit_error("duplicate package origin: package %s is not "
 					"newer than version %s already in repo for "
 					"origin %s", pkg_path, oversion, origin);
 			ret = EPKG_END;	/* keep what is already in the repo */
 			break;
 		}
 	}
 	else {
 		if (run_prepared_statement(DELETE, origin, origin) != SQLITE_DONE)
 			return (EPKG_FATAL); /* sqlite error */
 		ret = EPKG_OK;
 	}
 	return (ret);
 }
 int
 pkgdb_repo_cksum_exists(sqlite3 *sqlite, const char *cksum)
 {
 	if (run_prepared_statement(EXISTS, cksum) != SQLITE_ROW) {
 		ERROR_SQLITE(sqlite, SQL(EXISTS));
 		return (EPKG_FATAL);
 	}
 	if (sqlite3_column_int(STMT(EXISTS), 0) > 0) {
 		return (EPKG_OK);
 	}
 	return (EPKG_END);
 }
 int
 pkgdb_repo_add_package(struct pkg *pkg, const char *pkg_path,
 		sqlite3 *sqlite, bool forced)
 {
 	const char *name, *version, *origin, *comment, *desc;
 	const char *arch, *maintainer, *www, *prefix, *sum, *rpath;
 	const char *olddigest, *manifestdigest;
 	int64_t			 flatsize, pkgsize;
 	int64_t			 licenselogic;
 	int			 ret;
 	struct pkg_dep		*dep      = NULL;
 	struct pkg_option	*option   = NULL;
 	struct pkg_shlib	*shlib    = NULL;
 	const pkg_object	*obj, *licenses, *categories, *annotations;
 	pkg_iter		 it;
 	int64_t			 package_id;
 	pkg_get(pkg, PKG_ORIGIN, &origin, PKG_NAME, &name,
 			    PKG_VERSION, &version, PKG_COMMENT, &comment,
 			    PKG_DESC, &desc, PKG_ARCH, &arch,
 			    PKG_MAINTAINER, &maintainer, PKG_WWW, &www,
 			    PKG_PREFIX, &prefix, PKG_FLATSIZE, &flatsize,
 			    PKG_LICENSE_LOGIC, &licenselogic, PKG_CKSUM, &sum,
 			    PKG_PKGSIZE, &pkgsize, PKG_REPOPATH, &rpath,
 			    PKG_LICENSES, &licenses, PKG_CATEGORIES, &categories,
 			    PKG_ANNOTATIONS, &annotations, PKG_OLD_DIGEST, &olddigest,
 			    PKG_DIGEST, &manifestdigest);
 try_again:
 	if ((ret = run_prepared_statement(PKG, origin, name, version,
 			comment, desc, arch, maintainer, www, prefix,
 			pkgsize, flatsize, (int64_t)licenselogic, sum,
 			rpath, manifestdigest, olddigest)) != SQLITE_DONE) {
 		if (ret == SQLITE_CONSTRAINT) {
 			switch(maybe_delete_conflicting(origin,
 					version, pkg_path, forced)) {
 			case EPKG_FATAL: /* sqlite error */
 				ERROR_SQLITE(sqlite, SQL(PKG));
 				return (EPKG_FATAL);
 				break;
 			case EPKG_END: /* repo already has newer */
 				return (EPKG_END);
 				break;
 			default: /* conflict cleared, try again */
 				goto try_again;
 				break;
 			}
 		} else {
 			ERROR_SQLITE(sqlite, SQL(PKG));
 			return (EPKG_FATAL);
 		}
 	}
 	package_id = sqlite3_last_insert_rowid(sqlite);
 	if (run_prepared_statement (FTS_APPEND, package_id,
 			name, version, origin) != SQLITE_DONE) {
 		ERROR_SQLITE(sqlite, SQL(FTS_APPEND));
 		return (EPKG_FATAL);
 	}
 	dep = NULL;
 	while (pkg_deps(pkg, &dep) == EPKG_OK) {
 		if (run_prepared_statement(DEPS,
 				pkg_dep_origin(dep),
 				pkg_dep_name(dep),
 				pkg_dep_version(dep),
 				package_id) != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, SQL(DEPS));
 			return (EPKG_FATAL);
 		}
 	}
 	it = NULL;
 	while ((obj = pkg_object_iterate(categories, &it))) {
 		ret = run_prepared_statement(CAT1, pkg_object_string(obj));
 		if (ret == SQLITE_DONE)
 			ret = run_prepared_statement(CAT2, package_id,
 			    pkg_object_string(obj));
 		if (ret != SQLITE_DONE)
 		{
 			ERROR_SQLITE(sqlite, SQL(CAT2));
 			return (EPKG_FATAL);
 		}
 	}
 	it = NULL;
 	while ((obj = pkg_object_iterate(licenses, &it))) {
 		ret = run_prepared_statement(LIC1, pkg_object_string(obj));
 		if (ret == SQLITE_DONE)
 			ret = run_prepared_statement(LIC2, package_id,
 			    pkg_object_string(obj));
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, SQL(LIC2));
 			return (EPKG_FATAL);
 		}
 	}
 	option = NULL;
 	while (pkg_options(pkg, &option) == EPKG_OK) {
 		ret = run_prepared_statement(OPT1, pkg_option_opt(option));
 		if (ret == SQLITE_DONE)
 		    ret = run_prepared_statement(OPT2, pkg_option_opt(option),
 				pkg_option_value(option), package_id);
 		if(ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, SQL(OPT2));
 			return (EPKG_FATAL);
 		}
 	}
 	shlib = NULL;
 	while (pkg_shlibs_required(pkg, &shlib) == EPKG_OK) {
 		const char *shlib_name = pkg_shlib_name(shlib);
 		ret = run_prepared_statement(SHLIB1, shlib_name);
 		if (ret == SQLITE_DONE)
 			ret = run_prepared_statement(SHLIB_REQD, package_id,
 					shlib_name);
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, SQL(SHLIB_REQD));
 			return (EPKG_FATAL);
 		}
 	}
 	shlib = NULL;
 	while (pkg_shlibs_provided(pkg, &shlib) == EPKG_OK) {
 		const char *shlib_name = pkg_shlib_name(shlib);
 		ret = run_prepared_statement(SHLIB1, shlib_name);
 		if (ret == SQLITE_DONE)
 			ret = run_prepared_statement(SHLIB_PROV, package_id,
 					shlib_name);
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, SQL(SHLIB_PROV));
 			return (EPKG_FATAL);
 		}
 	}
 	it = NULL;
 	while ((obj = pkg_object_iterate(annotations, &it))) {
 		const char *note_tag = pkg_object_key(obj);
 		const char *note_val = pkg_object_string(obj);
 		ret = run_prepared_statement(ANNOTATE1, note_tag);
 		if (ret == SQLITE_DONE)
 			ret = run_prepared_statement(ANNOTATE1, note_val);
 		if (ret == SQLITE_DONE)
 			ret = run_prepared_statement(ANNOTATE2, package_id,
 				  note_tag, note_val);
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, SQL(ANNOTATE2));
 			return (EPKG_FATAL);
 		}
 	}
 	return (EPKG_OK);
 }
 int
 pkgdb_repo_remove_package(const char *origin)
 {
 	if (run_prepared_statement(DELETE, origin, origin) != SQLITE_DONE)
 		return (EPKG_FATAL); /* sqlite error */
 	return (EPKG_OK);
 }
 /* We want to replace some arbitrary number of instances of the placeholder
    %Q in the SQL with the name of the database. */
 static int
 substitute_into_sql(char *sqlbuf, size_t buflen, const char *fmt,
 		    const char *replacement)
 {
 	char	*f;
 	char	*f0;
 	char	*tofree;
 	char	*quoted;
 	size_t	 len;
 	int	 ret = EPKG_OK;
 	tofree = f = strdup(fmt);
 	if (tofree == NULL)
 		return (EPKG_FATAL); /* out of memory */
 	quoted = sqlite3_mprintf("%Q", replacement);
 	if (quoted == NULL) {
 		free(tofree);
 		return (EPKG_FATAL); /* out of memory */
 	}
 	sqlbuf[0] = '\0';
 	while ((f0 = strsep(&f, "%")) != NULL) {
 		len = strlcat(sqlbuf, f0, buflen);
 		if (len >= buflen) {
 			/* Overflowed the buffer */
 			ret = EPKG_FATAL;
 			break;
 		}
 		if (f == NULL)
 			break;	/* done */
 		if (f[0] == 'Q') {
 			len = strlcat(sqlbuf, quoted, buflen);
 			f++;	/* Jump the Q */
 		} else {
 			len = strlcat(sqlbuf, "%", buflen);
 		}
 		if (len >= buflen) {
 			/* Overflowed the buffer */
 			ret = EPKG_FATAL;
 			break;
 		}
 	}
 	free(tofree);
 	sqlite3_free(quoted);
 	return (ret);
 }
 static int
 set_repo_user_version(sqlite3 *sqlite, const char *database, int reposcver)
 {
 	int		 retcode = EPKG_OK;
 	char		 sql[BUFSIZ];
 	char		*errmsg;
 	const char	*fmt = "PRAGMA %Q.user_version = %d;" ;
 	assert(database != NULL);
 	sqlite3_snprintf(sizeof(sql), sql, fmt, database, reposcver);
 	if (sqlite3_exec(sqlite, sql, NULL, NULL, &errmsg) != SQLITE_OK) {
 		pkg_emit_error("sqlite: %s", errmsg);
 		sqlite3_free(errmsg);
 		retcode = EPKG_FATAL;
 	}
 	return (retcode);
 }
 static int
 apply_repo_change(struct pkgdb *db, const char *database,
 		  const struct repo_changes *repo_changes, const char *updown,
 		  int version, int *next_version)
 {
 	const struct repo_changes	*change;
 	bool			 found = false, in_trans = false;
 	int			 ret = EPKG_OK;
 	char			 sql[8192];
 	char			*errmsg;
 	for (change = repo_changes; change->version != -1; change++) {
 		if (change->version == version) {
 			found = true;
 			break;
 		}
 	}
 	if (!found) {
 		pkg_emit_error("Failed to %s \"%s\" repo schema "
 			" version %d (target version %d) "
 			"-- change not found", updown, database, version,
 			REPO_SCHEMA_VERSION);
 		return (EPKG_FATAL);
 	}
 	/* substitute the repo database name */
 	ret = substitute_into_sql(sql, sizeof(sql), change->sql, database);
 	/* begin transaction */
 	if (ret == EPKG_OK) {
 		in_trans = true;
 		ret = pkgdb_transaction_begin(db->sqlite, "SCHEMA");
 	}
 	/* apply change */
 	if (ret == EPKG_OK) {
 		pkg_debug(4, "Pkgdb: running '%s'", sql);
 		ret = sqlite3_exec(db->sqlite, sql, NULL, NULL, &errmsg);
 		if (ret != SQLITE_OK) {
 			pkg_emit_error("sqlite: %s", errmsg);
 			sqlite3_free(errmsg);
 			ret = EPKG_FATAL;
 		}
 	}
 	/* update repo user_version */
 	if (ret == EPKG_OK) {
 		*next_version = change->next_version;
 		ret = set_repo_user_version(db->sqlite, database, *next_version);
 	}
 	/* commit or rollback */
 	if (in_trans) {
 		if (ret != EPKG_OK)
 			pkgdb_transaction_rollback(db->sqlite, "SCHEMA");
 		if (pkgdb_transaction_commit(db->sqlite, "SCHEMA") != EPKG_OK)
 			ret = EPKG_FATAL;
 	}
 	if (ret == EPKG_OK) {
 		pkg_emit_notice("Repo \"%s\" %s schema %d to %d: %s",
 				database, updown, version,
 				change->next_version, change->message);
 	}
 	return (ret);
 }
 static int
 upgrade_repo_schema(struct pkgdb *db, const char *database, int current_version)
 {
 	int version;
 	int next_version;
 	int ret = EPKG_OK;
 	for (version = current_version;
 	     version < REPO_SCHEMA_VERSION;
 	     version = next_version)  {
 		ret = apply_repo_change(db, database, repo_upgrades,
 					"upgrade", version, &next_version);
 		if (ret != EPKG_OK)
 			break;
 		pkg_debug(1, "Upgrading repo database schema from %d to %d",
 				version, next_version);
 	}
 	return (ret);
 }
 static int
 downgrade_repo_schema(struct pkgdb *db, const char *database, int current_version)
 {
 	int version;
 	int next_version;
 	int ret = EPKG_OK;
 	for (version = current_version;
 	     version > REPO_SCHEMA_VERSION;
 	     version = next_version)  {
 		ret = apply_repo_change(db, database, repo_downgrades,
 					"downgrade", version, &next_version);
 		if (ret != EPKG_OK)
 			break;
 		pkg_debug(1, "Downgrading repo database schema from %d to %d",
 				version, next_version);
 	}
 	return (ret);
 }
 int
 pkgdb_repo_check_version(struct pkgdb *db, const char *database)
 {
 	int reposcver;
 	int repomajor;
 	int ret;
 	assert(db != NULL);
 	assert(database != NULL);
 	if ((ret = get_repo_user_version(db->sqlite, database, &reposcver))
 	    != EPKG_OK)
 		return (ret);	/* sqlite error */
 	/*
 	 * If the local pkgng uses a repo schema behind that used to
 	 * create the repo, we may still be able use it for reading
 	 * (ie pkg install), but pkg repo can't do an incremental
 	 * update unless the actual schema matches the compiled in
 	 * schema version.
 	 *
 	 * Use a major - minor version schema: as the user_version
 	 * PRAGMA takes an integer version, encode this as MAJOR *
 	 * 1000 + MINOR.
 	 *
 	 * So long as the major versions are the same, the local pkgng
 	 * should be compatible with any repo created by a more recent
 	 * pkgng, although it may need some modification of the repo
 	 * schema
 	 */
 	/* --- Temporary ---- Grandfather in the old repo schema
 	   version so this patch doesn't immediately invalidate all
 	   the repos out there */
 	if (reposcver == 2)
 		reposcver = 2000;
 	if (reposcver == 3)
 		reposcver = 2001;
 	repomajor = reposcver / 1000;
 	if (repomajor < REPO_SCHEMA_MAJOR) {
 		pkg_emit_error("Repo %s (schema version %d) is too old - "
 		    "need at least schema %d", database, reposcver,
 		    REPO_SCHEMA_MAJOR * 1000);
 		return (EPKG_REPOSCHEMA);
 	}
 	if (repomajor > REPO_SCHEMA_MAJOR) {
 		pkg_emit_error("Repo %s (schema version %d) is too new - "
 		    "we can accept at most schema %d", database, reposcver,
 		    ((REPO_SCHEMA_MAJOR + 1) * 1000) - 1);
 		return (EPKG_REPOSCHEMA);
 	}
 	/* This is a repo schema version we can work with */
 	ret = EPKG_OK;
 	if (reposcver < REPO_SCHEMA_VERSION) {
 		if (sqlite3_db_readonly(db->sqlite, database)) {
 			pkg_emit_error("Repo %s needs schema upgrade from "
 			"%d to %d but it is opened readonly", database,
 			       reposcver, REPO_SCHEMA_VERSION
 			);
 			ret = EPKG_FATAL;
 		} else
 			ret = upgrade_repo_schema(db, database, reposcver);
 	} else if (reposcver > REPO_SCHEMA_VERSION) {
 		if (sqlite3_db_readonly(db->sqlite, database)) {
 			pkg_emit_error("Repo %s needs schema downgrade from "
 			"%d to %d but it is opened readonly", database,
 			       reposcver, REPO_SCHEMA_VERSION
 			);
 			ret = EPKG_FATAL;
 		} else
 			ret = downgrade_repo_schema(db, database, reposcver);
 	}
 	return (ret);
 }
 struct pkgdb_it *
 pkgdb_repo_origins(sqlite3 *sqlite)
 {
 	sqlite3_stmt *stmt = NULL;
 	int ret;
 	static struct pkgdb repodb;
 	const char query_sql[] = ""
 		"SELECT id, origin, name, name || '~' || origin as uniqueid, version, comment, "
 		"prefix, desc, arch, maintainer, www, "
 		"licenselogic, flatsize, pkgsize, "
 		"cksum, path AS repopath, manifestdigest "
 		"FROM packages "
 		"ORDER BY origin;";
 	ret = sqlite3_prepare_v2(sqlite, query_sql, -1,
 			&stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, query_sql);
 		return (NULL);
 	}
 	repodb.sqlite = sqlite;
 	repodb.type = PKGDB_REMOTE;
 	return pkgdb_it_new(&repodb, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE);
 }
 int
 pkgdb_repo_register_conflicts(const char *origin, char **conflicts,
 		int conflicts_num, sqlite3 *sqlite)
 {
 	const char clean_conflicts_sql[] = ""
 			"DELETE FROM pkg_conflicts "
 			"WHERE package_id = ?1;";
 	const char select_id_sql[] = ""
 			"SELECT id FROM packages "
 			"WHERE origin = ?1;";
 	const char insert_conflict_sql[] = ""
 			"INSERT INTO pkg_conflicts "
 			"(package_id, conflict_id) "
 			"VALUES (?1, ?2);";
 	sqlite3_stmt *stmt = NULL;
 	int ret, i;
 	int64_t origin_id, conflict_id;
 	pkg_debug(4, "pkgdb_repo_register_conflicts: running '%s'", select_id_sql);
 	if (sqlite3_prepare_v2(sqlite, select_id_sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, select_id_sql);
 		return (EPKG_FATAL);
 	}
 	sqlite3_bind_text(stmt, 1, origin, -1, SQLITE_TRANSIENT);
 	ret = sqlite3_step(stmt);
 	if (ret == SQLITE_ROW) {
 		origin_id = sqlite3_column_int64(stmt, 0);
 	}
 	else {
 		ERROR_SQLITE(sqlite, select_id_sql);
 		return (EPKG_FATAL);
 	}
 	sqlite3_finalize(stmt);
 	pkg_debug(4, "pkgdb_repo_register_conflicts: running '%s'", clean_conflicts_sql);
 	if (sqlite3_prepare_v2(sqlite, clean_conflicts_sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, clean_conflicts_sql);
 		return (EPKG_FATAL);
 	}
 	sqlite3_bind_int64(stmt, 1, origin_id);
 	/* Ignore cleanup result */
 	(void)sqlite3_step(stmt);
 	sqlite3_finalize(stmt);
 	for (i = 0; i < conflicts_num; i ++) {
 		/* Select a conflict */
 		pkg_debug(4, "pkgdb_repo_register_conflicts: running '%s'", select_id_sql);
 		if (sqlite3_prepare_v2(sqlite, select_id_sql, -1, &stmt, NULL) != SQLITE_OK) {
 			ERROR_SQLITE(sqlite, select_id_sql);
 			return (EPKG_FATAL);
 		}
 		sqlite3_bind_text(stmt, 1, conflicts[i], -1, SQLITE_TRANSIENT);
 		ret = sqlite3_step(stmt);
 		if (ret == SQLITE_ROW) {
 			conflict_id = sqlite3_column_int64(stmt, 0);
 		}
 		else {
 			ERROR_SQLITE(sqlite, select_id_sql);
 			return (EPKG_FATAL);
 		}
 		sqlite3_finalize(stmt);
 		/* Insert a pair */
 		pkg_debug(4, "pkgdb_repo_register_conflicts: running '%s'", insert_conflict_sql);
 		if (sqlite3_prepare_v2(sqlite, insert_conflict_sql, -1, &stmt, NULL) != SQLITE_OK) {
 			ERROR_SQLITE(sqlite, insert_conflict_sql);
 			return (EPKG_FATAL);
 		}
 		sqlite3_bind_int64(stmt, 1, origin_id);
 		sqlite3_bind_int64(stmt, 2, conflict_id);
 		ret = sqlite3_step(stmt);
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, insert_conflict_sql);
 			return (EPKG_FATAL);
 		}
 		sqlite3_finalize(stmt);
 	}
 	return (EPKG_OK);
 }
 const char *
 pkgdb_get_reponame(struct pkgdb *db, const char *repo)
 {
 	const char	*reponame = NULL;
 	struct pkg_repo	*r;
 	assert(db->type == PKGDB_REMOTE);
 	if (repo != NULL) {
 		if ((r = pkg_repo_find_ident(repo)) == NULL) {
 			pkg_emit_error("repository '%s' does not exist", repo);
 			return (NULL);
 		}
 		reponame = pkg_repo_name(r);
 		if (!pkgdb_is_attached(db->sqlite, reponame)) {
 			pkg_emit_error("repository '%s' does not exist", repo);
 			return (NULL);
 		}
 	}
 	return (reponame);
 }
 struct pkgdb_it *
 pkgdb_rquery(struct pkgdb *db, const char *pattern, match_t match,
     const char *repo)
 {
 	sqlite3_stmt	*stmt = NULL;
 	struct sbuf	*sql = NULL;
 	const char	*reponame = NULL;
 	const char	*comp = NULL;
 	int		 ret;
 	char		 basesql[BUFSIZ] = ""
 		"SELECT id, origin, name, name || '~' || origin as uniqueid, version, comment, "
 		"prefix, desc, arch, maintainer, www, "
 		"licenselogic, flatsize, pkgsize, "
 		"cksum, manifestdigest, path AS repopath, '%1$s' AS dbname "
 		"FROM '%1$s'.packages p";
 	assert(db != NULL);
 	assert(match == MATCH_ALL || (pattern != NULL && pattern[0] != '\0'));
 	/*
 	 * If we have no remote repos loaded, we just return nothing instead of failing
 	 * an assert deep inside pkgdb_get_reponame
 	 */
 	if (db->type != PKGDB_REMOTE)
 		return (NULL);
 	reponame = pkgdb_get_reponame(db, repo);
 	sql = sbuf_new_auto();
 	comp = pkgdb_get_pattern_query(pattern, match);
 	if (comp && comp[0])
 		strlcat(basesql, comp, sizeof(basesql));
 	/*
 	 * Working on multiple remote repositories
 	 */
 	if (reponame == NULL) {
 		/* duplicate the query via UNION for all the attached
 		 * databases */
 		ret = pkgdb_sql_all_attached(db->sqlite, sql,
 		    basesql, " UNION ALL ");
 		if (ret != EPKG_OK) {
 			sbuf_delete(sql);
 			return (NULL);
 		}
 	} else
 		sbuf_printf(sql, basesql, reponame, reponame);
 	sbuf_cat(sql, " ORDER BY name;");
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s' query for %s", sbuf_get(sql), pattern);
 	ret = sqlite3_prepare_v2(db->sqlite, sbuf_get(sql), sbuf_size(sql), &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	if (match != MATCH_ALL && match != MATCH_CONDITION)
 		sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_rquery_provide(struct pkgdb *db, const char *provide, const char *repo)
 {
 	sqlite3_stmt	*stmt;
 	struct sbuf	*sql = NULL;
 	const char	*reponame = NULL;
 	int		 ret;
 	const char	 basesql[] = ""
 			"SELECT p.id, p.origin, p.name, p.version, p.comment, "
 			"p.name || '~' || p.origin as uniqueid, "
 			"p.prefix, p.desc, p.arch, p.maintainer, p.www, "
 			"p.licenselogic, p.flatsize, p.pkgsize, "
 			"p.cksum, p.manifestdigest, p.path AS repopath, '%1$s' AS dbname "
 			"FROM '%1$s'.packages AS p, '%1$s'.pkg_provides AS pp, "
 			"'%1$s'.provides AS pr "
 			"WHERE p.id = pp.package_id "
 			"AND pp.provide_id = pr.id "
 			"AND pr.name = ?1;";
 	assert(db != NULL);
 	reponame = pkgdb_get_reponame(db, repo);
 	sql = sbuf_new_auto();
 	/*
 	 * Working on multiple remote repositories
 	 */
 	if (reponame == NULL) {
 		/* duplicate the query via UNION for all the attached
 		 * databases */
 		ret = pkgdb_sql_all_attached(db->sqlite, sql,
 				basesql, " UNION ALL ");
 		if (ret != EPKG_OK) {
 			sbuf_delete(sql);
 			return (NULL);
 		}
 	} else
 		sbuf_printf(sql, basesql, reponame);
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s'", sbuf_get(sql));
 	ret = sqlite3_prepare_v2(db->sqlite, sbuf_get(sql), -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	sqlite3_bind_text(stmt, 1, provide, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_find_shlib_provide(struct pkgdb *db, const char *require, const char *repo)
 {
 	sqlite3_stmt	*stmt;
 	struct sbuf	*sql = NULL;
 	const char	*reponame = NULL;
 	int		 ret;
 	const char	 basesql[] = ""
 			"SELECT p.id, p.origin, p.name, p.version, p.comment, "
 			"p.name || '~' || p.origin as uniqueid, "
 			"p.prefix, p.desc, p.arch, p.maintainer, p.www, "
 			"p.licenselogic, p.flatsize, p.pkgsize, "
 			"p.cksum, p.manifestdigest, p.path AS repopath, '%1$s' AS dbname "
 			"FROM '%1$s'.packages AS p INNER JOIN '%1$s'.pkg_shlibs_provided AS ps ON "
 			"p.id = ps.package_id "
 			"WHERE ps.shlib_id IN (SELECT id FROM '%1$s'.shlibs WHERE "
 			"name BETWEEN ?1 AND ?1 || '.9');";
 	assert(db != NULL);
 	reponame = pkgdb_get_reponame(db, repo);
 	sql = sbuf_new_auto();
 	/*
 	 * Working on multiple remote repositories
 	 */
 	if (reponame == NULL) {
 		/* duplicate the query via UNION for all the attached
 		 * databases */
 		ret = pkgdb_sql_all_attached(db->sqlite, sql,
 				basesql, " UNION ALL ");
 		if (ret != EPKG_OK) {
 			sbuf_delete(sql);
 			return (NULL);
 		}
 	} else
 		sbuf_printf(sql, basesql, reponame);
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s'", sbuf_get(sql));
 	ret = sqlite3_prepare_v2(db->sqlite, sbuf_get(sql), -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	sqlite3_bind_text(stmt, 1, require, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE));
 }
 struct pkgdb_it *
 pkgdb_find_shlib_require(struct pkgdb *db, const char *provide, const char *repo)
 {
 	sqlite3_stmt	*stmt;
 	struct sbuf	*sql = NULL;
 	const char	*reponame = NULL;
 	int		 ret;
 	const char	 basesql[] = ""
 			"SELECT p.id, p.origin, p.name, p.version, p.comment, "
 			"p.name || '~' || p.origin as uniqueid, "
 			"p.prefix, p.desc, p.arch, p.maintainer, p.www, "
 			"p.licenselogic, p.flatsize, p.pkgsize, "
 			"p.cksum, p.manifestdigest, p.path AS repopath, '%1$s' AS dbname "
 			"FROM '%1$s'.packages AS p INNER JOIN '%1$s'.pkg_shlibs_required AS ps ON "
 			"p.id = ps.package_id "
 			"WHERE ps.shlib_id = (SELECT id FROM '%1$s'.shlibs WHERE name=?1);";
 	assert(db != NULL);
 	reponame = pkgdb_get_reponame(db, repo);
 	sql = sbuf_new_auto();
 	/*
 	 * Working on multiple remote repositories
 	 */
 	if (reponame == NULL) {
 		/* duplicate the query via UNION for all the attached
 		 * databases */
 		ret = pkgdb_sql_all_attached(db->sqlite, sql,
 				basesql, " UNION ALL ");
 		if (ret != EPKG_OK) {
 			sbuf_delete(sql);
 			return (NULL);
 		}
 	} else
 		sbuf_printf(sql, basesql, reponame);
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s'", sbuf_get(sql));
 	ret = sqlite3_prepare_v2(db->sqlite, sbuf_get(sql), -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	sqlite3_bind_text(stmt, 1, provide, -1, SQLITE_TRANSIENT);
 	return (pkgdb_it_new(db, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE));
 }

modified libpkg/private/pkg.h

@@ -320,8 +320,51 @@ struct pkg_repo_meta {

 	time_t eol;
 };
 struct pkg_repo_it;
 struct pkg_repo;
 struct pkg_repo_it_ops {
 	int (*next)(struct pkg_repo_it *it, struct pkg **pkg_p, unsigned flags);
 	void (*free)(struct pkg_repo_it *it);
 	void (*reset)(struct pkg_repo_it *it);
 };
 struct pkg_repo_it {
 	struct pkg_repo *repo;
 	struct pkg_repo_it_ops *ops;
 	int flags;
 	void *data;
 };
 struct pkg_repo_ops {
 	const char *type;
 	/* Accessing repo */
 	int (*init)(struct pkg_repo *);
 	int (*access)(struct pkg_repo *, unsigned);
 	int (*open)(struct pkg_repo *, unsigned);
 	int (*create)(struct pkg_repo *);
 	int (*close)(struct pkg_repo *, bool);
 	/* Updating repo */
 	int (*update)(struct pkg_repo *, bool);
 	/* Query repo */
 	struct pkg_repo_it * (*query)(struct pkg_repo *,
 					const char *, match_t);
 	struct pkg_repo_it * (*shlib_required)(struct pkg_repo *,
 					const char *);
 	struct pkg_repo_it * (*shlib_provided)(struct pkg_repo *,
 					const char *);
 	struct pkg_repo_it * (*search)(struct pkg_repo *, const char *, match_t,
 					pkgdb_field field, pkgdb_field sort);
 	/* Fetch package from repo */
 	int (*fetch_pkg)(struct pkg_repo *, struct pkg *);
 };
 struct pkg_repo {
 	repo_t type;
 	struct pkg_repo_ops *ops;
 	char *name;
 	char *url;
 	char *pubkey;

@@ -345,10 +388,11 @@ struct pkg_repo {

 	struct pkg_repo_meta *meta;
 	int (*update)(struct pkg_repo *, bool);
 	bool enable;
 	UT_hash_handle hh;
 	/* Opaque repository data */
 	void *priv;
 };
 /* sql helpers */

@@ -503,8 +547,6 @@ const char* packing_format_to_string(pkg_formats format);

 int pkg_delete_files(struct pkg *pkg, unsigned force);
 int pkg_delete_dirs(struct pkgdb *db, struct pkg *pkg, bool force);
 int pkgdb_is_dir_used(struct pkgdb *db, const char *dir, int64_t *res);
 int pkg_conflicts_request_resolve(struct pkg_jobs *j);
 int pkg_conflicts_append_pkg(struct pkg *p, struct pkg_jobs *j);
 int pkg_conflicts_integrity_check(struct pkg_jobs *j);

@@ -525,23 +567,6 @@ int sql_exec(sqlite3 *, const char *, ...);

 int get_pragma(sqlite3 *, const char *sql, int64_t *res, bool silence);
 int get_sql_string(sqlite3 *, const char *sql, char **res);
 int pkgdb_load_deps(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_rdeps(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_files(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_dirs(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_scripts(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_options(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_mtree(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_category(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_license(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_user(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_group(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_shlib_required(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_shlib_provided(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_annotations(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_conflicts(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_load_provides(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_register_pkg(struct pkgdb *db, struct pkg *pkg, int complete, int forced);
 int pkgdb_update_shlibs_required(struct pkg *pkg, int64_t package_id, sqlite3 *s);
 int pkgdb_update_shlibs_provided(struct pkg *pkg, int64_t package_id, sqlite3 *s);

@@ -557,7 +582,7 @@ int pkg_emit_filelist(struct pkg *, FILE *);

 int do_extract_mtree(char *mtree, const char *prefix);
 int pkg_repo_update_binary_pkgs(struct pkg_repo *repo, bool force);
 int pkg_repo_binary_update(struct pkg_repo *repo, bool force);
 bool ucl_object_emit_sbuf(const ucl_object_t *obj, enum ucl_emitter emit_type,
     struct sbuf **buf);

modified libpkg/private/pkgdb.h

@@ -34,18 +34,39 @@

 struct pkgdb {
 	sqlite3		*sqlite;
 	pkgdb_t		 type;
 	int		 lock_count;
 	bool		 prstmt_initialized;
 	struct _pkg_repo_list_item {
 		struct pkg_repo *repo;
 		struct _pkg_repo_list_item *next;
 	} *repos;
 };
 struct pkgdb_it {
 	struct pkgdb	*db;
 enum pkgdb_iterator_type {
 	PKGDB_IT_LOCAL = 0,
 	PKGDB_IT_REPO
 };
 struct pkgdb_sqlite_it {
 	sqlite3	*sqlite;
 	sqlite3_stmt	*stmt;
 	short	type;
 	short	flags;
 	short	finished;
 	short	pkg_type;
 };
 struct pkg_repo_it;
 struct pkgdb_it {
 	enum pkgdb_iterator_type type;
 	struct pkgdb *db;
 	union _un_pkg_it {
 		struct _pkg_repo_it_set {
 			struct pkg_repo_it *it;
 			struct _pkg_repo_it_set *next;
 		} *remote;
 		struct pkgdb_sqlite_it local;
 	} un;
 };
 #define PKGDB_IT_FLAG_CYCLED (0x1)

@@ -63,85 +84,14 @@ int pkgdb_transaction_begin(sqlite3 *sqlite, const char *savepoint);

 int pkgdb_transaction_commit(sqlite3 *sqlite, const char *savepoint);
 int pkgdb_transaction_rollback(sqlite3 *sqlite, const char *savepoint);
 struct pkgdb_it *pkgdb_it_new(struct pkgdb *db, sqlite3_stmt *s, int type, short flags);
 struct pkgdb_it *pkgdb_it_new_sqlite(struct pkgdb *db, sqlite3_stmt *s,
 	int type, short flags);
 struct pkgdb_it *pkgdb_it_new_repo(struct pkgdb *db);
 void pkgdb_it_repo_attach(struct pkgdb_it *it, struct pkg_repo_it *rit);
 void pkgshell_open(const char **r);
 /**
  * Open repodb for specified path
  * @param repodb path of repodb
  * @param force create repository if not exists
  * @param sqlite destination db pointer
  * @param incremental if this param is set to false, then database was re-created
  *  and thus needs to be updated
  * @return EPKG_OK if succeed
  */
 int pkgdb_repo_open(const char *repodb, bool force, sqlite3 **sqlite,
 	bool *incremental);
 /**
  * Init repository for pkgdb_repo* functions
  * @param sqlite sqlite object
  * @return EPKG_OK if succeed
  */
 int pkgdb_repo_init(sqlite3 *sqlite);
 /**
  * Finalize prepared statements for a repo
  */
 void pkgdb_repo_finalize_statements(void);
 /**
  * Close repodb and commit/rollback transaction started
  * @param sqlite sqlite pointer
  * @param commit commit transaction if true, rollback otherwise
  * @return EPKG_OK if succeed
  */
 int pkgdb_repo_close(sqlite3 *sqlite, bool commit);
 /**
  * Check whether a package with the cehcksum specified exists in pkg_repo
  * @param sqlite sqlite pointer
  * @param cksum sha256 printed checksum
  * @return EPKG_OK if checksum exists, EPKG_END if not and EPKG_FATAL if error occurred
  */
 int pkgdb_repo_cksum_exists(sqlite3 *sqlite, const char *cksum);
 /**
  * Add a package to pkg_repo
  * @param pkg package structure
  * @param pkg_path path triggered package addition
  * @param sqlite sqlite pointer
  * @param forced force adding of package even if it is outdated
  * @return EPKG_OK if package added, EPKG_END if package already exists and is newer than
  * inserted one, EPKG_FATAL if error occurred
  */
 int pkgdb_repo_add_package(struct pkg *pkg, const char *pkg_path,
 		sqlite3 *sqlite, bool forced);
 /**
  * Remove specified pkg from repo
  * @param origin the origin of package to remove
  * @return EPKG_OK if succeeded
  */
 int pkgdb_repo_remove_package(const char *origin);
 /**
  * Upgrade repo db version if required
  * @param db package database object
  * @param database name of database
  * @return EPKG_OK if succeeded
  */
 int pkgdb_repo_check_version(struct pkgdb *db, const char *database);
 /**
  * Returns a list of all packages sorted by origin
  * @param sqlite database
  * @return new iterator
  */
 struct pkgdb_it *pkgdb_repo_origins(sqlite3 *sqlite);
 /**
  * Register a conflicts list in a repo
  * @param origin the origin of a package
  * @param conflicts a list of conflicts origins

@@ -153,26 +103,6 @@ int pkgdb_repo_register_conflicts(const char *origin, char **conflicts,

 		int conflicts_num, sqlite3 *sqlite);
 /**
  * Execute SQL statement on all attached databases
  * @param s
  * @param sql
  * @param multireposql
  * @param compound
  * @return
  */
 int
 pkgdb_sql_all_attached(sqlite3 *s, struct sbuf *sql, const char *multireposql,
     const char *compound);
 /**
  * Get repository name
  * @param db
  * @param repo
  * @return
  */
 const char *pkgdb_get_reponame(struct pkgdb *db, const char *repo);
 /**
  * Get query for the specified match type
  * @param pattern
  * @param match

@@ -181,21 +111,13 @@ const char *pkgdb_get_reponame(struct pkgdb *db, const char *repo);

 const char * pkgdb_get_pattern_query(const char *pattern, match_t match);
 /**
  * Returns whether the specified database is attached
  * @param s
  * @param name
  * @return
  */
 bool pkgdb_is_attached(sqlite3 *s, const char *name);
 /**
  * Find provides for a specified require in repos
  * @param db
  * @param provide
  * @param repo
  * @return
  */
 struct pkgdb_it *pkgdb_find_shlib_require(struct pkgdb *db,
 struct pkgdb_it *pkgdb_repo_shlib_require(struct pkgdb *db,
 		const char *provide, const char *repo);
 /**
  * Find requires for a specified provide in repos

@@ -204,7 +126,7 @@ struct pkgdb_it *pkgdb_find_shlib_require(struct pkgdb *db,

  * @param repo
  * @return
  */
 struct pkgdb_it *pkgdb_find_shlib_provide(struct pkgdb *db,
 struct pkgdb_it *pkgdb_repo_shlib_provide(struct pkgdb *db,
 		const char *require, const char *repo);
 /**

@@ -225,4 +147,24 @@ int pkgdb_begin_solver(struct pkgdb *db);

  */
 int pkgdb_end_solver(struct pkgdb *db);
 /**
  * Check access mode for the specified file
  * @param mode
  * @param dbdir
  * @param dbname
  * @return
  */
 int pkgdb_check_access(unsigned mode, const char* dbdir, const char *dbname);
 /*
  * SQLite utility functions
  */
 void pkgdb_regex(sqlite3_context *ctx, int argc, sqlite3_value **argv);
 void pkgdb_regex_delete(void *p);
 void pkgdb_split_uid(sqlite3_context *ctx, int argc, sqlite3_value **argv);
 void pkgdb_split_version(sqlite3_context *ctx, int argc, sqlite3_value **argv);
 void pkgdb_now(sqlite3_context *ctx, int argc, __unused sqlite3_value **argv);
 void pkgdb_myarch(sqlite3_context *ctx, int argc, sqlite3_value **argv);
 int pkgdb_sqlcmd_init(sqlite3 *db, const char **err, const void *noused);
 #endif

modified libpkg/private/repodb.h

@@ -40,452 +40,5 @@ static const char repo_digests_archive[] = "digests";

 static const char repo_conflicts_file[] = "conflicts";
 static const char repo_conflicts_archive[] = "conflicts";
 static const char initsql[] = ""
 	"CREATE TABLE packages ("
 	    "id INTEGER PRIMARY KEY,"
 	    "origin TEXT UNIQUE,"
 	    "name TEXT NOT NULL,"
 	    "version TEXT NOT NULL,"
 	    "comment TEXT NOT NULL,"
 	    "desc TEXT NOT NULL,"
 	    "osversion TEXT,"
 	    "arch TEXT NOT NULL,"
 	    "maintainer TEXT NOT NULL,"
 	    "www TEXT,"
 	    "prefix TEXT NOT NULL,"
 	    "pkgsize INTEGER NOT NULL,"
 	    "flatsize INTEGER NOT NULL,"
 	    "licenselogic INTEGER NOT NULL,"
 	    "cksum TEXT NOT NULL,"
 	    /* relative path to the package in the repository */
 	    "path TEXT NOT NULL,"
 	    "pkg_format_version INTEGER,"
 	    "manifestdigest TEXT NULL,"
 	    "olddigest TEXT NULL"
 	");"
 	"CREATE TABLE deps ("
 	    "origin TEXT,"
 	    "name TEXT,"
 	    "version TEXT,"
 	    "package_id INTEGER REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "UNIQUE(package_id, origin)"
 	");"
 	"CREATE TABLE categories ("
 	    "id INTEGER PRIMARY KEY, "
 	    "name TEXT NOT NULL UNIQUE "
 	");"
 	"CREATE TABLE pkg_categories ("
 	    "package_id INTEGER REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "category_id INTEGER REFERENCES categories(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, category_id)"
 	");"
 	"CREATE TABLE licenses ("
 	    "id INTEGER PRIMARY KEY,"
 	    "name TEXT NOT NULL UNIQUE"
 	");"
 	"CREATE TABLE pkg_licenses ("
 	    "package_id INTEGER REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "license_id INTEGER REFERENCES licenses(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, license_id)"
 	");"
 	"CREATE TABLE option ("
 		"option_id INTEGER PRIMARY KEY,"
 		"option TEXT NOT NULL UNIQUE"
 	");"
 	"CREATE TABLE option_desc ("
 		"option_desc_id INTEGER PRIMARY KEY,"
 		"option_desc TEXT NOT NULL UNIQUE"
 	");"
 	"CREATE TABLE pkg_option ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"value TEXT NOT NULL,"
 		"PRIMARY KEY(package_id, option_id)"
 	");"
 	"CREATE TABLE pkg_option_desc ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"option_desc_id INTEGER NOT NULL "
 			"REFERENCES option_desc(option_desc_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"PRIMARY KEY(package_id, option_id)"
 	");"
 	"CREATE TABLE pkg_option_default ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"default_value TEXT NOT NULL,"
 		"PRIMARY KEY(package_id, option_id)"
 	");"
 	"CREATE TABLE shlibs ("
 	    "id INTEGER PRIMARY KEY,"
 	    "name TEXT NOT NULL UNIQUE "
 	");"
 	"CREATE TABLE pkg_shlibs_required ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, shlib_id)"
 	");"
 	"CREATE TABLE pkg_shlibs_provided ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, shlib_id)"
 	");"
 	"CREATE TABLE annotation ("
 	    "annotation_id INTEGER PRIMARY KEY,"
 	    "annotation TEXT NOT NULL UNIQUE"
 	");"
 	"CREATE TABLE pkg_annotation ("
 	    "package_id INTERGER REFERENCES packages(id)"
 	    " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	    "tag_id INTEGER NOT NULL REFERENCES annotation(annotation_id)"
 	    " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	    "value_id INTEGER NOT NULL REFERENCES annotation(annotation_id)"
 	    " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	    "UNIQUE (package_id, tag_id)"
 	");"
 	"CREATE TABLE pkg_conflicts ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "conflict_id INTEGER NOT NULL,"
 	    "UNIQUE(package_id, conflict_id)"
 	");"
 	"CREATE TABLE provides("
 	"    id INTEGER PRIMARY KEY,"
 	"    provide TEXT NOT NULL"
 	");"
 	"CREATE TABLE pkg_provides ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "provide_id INTEGER NOT NULL REFERENCES provides(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, provide_id)"
 	");"
 	"CREATE INDEX packages_origin ON packages(origin COLLATE NOCASE);"
 	"CREATE INDEX packages_name ON packages(name COLLATE NOCASE);"
 	"CREATE INDEX packages_uid_nocase ON packages(name COLLATE NOCASE, origin COLLATE NOCASE);"
 	"CREATE INDEX packages_version_nocase ON packages(name COLLATE NOCASE, version);"
 	"CREATE INDEX packages_uid ON packages(name, origin);"
 	"CREATE INDEX packages_version ON packages(name, version);"
 	"CREATE UNIQUE INDEX packages_digest ON packages(manifestdigest);"
 	/* FTS search table */
 	"CREATE VIRTUAL TABLE pkg_search USING fts4(id, name, origin);"
 	"PRAGMA user_version=%d;"
 	;
 struct repo_changes {
 	int version;		/* The repo schema this change applies to */
 	int next_version;	/* The repo schema this change creates */
 	const char *message;
 	const char *sql;
 };
 /* How to upgrade an older repo to match what the current system
    expects */
 static const struct repo_changes repo_upgrades[] = {
 	{2001,
 ,
 	 "Modify shlib tracking to add \'provided\' capability",
 	 "CREATE TABLE %Q.pkg_shlibs_required ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE CASCADE,"
 		"shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 		" ON DELETE RESTRICT ON UPDATE RESTRICT,"
 		"UNIQUE (package_id, shlib_id)"
 	 ");"
 	 "CREATE TABLE %Q.pkg_shlibs_provided ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE CASCADE,"
 		"shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 		" ON DELETE RESTRICT ON UPDATE RESTRICT,"
 		"UNIQUE (package_id, shlib_id)"
 	 ");"
 	 "INSERT INTO %Q.pkg_shlibs_required (package_id, shlib_id)"
 		" SELECT package_id, shlib_id FROM %Q.pkg_shlibs;"
 	 "DROP TABLE %Q.pkg_shlibs;"
 	},
 	{2002,
 ,
 	 "Add abstract metadata capability",
 	 "CREATE TABLE %Q.abstract ("
 		"abstract_id INTEGER PRIMARY KEY,"
 		"abstract TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE %Q.pkg_abstract ("
 		"package_id INTERGER REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE RESTRICT,"
 		"key_id INTEGER NOT NULL REFERENCES abstract(abstract_id)"
 		" ON DELETE CASCADE ON UPDATE RESTRICT,"
 		"value_id INTEGER NOT NULL REFERENCES abstract(abstract_id)"
 		" ON DELETE CASCADE ON UPDATE RESTRICT"
 	 ");"
 	},
 	{2003,
 ,
 	"Add manifest digest field",
 	"ALTER TABLE %Q.packages ADD COLUMN manifestdigest TEXT NULL;"
 	"CREATE INDEX IF NOT EXISTS %Q.pkg_digest_id ON packages(origin, manifestdigest);"
 	},
 	{2004,
 ,
 	 "Rename 'abstract metadata' to 'annotations'",
 	 "CREATE TABLE %Q.annotation ("
 	        "annotation_id INTEGER PRIMARY KEY,"
 	        "annotation TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE %Q.pkg_annotation ("
 	        "package_id INTEGER REFERENCES packages(id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "tag_id INTEGER NOT NULL REFERENCES annotation(annotation_id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "value_id INTEGER NOT NULL REFERENCES annotation(annotation_id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "UNIQUE (package_id, tag_id)"
 	 ");"
 	 "INSERT INTO %Q.annotation (annotation_id, annotation)"
 	        " SELECT abstract_id, abstract FROM %Q.abstract;"
 	 "INSERT INTO %Q.pkg_annotation (package_id,tag_id,value_id)"
 	        " SELECT package_id,key_id,value_id FROM %Q.pkg_abstract;"
 	 "DROP TABLE pkg_abstract;"
 	 "DROP TABLE abstract;"
 	},
 	{2005,
 ,
 	 "Add capability to track option descriptions and defaults",
 	 "CREATE TABLE %Q.option ("
 		"option_id INTEGER PRIMARY KEY,"
 		"option TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE %Q.option_desc ("
 		"option_desc_id INTEGER PRIMARY KEY,"
 		"option_desc TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE %Q.pkg_option ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"value TEXT NOT NULL,"
 		"PRIMARY KEY(package_id, option_id)"
 	 ");"
 	 "CREATE TABLE %Q.pkg_option_desc ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"option_desc_id INTEGER NOT NULL "
 			"REFERENCES option_desc(option_desc_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"PRIMARY KEY(package_id, option_id)"
 	 ");"
 	 "CREATE TABLE %Q.pkg_option_default ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"default_value TEXT NOT NULL,"
 		"PRIMARY KEY(package_id, option_id)"
 	 ");"
 	 "INSERT INTO %Q.option (option) "
 		"SELECT DISTINCT option FROM %Q.options;"
 	 "INSERT INTO %Q.pkg_option(package_id, option_id, value) "
 		"SELECT package_id, option_id, value "
 		"FROM %Q.options oo JOIN %Q.option o "
 			"ON (oo.option = o.option);"
 	 "DROP TABLE %Q.options;",
 	},
 	{2006,
 ,
 	 "Add conflicts and provides",
 	"CREATE TABLE %Q.pkg_conflicts ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "conflict_id INTEGER NOT NULL,"
 	    "UNIQUE(package_id, conflict_id)"
 	");"
 	"CREATE TABLE %Q.provides("
 	"    id INTEGER PRIMARY KEY,"
 	"    provide TEXT NOT NULL"
 	");"
 	"CREATE TABLE %Q.pkg_provides ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "provide_id INTEGER NOT NULL REFERENCES provides(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, provide_id)"
 	");"
 	},
 	{2007,
 ,
 	 "Add FTS index",
 	 "CREATE VIRTUAL TABLE %Q.pkg_search USING fts4(id, name, origin);"
 	 "INSERT INTO %Q.pkg_search SELECT id, name || '-' || version, origin FROM %Q.packages;"
 	 "CREATE INDEX %Q.packages_origin ON packages(origin COLLATE NOCASE);"
 	 "CREATE INDEX %Q.packages_name ON packages(name COLLATE NOCASE);"
 	},
 	{2008,
 ,
 	 "Optimize indicies",
 	 "CREATE INDEX IF NOT EXISTS %Q.packages_uid_nocase ON packages(name COLLATE NOCASE, origin COLLATE NOCASE);"
 	 "CREATE INDEX IF NOT EXISTS %Q.packages_version_nocase ON packages(name COLLATE NOCASE, version);"
 	 "CREATE INDEX IF NOT EXISTS %Q.packages_uid ON packages(name, origin);"
 	 "CREATE INDEX IF NOT EXISTS %Q.packages_version ON packages(name, version);"
 	 "CREATE UNIQUE INDEX IF NOT EXISTS %Q.packages_digest ON packages(manifestdigest);"
 	},
 	{2009,
 ,
 	 "Add legacy digest field",
 	 "ALTER TABLE %Q.packages ADD COLUMN olddigest TEXT NULL;"
 	 "UPDATE %Q.packages SET olddigest=manifestdigest WHERE olddigest=NULL;"
 	},
 	/* Mark the end of the array */
 	{ -1, -1, NULL, NULL, }
 };
 /* How to downgrade a newer repo to match what the current system
    expects */
 static const struct repo_changes repo_downgrades[] = {
 	{2010,
 ,
 	 "Drop olddigest field",
 	 "ALTER TABLE %Q.packages REMOVE COLUMN olddigest;"
 	},
 	{2009,
 ,
 	 "Drop indicies",
 	 "DROP INDEX %Q.packages_uid_nocase;"
 	 "DROP INDEX %Q.packages_version_nocase;"
 	 "DROP INDEX %Q.packages_uid;"
 	 "DROP INDEX %Q.packages_version;"
 	 "DROP INDEX %Q.packages_digest;"
 	},
 	{2008,
 ,
 	 "Drop FTS index",
 	 "DROP TABLE %Q.pkg_search;"
 	},
 	{2007,
 ,
 	 "Revert conflicts and provides creation",
 	 "DROP TABLE %Q.pkg_provides;"
 	 "DROP TABLE %Q.provides;"
 	 "DROP TABLE %Q.conflicts;"
 	},
 	{2006,
 ,
 	 "Revert addition of extra options related data",
 	 "CREATE TABLE %Q.options ("
 		"package_id INTEGER REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option TEXT,"
 		"value TEXT,"
 		"PRIMARY KEY(package_id,option)"
 	 ");"
 	 "INSERT INTO %Q.options (package_id, option, value) "
 		 "SELECT package_id, option, value "
 		"FROM %Q.pkg_option JOIN %Q.option USING(option_id);"
 	 "DROP TABLE pkg_option;"
 	 "DROP TABLE pkg_option_default;"
 	 "DROP TABLE option;"
 	 "DROP TABLE pkg_option_desc;"
 	 "DROP TABLE option_desc;",
 	},
 	{2005,
 ,
 	 "Revert rename of 'abstract metadata' to 'annotations'",
 	 "CREATE TABLE %Q.abstract ("
 	        "abstract_id INTEGER PRIMARY KEY,"
 	        "abstract TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE %Q.pkg_abstract ("
 	        "package_id INTEGER REFERENCES packages(id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "key_id INTEGER NOT NULL REFERENCES abstract(abstract_id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "value_id INTEGER NOT NULL REFERENCES abstract(abstract_id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT"
 	 ");"
 	 "INSERT INTO %Q.abstract (abstract_id, abstract)"
 	        " SELECT annotation_id, annotation FROM %Q.annotation;"
 	 "INSERT INTO %Q.pkg_abstract (package_id,key_id,value_id)"
 	        " SELECT package_id,tag_id,value_id FROM %Q.pkg_annotation;"
 	 "DROP TABLE %Q.pkg_annotation;"
 	 "DROP TABLE %Q.annotation;"
 	},
 	{2004,
 ,
 	 "Drop manifest digest index",
 	 "DROP INDEX %Q.pkg_digest_id;"
 	},
 	{2003,
 ,
 	 "Drop abstract metadata",
 	 "DROP TABLE %Q.pkg_abstract;"
 	 "DROP TABLE %Q.abstract;"
 	},
 	{2002,
 ,
 	 "Drop \'shlibs provided\' but retain \'shlibs required\'",
 	 "CREATE TABLE %Q.pkg_shlibs_required ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE CASCADE,"
 		"shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 		" ON DELETE RESTRICT ON UPDATE RESTRICT,"
 		"UNIQUE (package_id, shlib_id)"
 	 ");"
 	 "CREATE TABLE %Q.pkg_shlibs ("
 		"package_id INTEGER REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE CASCADE,"
 		"shlib_id INTEGER REFERENCES shlibs(id)"
 		" ON DELETE RESTRICT ON UPDATE RESTRICT,"
 		"PRIMARY KEY (package_id, shlib_id)"
 	 ");"
 	 "INSERT INTO %Q.pkg_shlibs (package_id, shlib_id)"
 		" SELECT package_id, shlib_id FROM %Q.pkg_shlibs_required;"
 	 "DELETE FROM %Q.shlibs WHERE id NOT IN"
 		" (SELECT shlib_id FROM %Q.pkg_shlibs);"
 	 "DROP TABLE %Q.pkg_shlibs_provided;"
 	 "DROP TABLE %Q.pkg_shlibs_required;"
 	},
 	/* Mark the end of the array */
 	{ -1, -1, NULL, NULL, }
 };
 #endif	/* _REPODB */

added libpkg/repo/Makefile.am

@@ -0,0 +1 @@

+	`SUBDIRS = @REPOS@`
	`\ No newline at end of file`

added libpkg/repo/binary/Makefile.am

@@ -0,0 +1,21 @@

 pkg_common_cflags=	-I$(top_srcdir)/libpkg \
 			@LIBSBUF_INCLUDE@ \
 			@LDNS_CFLAGS@ \
 			-I$(top_srcdir)/external/expat/lib \
 			-I$(top_srcdir)/external/libucl/include \
 			-I$(top_srcdir)/external/uthash \
 			-I$(top_srcdir)/external/sqlite \
 			-Wno-pointer-sign
 librepo_binary_la_SOURCES=	binary.c \
 							common.c \
 							init.c \
 							query.c \
 							update.c
 librepo_binary_la_CFLAGS=	$(pkg_common_cflags) -shared
 librepo_binary_static_la_LDFLAGS=	-all-static
 librepo_binary_static_la_SOURCES=	$(librepo_binary_la_SOURCES)
 librepo_binary_static_la_CFLAGS=	$(pkg_common_cflags) -static
 noinst_LTLIBRARIES= librepo_binary.la librepo_binary_static.la
 \ No newline at end of file

added libpkg/repo/binary/binary.c

@@ -0,0 +1,39 @@

 /* Copyright (c) 2014, Vsevolod Stakhov
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *       * Redistributions of source code must retain the above copyright
  *         notice, this list of conditions and the following disclaimer.
  *       * Redistributions in binary form must reproduce the above copyright
  *         notice, this list of conditions and the following disclaimer in the
  *         documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "binary.h"
 struct pkg_repo_ops pkg_repo_binary_ops = {
 	.type = "binary",
 	.init = pkg_repo_binary_init,
 	.access = pkg_repo_binary_access,
 	.open = pkg_repo_binary_open,
 	.create = pkg_repo_binary_create,
 	.close = pkg_repo_binary_close,
 	.update = pkg_repo_binary_update,
 	.query = pkg_repo_binary_query,
 	.shlib_provided = pkg_repo_binary_shlib_provide,
 	.shlib_required = pkg_repo_binary_shlib_require,
 	.search = pkg_repo_binary_search,
 	.fetch_pkg = NULL
 };

added libpkg/repo/binary/binary.h

@@ -0,0 +1,51 @@

 /* Copyright (c) 2014, Vsevolod Stakhov
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *       * Redistributions of source code must retain the above copyright
  *         notice, this list of conditions and the following disclaimer.
  *       * Redistributions in binary form must reproduce the above copyright
  *         notice, this list of conditions and the following disclaimer in the
  *         documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #ifndef BINARY_H_
 #define BINARY_H_
 #include "pkg.h"
 #include "private/pkg.h"
 #define PRIV_GET(repo) (sqlite3 *)(repo)->priv;
 extern struct pkg_repo_ops pkg_repo_binary_ops;
 int pkg_repo_binary_update(struct pkg_repo *repo, bool force);
 int pkg_repo_binary_init(struct pkg_repo *repo);
 int pkg_repo_binary_close(struct pkg_repo *repo, bool commit);
 int pkg_repo_binary_access(struct pkg_repo *repo, unsigned mode);
 int pkg_repo_binary_create(struct pkg_repo *repo);
 int pkg_repo_binary_open(struct pkg_repo *repo, unsigned mode);
 struct pkg_repo_it *pkg_repo_binary_query(struct pkg_repo *repo,
 	const char *pattern, match_t match);
 struct pkg_repo_it *pkg_repo_binary_shlib_provide(struct pkg_repo *repo,
 	const char *require);
 struct pkg_repo_it *pkg_repo_binary_shlib_require(struct pkg_repo *repo,
 	const char *provide);
 struct pkg_repo_it *pkg_repo_binary_search(struct pkg_repo *repo,
 	const char *pattern, match_t match,
     pkgdb_field field, pkgdb_field sort);
 #endif /* BINARY_H_ */

added libpkg/repo/binary/binary_private.h

@@ -0,0 +1,523 @@

 /* Copyright (c) 2014, Vsevolod Stakhov
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *       * Redistributions of source code must retain the above copyright
  *         notice, this list of conditions and the following disclaimer.
  *       * Redistributions in binary form must reproduce the above copyright
  *         notice, this list of conditions and the following disclaimer in the
  *         documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #ifndef INIT_PRIVATE_H_
 #define INIT_PRIVATE_H_
 #include <sqlite3.h>
 #include "pkg.h"
 #include "private/pkg.h"
 static const char binary_repo_initsql[] = ""
 	"CREATE TABLE packages ("
 	    "id INTEGER PRIMARY KEY,"
 	    "origin TEXT UNIQUE,"
 	    "name TEXT NOT NULL,"
 	    "version TEXT NOT NULL,"
 	    "comment TEXT NOT NULL,"
 	    "desc TEXT NOT NULL,"
 	    "osversion TEXT,"
 	    "arch TEXT NOT NULL,"
 	    "maintainer TEXT NOT NULL,"
 	    "www TEXT,"
 	    "prefix TEXT NOT NULL,"
 	    "pkgsize INTEGER NOT NULL,"
 	    "flatsize INTEGER NOT NULL,"
 	    "licenselogic INTEGER NOT NULL,"
 	    "cksum TEXT NOT NULL,"
 	    /* relative path to the package in the repository */
 	    "path TEXT NOT NULL,"
 	    "pkg_format_version INTEGER,"
 	    "manifestdigest TEXT NULL,"
 	    "olddigest TEXT NULL"
 	");"
 	"CREATE TABLE deps ("
 	    "origin TEXT,"
 	    "name TEXT,"
 	    "version TEXT,"
 	    "package_id INTEGER REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "UNIQUE(package_id, origin)"
 	");"
 	"CREATE TABLE categories ("
 	    "id INTEGER PRIMARY KEY, "
 	    "name TEXT NOT NULL UNIQUE "
 	");"
 	"CREATE TABLE pkg_categories ("
 	    "package_id INTEGER REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "category_id INTEGER REFERENCES categories(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, category_id)"
 	");"
 	"CREATE TABLE licenses ("
 	    "id INTEGER PRIMARY KEY,"
 	    "name TEXT NOT NULL UNIQUE"
 	");"
 	"CREATE TABLE pkg_licenses ("
 	    "package_id INTEGER REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "license_id INTEGER REFERENCES licenses(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, license_id)"
 	");"
 	"CREATE TABLE option ("
 		"option_id INTEGER PRIMARY KEY,"
 		"option TEXT NOT NULL UNIQUE"
 	");"
 	"CREATE TABLE option_desc ("
 		"option_desc_id INTEGER PRIMARY KEY,"
 		"option_desc TEXT NOT NULL UNIQUE"
 	");"
 	"CREATE TABLE pkg_option ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"value TEXT NOT NULL,"
 		"PRIMARY KEY(package_id, option_id)"
 	");"
 	"CREATE TABLE pkg_option_desc ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"option_desc_id INTEGER NOT NULL "
 			"REFERENCES option_desc(option_desc_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"PRIMARY KEY(package_id, option_id)"
 	");"
 	"CREATE TABLE pkg_option_default ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"default_value TEXT NOT NULL,"
 		"PRIMARY KEY(package_id, option_id)"
 	");"
 	"CREATE TABLE shlibs ("
 	    "id INTEGER PRIMARY KEY,"
 	    "name TEXT NOT NULL UNIQUE "
 	");"
 	"CREATE TABLE pkg_shlibs_required ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, shlib_id)"
 	");"
 	"CREATE TABLE pkg_shlibs_provided ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, shlib_id)"
 	");"
 	"CREATE TABLE annotation ("
 	    "annotation_id INTEGER PRIMARY KEY,"
 	    "annotation TEXT NOT NULL UNIQUE"
 	");"
 	"CREATE TABLE pkg_annotation ("
 	    "package_id INTERGER REFERENCES packages(id)"
 	    " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	    "tag_id INTEGER NOT NULL REFERENCES annotation(annotation_id)"
 	    " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	    "value_id INTEGER NOT NULL REFERENCES annotation(annotation_id)"
 	    " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	    "UNIQUE (package_id, tag_id)"
 	");"
 	"CREATE TABLE pkg_conflicts ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "conflict_id INTEGER NOT NULL,"
 	    "UNIQUE(package_id, conflict_id)"
 	");"
 	"CREATE TABLE provides("
 	"    id INTEGER PRIMARY KEY,"
 	"    provide TEXT NOT NULL"
 	");"
 	"CREATE TABLE pkg_provides ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "provide_id INTEGER NOT NULL REFERENCES provides(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, provide_id)"
 	");"
 	"CREATE INDEX packages_origin ON packages(origin COLLATE NOCASE);"
 	"CREATE INDEX packages_name ON packages(name COLLATE NOCASE);"
 	"CREATE INDEX packages_uid_nocase ON packages(name COLLATE NOCASE, origin COLLATE NOCASE);"
 	"CREATE INDEX packages_version_nocase ON packages(name COLLATE NOCASE, version);"
 	"CREATE INDEX packages_uid ON packages(name, origin);"
 	"CREATE INDEX packages_version ON packages(name, version);"
 	"CREATE UNIQUE INDEX packages_digest ON packages(manifestdigest);"
 	/* FTS search table */
 	"CREATE VIRTUAL TABLE pkg_search USING fts4(id, name, origin);"
 	"PRAGMA user_version=%d;"
 	;
 struct repo_changes {
 	int version;		/* The repo schema this change applies to */
 	int next_version;	/* The repo schema this change creates */
 	const char *message;
 	const char *sql;
 };
 /* How to upgrade an older repo to match what the current system
    expects */
 static const struct repo_changes repo_upgrades[] = {
 	{2001,
 ,
 	 "Modify shlib tracking to add \'provided\' capability",
 	 "CREATE TABLE pkg_shlibs_required ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE CASCADE,"
 		"shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 		" ON DELETE RESTRICT ON UPDATE RESTRICT,"
 		"UNIQUE (package_id, shlib_id)"
 	 ");"
 	 "CREATE TABLE pkg_shlibs_provided ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE CASCADE,"
 		"shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 		" ON DELETE RESTRICT ON UPDATE RESTRICT,"
 		"UNIQUE (package_id, shlib_id)"
 	 ");"
 	 "INSERT INTO pkg_shlibs_required (package_id, shlib_id)"
 		" SELECT package_id, shlib_id FROM pkg_shlibs;"
 	 "DROP TABLE pkg_shlibs;"
 	},
 	{2002,
 ,
 	 "Add abstract metadata capability",
 	 "CREATE TABLE abstract ("
 		"abstract_id INTEGER PRIMARY KEY,"
 		"abstract TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE pkg_abstract ("
 		"package_id INTERGER REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE RESTRICT,"
 		"key_id INTEGER NOT NULL REFERENCES abstract(abstract_id)"
 		" ON DELETE CASCADE ON UPDATE RESTRICT,"
 		"value_id INTEGER NOT NULL REFERENCES abstract(abstract_id)"
 		" ON DELETE CASCADE ON UPDATE RESTRICT"
 	 ");"
 	},
 	{2003,
 ,
 	"Add manifest digest field",
 	"ALTER TABLE packages ADD COLUMN manifestdigest TEXT NULL;"
 	"CREATE INDEX IF NOT EXISTS pkg_digest_id ON packages(origin, manifestdigest);"
 	},
 	{2004,
 ,
 	 "Rename 'abstract metadata' to 'annotations'",
 	 "CREATE TABLE annotation ("
 	        "annotation_id INTEGER PRIMARY KEY,"
 	        "annotation TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE pkg_annotation ("
 	        "package_id INTEGER REFERENCES packages(id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "tag_id INTEGER NOT NULL REFERENCES annotation(annotation_id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "value_id INTEGER NOT NULL REFERENCES annotation(annotation_id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "UNIQUE (package_id, tag_id)"
 	 ");"
 	 "INSERT INTO annotation (annotation_id, annotation)"
 	        " SELECT abstract_id, abstract FROM abstract;"
 	 "INSERT INTO pkg_annotation (package_id,tag_id,value_id)"
 	        " SELECT package_id,key_id,value_id FROM pkg_abstract;"
 	 "DROP TABLE pkg_abstract;"
 	 "DROP TABLE abstract;"
 	},
 	{2005,
 ,
 	 "Add capability to track option descriptions and defaults",
 	 "CREATE TABLE option ("
 		"option_id INTEGER PRIMARY KEY,"
 		"option TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE option_desc ("
 		"option_desc_id INTEGER PRIMARY KEY,"
 		"option_desc TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE pkg_option ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"value TEXT NOT NULL,"
 		"PRIMARY KEY(package_id, option_id)"
 	 ");"
 	 "CREATE TABLE pkg_option_desc ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"option_desc_id INTEGER NOT NULL "
 			"REFERENCES option_desc(option_desc_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"PRIMARY KEY(package_id, option_id)"
 	 ");"
 	 "CREATE TABLE pkg_option_default ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option_id INTEGER NOT NULL REFERENCES option(option_id) "
 			"ON DELETE RESTRICT ON UPDATE CASCADE,"
 		"default_value TEXT NOT NULL,"
 		"PRIMARY KEY(package_id, option_id)"
 	 ");"
 	 "INSERT INTO option (option) "
 		"SELECT DISTINCT option FROM options;"
 	 "INSERT INTO pkg_option(package_id, option_id, value) "
 		"SELECT package_id, option_id, value "
 		"FROM options oo JOIN option o "
 			"ON (oo.option = o.option);"
 	 "DROP TABLE options;",
 	},
 	{2006,
 ,
 	 "Add conflicts and provides",
 	"CREATE TABLE pkg_conflicts ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "conflict_id INTEGER NOT NULL,"
 	    "UNIQUE(package_id, conflict_id)"
 	");"
 	"CREATE TABLE provides("
 	"    id INTEGER PRIMARY KEY,"
 	"    provide TEXT NOT NULL"
 	");"
 	"CREATE TABLE pkg_provides ("
 	    "package_id INTEGER NOT NULL REFERENCES packages(id)"
 	    "  ON DELETE CASCADE ON UPDATE CASCADE,"
 	    "provide_id INTEGER NOT NULL REFERENCES provides(id)"
 	    "  ON DELETE RESTRICT ON UPDATE RESTRICT,"
 	    "UNIQUE(package_id, provide_id)"
 	");"
 	},
 	{2007,
 ,
 	 "Add FTS index",
 	 "CREATE VIRTUAL TABLE pkg_search USING fts4(id, name, origin);"
 	 "INSERT INTO pkg_search SELECT id, name || '-' || version, origin FROM packages;"
 	 "CREATE INDEX packages_origin ON packages(origin COLLATE NOCASE);"
 	 "CREATE INDEX packages_name ON packages(name COLLATE NOCASE);"
 	},
 	{2008,
 ,
 	 "Optimize indicies",
 	 "CREATE INDEX IF NOT EXISTS packages_uid_nocase ON packages(name COLLATE NOCASE, origin COLLATE NOCASE);"
 	 "CREATE INDEX IF NOT EXISTS packages_version_nocase ON packages(name COLLATE NOCASE, version);"
 	 "CREATE INDEX IF NOT EXISTS packages_uid ON packages(name, origin);"
 	 "CREATE INDEX IF NOT EXISTS packages_version ON packages(name, version);"
 	 "CREATE UNIQUE INDEX IF NOT EXISTS packages_digest ON packages(manifestdigest);"
 	},
 	{2009,
 ,
 	 "Add legacy digest field",
 	 "ALTER TABLE packages ADD COLUMN olddigest TEXT NULL;"
 	 "UPDATE packages SET olddigest=manifestdigest WHERE olddigest=NULL;"
 	},
 	/* Mark the end of the array */
 	{ -1, -1, NULL, NULL, }
 };
 /* How to downgrade a newer repo to match what the current system
    expects */
 static const struct repo_changes repo_downgrades[] = {
 	{2010,
 ,
 	 "Drop olddigest field",
 	 "ALTER TABLE packages REMOVE COLUMN olddigest;"
 	},
 	{2009,
 ,
 	 "Drop indicies",
 	 "DROP INDEX packages_uid_nocase;"
 	 "DROP INDEX packages_version_nocase;"
 	 "DROP INDEX packages_uid;"
 	 "DROP INDEX packages_version;"
 	 "DROP INDEX packages_digest;"
 	},
 	{2008,
 ,
 	 "Drop FTS index",
 	 "DROP TABLE pkg_search;"
 	},
 	{2007,
 ,
 	 "Revert conflicts and provides creation",
 	 "DROP TABLE pkg_provides;"
 	 "DROP TABLE provides;"
 	 "DROP TABLE conflicts;"
 	},
 	{2006,
 ,
 	 "Revert addition of extra options related data",
 	 "CREATE TABLE options ("
 		"package_id INTEGER REFERENCES packages(id) "
 			"ON DELETE CASCADE ON UPDATE CASCADE,"
 		"option TEXT,"
 		"value TEXT,"
 		"PRIMARY KEY(package_id,option)"
 	 ");"
 	 "INSERT INTO options (package_id, option, value) "
 		 "SELECT package_id, option, value "
 		"FROM pkg_option JOIN option USING(option_id);"
 	 "DROP TABLE pkg_option;"
 	 "DROP TABLE pkg_option_default;"
 	 "DROP TABLE option;"
 	 "DROP TABLE pkg_option_desc;"
 	 "DROP TABLE option_desc;",
 	},
 	{2005,
 ,
 	 "Revert rename of 'abstract metadata' to 'annotations'",
 	 "CREATE TABLE abstract ("
 	        "abstract_id INTEGER PRIMARY KEY,"
 	        "abstract TEXT NOT NULL UNIQUE"
 	 ");"
 	 "CREATE TABLE pkg_abstract ("
 	        "package_id INTEGER REFERENCES packages(id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "key_id INTEGER NOT NULL REFERENCES abstract(abstract_id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT,"
 	        "value_id INTEGER NOT NULL REFERENCES abstract(abstract_id)"
 	        " ON DELETE CASCADE ON UPDATE RESTRICT"
 	 ");"
 	 "INSERT INTO abstract (abstract_id, abstract)"
 	        " SELECT annotation_id, annotation FROM annotation;"
 	 "INSERT INTO pkg_abstract (package_id,key_id,value_id)"
 	        " SELECT package_id,tag_id,value_id FROM pkg_annotation;"
 	 "DROP TABLE pkg_annotation;"
 	 "DROP TABLE annotation;"
 	},
 	{2004,
 ,
 	 "Drop manifest digest index",
 	 "DROP INDEX pkg_digest_id;"
 	},
 	{2003,
 ,
 	 "Drop abstract metadata",
 	 "DROP TABLE pkg_abstract;"
 	 "DROP TABLE abstract;"
 	},
 	{2002,
 ,
 	 "Drop \'shlibs provided\' but retain \'shlibs required\'",
 	 "CREATE TABLE pkg_shlibs_required ("
 		"package_id INTEGER NOT NULL REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE CASCADE,"
 		"shlib_id INTEGER NOT NULL REFERENCES shlibs(id)"
 		" ON DELETE RESTRICT ON UPDATE RESTRICT,"
 		"UNIQUE (package_id, shlib_id)"
 	 ");"
 	 "CREATE TABLE pkg_shlibs ("
 		"package_id INTEGER REFERENCES packages(id)"
 		" ON DELETE CASCADE ON UPDATE CASCADE,"
 		"shlib_id INTEGER REFERENCES shlibs(id)"
 		" ON DELETE RESTRICT ON UPDATE RESTRICT,"
 		"PRIMARY KEY (package_id, shlib_id)"
 	 ");"
 	 "INSERT INTO pkg_shlibs (package_id, shlib_id)"
 		" SELECT package_id, shlib_id FROM pkg_shlibs_required;"
 	 "DELETE FROM shlibs WHERE id NOT IN"
 		" (SELECT shlib_id FROM pkg_shlibs);"
 	 "DROP TABLE pkg_shlibs_provided;"
 	 "DROP TABLE pkg_shlibs_required;"
 	},
 	/* Mark the end of the array */
 	{ -1, -1, NULL, NULL, }
 };
 /* The package repo schema major revision */
 #define REPO_SCHEMA_MAJOR 2
 /* The package repo schema minor revision.
    Minor schema changes don't prevent older pkgng
    versions accessing the repo. */
 #define REPO_SCHEMA_MINOR 10
 /* REPO_SCHEMA_VERSION=2007 */
 #define REPO_SCHEMA_VERSION (REPO_SCHEMA_MAJOR * 1000 + REPO_SCHEMA_MINOR)
 #define REPO_NAME_PREFIX "repo-"
 typedef enum _sql_prstmt_index {
 	PKG = 0,
 	DEPS,
 	CAT1,
 	CAT2,
 	LIC1,
 	LIC2,
 	OPT1,
 	OPT2,
 	SHLIB1,
 	SHLIB_REQD,
 	SHLIB_PROV,
 	ANNOTATE1,
 	ANNOTATE2,
 	EXISTS,
 	VERSION,
 	DELETE,
 	FTS_APPEND,
 	PRSTMT_LAST,
 } sql_prstmt_index;
 int pkg_repo_binary_init_prstatements(sqlite3 *sqlite);
 int pkg_repo_binary_run_prstatement(sql_prstmt_index s, ...);
 const char * pkg_repo_binary_sql_prstatement(sql_prstmt_index s);
 sqlite3_stmt* pkg_repo_binary_stmt_prstatement(sql_prstmt_index s);
 void pkg_repo_binary_finalize_prstatements(void);
 /*
  * Warning: returns a pointer to static array
  */
 const char * pkg_repo_binary_get_filename(const char *name);
 #endif /* INIT_PRIVATE_H_ */

added libpkg/repo/binary/common.c

@@ -0,0 +1,244 @@

 /* Copyright (c) 2014, Vsevolod Stakhov
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *       * Redistributions of source code must retain the above copyright
  *         notice, this list of conditions and the following disclaimer.
  *       * Redistributions in binary form must reproduce the above copyright
  *         notice, this list of conditions and the following disclaimer in the
  *         documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdbool.h>
 #include <string.h>
 #include <sqlite3.h>
 #include "pkg.h"
 #include "private/event.h"
 #include "private/pkg.h"
 #include "private/pkgdb.h"
 #include "private/utils.h"
 #include "binary_private.h"
 static sql_prstmt sql_prepared_statements[PRSTMT_LAST] = {
 	[PKG] = {
 		NULL,
 		"INSERT INTO packages ("
 		"origin, name, version, comment, desc, arch, maintainer, www, "
 		"prefix, pkgsize, flatsize, licenselogic, cksum, path, manifestdigest, olddigest"
 		")"
 		"VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
 		"TTTTTTTTTIIITTTT",
 	},
 	[DEPS] = {
 		NULL,
 		"INSERT INTO deps (origin, name, version, package_id) "
 		"VALUES (?1, ?2, ?3, ?4)",
 		"TTTI",
 	},
 	[CAT1] = {
 		NULL,
 		"INSERT OR IGNORE INTO categories(name) VALUES(?1)",
 		"T",
 	},
 	[CAT2] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_categories(package_id, category_id) "
 		"VALUES (?1, (SELECT id FROM categories WHERE name = ?2))",
 		"IT",
 	},
 	[LIC1] = {
 		NULL,
 		"INSERT OR IGNORE INTO licenses(name) VALUES(?1)",
 		"T",
 	},
 	[LIC2] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_licenses(package_id, license_id) "
 		"VALUES (?1, (SELECT id FROM licenses WHERE name = ?2))",
 		"IT",
 	},
 	[OPT1] = {
 		NULL,
 		"INSERT OR IGNORE INTO option(option) "
 		"VALUES (?1)",
 		"T",
 	},
 	[OPT2] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_option (option_id, value, package_id) "
 		"VALUES (( SELECT option_id FROM option WHERE option = ?1), ?2, ?3)",
 		"TTI",
 	},
 	[SHLIB1] = {
 		NULL,
 		"INSERT OR IGNORE INTO shlibs(name) VALUES(?1)",
 		"T",
 	},
 	[SHLIB_REQD] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_shlibs_required(package_id, shlib_id) "
 		"VALUES (?1, (SELECT id FROM shlibs WHERE name = ?2))",
 		"IT",
 	},
 	[SHLIB_PROV] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_shlibs_provided(package_id, shlib_id) "
 		"VALUES (?1, (SELECT id FROM shlibs WHERE name = ?2))",
 		"IT",
 	},
 	[EXISTS] = {
 		NULL,
 		"SELECT count(*) FROM packages WHERE cksum=?1",
 		"T",
 	},
 	[ANNOTATE1] = {
 		NULL,
 		"INSERT OR IGNORE INTO annotation(annotation) "
 		"VALUES (?1)",
 		"T",
 	},
 	[ANNOTATE2] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_annotation(package_id, tag_id, value_id) "
 		"VALUES (?1,"
 		" (SELECT annotation_id FROM annotation WHERE annotation=?2),"
 		" (SELECT annotation_id FROM annotation WHERE annotation=?3))",
 		"ITT",
 	},
 	[VERSION] = {
 		NULL,
 		"SELECT version FROM packages WHERE origin=?1",
 		"T",
 	},
 	[DELETE] = {
 		NULL,
 		"DELETE FROM packages WHERE origin=?1;"
 		"DELETE FROM pkg_search WHERE origin=?1;",
 		"TT",
 	},
 	[FTS_APPEND] = {
 		NULL,
 		"INSERT OR ROLLBACK INTO pkg_search(id, name, origin) "
 		"VALUES (?1, ?2 || '-' || ?3, ?4);",
 		"ITTT"
 	}
 	/* PRSTMT_LAST */
 };
 const char *
 pkg_repo_binary_sql_prstatement(sql_prstmt_index s)
 {
 	if (s < PRSTMT_LAST)
 		return (sql_prepared_statements[s].sql);
 	else
 		return ("unknown");
 }
 sqlite3_stmt*
 pkg_repo_binary_stmt_prstatement(sql_prstmt_index s)
 {
 	if (s < PRSTMT_LAST)
 		return (sql_prepared_statements[s].stmt);
 	else
 		return (NULL);
 }
 int
 pkg_repo_binary_init_prstatements(sqlite3 *sqlite)
 {
 	sql_prstmt_index i, last;
 	int ret;
 	last = PRSTMT_LAST;
 	for (i = 0; i < last; i++) {
 		ret = sqlite3_prepare_v2(sqlite, SQL(i), -1, &STMT(i), NULL);
 		if (ret != SQLITE_OK) {
 			ERROR_SQLITE(sqlite, SQL(i));
 			return (EPKG_FATAL);
 		}
 	}
 	return (EPKG_OK);
 }
 int
 pkg_repo_binary_run_prstatement(sql_prstmt_index s, ...)
 {
 	int retcode;	/* Returns SQLITE error code */
 	va_list ap;
 	sqlite3_stmt *stmt;
 	int i;
 	const char *argtypes;
 	stmt = STMT(s);
 	argtypes = sql_prepared_statements[s].argtypes;
 	sqlite3_reset(stmt);
 	va_start(ap, s);
 	for (i = 0; argtypes[i] != '\0'; i++)
 	{
 		switch (argtypes[i]) {
 		case 'T':
 			sqlite3_bind_text(stmt, i + 1, va_arg(ap, const char*),
 			    -1, SQLITE_STATIC);
 			break;
 		case 'I':
 			sqlite3_bind_int64(stmt, i + 1, va_arg(ap, int64_t));
 			break;
 		}
 	}
 	va_end(ap);
 	retcode = sqlite3_step(stmt);
 	return (retcode);
 }
 const char *
 pkg_repo_binary_get_filename(const char *name)
 {
 	static char reponame[MAXPATHLEN];
 	snprintf(reponame, sizeof(reponame), REPO_NAME_PREFIX "%s.sqlite",
 			name);
 	return (reponame);
 }
 void
 pkg_repo_binary_finalize_prstatements(void)
 {
 	sql_prstmt_index i, last;
 	last = PRSTMT_LAST;
 	for (i = 0; i < last; i++)
 	{
 		if (STMT(i) != NULL) {
 			sqlite3_finalize(STMT(i));
 			STMT(i) = NULL;
 		}
 	}
 	return;
 }

added libpkg/repo/binary/init.c

@@ -0,0 +1,489 @@

 /* Copyright (c) 2014, Vsevolod Stakhov
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *       * Redistributions of source code must retain the above copyright
  *         notice, this list of conditions and the following disclaimer.
  *       * Redistributions in binary form must reproduce the above copyright
  *         notice, this list of conditions and the following disclaimer in the
  *         documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/param.h>
 #include <sys/mount.h>
 #include <assert.h>
 #include <errno.h>
 #include <regex.h>
 #include <grp.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdbool.h>
 #include <string.h>
 #include <unistd.h>
 #include <libgen.h>
 #include <sqlite3.h>
 #include "pkg.h"
 #include "private/event.h"
 #include "private/pkg.h"
 #include "private/pkgdb.h"
 #include "private/utils.h"
 #include "binary.h"
 #include "binary_private.h"
 static void
 sqlite_file_exists(sqlite3_context *ctx, int argc, sqlite3_value **argv)
 {
 	char	 fpath[MAXPATHLEN];
 	sqlite3	*db = sqlite3_context_db_handle(ctx);
 	char	*path = dirname(sqlite3_db_filename(db, "main"));
 	char	 cksum[SHA256_DIGEST_LENGTH * 2 +1];
 	if (argc != 2) {
 		sqlite3_result_error(ctx, "file_exists needs two argument", -1);
 		return;
 	}
 	snprintf(fpath, sizeof(fpath), "%s/%s", path, sqlite3_value_text(argv[0]));
 	if (access(fpath, R_OK) == 0) {
 		sha256_file(fpath, cksum);
 		if (strcmp(cksum, sqlite3_value_text(argv[1])) == 0)
 			sqlite3_result_int(ctx, 1);
 		else
 			sqlite3_result_int(ctx, 0);
 	} else {
 		sqlite3_result_int(ctx, 0);
 	}
 }
 static int
 pkg_repo_binary_get_user_version(sqlite3 *sqlite, int *reposcver)
 {
 	sqlite3_stmt *stmt;
 	int retcode;
 	const char *sql = "PRAGMA user_version;";
 	if (sqlite3_prepare_v2(sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sql);
 		return (EPKG_FATAL);
 	}
 	if (sqlite3_step(stmt) == SQLITE_ROW) {
 		*reposcver = sqlite3_column_int(stmt, 0);
 		retcode = EPKG_OK;
 	} else {
 		*reposcver = -1;
 		retcode = EPKG_FATAL;
 	}
 	sqlite3_finalize(stmt);
 	return (retcode);
 }
 static int
 pkg_repo_binary_set_version(sqlite3 *sqlite, int reposcver)
 {
 	int		 retcode = EPKG_OK;
 	char		*errmsg;
 	const char	*sql = "PRAGMA user_version = %d;" ;
 	if (sqlite3_exec(sqlite, sql, NULL, NULL, &errmsg) != SQLITE_OK) {
 		pkg_emit_error("sqlite: %s", errmsg);
 		sqlite3_free(errmsg);
 		retcode = EPKG_FATAL;
 	}
 	return (retcode);
 }
 static int
 pkg_repo_binary_apply_change(struct pkg_repo *repo, sqlite3 *sqlite,
 		  const struct repo_changes *repo_changes, const char *updown,
 		  int version, int *next_version)
 {
 	const struct repo_changes	*change;
 	bool			 found = false, in_trans = false;
 	int			 ret = EPKG_OK;
 	char			*errmsg;
 	for (change = repo_changes; change->version != -1; change++) {
 		if (change->version == version) {
 			found = true;
 			break;
 		}
 	}
 	if (!found) {
 		pkg_emit_error("Failed to %s \"%s\" repo schema "
 			" version %d (target version %d) "
 			"-- change not found", updown, repo->name, version,
 			REPO_SCHEMA_VERSION);
 		return (EPKG_FATAL);
 	}
 	/* begin transaction */
 	in_trans = true;
 	ret = pkgdb_transaction_begin(sqlite, "SCHEMA");
 	/* apply change */
 	if (ret == EPKG_OK) {
 		pkg_debug(4, "Pkgdb: running '%s'", change->sql);
 		ret = sqlite3_exec(sqlite, change->sql, NULL, NULL, &errmsg);
 		if (ret != SQLITE_OK) {
 			pkg_emit_error("sqlite: %s", errmsg);
 			sqlite3_free(errmsg);
 			ret = EPKG_FATAL;
 		}
 	}
 	/* update repo user_version */
 	if (ret == EPKG_OK) {
 		*next_version = change->next_version;
 		ret = pkg_repo_binary_set_version(sqlite, *next_version);
 	}
 	/* commit or rollback */
 	if (in_trans) {
 		if (ret != EPKG_OK)
 			pkgdb_transaction_rollback(sqlite, "SCHEMA");
 		if (pkgdb_transaction_commit(sqlite, "SCHEMA") != EPKG_OK)
 			ret = EPKG_FATAL;
 	}
 	if (ret == EPKG_OK) {
 		pkg_emit_notice("Repo \"%s\" %s schema %d to %d: %s",
 				repo->name, updown, version,
 				change->next_version, change->message);
 	}
 	return (ret);
 }
 static int
 pkg_repo_binary_upgrade(struct pkg_repo *repo, sqlite3 *sqlite, int current_version)
 {
 	int version;
 	int next_version;
 	int ret = EPKG_OK;
 	for (version = current_version;
 	     version < REPO_SCHEMA_VERSION;
 	     version = next_version)  {
 		ret = pkg_repo_binary_apply_change(repo, sqlite, repo_upgrades,
 					"upgrade", version, &next_version);
 		if (ret != EPKG_OK)
 			break;
 		pkg_debug(1, "Upgrading repo database schema from %d to %d",
 				version, next_version);
 	}
 	return (ret);
 }
 static int
 pkg_repo_binary_downgrade(struct pkg_repo *repo, sqlite3 *sqlite, int current_version)
 {
 	int version;
 	int next_version;
 	int ret = EPKG_OK;
 	for (version = current_version;
 	     version > REPO_SCHEMA_VERSION;
 	     version = next_version)  {
 		ret = pkg_repo_binary_apply_change(repo, sqlite, repo_downgrades,
 					"downgrade", version, &next_version);
 		if (ret != EPKG_OK)
 			break;
 		pkg_debug(1, "Downgrading repo database schema from %d to %d",
 				version, next_version);
 	}
 	return (ret);
 }
 int
 pkg_repo_binary_check_version(struct pkg_repo *repo, sqlite3 *sqlite)
 {
 	int reposcver;
 	int repomajor;
 	int ret;
 	if ((ret = pkg_repo_binary_get_user_version(sqlite, &reposcver))
 	    != EPKG_OK)
 		return (ret);	/* sqlite error */
 	/*
 	 * If the local pkgng uses a repo schema behind that used to
 	 * create the repo, we may still be able use it for reading
 	 * (ie pkg install), but pkg repo can't do an incremental
 	 * update unless the actual schema matches the compiled in
 	 * schema version.
 	 *
 	 * Use a major - minor version schema: as the user_version
 	 * PRAGMA takes an integer version, encode this as MAJOR *
 	 * 1000 + MINOR.
 	 *
 	 * So long as the major versions are the same, the local pkgng
 	 * should be compatible with any repo created by a more recent
 	 * pkgng, although it may need some modification of the repo
 	 * schema
 	 */
 	/* --- Temporary ---- Grandfather in the old repo schema
 	   version so this patch doesn't immediately invalidate all
 	   the repos out there */
 	if (reposcver == 2)
 		reposcver = 2000;
 	if (reposcver == 3)
 		reposcver = 2001;
 	repomajor = reposcver / 1000;
 	if (repomajor < REPO_SCHEMA_MAJOR) {
 		pkg_emit_error("Repo %s (schema version %d) is too old - "
 		    "need at least schema %d", repo->name, reposcver,
 		    REPO_SCHEMA_MAJOR * 1000);
 		return (EPKG_REPOSCHEMA);
 	}
 	if (repomajor > REPO_SCHEMA_MAJOR) {
 		pkg_emit_error("Repo %s (schema version %d) is too new - "
 		    "we can accept at most schema %d", repo->name, reposcver,
 		    ((REPO_SCHEMA_MAJOR + 1) * 1000) - 1);
 		return (EPKG_REPOSCHEMA);
 	}
 	/* This is a repo schema version we can work with */
 	ret = EPKG_OK;
 	if (reposcver < REPO_SCHEMA_VERSION) {
 		if (sqlite3_db_readonly(sqlite, "main")) {
 			pkg_emit_error("Repo %s needs schema upgrade from "
 			    "%d to %d but it is opened readonly", repo->name,
 			    reposcver, REPO_SCHEMA_VERSION);
 			ret = EPKG_FATAL;
 		} else
 			ret = pkg_repo_binary_upgrade(repo, sqlite, reposcver);
 	} else if (reposcver > REPO_SCHEMA_VERSION) {
 		if (sqlite3_db_readonly(sqlite, "main")) {
 			pkg_emit_error("Repo %s needs schema downgrade from "
 			"%d to %d but it is opened readonly", repo->name,
 			       reposcver, REPO_SCHEMA_VERSION
 			);
 			ret = EPKG_FATAL;
 		} else
 			ret = pkg_repo_binary_downgrade(repo, sqlite, reposcver);
 	}
 	return (ret);
 }
 int
 pkg_repo_binary_open(struct pkg_repo *repo, unsigned mode)
 {
 	char filepath[MAXPATHLEN];
 	const char *dbdir = NULL;
 	sqlite3 *sqlite = NULL;
 	int flags;
 	int64_t res;
 	sqlite3_initialize();
 	dbdir = pkg_object_string(pkg_config_get("PKG_DBDIR"));
 	snprintf(filepath, sizeof(filepath), "%s/%s",
 		dbdir, pkg_repo_binary_get_filename(pkg_repo_name(repo)));
 	/* Always want read mode here */
 	if (access(filepath, R_OK | mode) != 0)
 		return (EPKG_ENOACCESS);
 	flags = (mode & W_OK) != 0 ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY;
 	if (sqlite3_open_v2(filepath, &sqlite, flags, NULL) != SQLITE_OK)
 		return (EPKG_FATAL);
 	/* Sanitise sqlite database */
 	if (get_pragma(sqlite, "SELECT count(name) FROM sqlite_master "
 		"WHERE type='table' AND name='repodata';", &res, false) != EPKG_OK) {
 		pkg_emit_error("Unable to query repository");
 		sqlite3_close(sqlite);
 		return (EPKG_FATAL);
 	}
 	if (res != 1) {
 		pkg_emit_notice("Repository %s contains no repodata table, "
 			"need to re-create database", repo->name);
 		sqlite3_close(sqlite);
 		return (EPKG_FATAL);
 	}
 	/* Check package site */
 	char *req = sqlite3_mprintf("select count(key) from repodata "
 		"WHERE key = \"packagesite\" and value = '%q'", pkg_repo_url(repo));
 	res = 0;
 	get_pragma(sqlite, req, &res, true);
 	sqlite3_free(req);
 	if (res != 1) {
 		pkg_emit_notice("Repository %s has a wrong packagesite, need to "
 			"re-create database", repo->name);
 		sqlite3_close(sqlite);
 		return (EPKG_FATAL);
 	}
 	/* Check version */
 	if (pkg_repo_binary_check_version(repo, sqlite) != EPKG_OK) {
 		pkg_emit_error("need to re-create repo %s to upgrade schema version",
 			repo->name);
 		sqlite3_close(sqlite);
 		if (mode & W_OK)
 			unlink(filepath);
 		return (EPKG_REPOSCHEMA);
 	}
 	repo->priv = sqlite;
 	return (EPKG_OK);
 }
 int
 pkg_repo_binary_create(struct pkg_repo *repo)
 {
 	char filepath[MAXPATHLEN];
 	const char *dbdir = NULL;
 	sqlite3 *sqlite = NULL;
 	int retcode;
 	sqlite3_initialize();
 	dbdir = pkg_object_string(pkg_config_get("PKG_DBDIR"));
 	snprintf(filepath, sizeof(filepath), "%s/%s",
 		dbdir, pkg_repo_binary_get_filename(pkg_repo_name(repo)));
 	/* Should never ever happen */
 	if (access(filepath, R_OK) == 0)
 		return (EPKG_CONFLICT);
 	/* Open for read/write/create */
 	if (sqlite3_open(filepath, &sqlite) != SQLITE_OK)
 		return (EPKG_FATAL);
 	retcode = sql_exec(sqlite, binary_repo_initsql, REPO_SCHEMA_VERSION);
 	if (retcode == EPKG_OK) {
 		sqlite3_stmt *stmt;
 		const char sql[] = ""
 						"INSERT OR REPLACE INTO repodata (key, value) "
 						"VALUES (\"packagesite\", ?1);";
 		/* register the packagesite */
 		if (sql_exec(sqlite, "CREATE TABLE IF NOT EXISTS repodata ("
 			"   key TEXT UNIQUE NOT NULL,"
 			"   value TEXT NOT NULL"
 			");") != EPKG_OK) {
 			pkg_emit_error("Unable to register the packagesite in the "
 				"database");
 			retcode = EPKG_FATAL;
 			goto cleanup;
 		}
 		if (sqlite3_prepare_v2(sqlite, sql, -1, &stmt, NULL) != SQLITE_OK) {
 			ERROR_SQLITE(sqlite, sql);
 			retcode = EPKG_FATAL;
 			goto cleanup;
 		}
 		sqlite3_bind_text(stmt, 1, pkg_repo_url(repo), -1, SQLITE_STATIC);
 		if (sqlite3_step(stmt) != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, sql);
 			sqlite3_finalize(stmt);
 			retcode = EPKG_FATAL;
 			goto cleanup;
 		}
 		sqlite3_finalize(stmt);
 	}
 cleanup:
 	sqlite3_close(sqlite);
 	return (retcode);
 }
 int
 pkg_repo_binary_init(struct pkg_repo *repo)
 {
 	int retcode = EPKG_OK;
 	sqlite3 *sqlite = PRIV_GET(repo);
 	sqlite3_create_function(sqlite, "file_exists", 2, SQLITE_ANY, NULL,
 		    sqlite_file_exists, NULL, NULL);
 	retcode = sql_exec(sqlite, "PRAGMA synchronous=default");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = sql_exec(sqlite, "PRAGMA foreign_keys=on");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	pkgdb_sqlcmd_init(sqlite, NULL, NULL);
 	retcode = pkg_repo_binary_init_prstatements(sqlite);
 	if (retcode != EPKG_OK)
 		return (retcode);
 	repo->priv = sqlite;
 	return (EPKG_OK);
 }
 int
 pkg_repo_binary_close(struct pkg_repo *repo, bool commit)
 {
 	int retcode = EPKG_OK;
 	sqlite3 *sqlite = PRIV_GET(repo);
 	if (sqlite == NULL)
 		return (retcode);
 	if (commit) {
 		if (pkgdb_transaction_commit(sqlite, NULL) != SQLITE_OK)
 			retcode = EPKG_FATAL;
 	}
 	pkg_repo_binary_finalize_prstatements();
 	sqlite3_free(sqlite);
 	repo->priv = NULL;
 	return (retcode);
 }
 int
 pkg_repo_binary_access(struct pkg_repo *repo, unsigned mode)
 {
 	const pkg_object	*o;
 	const char		*dbdir;
 	int			 ret = EPKG_OK;
 	o = pkg_config_get("PKG_DBDIR");
 	dbdir = pkg_object_string(o);
 	ret = pkgdb_check_access(mode, dbdir,
 		pkg_repo_binary_get_filename(pkg_repo_name(repo)));
 	return (ret);
 }

added libpkg/repo/binary/query.c

@@ -0,0 +1,355 @@

 /* Copyright (c) 2014, Vsevolod Stakhov
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *       * Redistributions of source code must retain the above copyright
  *         notice, this list of conditions and the following disclaimer.
  *       * Redistributions in binary form must reproduce the above copyright
  *         notice, this list of conditions and the following disclaimer in the
  *         documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <assert.h>
 #include <errno.h>
 #include <regex.h>
 #include <grp.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdbool.h>
 #include <string.h>
 #include <unistd.h>
 #include <libgen.h>
 #include <sqlite3.h>
 #include "pkg.h"
 #include "private/event.h"
 #include "private/pkg.h"
 #include "private/pkgdb.h"
 #include "private/utils.h"
 #include "binary.h"
 static struct pkg_repo_it* pkg_repo_binary_it_new(struct pkg_repo *repo,
 	sqlite3_stmt *s, short flags);
 static int pkg_repo_binary_it_next(struct pkg_repo_it *it, struct pkg **pkg_p, unsigned flags);
 static void pkg_repo_binary_it_free(struct pkg_repo_it *it);
 static void pkg_repo_binary_it_reset(struct pkg_repo_it *it);
 static struct pkg_repo_it_ops pkg_repo_binary_it_ops = {
 	.next = pkg_repo_binary_it_next,
 	.free = pkg_repo_binary_it_free,
 	.reset = pkg_repo_binary_it_reset
 };
 static struct pkg_repo_it*
 pkg_repo_binary_it_new(struct pkg_repo *repo, sqlite3_stmt *s, short flags)
 {
 	struct pkg_repo_it *it;
 	struct pkgdb fakedb;
 	it = malloc(sizeof(*it));
 	if (it == NULL) {
 		pkg_emit_errno("malloc", "pkg_repo_it");
 		sqlite3_finalize(s);
 		return (NULL);
 	}
 	it->ops = &pkg_repo_binary_it_ops;
 	it->flags = flags;
 	it->repo = repo;
 	fakedb.sqlite = PRIV_GET(repo);
 	it->data = pkgdb_it_new_sqlite(&fakedb, s, PKG_REMOTE, flags);
 	if (it->data == NULL) {
 		free(it);
 		return (NULL);
 	}
 	return (it);
 }
 static int
 pkg_repo_binary_it_next(struct pkg_repo_it *it, struct pkg **pkg_p, unsigned flags)
 {
 	return (pkgdb_it_next(it->data, pkg_p, flags));
 }
 static void
 pkg_repo_binary_it_free(struct pkg_repo_it *it)
 {
 	pkgdb_it_free(it->data);
 	free(it);
 }
 static void
 pkg_repo_binary_it_reset(struct pkg_repo_it *it)
 {
 	pkgdb_it_reset(it->data);
 }
 struct pkg_repo_it *
 pkg_repo_binary_query(struct pkg_repo *repo, const char *pattern, match_t match)
 {
 	sqlite3 *sqlite = PRIV_GET(repo);
 	sqlite3_stmt	*stmt = NULL;
 	struct sbuf	*sql = NULL;
 	const char	*comp = NULL;
 	int		 ret;
 	char		 basesql[BUFSIZ] = ""
 		"SELECT id, origin, name, name || '~' || origin as uniqueid, version, comment, "
 		"prefix, desc, arch, maintainer, www, "
 		"licenselogic, flatsize, pkgsize, "
 		"cksum, manifestdigest, path AS repopath, '%s' AS dbname "
 		"FROM packages";
 	assert(match == MATCH_ALL || (pattern != NULL && pattern[0] != '\0'));
 	sql = sbuf_new_auto();
 	comp = pkgdb_get_pattern_query(pattern, match);
 	if (comp && comp[0])
 		strlcat(basesql, comp, sizeof(basesql));
 	sbuf_printf(sql, basesql, repo->name);
 	sbuf_cat(sql, " ORDER BY name;");
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s' query for %s", sbuf_get(sql), pattern);
 	ret = sqlite3_prepare_v2(sqlite, sbuf_get(sql), sbuf_size(sql), &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	if (match != MATCH_ALL && match != MATCH_CONDITION)
 		sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
 	return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
 }
 struct pkg_repo_it *
 pkg_repo_binary_shlib_provide(struct pkg_repo *repo, const char *require)
 {
 	sqlite3_stmt	*stmt;
 	sqlite3 *sqlite = PRIV_GET(repo);
 	struct sbuf	*sql = NULL;
 	int		 ret;
 	const char	 basesql[] = ""
 			"SELECT p.id, p.origin, p.name, p.version, p.comment, "
 			"p.name || '~' || p.origin as uniqueid, "
 			"p.prefix, p.desc, p.arch, p.maintainer, p.www, "
 			"p.licenselogic, p.flatsize, p.pkgsize, "
 			"p.cksum, p.manifestdigest, p.path AS repopath, '%s' AS dbname "
 			"FROM packages AS p INNER JOIN pkg_shlibs_provided AS ps ON "
 			"p.id = ps.package_id "
 			"WHERE ps.shlib_id IN (SELECT id FROM shlibs WHERE "
 			"name BETWEEN ?1 AND ?1 || '.9');";
 	sql = sbuf_new_auto();
 	sbuf_printf(sql, basesql, repo->name);
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s'", sbuf_get(sql));
 	ret = sqlite3_prepare_v2(sqlite, sbuf_get(sql), -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	sqlite3_bind_text(stmt, 1, require, -1, SQLITE_TRANSIENT);
 	return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
 }
 struct pkg_repo_it *
 pkg_repo_binary_shlib_require(struct pkg_repo *repo, const char *provide)
 {
 	sqlite3_stmt	*stmt;
 	sqlite3 *sqlite = PRIV_GET(repo);
 	struct sbuf	*sql = NULL;
 	int		 ret;
 	const char	 basesql[] = ""
 			"SELECT p.id, p.origin, p.name, p.version, p.comment, "
 			"p.name || '~' || p.origin as uniqueid, "
 			"p.prefix, p.desc, p.arch, p.maintainer, p.www, "
 			"p.licenselogic, p.flatsize, p.pkgsize, "
 			"p.cksum, p.manifestdigest, p.path AS repopath, '%s' AS dbname "
 			"FROM packages AS p INNER JOIN pkg_shlibs_required AS ps ON "
 			"p.id = ps.package_id "
 			"WHERE ps.shlib_id = (SELECT id FROM shlibs WHERE name=?1);";
 	sql = sbuf_new_auto();
 	sbuf_printf(sql, basesql, repo->name);
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s'", sbuf_get(sql));
 	ret = sqlite3_prepare_v2(sqlite, sbuf_get(sql), -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	sqlite3_bind_text(stmt, 1, provide, -1, SQLITE_TRANSIENT);
 	return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
 }
 static const char *
 pkg_repo_binary_search_how(match_t match)
 {
 	const char	*how = NULL;
 	switch (match) {
 	case MATCH_ALL:
 		how = NULL;
 		break;
 	case MATCH_EXACT:
 		if (pkgdb_case_sensitive())
 			how = "%s = ?1";
 		else
 			how = "%s = ?1 COLLATE NOCASE";
 		break;
 	case MATCH_GLOB:
 		how = "%s GLOB ?1";
 		break;
 	case MATCH_REGEX:
 		how = "%s REGEXP ?1";
 		break;
 	case MATCH_CONDITION:
 		/* Should not be called by pkgdb_get_match_how(). */
 		assert(0);
 		break;
 	case MATCH_FTS:
 		how = "id IN (SELECT id FROM pkg_search WHERE %s MATCH ?1)";
 		break;
 	}
 	return (how);
 }
 static int
 pkg_repo_binary_build_search_query(struct sbuf *sql, match_t match,
     pkgdb_field field, pkgdb_field sort)
 {
 	const char	*how = NULL;
 	const char	*what = NULL;
 	const char	*orderby = NULL;
 	how = pkg_repo_binary_search_how(match);
 	switch (field) {
 	case FIELD_NONE:
 		what = NULL;
 		break;
 	case FIELD_ORIGIN:
 		what = "origin";
 		break;
 	case FIELD_NAME:
 		what = "name";
 		break;
 	case FIELD_NAMEVER:
 		what = "name || '-' || version";
 		break;
 	case FIELD_COMMENT:
 		what = "comment";
 		break;
 	case FIELD_DESC:
 		what = "desc";
 		break;
 	}
 	if (what != NULL && how != NULL)
 		sbuf_printf(sql, how, what);
 	switch (sort) {
 	case FIELD_NONE:
 		orderby = NULL;
 		break;
 	case FIELD_ORIGIN:
 		orderby = " ORDER BY origin";
 		break;
 	case FIELD_NAME:
 		orderby = " ORDER BY name";
 		break;
 	case FIELD_NAMEVER:
 		orderby = " ORDER BY name, version";
 		break;
 	case FIELD_COMMENT:
 		orderby = " ORDER BY comment";
 		break;
 	case FIELD_DESC:
 		orderby = " ORDER BY desc";
 		break;
 	}
 	if (orderby != NULL)
 		sbuf_cat(sql, orderby);
 	return (EPKG_OK);
 }
 struct pkg_repo_it *
 pkg_repo_binary_search(struct pkg_repo *repo, const char *pattern, match_t match,
     pkgdb_field field, pkgdb_field sort)
 {
 	sqlite3 *sqlite = PRIV_GET(repo);
 	sqlite3_stmt	*stmt = NULL;
 	struct sbuf	*sql = NULL;
 	int		 ret;
 	const char	*multireposql = ""
 		"SELECT id, origin, name, version, comment, "
 		"prefix, desc, arch, maintainer, www, "
 		"licenselogic, flatsize, pkgsize, "
 		"cksum, path AS repopath, '%1$s' AS dbname "
 		"FROM packages ";
 	assert(pattern != NULL && pattern[0] != '\0');
 	sql = sbuf_new_auto();
 	sbuf_printf(sql, multireposql, repo->name);
 	/* close the UNIONs and build the search query */
 	sbuf_cat(sql, "WHERE ");
 	pkg_repo_binary_build_search_query(sql, match, field, sort);
 	sbuf_cat(sql, ";");
 	sbuf_finish(sql);
 	pkg_debug(4, "Pkgdb: running '%s'", sbuf_get(sql));
 	ret = sqlite3_prepare_v2(sqlite, sbuf_get(sql), -1, &stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, sbuf_get(sql));
 		sbuf_delete(sql);
 		return (NULL);
 	}
 	sbuf_delete(sql);
 	sqlite3_bind_text(stmt, 1, pattern, -1, SQLITE_TRANSIENT);
 	return (pkg_repo_binary_it_new(repo, stmt, PKGDB_IT_FLAG_ONCE));
 }

added libpkg/repo/binary/update.c

@@ -0,0 +1,837 @@

 /*
  * Copyright (c) 2014, Vsevolod Stakhov
  * Copyright (c) 2012-2014 Baptiste Daroussin <bapt@FreeBSD.org>
  * Copyright (c) 2012 Julien Laffaye <jlaffaye@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *       * Redistributions of source code must retain the above copyright
  *         notice, this list of conditions and the following disclaimer.
  *       * Redistributions in binary form must reproduce the above copyright
  *         notice, this list of conditions and the following disclaimer in the
  *         documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <sys/mman.h>
 #define _WITH_GETLINE
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 #include <errno.h>
 #include <limits.h>
 #include <archive.h>
 #include <archive_entry.h>
 #include "pkg.h"
 #include "private/event.h"
 #include "private/utils.h"
 #include "private/pkgdb.h"
 #include "private/repodb.h"
 #include "private/pkg.h"
 #include "binary.h"
 #include "binary_private.h"
 static int
 pkg_repo_binary_delete_conflicting(const char *origin, const char *version,
 			 const char *pkg_path, bool forced)
 {
 	int ret = EPKG_FATAL;
 	const char *oversion;
 	if (pkg_repo_binary_run_prstatement(VERSION, origin) != SQLITE_ROW)
 		return (EPKG_FATAL); /* sqlite error */
 	oversion = sqlite3_column_text(pkg_repo_binary_stmt_prstatement(VERSION), 0);
 	if (!forced) {
 		switch(pkg_version_cmp(oversion, version)) {
 		case -1:
 			pkg_emit_error("duplicate package origin: replacing older "
 					"version %s in repo with package %s for "
 					"origin %s", oversion, pkg_path, origin);
 			if (pkg_repo_binary_run_prstatement(DELETE, origin, origin) != SQLITE_DONE)
 				return (EPKG_FATAL); /* sqlite error */
 			ret = EPKG_OK;	/* conflict cleared */
 			break;
 		case 0:
 		case 1:
 			pkg_emit_error("duplicate package origin: package %s is not "
 					"newer than version %s already in repo for "
 					"origin %s", pkg_path, oversion, origin);
 			ret = EPKG_END;	/* keep what is already in the repo */
 			break;
 		}
 	}
 	else {
 		if (pkg_repo_binary_run_prstatement(DELETE, origin, origin) != SQLITE_DONE)
 			return (EPKG_FATAL); /* sqlite error */
 		ret = EPKG_OK;
 	}
 	return (ret);
 }
 static int
 pkg_repo_binary_add_pkg(struct pkg *pkg, const char *pkg_path,
 		sqlite3 *sqlite, bool forced)
 {
 	const char *name, *version, *origin, *comment, *desc;
 	const char *arch, *maintainer, *www, *prefix, *sum, *rpath;
 	const char *olddigest, *manifestdigest;
 	int64_t			 flatsize, pkgsize;
 	int64_t			 licenselogic;
 	int			 ret;
 	struct pkg_dep		*dep      = NULL;
 	struct pkg_option	*option   = NULL;
 	struct pkg_shlib	*shlib    = NULL;
 	const pkg_object	*obj, *licenses, *categories, *annotations;
 	pkg_iter		 it;
 	int64_t			 package_id;
 	pkg_get(pkg, PKG_ORIGIN, &origin, PKG_NAME, &name,
 			    PKG_VERSION, &version, PKG_COMMENT, &comment,
 			    PKG_DESC, &desc, PKG_ARCH, &arch,
 			    PKG_MAINTAINER, &maintainer, PKG_WWW, &www,
 			    PKG_PREFIX, &prefix, PKG_FLATSIZE, &flatsize,
 			    PKG_LICENSE_LOGIC, &licenselogic, PKG_CKSUM, &sum,
 			    PKG_PKGSIZE, &pkgsize, PKG_REPOPATH, &rpath,
 			    PKG_LICENSES, &licenses, PKG_CATEGORIES, &categories,
 			    PKG_ANNOTATIONS, &annotations, PKG_OLD_DIGEST, &olddigest,
 			    PKG_DIGEST, &manifestdigest);
 try_again:
 	if ((ret = pkg_repo_binary_run_prstatement(PKG, origin, name, version,
 			comment, desc, arch, maintainer, www, prefix,
 			pkgsize, flatsize, (int64_t)licenselogic, sum,
 			rpath, manifestdigest, olddigest)) != SQLITE_DONE) {
 		if (ret == SQLITE_CONSTRAINT) {
 			switch(pkg_repo_binary_delete_conflicting(origin,
 					version, pkg_path, forced)) {
 			case EPKG_FATAL: /* sqlite error */
 				ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(PKG));
 				return (EPKG_FATAL);
 				break;
 			case EPKG_END: /* repo already has newer */
 				return (EPKG_END);
 				break;
 			default: /* conflict cleared, try again */
 				goto try_again;
 				break;
 			}
 		} else {
 			ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(PKG));
 			return (EPKG_FATAL);
 		}
 	}
 	package_id = sqlite3_last_insert_rowid(sqlite);
 	if (pkg_repo_binary_run_prstatement (FTS_APPEND, package_id,
 			name, version, origin) != SQLITE_DONE) {
 		ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(FTS_APPEND));
 		return (EPKG_FATAL);
 	}
 	dep = NULL;
 	while (pkg_deps(pkg, &dep) == EPKG_OK) {
 		if (pkg_repo_binary_run_prstatement(DEPS,
 				pkg_dep_origin(dep),
 				pkg_dep_name(dep),
 				pkg_dep_version(dep),
 				package_id) != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(DEPS));
 			return (EPKG_FATAL);
 		}
 	}
 	it = NULL;
 	while ((obj = pkg_object_iterate(categories, &it))) {
 		ret = pkg_repo_binary_run_prstatement(CAT1, pkg_object_string(obj));
 		if (ret == SQLITE_DONE)
 			ret = pkg_repo_binary_run_prstatement(CAT2, package_id,
 			    pkg_object_string(obj));
 		if (ret != SQLITE_DONE)
 		{
 			ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(CAT2));
 			return (EPKG_FATAL);
 		}
 	}
 	it = NULL;
 	while ((obj = pkg_object_iterate(licenses, &it))) {
 		ret = pkg_repo_binary_run_prstatement(LIC1, pkg_object_string(obj));
 		if (ret == SQLITE_DONE)
 			ret = pkg_repo_binary_run_prstatement(LIC2, package_id,
 			    pkg_object_string(obj));
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(LIC2));
 			return (EPKG_FATAL);
 		}
 	}
 	option = NULL;
 	while (pkg_options(pkg, &option) == EPKG_OK) {
 		ret = pkg_repo_binary_run_prstatement(OPT1, pkg_option_opt(option));
 		if (ret == SQLITE_DONE)
 		    ret = pkg_repo_binary_run_prstatement(OPT2, pkg_option_opt(option),
 				pkg_option_value(option), package_id);
 		if(ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(OPT2));
 			return (EPKG_FATAL);
 		}
 	}
 	shlib = NULL;
 	while (pkg_shlibs_required(pkg, &shlib) == EPKG_OK) {
 		const char *shlib_name = pkg_shlib_name(shlib);
 		ret = pkg_repo_binary_run_prstatement(SHLIB1, shlib_name);
 		if (ret == SQLITE_DONE)
 			ret = pkg_repo_binary_run_prstatement(SHLIB_REQD, package_id,
 					shlib_name);
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(SHLIB_REQD));
 			return (EPKG_FATAL);
 		}
 	}
 	shlib = NULL;
 	while (pkg_shlibs_provided(pkg, &shlib) == EPKG_OK) {
 		const char *shlib_name = pkg_shlib_name(shlib);
 		ret = pkg_repo_binary_run_prstatement(SHLIB1, shlib_name);
 		if (ret == SQLITE_DONE)
 			ret = pkg_repo_binary_run_prstatement(SHLIB_PROV, package_id,
 					shlib_name);
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(SHLIB_PROV));
 			return (EPKG_FATAL);
 		}
 	}
 	it = NULL;
 	while ((obj = pkg_object_iterate(annotations, &it))) {
 		const char *note_tag = pkg_object_key(obj);
 		const char *note_val = pkg_object_string(obj);
 		ret = pkg_repo_binary_run_prstatement(ANNOTATE1, note_tag);
 		if (ret == SQLITE_DONE)
 			ret = pkg_repo_binary_run_prstatement(ANNOTATE1, note_val);
 		if (ret == SQLITE_DONE)
 			ret = pkg_repo_binary_run_prstatement(ANNOTATE2, package_id,
 				  note_tag, note_val);
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, pkg_repo_binary_sql_prstatement(ANNOTATE2));
 			return (EPKG_FATAL);
 		}
 	}
 	return (EPKG_OK);
 }
 int
 pkgdb_repo_remove_package(const char *origin)
 {
 	if (pkg_repo_binary_run_prstatement(DELETE, origin, origin) != SQLITE_DONE)
 		return (EPKG_FATAL); /* sqlite error */
 	return (EPKG_OK);
 }
 static int
 pkg_repo_binary_register_conflicts(const char *origin, char **conflicts,
 		int conflicts_num, sqlite3 *sqlite)
 {
 	const char clean_conflicts_sql[] = ""
 			"DELETE FROM pkg_conflicts "
 			"WHERE package_id = ?1;";
 	const char select_id_sql[] = ""
 			"SELECT id FROM packages "
 			"WHERE origin = ?1;";
 	const char insert_conflict_sql[] = ""
 			"INSERT INTO pkg_conflicts "
 			"(package_id, conflict_id) "
 			"VALUES (?1, ?2);";
 	sqlite3_stmt *stmt = NULL;
 	int ret, i;
 	int64_t origin_id, conflict_id;
 	pkg_debug(4, "pkgdb_repo_register_conflicts: running '%s'", select_id_sql);
 	if (sqlite3_prepare_v2(sqlite, select_id_sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, select_id_sql);
 		return (EPKG_FATAL);
 	}
 	sqlite3_bind_text(stmt, 1, origin, -1, SQLITE_TRANSIENT);
 	ret = sqlite3_step(stmt);
 	if (ret == SQLITE_ROW) {
 		origin_id = sqlite3_column_int64(stmt, 0);
 	}
 	else {
 		ERROR_SQLITE(sqlite, select_id_sql);
 		return (EPKG_FATAL);
 	}
 	sqlite3_finalize(stmt);
 	pkg_debug(4, "pkgdb_repo_register_conflicts: running '%s'", clean_conflicts_sql);
 	if (sqlite3_prepare_v2(sqlite, clean_conflicts_sql, -1, &stmt, NULL) != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, clean_conflicts_sql);
 		return (EPKG_FATAL);
 	}
 	sqlite3_bind_int64(stmt, 1, origin_id);
 	/* Ignore cleanup result */
 	(void)sqlite3_step(stmt);
 	sqlite3_finalize(stmt);
 	for (i = 0; i < conflicts_num; i ++) {
 		/* Select a conflict */
 		pkg_debug(4, "pkgdb_repo_register_conflicts: running '%s'", select_id_sql);
 		if (sqlite3_prepare_v2(sqlite, select_id_sql, -1, &stmt, NULL) != SQLITE_OK) {
 			ERROR_SQLITE(sqlite, select_id_sql);
 			return (EPKG_FATAL);
 		}
 		sqlite3_bind_text(stmt, 1, conflicts[i], -1, SQLITE_TRANSIENT);
 		ret = sqlite3_step(stmt);
 		if (ret == SQLITE_ROW) {
 			conflict_id = sqlite3_column_int64(stmt, 0);
 		}
 		else {
 			ERROR_SQLITE(sqlite, select_id_sql);
 			return (EPKG_FATAL);
 		}
 		sqlite3_finalize(stmt);
 		/* Insert a pair */
 		pkg_debug(4, "pkgdb_repo_register_conflicts: running '%s'", insert_conflict_sql);
 		if (sqlite3_prepare_v2(sqlite, insert_conflict_sql, -1, &stmt, NULL) != SQLITE_OK) {
 			ERROR_SQLITE(sqlite, insert_conflict_sql);
 			return (EPKG_FATAL);
 		}
 		sqlite3_bind_int64(stmt, 1, origin_id);
 		sqlite3_bind_int64(stmt, 2, conflict_id);
 		ret = sqlite3_step(stmt);
 		if (ret != SQLITE_DONE) {
 			ERROR_SQLITE(sqlite, insert_conflict_sql);
 			return (EPKG_FATAL);
 		}
 		sqlite3_finalize(stmt);
 	}
 	return (EPKG_OK);
 }
 static int
 pkg_repo_binary_add_from_manifest(char *buf, const char *origin, const char *digest,
 		long offset, sqlite3 *sqlite,
 		struct pkg_manifest_key **keys, struct pkg **p, bool is_legacy,
 		struct pkg_repo *repo)
 {
 	int rc = EPKG_OK;
 	struct pkg *pkg;
 	const char *local_origin, *pkg_arch;
 	if (*p == NULL) {
 		rc = pkg_new(p, PKG_REMOTE);
 		if (rc != EPKG_OK)
 			return (EPKG_FATAL);
 	} else {
 		pkg_reset(*p, PKG_REMOTE);
 	}
 	pkg = *p;
 	pkg_manifest_keys_new(keys);
 	rc = pkg_parse_manifest(pkg, buf, offset, *keys);
 	if (rc != EPKG_OK) {
 		goto cleanup;
 	}
 	rc = pkg_is_valid(pkg);
 	if (rc != EPKG_OK) {
 		goto cleanup;
 	}
 	/* Ensure that we have a proper origin and arch*/
 	pkg_get(pkg, PKG_ORIGIN, &local_origin, PKG_ARCH, &pkg_arch);
 	if (local_origin == NULL || strcmp(local_origin, origin) != 0) {
 		pkg_emit_error("manifest contains origin %s while we wanted to add origin %s",
 				local_origin ? local_origin : "NULL", origin);
 		rc = EPKG_FATAL;
 		goto cleanup;
 	}
 	if (pkg_arch == NULL || !is_valid_abi(pkg_arch, true)) {
 		rc = EPKG_FATAL;
 		goto cleanup;
 	}
 	pkg_set(pkg, PKG_REPONAME, repo->name);
 	if (is_legacy) {
 		pkg_set(pkg, PKG_OLD_DIGEST, digest);
 		pkg_checksum_calculate(pkg, NULL);
 	}
 	else {
 		pkg_set(pkg, PKG_DIGEST, digest);
 	}
 	rc = pkg_repo_binary_add_pkg(pkg, NULL, sqlite, true);
 cleanup:
 	return (rc);
 }
 struct pkg_increment_task_item {
 	char *origin;
 	char *digest;
 	char *olddigest;
 	long offset;
 	long length;
 	UT_hash_handle hh;
 };
 static void
 pkg_repo_binary_update_item_new(struct pkg_increment_task_item **head, const char *origin,
 		const char *digest, long offset, long length)
 {
 	struct pkg_increment_task_item *item;
 	item = calloc(1, sizeof(struct pkg_increment_task_item));
 	item->origin = strdup(origin);
 	if (digest == NULL)
 		digest = "";
 	item->digest = strdup(digest);
 	item->offset = offset;
 	item->length = length;
 	HASH_ADD_KEYPTR(hh, *head, item->origin, strlen(item->origin), item);
 }
 static void __unused
 pkg_repo_binary_parse_conflicts(FILE *f, sqlite3 *sqlite)
 {
 	size_t linecap = 0;
 	ssize_t linelen;
 	char *linebuf = NULL, *p, **deps;
 	const char *origin, *pdep;
 	int ndep, i;
 	const char conflicts_clean_sql[] = ""
 			"DELETE FROM pkg_conflicts;";
 	pkg_debug(4, "pkg_parse_conflicts_file: running '%s'", conflicts_clean_sql);
 	(void)sql_exec(sqlite, conflicts_clean_sql);
 	while ((linelen = getline(&linebuf, &linecap, f)) > 0) {
 		p = linebuf;
 		origin = strsep(&p, ":");
 		/* Check dependencies number */
 		pdep = p;
 		ndep = 1;
 		while (*pdep != '\0') {
 			if (*pdep == ',')
 				ndep ++;
 			pdep ++;
 		}
 		deps = malloc(sizeof(char *) * ndep);
 		for (i = 0; i < ndep; i ++) {
 			deps[i] = strsep(&p, ",\n");
 		}
 		pkg_repo_binary_register_conflicts(origin, deps, ndep, sqlite);
 		free(deps);
 	}
 	if (linebuf != NULL)
 		free(linebuf);
 }
 sqlite3_stmt *
 pkg_repo_binary_get_origins(sqlite3 *sqlite)
 {
 	sqlite3_stmt *stmt = NULL;
 	int ret;
 	const char query_sql[] = ""
 		"SELECT id, origin, name, name || '~' || origin as uniqueid, version, comment, "
 		"prefix, desc, arch, maintainer, www, "
 		"licenselogic, flatsize, pkgsize, "
 		"cksum, path AS repopath, manifestdigest "
 		"FROM packages "
 		"ORDER BY origin;";
 	pkg_debug(4, "binary_repo: running '%s'", query_sql);
 	ret = sqlite3_prepare_v2(sqlite, query_sql, -1,
 			&stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(sqlite, query_sql);
 		return (NULL);
 	}
 	return (stmt);
 }
 static int
 pkg_repo_binary_update_incremental(const char *name, struct pkg_repo *repo,
 	time_t *mtime)
 {
 	FILE *fmanifest = NULL, *fdigests = NULL /*, *fconflicts = NULL*/;
 	struct pkg *pkg = NULL;
 	int rc = EPKG_FATAL;
 	sqlite3 *sqlite = PRIV_GET(repo);
 	sqlite3_stmt *stmt;
 	const char *origin, *digest, *offset, *length;
 	char *linebuf = NULL, *p;
 	int updated = 0, removed = 0, added = 0, processed = 0, pushed = 0;
 	long num_offset, num_length;
 	time_t local_t = *mtime;
 	time_t digest_t;
 	time_t packagesite_t;
 	struct pkg_increment_task_item *ldel = NULL, *ladd = NULL,
 			*item, *tmp_item;
 	struct pkg_manifest_key *keys = NULL;
 	size_t linecap = 0;
 	ssize_t linelen;
 	char *map = MAP_FAILED;
 	size_t len = 0;
 	int hash_it = 0;
 	bool in_trans = false, new_repo = true, legacy_repo = false;
 	/* Required for making iterator */
 	struct pkgdb_it *it = NULL;
 	struct pkgdb fakedb;
 	pkg_debug(1, "Pkgrepo, begin incremental update of '%s'", name);
 	if (pkg_repo_fetch_meta(repo, NULL) == EPKG_FATAL)
 		pkg_emit_notice("repository %s has no meta file, using "
 		    "default settings", repo->name);
 	fdigests = pkg_repo_fetch_remote_extract_tmp(repo,
 			repo->meta->digests, &local_t, &rc);
 	if (fdigests == NULL) {
 		if (rc == EPKG_FATAL)
 			/* Destroy repo completely */
 			if (new_repo)
 				unlink(name);
 		goto cleanup;
 	}
 	digest_t = local_t;
 	local_t = *mtime;
 	fmanifest = pkg_repo_fetch_remote_extract_tmp(repo,
 			repo->meta->manifests, &local_t, &rc);
 	if (fmanifest == NULL) {
 		if (rc == EPKG_FATAL)
 			/* Destroy repo completely */
 			if (new_repo)
 				unlink(name);
 		goto cleanup;
 	}
 	packagesite_t = digest_t;
 	*mtime = packagesite_t > digest_t ? packagesite_t : digest_t;
 	/*fconflicts = repo_fetch_remote_extract_tmp(repo,
 			repo_conflicts_archive, "txz", &local_t,
 			&rc, repo_conflicts_file);*/
 	fseek(fmanifest, 0, SEEK_END);
 	len = ftell(fmanifest);
 	/* Detect whether we have legacy repo */
 	if ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
 		p = linebuf;
 		origin = strsep(&p, ":");
 		digest = strsep(&p, ":");
 		if (digest == NULL) {
 			pkg_emit_error("invalid digest file format");
 			rc = EPKG_FATAL;
 			goto cleanup;
 		}
 		if (!pkg_checksum_is_valid(digest, strlen(digest))) {
 			legacy_repo = true;
 			pkg_debug(1, "repository '%s' has a legacy digests format", repo->name);
 		}
 	}
 	fseek(fdigests, 0, SEEK_SET);
 	/* Load local repo data */
 	stmt = pkg_repo_binary_get_origins(sqlite);
 	if (stmt == NULL) {
 		rc = EPKG_FATAL;
 		goto cleanup;
 	}
 	fakedb.sqlite = sqlite;
 	it = pkgdb_it_new_sqlite(&fakedb, stmt, PKG_REMOTE, PKGDB_IT_FLAG_ONCE);
 	if (it != NULL) {
 		while (pkgdb_it_next(it, &pkg, PKG_LOAD_BASIC) == EPKG_OK) {
 			pkg_get(pkg, PKG_ORIGIN, &origin, legacy_repo ? PKG_OLD_DIGEST : PKG_DIGEST,
 							&digest);
 			pkg_repo_binary_update_item_new(&ldel, origin, digest, 4, 0);
 		}
 		pkgdb_it_free(it);
 	}
 	pkg_debug(1, "Pkgrepo, reading new packagesite.yaml for '%s'", name);
 	/* load the while digests */
 	while ((linelen = getline(&linebuf, &linecap, fdigests)) > 0) {
 		p = linebuf;
 		origin = strsep(&p, ":");
 		digest = strsep(&p, ":");
 		offset = strsep(&p, ":");
 		/* files offset */
 		strsep(&p, ":");
 		length = strsep(&p, ":");
 		if (origin == NULL || digest == NULL ||
 				offset == NULL) {
 			pkg_emit_error("invalid digest file format");
 			rc = EPKG_FATAL;
 			goto cleanup;
 		}
 		errno = 0;
 		num_offset = (long)strtoul(offset, NULL, 10);
 		if (errno != 0) {
 			pkg_emit_errno("strtoul", "digest format error");
 			rc = EPKG_FATAL;
 			goto cleanup;
 		}
 		if (length != NULL) {
 			errno = 0;
 			num_length = (long)strtoul(length, NULL, 10);
 			if (errno != 0) {
 				pkg_emit_errno("strtoul", "digest format error");
 				rc = EPKG_FATAL;
 				goto cleanup;
 			}
 		}
 		else {
 			num_length = 0;
 		}
 		processed++;
 		HASH_FIND_STR(ldel, origin, item);
 		if (item == NULL) {
 			added++;
 			pkg_repo_binary_update_item_new(&ladd, origin, digest, num_offset,
 					num_length);
 		} else {
 			if (strcmp(digest, item->digest) == 0) {
 				free(item->origin);
 				free(item->digest);
 				HASH_DEL(ldel, item);
 				free(item);
 				item = NULL;
 			} else {
 				free(item->origin);
 				free(item->digest);
 				HASH_DEL(ldel, item);
 				free(item);
 				item = NULL;
 				pkg_repo_binary_update_item_new(&ladd, origin, digest,
 						num_offset, num_length);
 				updated++;
 			}
 		}
 	}
 	rc = EPKG_OK;
 	pkg_debug(1, "Pkgrepo, removing old entries for '%s'", name);
 	in_trans = true;
 	rc = pkgdb_transaction_begin(sqlite, "REPO");
 	if (rc != EPKG_OK)
 		goto cleanup;
 	removed = HASH_COUNT(ldel);
 	hash_it = 0;
 	pkg_emit_progress_start("Removing expired entries");
 	HASH_ITER(hh, ldel, item, tmp_item) {
 		pkg_emit_progress_tick(++hash_it, removed);
 		if (rc == EPKG_OK) {
 			rc = pkgdb_repo_remove_package(item->origin);
 		}
 		free(item->origin);
 		free(item->digest);
 		HASH_DEL(ldel, item);
 		free(item);
 	}
 	pkg_debug(1, "Pkgrepo, pushing new entries for '%s'", name);
 	pkg = NULL;
 	if (len > 0 && len < SSIZE_MAX) {
 		map = mmap(NULL, len, PROT_READ, MAP_SHARED, fileno(fmanifest), 0);
 		fclose(fmanifest);
 	} else {
 		if (len == 0)
 			pkg_emit_error("Empty catalog");
 		else
 			pkg_emit_error("Catalog too large");
 		goto cleanup;
 	}
 	hash_it = 0;
 	pushed = HASH_COUNT(ladd);
 	pkg_emit_progress_start("Adding new entries");
 	HASH_ITER(hh, ladd, item, tmp_item) {
 		pkg_emit_progress_tick(++hash_it, pushed);
 		if (rc == EPKG_OK) {
 			if (item->length != 0) {
 				rc = pkg_repo_binary_add_from_manifest(map + item->offset, item->origin,
 				    item->digest, item->length, sqlite, &keys, &pkg, legacy_repo,
 				    repo);
 			}
 			else {
 				rc = pkg_repo_binary_add_from_manifest(map + item->offset, item->origin,
 				    item->digest, len - item->offset, sqlite, &keys, &pkg,
 				    legacy_repo, repo);
 			}
 		}
 		free(item->origin);
 		free(item->digest);
 		HASH_DEL(ladd, item);
 		free(item);
 	}
 	pkg_manifest_keys_free(keys);
 	pkg_emit_incremental_update(updated, removed, added, processed);
 cleanup:
 	if (in_trans) {
 		if (rc != EPKG_OK)
 			pkgdb_transaction_rollback(sqlite, "REPO");
 		if (pkgdb_transaction_commit(sqlite, "REPO") != EPKG_OK)
 			rc = EPKG_FATAL;
 	}
 	if (pkg != NULL)
 		pkg_free(pkg);
 	if (map == MAP_FAILED && fmanifest)
 		fclose(fmanifest);
 	if (fdigests)
 		fclose(fdigests);
 	/* if (fconflicts)
 		fclose(fconflicts);*/
 	if (map != MAP_FAILED)
 		munmap(map, len);
 	if (linebuf != NULL)
 		free(linebuf);
 	repo->ops->close(repo, false);
 	return (rc);
 }
 int
 pkg_repo_binary_update(struct pkg_repo *repo, bool force)
 {
 	char filepath[MAXPATHLEN];
 	const char *dbdir = NULL;
 	struct stat st;
 	time_t t = 0;
 	int res = EPKG_FATAL;
 	bool got_meta = false;
 	sqlite3_initialize();
 	if (!pkg_repo_enabled(repo))
 		return (EPKG_OK);
 	dbdir = pkg_object_string(pkg_config_get("PKG_DBDIR"));
 	pkg_debug(1, "PkgRepo: verifying update for %s", pkg_repo_name(repo));
 	snprintf(filepath, sizeof(filepath), "%s/%s.meta", dbdir, pkg_repo_name(repo));
 	if (stat(filepath, &st) != -1) {
 		t = force ? 0 : st.st_mtime;
 		got_meta = true;
 	}
 	snprintf(filepath, sizeof(filepath), "%s/%s", dbdir,
 		pkg_repo_binary_get_filename(pkg_repo_name(repo)));
 	if (stat(filepath, &st) != -1) {
 		if (!got_meta && !force)
 			t = st.st_mtime;
 	}
 	if (t != 0) {
 		/* Try to open repo */
 		if (repo->ops->open(repo, R_OK|W_OK) != EPKG_OK) {
 			/* Try to re-create it */
 			unlink(filepath);
 			if (repo->ops->create(repo) != EPKG_OK) {
 				pkg_emit_notice("Unable to create repository %s", repo->name);
 				goto cleanup;
 			}
 			t = 0;
 			if (repo->ops->open(repo, R_OK|W_OK) != EPKG_OK) {
 				pkg_emit_notice("Unable to open created repository %s", repo->name);
 				goto cleanup;
 			}
 		}
 	}
 	else {
 		/* [Re]create repo */
 		unlink(filepath);
 		if (repo->ops->create(repo) != EPKG_OK) {
 			pkg_emit_notice("Unable to create repository %s", repo->name);
 			goto cleanup;
 		}
 		if (repo->ops->open(repo, R_OK|W_OK) != EPKG_OK) {
 			pkg_emit_notice("Unable to open created repository %s", repo->name);
 			goto cleanup;
 		}
 	}
 	repo->ops->init(repo);
 	res = pkg_repo_binary_update_incremental(filepath, repo, &t);
 	if (res != EPKG_OK && res != EPKG_UPTODATE) {
 		pkg_emit_notice("Unable to update repository %s", repo->name);
 		goto cleanup;
 	}
 cleanup:
 	/* Set mtime from http request if possible */
 	if (t != 0 && res == EPKG_OK) {
 		struct timeval ftimes[2] = {
 			{
 			.tv_sec = t,
 			.tv_usec = 0
 			},
 			{
 			.tv_sec = t,
 			.tv_usec = 0
 			}
 		};
 		if (got_meta)
 			snprintf(filepath, sizeof(filepath), "%s/%s.meta", dbdir, pkg_repo_name(repo));
 		utimes(filepath, ftimes);
 	}
 	return (res);
 }

modified src/Makefile.am

@@ -43,7 +43,8 @@ pkg_CFLAGS= -I$(top_srcdir)/libpkg \

 			-I$(top_srcdir)/external/expat/lib \
 			-DGITHASH=\"$(GIT_HEAD)\"
 pkg_static_SOURCES=
 pkg_static_LDADD=	$(top_builddir)/libpkg/libpkg_static.la \
 pkg_static_LDADD= @REPOS_LDADD_STATIC@ \
 			$(top_builddir)/libpkg/libpkg_static.la \
 			$(top_builddir)/external/libexpat_static.la \
 			$(pkg_OBJECTS) \
 			@LIBJAIL_LIB@ \

modified src/clean.c

@@ -255,7 +255,7 @@ exec_clean(int argc, char **argv)

 		}
 		if (sumlist == NULL && !sumloaded) {
 			it = pkgdb_search(db, "*", MATCH_GLOB, FIELD_NAME, FIELD_NONE, NULL);
 			it = pkgdb_repo_search(db, "*", MATCH_GLOB, FIELD_NAME, FIELD_NONE, NULL);
 			while (pkgdb_it_next(it, &p, PKG_LOAD_BASIC) == EPKG_OK) {
 				pkg_get(p, PKG_CKSUM, &sum);
 				slen = MIN(strlen(sum), PKG_FILE_CKSUM_CHARS);

modified src/main.c

@@ -325,7 +325,7 @@ show_repository_info(void)

 		}
 		printf("  %s: { \n    %-16s: \"%s\",\n    %-16s: %s",
 		    pkg_repo_ident(repo),
 		    pkg_repo_name(repo),
                     "url", pkg_repo_url(repo),
 		    "enabled", pkg_repo_enabled(repo) ? "yes" : "no");
 		if (pkg_repo_mirror_type(repo) != NOMIRROR)

modified src/rquery.c

@@ -232,7 +232,7 @@ exec_rquery(int argc, char **argv)

 		const char *condition_sql = NULL;
 		if (match == MATCH_CONDITION && sqlcond)
 			condition_sql = sbuf_data(sqlcond);
 		if ((it = pkgdb_rquery(db, condition_sql, match, reponame)) == NULL)
 		if ((it = pkgdb_repo_query(db, condition_sql, match, reponame)) == NULL)
 			return (EX_IOERR);
 		while ((ret = pkgdb_it_next(it, &pkg, query_flags)) == EPKG_OK) {

@@ -250,7 +250,7 @@ exec_rquery(int argc, char **argv)

 		for (i = (index_output ? 0 : 1); i < argc; i++) {
 			pkgname = argv[i];
 			if ((it = pkgdb_rquery(db, pkgname, match, reponame)) == NULL)
 			if ((it = pkgdb_repo_query(db, pkgname, match, reponame)) == NULL)
 				return (EX_IOERR);
 			while ((ret = pkgdb_it_next(it, &pkg, query_flags)) == EPKG_OK) {

modified src/search.c

@@ -414,7 +414,7 @@ exec_search(int argc, char **argv)

 	if (pkgdb_open_all(&db, PKGDB_REMOTE, reponame) != EPKG_OK)
 		return (EX_IOERR);
 	if ((it = pkgdb_search(db, pattern, match, search, search,
 	if ((it = pkgdb_repo_search(db, pattern, match, search, search,
 	    reponame)) == NULL) {
 		pkgdb_close(db);
 		return (EX_IOERR);

modified src/shlib.c

@@ -79,7 +79,7 @@ pkgs_providing_lib(struct pkgdb *db, const char *libname)

 	int		 ret = EPKG_OK;
 	int		 count = 0;
 	if ((it = pkgdb_query_shlib_provided(db, libname)) == NULL) {
 	if ((it = pkgdb_query_shlib_provide(db, libname)) == NULL) {
 		return (EPKG_FATAL);
 	}

@@ -113,7 +113,7 @@ pkgs_requiring_lib(struct pkgdb *db, const char *libname)

 	int		 ret = EPKG_OK;
 	int		 count = 0;
 	if ((it = pkgdb_query_shlib_required(db, libname)) == NULL) {
 	if ((it = pkgdb_query_shlib_require(db, libname)) == NULL) {
 		return (EPKG_FATAL);
 	}

modified src/update.c

@@ -71,7 +71,7 @@ pkgcli_update(bool force, const char *reponame)

 	while (pkg_repos(&r) == EPKG_OK) {
 		if (reponame != NULL) {
 			if (strcmp(pkg_repo_ident(r), reponame) != 0)
 			if (strcmp(pkg_repo_name(r), reponame) != 0)
 				continue;
 		} else {
 			if (!pkg_repo_enabled(r))

@@ -83,7 +83,7 @@ pkgcli_update(bool force, const char *reponame)

 			if (!quiet)
 				printf("%s repository catalogue is "
 				       "up-to-date, no need to fetch "
 				       "fresh copy\n", pkg_repo_ident(r));
 				       "fresh copy\n", pkg_repo_name(r));
 				retcode = EPKG_OK;
 		}
 		if (retcode != EPKG_OK)

modified src/version.c

@@ -476,7 +476,7 @@ do_source_remote(unsigned int opt, char limchar, char *pattern, match_t match,

 		    strcmp(origin, matchorigin) != 0)
 			continue;
 		it_remote = pkgdb_rquery(db, origin, MATCH_EXACT, reponame);
 		it_remote = pkgdb_repo_query(db, origin, MATCH_EXACT, reponame);
 		if (it_remote == NULL) {
 			retcode = EX_IOERR;
 			goto cleanup;