c50aa93 More steps to incremental update. — HardenedBSD-pkg

modified libpkg/pkg_repo.c

@@ -246,7 +246,10 @@ pkg_create_repo(char *path, bool force, void (progress)(struct pkg *pkg, void *d

 	pack_extract(repopack, repo_db_file, repodb);
 	if ((retcode = pkgdb_repo_init(repodb, force, &sqlite)) != EPKG_OK)
 	if ((retcode = pkgdb_repo_open(repodb, force, &sqlite)) != EPKG_OK)
 		goto cleanup;
 	if ((retcode = pkgdb_repo_init(sqlite)) != EPKG_OK)
 		goto cleanup;
 	thd_data.root_path = path;

@@ -340,7 +343,6 @@ pkg_create_repo(char *path, bool force, void (progress)(struct pkg *pkg, void *d

 cleanup:
 	if (pkgdb_repo_close(sqlite, retcode == EPKG_OK) != EPKG_OK) {
 		retcode = EPKG_FATAL;
 		goto cleanup;
 	}
 	LL_FOREACH_SAFE(dlist, cur_dig, dtmp) {
 		if (retcode == EPKG_OK) {

modified libpkg/pkgdb.c

@@ -1000,7 +1000,7 @@ pkgdb_open(struct pkgdb **db_p, pkgdb_t type)

 		/* Create the diretory if it doesn't exists */
 		if (createdir && mkdirs(dbdir) != EPKG_OK)
 				return (EPKG_FATAL);
 			return (EPKG_FATAL);
 		sqlite3_initialize();
 		if (sqlite3_open(localpath, &db->sqlite) != SQLITE_OK) {

@@ -1274,9 +1274,15 @@ pkgdb_it_next(struct pkgdb_it *it, struct pkg **pkg_p, unsigned flags)

 		for (i = 0; load_on_flag[i].load != NULL; i++) {
 			if (flags & load_on_flag[i].flag) {
 				ret = load_on_flag[i].load(it->db, pkg);
 				if (ret != EPKG_OK)
 					return (ret);
 				if (it->db != NULL) {
 					ret = load_on_flag[i].load(it->db, pkg);
 					if (ret != EPKG_OK)
 						return (ret);
 				}
 				else {
 					pkg_emit_error("invalid iterator passed to pkgdb_it_next");
 					return (EPKG_FATAL);
 				}
 			}
 		}

@@ -1284,7 +1290,10 @@ pkgdb_it_next(struct pkgdb_it *it, struct pkg **pkg_p, unsigned flags)

 	case SQLITE_DONE:
 		return (EPKG_END);
 	default:
 		ERROR_SQLITE(it->db->sqlite);
 		if (it->db)
 			ERROR_SQLITE(it->db->sqlite);
 		else
 			pkg_emit_error("invalid iterator passed to pkgdb_it_next");
 		return (EPKG_FATAL);
 	}
 }

modified libpkg/pkgdb_repo.c

@@ -303,7 +303,7 @@ finalize_prepared_statements(void)

 }
 int
 pkgdb_repo_init(const char *repodb, bool force, sqlite3 **sqlite)
 pkgdb_repo_open(const char *repodb, bool force, sqlite3 **sqlite)
 {
 	bool incremental = false;
 	bool db_not_open;

@@ -324,9 +324,9 @@ pkgdb_repo_init(const char *repodb, bool force, sqlite3 **sqlite)

 		db_not_open = false;
 		/* If the schema is too old, or we're forcing a full
 		   update, then we cannot do an incremental update.
 		   Delete the existing repo, and promote this to a
 		   full update */
 			   update, then we cannot do an incremental update.
 			   Delete the existing repo, and promote this to a
 			   full update */
 		if (!incremental)
 			continue;
 		retcode = get_repo_user_version(*sqlite, "main", &reposcver);

@@ -335,8 +335,8 @@ pkgdb_repo_init(const char *repodb, bool force, sqlite3 **sqlite)

 		if (force || reposcver != REPO_SCHEMA_VERSION) {
 			if (reposcver != REPO_SCHEMA_VERSION)
 				pkg_emit_error("re-creating repo to upgrade schema version "
 				     "from %d to %d", reposcver,
 				     REPO_SCHEMA_VERSION);
 						"from %d to %d", reposcver,
 						REPO_SCHEMA_VERSION);
 			sqlite3_close(*sqlite);
 			unlink(repodb);
 			incremental = false;

@@ -344,46 +344,23 @@ pkgdb_repo_init(const char *repodb, bool force, sqlite3 **sqlite)

 		}
 	}
 	sqlite3_create_function(*sqlite, "file_exists", 2, SQLITE_ANY, NULL,
 	    file_exists, NULL, NULL);
 	retcode = sql_exec(*sqlite, "PRAGMA synchronous=off");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = sql_exec(*sqlite, "PRAGMA journal_mode=memory");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = sql_exec(*sqlite, "PRAGMA foreign_keys=on");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	if (!incremental) {
 		retcode = sql_exec(*sqlite, initsql, REPO_SCHEMA_VERSION);
 		if (retcode != EPKG_OK)
 			return (retcode);
 	}
 	retcode = initialize_prepared_statements(*sqlite);
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = pkgdb_transaction_begin(*sqlite, NULL);
 	if (retcode != EPKG_OK)
 		return (retcode);
 	/* remove anything that is no longer in the repository. */
 	if (incremental) {
 		const char *obsolete[] = {
 			"packages WHERE NOT FILE_EXISTS(path, cksum)",
 			"categories WHERE id NOT IN "
 				"packages WHERE NOT FILE_EXISTS(path, cksum)",
 				"categories WHERE id NOT IN "
 				"(SELECT category_id FROM pkg_categories)",
 			"licenses WHERE id NOT IN "
 				"licenses WHERE id NOT IN "
 				"(SELECT license_id FROM pkg_licenses)",
 			"shlibs WHERE id NOT IN "
 				"shlibs WHERE id NOT IN "
 				"(SELECT shlib_id FROM pkg_shlibs_required)"
 			        "AND id NOT IN "
 				"AND id NOT IN "
 				"(SELECT shlib_id FROM pkg_shlibs_provided)"
 		};
 		size_t num_objs = sizeof(obsolete) / sizeof(*obsolete);

@@ -395,6 +372,37 @@ pkgdb_repo_init(const char *repodb, bool force, sqlite3 **sqlite)

 }
 int
 pkgdb_repo_init(sqlite3 *sqlite)
 {
 	int retcode = EPKG_OK;
 	sqlite3_create_function(sqlite, "file_exists", 2, SQLITE_ANY, NULL,
 	    file_exists, NULL, NULL);
 	retcode = sql_exec(sqlite, "PRAGMA synchronous=off");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = sql_exec(sqlite, "PRAGMA journal_mode=memory");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = sql_exec(sqlite, "PRAGMA foreign_keys=on");
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = initialize_prepared_statements(sqlite);
 	if (retcode != EPKG_OK)
 		return (retcode);
 	retcode = pkgdb_transaction_begin(sqlite, NULL);
 	if (retcode != EPKG_OK)
 		return (retcode);
 	return (EPKG_OK);
 }
 int
 pkgdb_repo_close(sqlite3 *sqlite, bool commit)
 {
 	int retcode = EPKG_OK;

@@ -869,24 +877,31 @@ pkgdb_repo_check_version(struct pkgdb *db, const char *database)

 }
 struct pkgdb_it *
 pkgdb_repo_origins(struct pkgdb *db)
 pkgdb_repo_origins(sqlite3 *sqlite)
 {
 	sqlite3_stmt *stmt = NULL;
 	int ret;
 	struct pkgdb_it *it;
 	const char query_sql[] = ""
 		"SELECT origin, manifestdigest "
 		"FROM packages "
 		"ORDER BY origin;";
 	assert(db != NULL);
 	assert(db->type == PKGDB_REMOTE);
 	ret = sqlite3_prepare_v2(db->sqlite, query_sql, -1,
 	ret = sqlite3_prepare_v2(sqlite, query_sql, -1,
 			&stmt, NULL);
 	if (ret != SQLITE_OK) {
 		ERROR_SQLITE(db->sqlite);
 		ERROR_SQLITE(sqlite);
 		return (NULL);
 	}
 	if ((it = malloc(sizeof(struct pkgdb_it))) == NULL) {
 		pkg_emit_errno("malloc", "pkgdb_repo_origins");
 		sqlite3_finalize(stmt);
 		return (NULL);
 	}
 	return (pkgdb_it_new(db, stmt, PKG_REMOTE));
 	it->db = NULL;
 	it->stmt = stmt;
 	it->type = PKG_REMOTE;
 	return (it);
 }

modified libpkg/private/pkgdb.h

@@ -70,13 +70,20 @@ int pkgdb_release_lock(struct pkgdb *db);

 void pkgshell_open(const char **r);
 /**
  * Init repodb for specified path
  * Open repodb for specified path
  * @param repodb path of repodb
  * @param force create repository if not exists
  * @param sqlite destination db pointer
  * @return EPKG_OK if succeed
  */
 int pkgdb_repo_init(const char *repodb, bool force, sqlite3 **sqlite);
 int pkgdb_repo_open(const char *repodb, bool force, sqlite3 **sqlite);
 /**
  * Init repository for pkgdb_repo* functions
  * @param sqlite sqlite object
  * @return EPKG_OK if succeed
  */
 int pkgdb_repo_init(sqlite3 *sqlite);
 /**
  * Close repodb and commit/rollback transaction started

@@ -124,9 +131,9 @@ int pkgdb_repo_check_version(struct pkgdb *db, const char *database);

 /**
  * Returns a list of all packages sorted by origin
  * @param db database
  * @param sqlite database
  * @return new iterator
  */
 struct pkgdb_it *pkgdb_repo_origins(struct pkgdb *db);
 struct pkgdb_it *pkgdb_repo_origins(sqlite3 *sqlite);
 #endif

modified libpkg/update.c

@@ -31,6 +31,7 @@

 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 #include <errno.h>
 #include <archive.h>
 #include <archive_entry.h>

@@ -292,27 +293,67 @@ pkg_update_full(const char *repofile, const char *name, const char *packagesite,

 	return (rc);
 }
 static __unused int
 static int
 pkg_add_from_manifest(FILE *f, const char *origin, long offset,
 		const char *manifest_digest, sqlite3 *sqlite)
 {
 	int rc = EPKG_OK;
 	struct pkg *pkg;
 	const char *local_origin;
 	if (fseek(f, offset, SEEK_SET) == -1) {
 		pkg_emit_errno("fseek", "invalid manifest offset");
 		return (EPKG_FATAL);
 	}
 	rc = pkg_new(&pkg, PKG_REMOTE);
 	if (rc != EPKG_OK)
 		return (EPKG_FATAL);
 	rc = pkg_parse_manifest_file(pkg, f);
 	if (rc != EPKG_OK) {
 		rc = EPKG_FATAL;
 		goto cleanup;
 	}
 	/* Ensure that we have a proper origin */
 	pkg_get(pkg, PKG_ORIGIN, &local_origin);
 	if (local_origin == NULL || strcmp(local_origin, origin) != 0) {
 		pkg_emit_error("manifest contains origin %s while we wanted to add origin %s",
 				local_origin ? local_origin : "NULL", origin);
 		rc = EPKG_FATAL;
 		goto cleanup;
 	}
 	rc = pkgdb_repo_add_package(pkg, NULL, sqlite, manifest_digest, true);
 cleanup:
 	pkg_free(pkg);
 	return (rc);
 }
 static int
 pkg_update_incremental(const char *name, const char *packagesite, time_t *mtime)
 {
 	FILE *fmanifest = NULL, *fdigests = NULL;
 	int fd_manifest, fd_digests;
 	struct pkgdb *db = NULL;
 	sqlite3 *sqlite = NULL;
 	struct pkg *local_pkg;
 	int rc = EPKG_FATAL, ret;
 	const char *local_origin, *local_digest;
 	struct pkgdb_it *it;
 	char linebuf[1024], *digest_origin, *digest_digest, *digest_offset, *p;
 	int updated = 0, removed = 0, added = 0;
 	long num_offset;
 	if (pkgdb_open(&db, PKGDB_REMOTE) != EPKG_OK)
 	if ((rc = pkgdb_repo_open(name, true, &sqlite)) != EPKG_OK)
 		goto cleanup;
 	/* Initialize the remote remote */
 	if (pkgdb_remote_init(db, name) != EPKG_OK)
 	if ((rc = pkgdb_repo_init(sqlite)) != EPKG_OK)
 		goto cleanup;
 	it = pkgdb_repo_origins(db);
 	it = pkgdb_repo_origins(sqlite);
 	if (it == NULL) {
 		rc = EPKG_FATAL;
 		goto cleanup;

@@ -341,6 +382,9 @@ pkg_update_incremental(const char *name, const char *packagesite, time_t *mtime)

 		if (fgets(linebuf, sizeof(linebuf) - 1, fdigests) == NULL) {
 			while ((ret = pkgdb_it_next(it, &local_pkg, PKG_LOAD_BASIC)) == EPKG_OK) {
 				/* Remove packages */
 				rc = pkgdb_repo_remove_package(local_pkg);
 				if (rc != EPKG_OK)
 					goto cleanup;
 				removed ++;
 			}
 		}

@@ -354,6 +398,13 @@ pkg_update_incremental(const char *name, const char *packagesite, time_t *mtime)

 			rc = EPKG_FATAL;
 			goto cleanup;
 		}
 		errno = 0;
 		num_offset = (long)strtoul(digest_offset, NULL, 10);
 		if (errno != 0) {
 			pkg_emit_errno("strtoul", "digest format error");
 			rc = EPKG_FATAL;
 			goto cleanup;
 		}
 		/*
 		 * Now we have local and remote origins that are sorted,
 		 * so here are possible cases:

@@ -364,8 +415,10 @@ pkg_update_incremental(const char *name, const char *packagesite, time_t *mtime)

 		 */
 		ret = strcmp(local_origin, digest_origin);
 		if (ret == 0) {
 			if (strcmp(digest_digest, local_digest) == 0) {
 			if (strcmp(digest_digest, local_digest) != 0) {
 				/* Do upgrade */
 				rc = pkg_add_from_manifest(fmanifest, digest_origin,
 						num_offset, digest_digest, sqlite);
 				updated ++;
 			}
 			else {

@@ -380,6 +433,9 @@ pkg_update_incremental(const char *name, const char *packagesite, time_t *mtime)

 				if (strcmp(local_origin, digest_origin) <= 0) {
 					break;
 				}
 				rc = pkgdb_repo_remove_package(local_pkg);
 				if (rc != EPKG_OK)
 					goto cleanup;
 				removed ++;
 				ret = pkgdb_it_next(it, &local_pkg, PKG_LOAD_BASIC);
 			} while (ret == EPKG_OK);

@@ -389,6 +445,8 @@ pkg_update_incremental(const char *name, const char *packagesite, time_t *mtime)

 				if (strcmp(local_origin, digest_origin) < 0) {
 					/* Insert a package from manifest */
 					added ++;
 					rc = pkg_add_from_manifest(fmanifest, digest_origin,
 							num_offset, digest_digest, sqlite);
 				}
 				else {
 					break;

@@ -406,12 +464,22 @@ pkg_update_incremental(const char *name, const char *packagesite, time_t *mtime)

 					rc = EPKG_FATAL;
 					goto cleanup;
 				}
 				errno = 0;
 				num_offset = (long)strtoul(digest_offset, NULL, 10);
 				if (errno != 0) {
 					pkg_emit_errno("strtoul", "digest format error");
 					rc = EPKG_FATAL;
 					goto cleanup;
 				}
 			}
 		}
 	} while (ret == EPKG_OK);
 	rc = EPKG_OK;
 cleanup:
 	if (pkgdb_repo_close(sqlite, rc == EPKG_OK) != EPKG_OK) {
 		rc = EPKG_FATAL;
 	}
 	if (fmanifest)
 		fclose(fmanifest);
 	if (fdigests)

@@ -478,11 +546,22 @@ pkg_update(const char *name, const char *packagesite, bool force)

 		if (sqlite != NULL)
 			sqlite3_close(sqlite);
 	}
 	if ((res = pkg_update_full(repofile, name, packagesite, &t)) != EPKG_OK) {
 		return (res);
 	if (false) {
 		/* XXX: Still disabled by default */
 		res = pkg_update_incremental(name, packagesite, &t);
 		if (res != EPKG_OK) {
 			/* Still try to do full upgrade */
 			if ((res = pkg_update_full(repofile, name, packagesite, &t)) != EPKG_OK)
 				goto cleanup;
 		}
 	}
 	else {
 		if ((res = pkg_update_full(repofile, name, packagesite, &t)) != EPKG_OK)
 			goto cleanup;
 	}
 	res = EPKG_OK;
 cleanup:
 	/* Set mtime from http request if possible */
 	if (t != 0) {
 		struct timeval ftimes[2] = {

@@ -498,5 +577,5 @@ pkg_update(const char *name, const char *packagesite, bool force)

 		utimes(repofile, ftimes);
 	}
 	return (EPKG_OK);
 	return (res);
 }