e4adcd5 Improve cache rebuild with manifest mtime Perfs are not really here for the moment while pkgdb api isn't event driven — HardenedBSD-pkg

HardenedBSD Package Manager

Improve cache rebuild with manifest mtime Perfs are not really here for the moment while pkgdb api isn't event driven

Philippe Pepiot committed 15 years ago

commit e4adcd57b2968ae9152b98b7cdc57c0fb54f73e0
parent 245fe19490b109c61a14c5a17f4b01f3e1bd4333

5 files changed +190 -133

modified docs/cache_db.txt

@@ -11,13 +11,14 @@ key value

 (char*)"count"		(size_t)nb_packages
 [...]
 (size_t)N		(char*) package N name
 (size_t)N		(char*) package N name+version
 [...]
 note that packages are 0-indexed
 (char*)$name+$version	(size_t)N (package index)
 (char*)$name			(size_t)N (package index)
 (char*)$Nn				(char*)package N name+version
 (char*)$Nn				(char*)package N name
 (char*)$Nv				(char*)package N version
 (char*)$Nc				(char*)package N comment
 (char*)$Nd				(char*)package N desc

@@ -32,7 +33,9 @@ __future__

 (char*)$Ns				(char*)package N checksum (for a package)
 (char*)$NC				(char*)"$conflict0 $conflict1 ..." package N conflicts
 Why N -> name
 Why N -> name+version
 => to loop over all packages
 Why name -> N
 Why name+version -> N
 => to get package depends without re-looping over all packages
 Why name -> N
 => to get package depends with a version mismatch

modified libpkg/pkg.h

@@ -3,6 +3,7 @@

 #include <cdb.h>
 #include <stdio.h> /* for size_t */
 #include <sys/time.h> /* for time_t */
 #define PKGERR_NOT_INSTALLED    (1<<0) /* dep not register (partial pkg with only name_version set */
 #define PKGERR_VERSION_MISMATCH (1<<1) /* dep_version != require version */

@@ -18,6 +19,8 @@ struct pkg {

 	struct pkg **rdeps; /* null-terminated */
 	unsigned char errors; /* PKGERR_* */
 	size_t idx; /* index on pkgdb */
 	void *manifest;
 	time_t mtime; /* manifest mtime */
 };
 #define PKGDB_INIT_DEPS (1<<0)

modified libpkg/pkgdb.c

@@ -13,6 +13,7 @@

 #include "pkgdb.h"
 #include "pkgdb_cache.h"
 const char *
 pkgdb_get_dir(void)
 {

@@ -24,7 +25,7 @@ pkgdb_get_dir(void)

 	return pkg_dbdir;
 }
 static int
 int
 pkgdb_open(struct pkgdb *db)
 {
 	char path[MAXPATHLEN];

@@ -32,22 +33,20 @@ pkgdb_open(struct pkgdb *db)

 	snprintf(path, sizeof(path), "%s/pkgdb.cache", pkgdb_get_dir());
 	if ((fd = open(path, O_RDONLY)) == -1)
 		warn("open(%s):", path);
 	else
 	if ((fd = open(path, O_RDONLY)) != -1)
 		fd = cdb_init(&db->db, fd);
 	return (fd);
 }
 /* query formated string -> string ('\0'-terminated) */
 static const char *
 /* query formated using string key */
 const void *
 pkgdb_query(struct pkgdb *db, const char *fmt, ...)
 {
 	const char *string;
 	va_list args;
 	char key[BUFSIZ];
 	size_t len;
 	const void *val;
 	va_start(args, fmt);
 	len = vsnprintf(key, sizeof(key), fmt, args);

@@ -64,12 +63,12 @@ pkgdb_query(struct pkgdb *db, const char *fmt, ...)

 	if (cdb_find(&db->db, key, len) < 0)
 		return NULL;
 	db_get(string, &db->db);
 	return (string);
 	db_get(val, &db->db);
 	return (val);
 }
 /* query a pkg from db using index */
 static struct pkg *
 struct pkg *
 pkgdb_pkg_query(struct pkgdb *db, size_t idx)
 {
 	struct pkg *pkg;

@@ -82,25 +81,23 @@ pkgdb_pkg_query(struct pkgdb *db, size_t idx)

 	pkg = malloc(sizeof(*pkg));
 	pkg->idx = idx;
 	db_get(pkg->name, &db->db);
 	db_get(pkg->name_version, &db->db);
 	pkg->name_version = pkgdb_query(db, PKGDB_NAMEVER, idx);
 	pkg->name = pkgdb_query(db, PKGDB_NAME, idx);
 	pkg->version = pkgdb_query(db, PKGDB_VERSION, idx);
 	pkg->comment = pkgdb_query(db, PKGDB_COMMENT, idx);
 	pkg->desc    = pkgdb_query(db, PKGDB_DESC, idx);
 	pkg->origin  = pkgdb_query(db, PKGDB_ORIGIN, idx);
 	pkg->desc = pkgdb_query(db, PKGDB_DESC, idx);
 	pkg->origin = pkgdb_query(db, PKGDB_ORIGIN, idx);
 	return (pkg);
 }
 /* populate deps on pkg */
 static void
 void
 pkgdb_deps_query(struct pkgdb *db, struct pkg *pkg)
 {
 	struct cdb_find cdbf;
 	size_t count = 0, idx, j, klen;
 	const char *name_version;
 	char *name, *version;
 	size_t count = 0, j, klen;
 	char key[BUFSIZ];
 	struct pkg *dep;

@@ -122,35 +119,8 @@ pkgdb_deps_query(struct pkgdb *db, struct pkg *pkg)

 	j = 0;
 	while (cdb_findnext(&cdbf) > 0) {
 		db_get(name_version, &db->db);
 		name = strdup(name_version);
 		if ((version = strrchr(name, '-')) == NULL) {
 			free(name);
 			continue;
 		}
 		*(version++) = '\0';
 		/* get index */
 		if (cdb_find(&db->db, name, strlen(name)) <= 0) {
 			free(name);
 			continue;
 		}
 		free(name);
 		cdb_read(&db->db, &idx, sizeof(idx), cdb_datapos(&db->db));
 		/* get package */
 		if ((dep = pkgdb_pkg_query(db, idx)) == NULL) {
 			/* partial package */
 			dep = calloc(1, sizeof(*dep));
 			dep->name_version = name_version;
 			dep->errors |= PKGERR_NOT_INSTALLED;
 		} else { /* package exist */
 			if (strcmp(version, dep->version) != 0)
 				dep->errors |= PKGERR_VERSION_MISMATCH;
 		}
 		dep = calloc(1, sizeof(*dep));
 		db_get(dep->name_version, &db->db);
 		pkg->deps[j++] = dep;
 	}
 }

@@ -176,7 +146,7 @@ pkgdb_rdeps_query(struct pkgdb *db, struct pkg *pkg, size_t count)

 		for (deps = p->deps; *deps != NULL; deps++) {
 			if (!((*deps)->errors & PKGERR_NOT_INSTALLED)
 					&& strcmp((*deps)->name, pkg->name) == 0) {
 					&& strcmp((*deps)->name_version, pkg->name_version) == 0) {
 				pkg->rdeps[j] = p;
 				break;
 			}

@@ -227,8 +197,6 @@ pkg_match(struct pkg *pkg, const regex_t *re, const char *pattern, match_t match

 	return (matched);
 }
 #define LOCK_FILE "lock"
 /*
  * Acquire a lock to access the database.
  * If `writer' is set to 1, an exclusive lock is requested so it wont mess up

@@ -240,7 +208,7 @@ pkgdb_lock(struct pkgdb *db, int writer)

 	char fname[FILENAME_MAX];
 	int flags;
 	snprintf(fname, sizeof(fname), "%s/%s", pkgdb_get_dir(), LOCK_FILE);
 	snprintf(fname, sizeof(fname), "%s/%s", pkgdb_get_dir(), PKGDB_LOCK);
 	if ((db->lock_fd = open(fname, O_RDONLY | O_CREAT, S_IRUSR | S_IRGRP | S_IROTH)) < 0)
 		err(EXIT_FAILURE, "open(%s)", fname);

@@ -269,6 +237,9 @@ pkgdb_init(struct pkgdb *db, const char *pattern, match_t match, unsigned char f

 	pkgdb_cache_update(db);
 	if (pkgdb_open(db) == -1)
 		return;
 	pkgdb_lock(db, 0);
 	db->count = 0;

@@ -279,9 +250,6 @@ pkgdb_init(struct pkgdb *db, const char *pattern, match_t match, unsigned char f

 		return;
 	}
 	if (pkgdb_open(db) == -1)
 		return;
 	if (cdb_find(&db->db, PKGDB_COUNT, strlen(PKGDB_COUNT)) <= 0) {
 		warnx("corrupted database");
 		return;

modified libpkg/pkgdb.h

@@ -3,12 +3,14 @@

 #include <pkg.h>
 #define PKG_DBDIR "/var/db/pkg"
 #define PKGDB_NAMEVER "%zun"
 #define PKGDB_LOCK "lock"
 #define PKGDB_NAME    "%zun"
 #define PKGDB_VERSION "%zuv"
 #define PKGDB_COMMENT "%zuc"
 #define PKGDB_DESC    "%zud"
 #define PKGDB_ORIGIN  "%zuo"
 #define PKGDB_DEPS    "%zuD"
 #define PKGDB_MTIME   "%zum"
 #define PKGDB_COUNT   "count"
 /* quick cdb_get */

@@ -31,4 +33,9 @@ void pkgdb_init(struct pkgdb *, const char *pattern, match_t match, unsigned cha

 void pkgdb_free(struct pkgdb *db);
 size_t pkgdb_count(struct pkgdb *db);
 const void *pkgdb_query(struct pkgdb *db, const char *fmt, ...);
 struct pkg *pkgdb_pkg_query(struct pkgdb *db, size_t idx);
 void pkgdb_deps_query(struct pkgdb *db, struct pkg *pkg);
 int pkgdb_open(struct pkgdb *db);
 #endif

modified libpkg/pkgdb_cache.c

@@ -16,74 +16,170 @@

 #include "pkg_compat.h"
 #include "pkgdb_cache.h"
 /* add record formated string -> string */
 /* add record formated string */
 static int
 db_add(struct cdb_make *db, const char *val, const char *fmt, ...)
 pkgdb_vadd(struct cdb_make *db, const void *val, size_t vallen, const char *fmt, va_list args)
 {
 	char key[BUFSIZ];
 	va_list args;
 	size_t len;
 	if (db == NULL || key == NULL || val == NULL)
 		return (-1);
 	va_start(args, fmt);
 	len = vsnprintf(key, sizeof(key), fmt, args);
 	if (len != strlen(key)) {
 		warnx("key too long:");
 		vwarnx(fmt, args);
 		va_end(args);
 		return (-1);
 	}
 	/* record the last \0 */
 	return (cdb_make_add(db, key, len, val, vallen));
 }
 static int
 pkgdb_add(struct cdb_make *db, const void *val, size_t vallen, const char *fmt, ...)
 {
 	int ret;
 	va_list args;
 	va_start(args, fmt);
 	ret = pkgdb_vadd(db, val, vallen, fmt, args);
 	va_end(args);
 	/* record the last \0 */
 	return (cdb_make_add(db, key, len, val, strlen(val)+1));
 	return (ret);
 }
 static cJSON *
 pkgdb_cache_load_port(const char *pkg_dbdir, char *pkgname)
 /* add record formated string -> string (record the last \0 on value) */
 static int
 pkgdb_add_string(struct cdb_make *db, const char *val, const char *fmt, ...)
 {
 	int ret;
 	va_list args;
 	va_start(args, fmt);
 	ret = pkgdb_vadd(db, val, strlen(val)+1, fmt, args);
 	va_end(args);
 	return (ret);
 }
 static char*cJSON_GetString(cJSON*j,const char *k){/*cJSON style*/cJSON*n;if((n=cJSON_GetObjectItem(j,k)))return n->valuestring;return NULL;}
 static struct pkg *
 pkg_from_manifest(cJSON *manifest)
 {
 	cJSON *node, *array;
 	struct pkg *pkg;
 	char *dep_name;
 	char *dep_version;
 	int i, array_size, j;
 	pkg = calloc(1, sizeof(*pkg));
 	pkg->name = cJSON_GetString(manifest, "name");
 	pkg->version = cJSON_GetString(manifest, "version");
 	pkg->comment = cJSON_GetString(manifest, "comment");
 	pkg->origin = cJSON_GetString(manifest, "origin");
 	if (pkg->name == NULL || pkg->version == NULL ||
 			pkg->comment == NULL || pkg->origin == NULL) {
 		free(pkg);
 		pkg = NULL;
 	}
 	else {
 		pkg->desc = cJSON_GetString(manifest, "desc");
 		array = cJSON_GetObjectItem(manifest, "deps");
 		if (array && (array_size = cJSON_GetArraySize(array)) > 0) {
 			pkg->deps = calloc(array_size+1, sizeof(struct pkg *));
 			for (i = 0, j = 0; i < array_size; i++) {
 				if ((node = cJSON_GetArrayItem(array, i)) != NULL) {
 					dep_name = cJSON_GetString(node, "name");
 					dep_version = cJSON_GetString(node, "version");
 					if (dep_name != NULL && dep_version != NULL) {
 						pkg->deps[j] = calloc(1, sizeof(struct pkg));
 						pkg->deps[j]->name = dep_name;
 						pkg->deps[j]->version = dep_version;
 						j++;
 					}
 				}
 			}
 		}
 	}
 	return (pkg);
 }
 static struct pkg *
 pkg_from_dir(struct pkgdb *db, const char *pkg_dbdir, char *pkgname)
 {
 	cJSON *manifest;
 	char manifestpath[MAXPATHLEN];
 	char *buffer;
 	struct stat st;
 	struct pkg *pkg = NULL;
 	const size_t *idx;
 	const time_t *old_mtime = 0;
 	snprintf(manifestpath, sizeof(manifestpath), "%s/%s/+MANIFEST", pkg_dbdir,
 			 pkgname);
 			pkgname);
 	if (stat(manifestpath, &st) == -1) {
 		warn("stat(%s):", manifestpath);
 		return NULL;
 	}
 	/* compare with last manifest mtime */
 	if (cdb_fileno(&db->db) != -1 &&
 			(idx = pkgdb_query(db, "%s", pkgname)) != NULL &&
 			(old_mtime = pkgdb_query(db, PKGDB_MTIME, *idx)) != NULL &&
 			st.st_mtime == *old_mtime &&
 			(pkg = pkgdb_pkg_query(db, *idx)) != NULL) { /* jackpot */
 		pkgdb_deps_query(db, pkg);
 		pkg->mtime = st.st_mtime;
 		return (pkg);
 	}
 	if ((file_to_buffer(manifestpath, &buffer)) == -1) {
 		warnx("An error occured while trying to read "
 				"+MANIFEST for %s, falling back to old "
 				"+CONTENTS format", pkgname);
 		manifest = pkg_compat_convert_installed( pkg_dbdir, pkgname,
 				manifestpath);
 		return (manifest);
 	}
 	else {
 		manifest = cJSON_Parse(buffer);
 		free(buffer);
 	}
 	if ((manifest = cJSON_Parse(buffer)) == 0)
 	if (manifest == NULL) {
 		warnx("%s: Manifest corrputed, skipping", pkgname);
 		return NULL;
 	}
 	free(buffer);
 	if ((pkg = pkg_from_manifest(manifest)) == NULL)
 		cJSON_Delete(manifest);
 	else
 		pkg->mtime = st.st_mtime;
 	return (manifest);
 	return (pkg);
 }
 static void
 pkgdb_cache_rebuild(const char *pkg_dbdir, const char *cache_path)
 pkgdb_cache_rebuild(struct pkgdb *db, const char *pkg_dbdir, const char *cache_path)
 {
 	int fd;
 	char tmppath[MAXPATHLEN], name_version[FILENAME_MAX];
 	char tmppath[MAXPATHLEN], namever[FILENAME_MAX];
 	struct cdb_make cdb_make;
 	DIR *dir;
 	struct dirent *portsdir;
 	cJSON *manifest, *node, *subnode, *array;
 	size_t idx = 0, array_size, i;
 	struct pkg pkg;
 	struct pkg *pkg, **deps;
 	size_t idx;
 	snprintf(tmppath, sizeof(tmppath), "%s/pkgdb.cache-XXXXX", pkg_dbdir);

@@ -94,6 +190,9 @@ pkgdb_cache_rebuild(const char *pkg_dbdir, const char *cache_path)

 	cdb_make_start(&cdb_make, fd);
 	if (pkgdb_open(db) == -1)
 		cdb_fileno(&db->db) = -1;
 	/* Now go through pkg_dbdir rebuild the cache */
 	if ((dir = opendir(pkg_dbdir)) != NULL) {

@@ -104,72 +203,49 @@ pkgdb_cache_rebuild(const char *pkg_dbdir, const char *cache_path)

 				if (portsdir->d_type != DT_DIR)
 					continue;
 				manifest = pkgdb_cache_load_port(pkg_dbdir,
 						portsdir->d_name);
 				if (manifest == 0)
 					continue;
 				if ((node = cJSON_GetObjectItem(manifest, "name")) == NULL || node->valuestring == NULL)
 					continue;
 				pkg.name = node->valuestring;
 				if ((node = cJSON_GetObjectItem(manifest, "version")) == NULL || node->valuestring == NULL)
 					continue;
 				pkg.version = node->valuestring;
 				if ((node = cJSON_GetObjectItem(manifest, "comment")) == NULL || node->valuestring == NULL)
 					continue;
 				pkg.comment = node->valuestring;
 				pkg = pkg_from_dir(db, pkg_dbdir, portsdir->d_name);
 				if ((node = cJSON_GetObjectItem(manifest, "origin")) == NULL || node->valuestring == NULL)
 				if (pkg == NULL)
 					continue;
 				pkg.origin = node->valuestring;
 				/* index -> name */
 				cdb_make_add(&cdb_make, &idx, sizeof(idx), pkg.name, strlen(pkg.name)+1);
 				snprintf(namever, sizeof(namever), "%s-%s", pkg->name, pkg->version);
 				/* index -> namever */
 				cdb_make_add(&cdb_make, &idx, sizeof(idx), namever, strlen(namever)+1);
 				/* namever -> index */
 				cdb_make_add(&cdb_make, namever, strlen(namever), &idx, sizeof(idx));
 				/* name -> index */
 				cdb_make_add(&cdb_make, pkg.name, strlen(pkg.name), &idx, sizeof(idx));
 				snprintf(name_version, FILENAME_MAX, "%s-%s", pkg.name, pkg.version);
 				db_add(&cdb_make, name_version, PKGDB_NAMEVER, idx);
 				db_add(&cdb_make, pkg.comment, PKGDB_COMMENT, idx);
 				db_add(&cdb_make, pkg.origin, PKGDB_ORIGIN, idx);
 				if ((node = cJSON_GetObjectItem(manifest, "desc")) != NULL && node->valuestring != NULL)
 					db_add(&cdb_make, node->valuestring, PKGDB_DESC, idx);
 				array = cJSON_GetObjectItem(manifest, "deps");
 				if (array && (array_size = cJSON_GetArraySize(array)) > 0) {
 					for (i = 0; i < array_size; i++) {
 						if ((node = cJSON_GetArrayItem(array, i)) != NULL) {
 							if ((subnode = cJSON_GetObjectItem(node, "name")) == NULL || subnode->valuestring == NULL)
 								continue;
 							pkg.name = subnode->valuestring;
 							if ((subnode = cJSON_GetObjectItem(node, "version")) == NULL && subnode->valuestring == NULL)
 								continue;
 							pkg.version = subnode->valuestring;
 							snprintf(name_version, FILENAME_MAX, "%s-%s", pkg.name, pkg.version);
 							db_add(&cdb_make, name_version, PKGDB_DEPS, idx);
 						}
 				cdb_make_add(&cdb_make, pkg->name, strlen(pkg->name), &idx, sizeof(idx));
 				pkgdb_add_string(&cdb_make, pkg->name, PKGDB_NAME, idx);
 				pkgdb_add_string(&cdb_make, pkg->comment, PKGDB_COMMENT, idx);
 				pkgdb_add_string(&cdb_make, pkg->origin, PKGDB_ORIGIN, idx);
 				pkgdb_add_string(&cdb_make, pkg->desc, PKGDB_DESC, idx);
 				pkgdb_add(&cdb_make, &pkg->mtime, sizeof(pkg->mtime), PKGDB_MTIME, idx);
 				if (pkg->deps) {
 					for (deps = pkg->deps; *deps != NULL; deps++) {
 						snprintf(namever, sizeof(namever), "%s-%s", (*deps)->name, (*deps)->version);
 						pkgdb_add_string(&cdb_make, namever, PKGDB_DEPS, idx);
 						free(*deps);
 					}
 					free(pkg->deps);
 				}
 				cJSON_Delete(manifest);
 				cJSON_Delete(pkg->manifest);
 				free(pkg);
 				idx++;
 			}
 		}
 		closedir(dir);
 	}
 	/* close old db */
 	if (cdb_fileno(&db->db) != -1) {
 		close(cdb_fileno(&db->db));
 		cdb_free(&db->db);
 	}
 	/* record packages len */
 	cdb_make_add(&cdb_make, PKGDB_COUNT, strlen(PKGDB_COUNT), &idx, sizeof(idx));
 	cdb_make_finish(&cdb_make);
 	close(fd);

@@ -206,7 +282,7 @@ pkgdb_cache_update(struct pkgdb *db)

 	if (errno == ENOENT || dir_st.st_mtime > cache_st.st_mtime) {
 		pkgdb_lock(db, 1);
 		pkgdb_cache_rebuild(pkg_dbdir, cache_path);
 		pkgdb_cache_rebuild(db, pkg_dbdir, cache_path);
 		pkgdb_unlock(db);
 	}
 }