Radish alpha
H
HardenedBSD-pkg
rad:z3QDZAW2FAfuLvihrhiyDC9fAD8G9
HardenedBSD Package Manager
Radicle
Git
Commits › 3a527e6
update: drastically reduce memory usage
Baptiste Daroussin committed 1 year ago
commit 3a527e6682669f01947a0bdd67a87209be92c48f
parent 6ca1a87
6 files changed +616 -47
modified external/Makefile.autosetup
@@ -1,5 +1,5 @@
include @builddir@/mk/defs.mk
- 
DIRS=	blake2 picosat linenoise sqlite libucl liblua yxml libder libecc
+ 
DIRS=	blake2 picosat linenoise sqlite libucl liblua yxml libder libecc jsmn

@if libelf-internal

DIRS+=	libelf

@endif
added external/include/jsmn.h
@@ -0,0 +1,471 @@
+ 
/*
+ 
 * MIT License
+ 
 *
+ 
 * Copyright (c) 2010 Serge Zaitsev
+ 
 *
+ 
 * Permission is hereby granted, free of charge, to any person obtaining a copy
+ 
 * of this software and associated documentation files (the "Software"), to deal
+ 
 * in the Software without restriction, including without limitation the rights
+ 
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ 
 * copies of the Software, and to permit persons to whom the Software is
+ 
 * furnished to do so, subject to the following conditions:
+ 
 *
+ 
 * The above copyright notice and this permission notice shall be included in
+ 
 * all copies or substantial portions of the Software.
+ 
 *
+ 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ 
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ 
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ 
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ 
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ 
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ 
 * SOFTWARE.
+ 
 */
+ 
#ifndef JSMN_H
+ 
#define JSMN_H
+
+ 
#include <stddef.h>
+
+ 
#ifdef __cplusplus
+ 
extern "C" {
+ 
#endif
+
+ 
#ifdef JSMN_STATIC
+ 
#define JSMN_API static
+ 
#else
+ 
#define JSMN_API extern
+ 
#endif
+
+ 
/**
+ 
 * JSON type identifier. Basic types are:
+ 
 * 	o Object
+ 
 * 	o Array
+ 
 * 	o String
+ 
 * 	o Other primitive: number, boolean (true/false) or null
+ 
 */
+ 
typedef enum {
+ 
  JSMN_UNDEFINED = 0,
+ 
  JSMN_OBJECT = 1 << 0,
+ 
  JSMN_ARRAY = 1 << 1,
+ 
  JSMN_STRING = 1 << 2,
+ 
  JSMN_PRIMITIVE = 1 << 3
+ 
} jsmntype_t;
+
+ 
enum jsmnerr {
+ 
  /* Not enough tokens were provided */
+ 
  JSMN_ERROR_NOMEM = -1,
+ 
  /* Invalid character inside JSON string */
+ 
  JSMN_ERROR_INVAL = -2,
+ 
  /* The string is not a full JSON packet, more bytes expected */
+ 
  JSMN_ERROR_PART = -3
+ 
};
+
+ 
/**
+ 
 * JSON token description.
+ 
 * type		type (object, array, string etc.)
+ 
 * start	start position in JSON data string
+ 
 * end		end position in JSON data string
+ 
 */
+ 
typedef struct jsmntok {
+ 
  jsmntype_t type;
+ 
  int start;
+ 
  int end;
+ 
  int size;
+ 
#ifdef JSMN_PARENT_LINKS
+ 
  int parent;
+ 
#endif
+ 
} jsmntok_t;
+
+ 
/**
+ 
 * JSON parser. Contains an array of token blocks available. Also stores
+ 
 * the string being parsed now and current position in that string.
+ 
 */
+ 
typedef struct jsmn_parser {
+ 
  unsigned int pos;     /* offset in the JSON string */
+ 
  unsigned int toknext; /* next token to allocate */
+ 
  int toksuper;         /* superior token node, e.g. parent object or array */
+ 
} jsmn_parser;
+
+ 
/**
+ 
 * Create JSON parser over an array of tokens
+ 
 */
+ 
JSMN_API void jsmn_init(jsmn_parser *parser);
+
+ 
/**
+ 
 * Run JSON parser. It parses a JSON data string into and array of tokens, each
+ 
 * describing
+ 
 * a single JSON object.
+ 
 */
+ 
JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len,
+ 
                        jsmntok_t *tokens, const unsigned int num_tokens);
+
+ 
#ifndef JSMN_HEADER
+ 
/**
+ 
 * Allocates a fresh unused token from the token pool.
+ 
 */
+ 
static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens,
+ 
                                   const size_t num_tokens) {
+ 
  jsmntok_t *tok;
+ 
  if (parser->toknext >= num_tokens) {
+ 
    return NULL;
+ 
  }
+ 
  tok = &tokens[parser->toknext++];
+ 
  tok->start = tok->end = -1;
+ 
  tok->size = 0;
+ 
#ifdef JSMN_PARENT_LINKS
+ 
  tok->parent = -1;
+ 
#endif
+ 
  return tok;
+ 
}
+
+ 
/**
+ 
 * Fills token type and boundaries.
+ 
 */
+ 
static void jsmn_fill_token(jsmntok_t *token, const jsmntype_t type,
+ 
                            const int start, const int end) {
+ 
  token->type = type;
+ 
  token->start = start;
+ 
  token->end = end;
+ 
  token->size = 0;
+ 
}
+
+ 
/**
+ 
 * Fills next available token with JSON primitive.
+ 
 */
+ 
static int jsmn_parse_primitive(jsmn_parser *parser, const char *js,
+ 
                                const size_t len, jsmntok_t *tokens,
+ 
                                const size_t num_tokens) {
+ 
  jsmntok_t *token;
+ 
  int start;
+
+ 
  start = parser->pos;
+
+ 
  for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+ 
    switch (js[parser->pos]) {
+ 
#ifndef JSMN_STRICT
+ 
    /* In strict mode primitive must be followed by "," or "}" or "]" */
+ 
    case ':':
+ 
#endif
+ 
    case '\t':
+ 
    case '\r':
+ 
    case '\n':
+ 
    case ' ':
+ 
    case ',':
+ 
    case ']':
+ 
    case '}':
+ 
      goto found;
+ 
    default:
+ 
                   /* to quiet a warning from gcc*/
+ 
      break;
+ 
    }
+ 
    if (js[parser->pos] < 32 || js[parser->pos] >= 127) {
+ 
      parser->pos = start;
+ 
      return JSMN_ERROR_INVAL;
+ 
    }
+ 
  }
+ 
#ifdef JSMN_STRICT
+ 
  /* In strict mode primitive must be followed by a comma/object/array */
+ 
  parser->pos = start;
+ 
  return JSMN_ERROR_PART;
+ 
#endif
+
+ 
found:
+ 
  if (tokens == NULL) {
+ 
    parser->pos--;
+ 
    return 0;
+ 
  }
+ 
  token = jsmn_alloc_token(parser, tokens, num_tokens);
+ 
  if (token == NULL) {
+ 
    parser->pos = start;
+ 
    return JSMN_ERROR_NOMEM;
+ 
  }
+ 
  jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos);
+ 
#ifdef JSMN_PARENT_LINKS
+ 
  token->parent = parser->toksuper;
+ 
#endif
+ 
  parser->pos--;
+ 
  return 0;
+ 
}
+
+ 
/**
+ 
 * Fills next token with JSON string.
+ 
 */
+ 
static int jsmn_parse_string(jsmn_parser *parser, const char *js,
+ 
                             const size_t len, jsmntok_t *tokens,
+ 
                             const size_t num_tokens) {
+ 
  jsmntok_t *token;
+
+ 
  int start = parser->pos;
+
+ 
  /* Skip starting quote */
+ 
  parser->pos++;
+
+ 
  for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+ 
    char c = js[parser->pos];
+
+ 
    /* Quote: end of string */
+ 
    if (c == '\"') {
+ 
      if (tokens == NULL) {
+ 
        return 0;
+ 
      }
+ 
      token = jsmn_alloc_token(parser, tokens, num_tokens);
+ 
      if (token == NULL) {
+ 
        parser->pos = start;
+ 
        return JSMN_ERROR_NOMEM;
+ 
      }
+ 
      jsmn_fill_token(token, JSMN_STRING, start + 1, parser->pos);
+ 
#ifdef JSMN_PARENT_LINKS
+ 
      token->parent = parser->toksuper;
+ 
#endif
+ 
      return 0;
+ 
    }
+
+ 
    /* Backslash: Quoted symbol expected */
+ 
    if (c == '\\' && parser->pos + 1 < len) {
+ 
      int i;
+ 
      parser->pos++;
+ 
      switch (js[parser->pos]) {
+ 
      /* Allowed escaped symbols */
+ 
      case '\"':
+ 
      case '/':
+ 
      case '\\':
+ 
      case 'b':
+ 
      case 'f':
+ 
      case 'r':
+ 
      case 'n':
+ 
      case 't':
+ 
        break;
+ 
      /* Allows escaped symbol \uXXXX */
+ 
      case 'u':
+ 
        parser->pos++;
+ 
        for (i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0';
+ 
             i++) {
+ 
          /* If it isn't a hex character we have an error */
+ 
          if (!((js[parser->pos] >= 48 && js[parser->pos] <= 57) ||   /* 0-9 */
+ 
                (js[parser->pos] >= 65 && js[parser->pos] <= 70) ||   /* A-F */
+ 
                (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */
+ 
            parser->pos = start;
+ 
            return JSMN_ERROR_INVAL;
+ 
          }
+ 
          parser->pos++;
+ 
        }
+ 
        parser->pos--;
+ 
        break;
+ 
      /* Unexpected symbol */
+ 
      default:
+ 
        parser->pos = start;
+ 
        return JSMN_ERROR_INVAL;
+ 
      }
+ 
    }
+ 
  }
+ 
  parser->pos = start;
+ 
  return JSMN_ERROR_PART;
+ 
}
+
+ 
/**
+ 
 * Parse JSON string and fill tokens.
+ 
 */
+ 
JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len,
+ 
                        jsmntok_t *tokens, const unsigned int num_tokens) {
+ 
  int r;
+ 
  int i;
+ 
  jsmntok_t *token;
+ 
  int count = parser->toknext;
+
+ 
  for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+ 
    char c;
+ 
    jsmntype_t type;
+
+ 
    c = js[parser->pos];
+ 
    switch (c) {
+ 
    case '{':
+ 
    case '[':
+ 
      count++;
+ 
      if (tokens == NULL) {
+ 
        break;
+ 
      }
+ 
      token = jsmn_alloc_token(parser, tokens, num_tokens);
+ 
      if (token == NULL) {
+ 
        return JSMN_ERROR_NOMEM;
+ 
      }
+ 
      if (parser->toksuper != -1) {
+ 
        jsmntok_t *t = &tokens[parser->toksuper];
+ 
#ifdef JSMN_STRICT
+ 
        /* In strict mode an object or array can't become a key */
+ 
        if (t->type == JSMN_OBJECT) {
+ 
          return JSMN_ERROR_INVAL;
+ 
        }
+ 
#endif
+ 
        t->size++;
+ 
#ifdef JSMN_PARENT_LINKS
+ 
        token->parent = parser->toksuper;
+ 
#endif
+ 
      }
+ 
      token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY);
+ 
      token->start = parser->pos;
+ 
      parser->toksuper = parser->toknext - 1;
+ 
      break;
+ 
    case '}':
+ 
    case ']':
+ 
      if (tokens == NULL) {
+ 
        break;
+ 
      }
+ 
      type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY);
+ 
#ifdef JSMN_PARENT_LINKS
+ 
      if (parser->toknext < 1) {
+ 
        return JSMN_ERROR_INVAL;
+ 
      }
+ 
      token = &tokens[parser->toknext - 1];
+ 
      for (;;) {
+ 
        if (token->start != -1 && token->end == -1) {
+ 
          if (token->type != type) {
+ 
            return JSMN_ERROR_INVAL;
+ 
          }
+ 
          token->end = parser->pos + 1;
+ 
          parser->toksuper = token->parent;
+ 
          break;
+ 
        }
+ 
        if (token->parent == -1) {
+ 
          if (token->type != type || parser->toksuper == -1) {
+ 
            return JSMN_ERROR_INVAL;
+ 
          }
+ 
          break;
+ 
        }
+ 
        token = &tokens[token->parent];
+ 
      }
+ 
#else
+ 
      for (i = parser->toknext - 1; i >= 0; i--) {
+ 
        token = &tokens[i];
+ 
        if (token->start != -1 && token->end == -1) {
+ 
          if (token->type != type) {
+ 
            return JSMN_ERROR_INVAL;
+ 
          }
+ 
          parser->toksuper = -1;
+ 
          token->end = parser->pos + 1;
+ 
          break;
+ 
        }
+ 
      }
+ 
      /* Error if unmatched closing bracket */
+ 
      if (i == -1) {
+ 
        return JSMN_ERROR_INVAL;
+ 
      }
+ 
      for (; i >= 0; i--) {
+ 
        token = &tokens[i];
+ 
        if (token->start != -1 && token->end == -1) {
+ 
          parser->toksuper = i;
+ 
          break;
+ 
        }
+ 
      }
+ 
#endif
+ 
      break;
+ 
    case '\"':
+ 
      r = jsmn_parse_string(parser, js, len, tokens, num_tokens);
+ 
      if (r < 0) {
+ 
        return r;
+ 
      }
+ 
      count++;
+ 
      if (parser->toksuper != -1 && tokens != NULL) {
+ 
        tokens[parser->toksuper].size++;
+ 
      }
+ 
      break;
+ 
    case '\t':
+ 
    case '\r':
+ 
    case '\n':
+ 
    case ' ':
+ 
      break;
+ 
    case ':':
+ 
      parser->toksuper = parser->toknext - 1;
+ 
      break;
+ 
    case ',':
+ 
      if (tokens != NULL && parser->toksuper != -1 &&
+ 
          tokens[parser->toksuper].type != JSMN_ARRAY &&
+ 
          tokens[parser->toksuper].type != JSMN_OBJECT) {
+ 
#ifdef JSMN_PARENT_LINKS
+ 
        parser->toksuper = tokens[parser->toksuper].parent;
+ 
#else
+ 
        for (i = parser->toknext - 1; i >= 0; i--) {
+ 
          if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) {
+ 
            if (tokens[i].start != -1 && tokens[i].end == -1) {
+ 
              parser->toksuper = i;
+ 
              break;
+ 
            }
+ 
          }
+ 
        }
+ 
#endif
+ 
      }
+ 
      break;
+ 
#ifdef JSMN_STRICT
+ 
    /* In strict mode primitives are: numbers and booleans */
+ 
    case '-':
+ 
    case '0':
+ 
    case '1':
+ 
    case '2':
+ 
    case '3':
+ 
    case '4':
+ 
    case '5':
+ 
    case '6':
+ 
    case '7':
+ 
    case '8':
+ 
    case '9':
+ 
    case 't':
+ 
    case 'f':
+ 
    case 'n':
+ 
      /* And they must not be keys of the object */
+ 
      if (tokens != NULL && parser->toksuper != -1) {
+ 
        const jsmntok_t *t = &tokens[parser->toksuper];
+ 
        if (t->type == JSMN_OBJECT ||
+ 
            (t->type == JSMN_STRING && t->size != 0)) {
+ 
          return JSMN_ERROR_INVAL;
+ 
        }
+ 
      }
+ 
#else
+ 
    /* In non-strict mode every unquoted value is a primitive */
+ 
    default:
+ 
#endif
+ 
      r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens);
+ 
      if (r < 0) {
+ 
        return r;
+ 
      }
+ 
      count++;
+ 
      if (parser->toksuper != -1 && tokens != NULL) {
+ 
        tokens[parser->toksuper].size++;
+ 
      }
+ 
      break;
+
+ 
#ifdef JSMN_STRICT
+ 
    /* Unexpected char in strict mode */
+ 
    default:
+ 
      return JSMN_ERROR_INVAL;
+ 
#endif
+ 
    }
+ 
  }
+
+ 
  if (tokens != NULL) {
+ 
    for (i = parser->toknext - 1; i >= 0; i--) {
+ 
      /* Unmatched opened object or array */
+ 
      if (tokens[i].start != -1 && tokens[i].end == -1) {
+ 
        return JSMN_ERROR_PART;
+ 
      }
+ 
    }
+ 
  }
+
+ 
  return count;
+ 
}
+
+ 
/**
+ 
 * Creates a new parser based over a given buffer with an array of tokens
+ 
 * available.
+ 
 */
+ 
JSMN_API void jsmn_init(jsmn_parser *parser) {
+ 
  parser->pos = 0;
+ 
  parser->toknext = 0;
+ 
  parser->toksuper = -1;
+ 
}
+
+ 
#endif /* JSMN_HEADER */
+
+ 
#ifdef __cplusplus
+ 
}
+ 
#endif
+
+ 
#endif /* JSMN_H */
modified libpkg/Makefile.autosetup
@@ -58,7 +58,8 @@ SRCS= backup_lib.c \
	fetch_file.c \

	triggers.c \

	pkghash.c \
- 
	yuarel.c
+ 
	yuarel.c \
+ 
	json.c



LOCAL_CFLAGS=	-I$(top_srcdir)/compat \

		-I$(top_srcdir)/external/blake2 \
added libpkg/json.c
@@ -0,0 +1,38 @@
+ 
/*-
+ 
 * Copyright (c) 2024-2025 Baptiste Daroussin <bapt@FreeBSD.org>
+ 
 *
+ 
 * SPDX-License-Identifier: BSD-2-Clause
+ 
 */
+
+ 
#define JSMN_PARENT_LINKS 1
+ 
#define JSMN_STRICT 1
+ 
#include <jsmn.h>
+ 
#include <libpkg/private/json.h>
+ 
#include <stdio.h>
+
+ 
jsmntok_t *
+ 
jsmn_next(jsmntok_t *tok) {
+ 
	jsmntok_t *cur = tok;
+ 
	int cnt = tok->size;
+
+ 
	while (cnt--) {
+ 
		cur++;
+ 
		cur = jsmn_next(cur);
+ 
	}
+ 
	return (cur);
+ 
}
+
+ 
bool
+ 
jsmntok_stringeq(jsmntok_t *tok, const char *line, const char *str) {
+ 
	return (strncmp(str, line + tok->start, jsmn_toklen(tok)) == 0);
+ 
}
+
+ 
int
+ 
jsmntok_nextchild(jsmntok_t *tok, int tokcount, int parent, int me)
+ 
{
+ 
	for (int i = me + 1; i < tokcount; i++) {
+ 
		if ((tok + i)->parent == parent)
+ 
			return (i);
+ 
	}
+ 
	return (-1);
+ 
}
added libpkg/private/json.h
@@ -0,0 +1,19 @@
+ 
/*-
+ 
 * Copyright (c) 2024-2025 Baptiste Daroussin <bapt@FreeBSD.org>
+ 
 *
+ 
 * SPDX-License-Identifier: BSD-2-Clause
+ 
 */
+
+ 
#pragma once
+
+ 
#define JSMN_PARENT_LINKS 1
+ 
#define JSMN_HEADER 1
+ 
#define JSMN_STRICT 1
+ 
#include <jsmn.h>
+ 
#include <stdbool.h>
+ 
#include <string.h>
+
+ 
jsmntok_t *jsmn_next(jsmntok_t *tok);
+ 
#define jsmn_toklen(x) (x->end - x->start)
+ 
bool jsmntok_stringeq(jsmntok_t *tok, const char *line, const char *str);
+ 
int jsmntok_nextchild(jsmntok_t *tok, int tokcount, int parent, int me);
modified libpkg/repo/binary/update.c
@@ -30,6 +30,7 @@
#include <sys/param.h>

#include <sys/file.h>

#include <sys/time.h>
+ 
#include <sys/mman.h>



#include <stdio.h>

#include <stdlib.h>
@@ -47,6 +48,7 @@
#include "private/utils.h"

#include "private/pkgdb.h"

#include "private/pkg.h"
+ 
#include "private/json.h"

#include "binary.h"

#include "binary_private.h"
@@ -363,7 +365,7 @@ pkg_repo_binary_register_conflicts(const char *origin, char **conflicts,
}



static int
- 
pkg_repo_binary_add_from_ucl(sqlite3 *sqlite, ucl_object_t *o, struct pkg_repo *repo)
+ 
pkg_repo_binary_add_from_string(sqlite3 *sqlite, const char *str, size_t len, struct pkg_repo *repo)

{

	int rc = EPKG_OK;

	struct pkg *pkg;
@@ -373,10 +375,9 @@ pkg_repo_binary_add_from_ucl(sqlite3 *sqlite, ucl_object_t *o, struct pkg_repo *
	if (rc != EPKG_OK)

		return (EPKG_FATAL);
- 
	rc = pkg_parse_manifest_ucl(pkg, o);
- 
	if (rc != EPKG_OK) {
+ 
	rc = pkg_parse_manifest(pkg, str, len);
+ 
	if (rc != EPKG_OK)

		goto cleanup;
- 
	}



	if (pkg->digest == NULL || !pkg_checksum_is_valid(pkg->digest, strlen(pkg->digest)))

		pkg_checksum_calculate(pkg, NULL, false, true, false);
@@ -400,7 +401,6 @@ pkg_repo_binary_add_from_ucl(sqlite3 *sqlite, ucl_object_t *o, struct pkg_repo *
	rc = pkg_repo_binary_add_pkg(pkg, sqlite, true);



cleanup:
- 
	ucl_object_unref(o);

	pkg_free(pkg);



	return (rc);
@@ -499,20 +499,27 @@ rollback_repo(void *data)
	unlink(path);

}
- 
static void
- 
save_ucl(struct pkg_repo *repo, ucl_object_t *obj, const char* dst_name)
+ 
static int
+ 
dump_json(struct pkg_repo *repo, const char *line, jsmntok_t *tok, const char *dst_name)

{
- 
	if (obj == NULL)
- 
		return;
+ 
	if (tok->type != JSMN_ARRAY) {
+ 
		pkg_emit_error("Invalid %s, expecting an array", dst_name);
+ 
		return (1);
+ 
	}
+ 
	if (tok->size == 0) {
+ 
		return (1);
+ 
	}

	if (repo->dfd == -1 && pkg_repo_open(repo) == EPKG_FATAL)
- 
		return;
+ 
		return (0);

	int fd = openat(repo->dfd, dst_name, O_CREAT|O_TRUNC|O_RDWR, 0644);

	if (fd == -1) {
- 
		pkg_emit_errno("openat", "repo save_ucl");
- 
		return;
+ 
		pkg_emit_errno("openat", "repo dump_json");

	}
- 
	ucl_object_emit_fd(obj, UCL_EMIT_JSON_COMPACT, fd);
- 
	close(fd);
+ 
	FILE *f = fdopen(fd, "w");
+ 
	fprintf(f, "%.*s", jsmn_toklen(tok), line + tok->start);
+ 
	fclose(f);
+
+ 
	return (0);

}



static int
@@ -530,7 +537,7 @@ pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,
	size_t linecap = 0;

	ssize_t linelen, totallen = 0;

	struct pkg_repo_content prc;
- 
	ucl_object_t *data = NULL;
+ 
	struct stat st;



	pkg_debug(1, "Pkgrepo, begin update of '%s'", name);
@@ -556,19 +563,16 @@ pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,
		goto cleanup;



	if (rc == EPKG_OK) {
- 
		struct ucl_parser *p = ucl_parser_new(0);
- 
		if (!ucl_parser_add_fd(p, prc.data_fd)) {
- 
			pkg_emit_error("Error parsing data file: %s'",
- 
			    ucl_parser_get_error(p));
- 
			ucl_parser_free(p);
+ 
		fstat(prc.data_fd, &st);
+ 
		lseek(prc.data_fd, 0, SEEK_SET);
+ 
		linelen = st.st_size;
+ 
		line = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, prc.data_fd, 0);
+ 
		if (line == MAP_FAILED) {
+ 
			pkg_emit_errno("Error parsing data", "mmap");

			close(prc.data_fd);

			rc = EPKG_FATAL;

			goto cleanup;

		}
- 
		data = ucl_parser_get_object(p);
- 
		ucl_parser_free(p);
- 
		close(prc.data_fd);
- 
		/* XXX TODO check against a schema */

	} else {

		rc = pkg_repo_fetch_remote_extract_fd(repo, &prc);

		if (rc != EPKG_OK)
@@ -612,6 +616,57 @@ pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,
		goto cleanup;



	in_trans = true;
+ 
	if (line != NULL) {
+ 
		jsmn_parser p;
+ 
		jsmntok_t *tok;
+ 
		jsmn_init(&p);
+ 
		int tokcount = jsmn_parse(&p, line, linelen, NULL, 0);
+ 
		if (tokcount < 0) {
+ 
			pkg_emit_error("Invalid data");
+ 
			goto cleanup;
+ 
		}
+ 
		tok = xcalloc(tokcount, sizeof(*tok));
+ 
		jsmn_init(&p);
+ 
		tokcount = jsmn_parse(&p, line, linelen, tok, tokcount);
+ 
		if (tokcount < 0) {
+ 
			pkg_emit_error("Invalid data");
+ 
			goto cleanup;
+ 
		}
+ 
		tokcount = p.toknext;
+ 
		if (tok->type != JSMN_OBJECT) {
+ 
			pkg_emit_error("Invalid data (expecting a json object)");
+ 
			free(tok);
+ 
			goto cleanup;
+ 
		}
+ 
		int i = 0;
+ 
		while ((i = jsmntok_nextchild(tok, tokcount, 0, i)) > 0) {
+ 
			jsmntok_t *key = tok + i;
+ 
			jsmntok_t *value = tok + i +1;
+
+ 
			if (key->type != JSMN_STRING) {
+ 
				continue;
+ 
			}
+ 
			if (jsmntok_stringeq(key, line, "groups")) {
+ 
				dump_json(repo, line, value, "groups");
+ 
			} else if (jsmntok_stringeq(key, line, "expired_packages")) {
+ 
				dump_json(repo, line, value, "expired_packages");
+ 
			} else if (jsmntok_stringeq(key, line, "packages")) {
+ 
				if (value->type == JSMN_ARRAY) {
+ 
					int j = i + 1;
+ 
					while ((j = jsmntok_nextchild(tok, tokcount, i + 1, j)) > 0) {
+ 
						jsmntok_t *jobj = tok + j;
+ 
						cnt++;
+ 
						if ((cnt % 10) == 0)
+ 
							cancel = pkg_emit_progress_tick(cnt, value->size);
+ 
						rc = pkg_repo_binary_add_from_string(sqlite, line + jobj->start, jsmn_toklen(jobj), repo);
+ 
						if (rc != EPKG_OK || cancel != 0)
+ 
							break;
+ 
					}
+ 
					pkg_emit_progress_tick(cnt, value->size);
+ 
				}
+ 
			}
+ 
		}
+ 
	}

	if (f != NULL) {

		while ((linelen = getline(&line, &linecap, f)) > 0) {

			cnt++;
@@ -625,25 +680,6 @@ pkg_repo_binary_update_proceed(const char *name, struct pkg_repo *repo,
		}

		pkg_emit_progress_tick(prc.manifest_len, prc.manifest_len);

	}
- 
	if (data != NULL) {
- 
		ucl_object_t *pkgs = ucl_object_ref(ucl_object_find_key(data, "packages"));
- 
		int nbel = ucl_array_size(pkgs);
- 
		while (cnt < nbel) {
- 
			ucl_object_t *o = ucl_array_pop_first(pkgs);
- 
			cnt++;
- 
			if ((cnt % 10 ) == 0)
- 
				cancel = pkg_emit_progress_tick(cnt, nbel);
- 
			rc = pkg_repo_binary_add_from_ucl(sqlite, o, repo);
- 
			if (rc != EPKG_OK || cancel != 0)
- 
				break;
- 
		}
- 
		pkg_emit_progress_tick(cnt, nbel);
- 
		save_ucl(repo,
- 
		    ucl_object_ref(ucl_object_find_key(data, "groups")), "groups.ucl");
- 
		save_ucl(repo,
- 
		    ucl_object_ref(ucl_object_find_key(data, "expired_packages")),
- 
		    "expired_packages.ucl");
- 
	}



	if (rc == EPKG_OK)

		pkg_emit_incremental_update(repo->name, cnt);
@@ -677,9 +713,13 @@ cleanup:
		free(path);

	}

	pkg_unregister_cleanup_callback(rollback_repo, (void *)name);
- 
	free(line);
- 
	if (f != NULL)
+ 
	if (f != NULL) {
+ 
		free(line);

		fclose(f);
+ 
	} else {
+ 
		munmap(line, linelen);
+ 
		close(prc.data_fd);
+ 
	}



	return (rc);

}