Blame gio/gvdb/gvdb-builder.c

Packit ae235b
/*
Packit ae235b
 * Copyright © 2010 Codethink Limited
Packit ae235b
 *
Packit ae235b
 * This library is free software; you can redistribute it and/or
Packit ae235b
 * modify it under the terms of the GNU Lesser General Public
Packit ae235b
 * License as published by the Free Software Foundation; either
Packit ae235b
 * version 2.1 of the License, or (at your option) any later version.
Packit ae235b
 *
Packit ae235b
 * This library is distributed in the hope that it will be useful,
Packit ae235b
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Packit ae235b
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Packit ae235b
 * Lesser General Public License for more details.
Packit ae235b
 *
Packit ae235b
 * You should have received a copy of the GNU Lesser General Public
Packit ae235b
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
Packit ae235b
 *
Packit ae235b
 * Author: Ryan Lortie <desrt@desrt.ca>
Packit ae235b
 */
Packit ae235b
Packit ae235b
#include "gvdb-builder.h"
Packit ae235b
#include "gvdb-format.h"
Packit ae235b
Packit ae235b
#include <glib.h>
Packit ae235b
#include <fcntl.h>
Packit ae235b
#if !defined(G_OS_WIN32) || !defined(_MSC_VER)
Packit ae235b
#include <unistd.h>
Packit ae235b
#endif
Packit ae235b
#include <string.h>
Packit ae235b
Packit ae235b
Packit ae235b
struct _GvdbItem
Packit ae235b
{
Packit ae235b
  gchar *key;
Packit ae235b
  guint32 hash_value;
Packit ae235b
  guint32_le assigned_index;
Packit ae235b
  GvdbItem *parent;
Packit ae235b
  GvdbItem *sibling;
Packit ae235b
  GvdbItem *next;
Packit ae235b
Packit ae235b
  /* one of:
Packit ae235b
   * this:
Packit ae235b
   */
Packit ae235b
  GVariant *value;
Packit ae235b
Packit ae235b
  /* this: */
Packit ae235b
  GHashTable *table;
Packit ae235b
Packit ae235b
  /* or this: */
Packit ae235b
  GvdbItem *child;
Packit ae235b
};
Packit ae235b
Packit ae235b
static void
Packit ae235b
gvdb_item_free (gpointer data)
Packit ae235b
{
Packit ae235b
  GvdbItem *item = data;
Packit ae235b
Packit ae235b
  g_free (item->key);
Packit ae235b
Packit ae235b
  if (item->value)
Packit ae235b
    g_variant_unref (item->value);
Packit ae235b
Packit ae235b
  if (item->table)
Packit ae235b
    g_hash_table_unref (item->table);
Packit ae235b
Packit ae235b
  g_slice_free (GvdbItem, item);
Packit ae235b
}
Packit ae235b
Packit ae235b
GHashTable *
Packit ae235b
gvdb_hash_table_new (GHashTable  *parent,
Packit ae235b
                     const gchar *name_in_parent)
Packit ae235b
{
Packit ae235b
  GHashTable *table;
Packit ae235b
Packit ae235b
  table = g_hash_table_new_full (g_str_hash, g_str_equal,
Packit ae235b
                                 g_free, gvdb_item_free);
Packit ae235b
Packit ae235b
  if (parent)
Packit ae235b
    {
Packit ae235b
      GvdbItem *item;
Packit ae235b
Packit ae235b
      item = gvdb_hash_table_insert (parent, name_in_parent);
Packit ae235b
      gvdb_item_set_hash_table (item, table);
Packit ae235b
    }
Packit ae235b
Packit ae235b
  return table;
Packit ae235b
}
Packit ae235b
Packit ae235b
static guint32
Packit ae235b
djb_hash (const gchar *key)
Packit ae235b
{
Packit ae235b
  guint32 hash_value = 5381;
Packit ae235b
Packit ae235b
  while (*key)
Packit ae235b
    hash_value = hash_value * 33 + *(signed char *)key++;
Packit ae235b
Packit ae235b
  return hash_value;
Packit ae235b
}
Packit ae235b
Packit ae235b
GvdbItem *
Packit ae235b
gvdb_hash_table_insert (GHashTable  *table,
Packit ae235b
                        const gchar *key)
Packit ae235b
{
Packit ae235b
  GvdbItem *item;
Packit ae235b
Packit ae235b
  item = g_slice_new0 (GvdbItem);
Packit ae235b
  item->key = g_strdup (key);
Packit ae235b
  item->hash_value = djb_hash (key);
Packit ae235b
Packit ae235b
  g_hash_table_insert (table, g_strdup (key), item);
Packit ae235b
Packit ae235b
  return item;
Packit ae235b
}
Packit ae235b
Packit ae235b
void
Packit ae235b
gvdb_hash_table_insert_string (GHashTable  *table,
Packit ae235b
                               const gchar *key,
Packit ae235b
                               const gchar *value)
Packit ae235b
{
Packit ae235b
  GvdbItem *item;
Packit ae235b
Packit ae235b
  item = gvdb_hash_table_insert (table, key);
Packit ae235b
  gvdb_item_set_value (item, g_variant_new_string (value));
Packit ae235b
}
Packit ae235b
Packit ae235b
void
Packit ae235b
gvdb_item_set_value (GvdbItem *item,
Packit ae235b
                     GVariant *value)
Packit ae235b
{
Packit ae235b
  g_return_if_fail (!item->value && !item->table && !item->child);
Packit ae235b
Packit ae235b
  item->value = g_variant_ref_sink (value);
Packit ae235b
}
Packit ae235b
Packit ae235b
void
Packit ae235b
gvdb_item_set_hash_table (GvdbItem   *item,
Packit ae235b
                          GHashTable *table)
Packit ae235b
{
Packit ae235b
  g_return_if_fail (!item->value && !item->table && !item->child);
Packit ae235b
Packit ae235b
  item->table = g_hash_table_ref (table);
Packit ae235b
}
Packit ae235b
Packit ae235b
void
Packit ae235b
gvdb_item_set_parent (GvdbItem *item,
Packit ae235b
                      GvdbItem *parent)
Packit ae235b
{
Packit ae235b
  GvdbItem **node;
Packit ae235b
Packit ae235b
  g_return_if_fail (g_str_has_prefix (item->key, parent->key));
Packit ae235b
  g_return_if_fail (!parent->value && !parent->table);
Packit ae235b
  g_return_if_fail (!item->parent && !item->sibling);
Packit ae235b
Packit ae235b
  for (node = &parent->child; *node; node = &(*node)->sibling)
Packit ae235b
    if (strcmp ((*node)->key, item->key) > 0)
Packit ae235b
      break;
Packit ae235b
Packit ae235b
  item->parent = parent;
Packit ae235b
  item->sibling = *node;
Packit ae235b
  *node = item;
Packit ae235b
}
Packit ae235b
Packit ae235b
typedef struct
Packit ae235b
{
Packit ae235b
  GvdbItem **buckets;
Packit ae235b
  gint n_buckets;
Packit ae235b
} HashTable;
Packit ae235b
Packit ae235b
static HashTable *
Packit ae235b
hash_table_new (gint n_buckets)
Packit ae235b
{
Packit ae235b
  HashTable *table;
Packit ae235b
Packit ae235b
  table = g_slice_new (HashTable);
Packit ae235b
  table->buckets = g_new0 (GvdbItem *, n_buckets);
Packit ae235b
  table->n_buckets = n_buckets;
Packit ae235b
Packit ae235b
  return table;
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
hash_table_free (HashTable *table)
Packit ae235b
{
Packit ae235b
  g_free (table->buckets);
Packit ae235b
Packit ae235b
  g_slice_free (HashTable, table);
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
hash_table_insert (gpointer key,
Packit ae235b
                   gpointer value,
Packit ae235b
                   gpointer data)
Packit ae235b
{
Packit ae235b
  guint32 hash_value, bucket;
Packit ae235b
  HashTable *table = data;
Packit ae235b
  GvdbItem *item = value;
Packit ae235b
Packit ae235b
  hash_value = djb_hash (key);
Packit ae235b
  bucket = hash_value % table->n_buckets;
Packit ae235b
  item->next = table->buckets[bucket];
Packit ae235b
  table->buckets[bucket] = item;
Packit ae235b
}
Packit ae235b
Packit ae235b
static guint32_le
Packit ae235b
item_to_index (GvdbItem *item)
Packit ae235b
{
Packit ae235b
  if (item != NULL)
Packit ae235b
    return item->assigned_index;
Packit ae235b
Packit ae235b
  return guint32_to_le (-1u);
Packit ae235b
}
Packit ae235b
Packit ae235b
typedef struct
Packit ae235b
{
Packit ae235b
  GQueue *chunks;
Packit ae235b
  guint64 offset;
Packit ae235b
  gboolean byteswap;
Packit ae235b
} FileBuilder;
Packit ae235b
Packit ae235b
typedef struct
Packit ae235b
{
Packit ae235b
  gsize offset;
Packit ae235b
  gsize size;
Packit ae235b
  gpointer data;
Packit ae235b
} FileChunk;
Packit ae235b
Packit ae235b
static gpointer
Packit ae235b
file_builder_allocate (FileBuilder         *fb,
Packit ae235b
                       guint                alignment,
Packit ae235b
                       gsize                size,
Packit ae235b
                       struct gvdb_pointer *pointer)
Packit ae235b
{
Packit ae235b
  FileChunk *chunk;
Packit ae235b
Packit ae235b
  if (size == 0)
Packit ae235b
    return NULL;
Packit ae235b
Packit ae235b
  fb->offset += (-fb->offset) & (alignment - 1);
Packit ae235b
  chunk = g_slice_new (FileChunk);
Packit ae235b
  chunk->offset = fb->offset;
Packit ae235b
  chunk->size = size;
Packit ae235b
  chunk->data = g_malloc (size);
Packit ae235b
Packit ae235b
  pointer->start = guint32_to_le (fb->offset);
Packit ae235b
  fb->offset += size;
Packit ae235b
  pointer->end = guint32_to_le (fb->offset);
Packit ae235b
Packit ae235b
  g_queue_push_tail (fb->chunks, chunk);
Packit ae235b
Packit ae235b
  return chunk->data;
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
file_builder_add_value (FileBuilder         *fb,
Packit ae235b
                        GVariant            *value,
Packit ae235b
                        struct gvdb_pointer *pointer)
Packit ae235b
{
Packit ae235b
  GVariant *variant, *normal;
Packit ae235b
  gpointer data;
Packit ae235b
  gsize size;
Packit ae235b
Packit ae235b
  if (fb->byteswap)
Packit ae235b
    {
Packit ae235b
      value = g_variant_byteswap (value);
Packit ae235b
      variant = g_variant_new_variant (value);
Packit ae235b
      g_variant_unref (value);
Packit ae235b
    }
Packit ae235b
  else
Packit ae235b
    variant = g_variant_new_variant (value);
Packit ae235b
Packit ae235b
  normal = g_variant_get_normal_form (variant);
Packit ae235b
  g_variant_unref (variant);
Packit ae235b
Packit ae235b
  size = g_variant_get_size (normal);
Packit ae235b
  data = file_builder_allocate (fb, 8, size, pointer);
Packit ae235b
  g_variant_store (normal, data);
Packit ae235b
  g_variant_unref (normal);
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
file_builder_add_string (FileBuilder *fb,
Packit ae235b
                         const gchar *string,
Packit ae235b
                         guint32_le  *start,
Packit ae235b
                         guint16_le  *size)
Packit ae235b
{
Packit ae235b
  FileChunk *chunk;
Packit ae235b
  gsize length;
Packit ae235b
Packit ae235b
  length = strlen (string);
Packit ae235b
Packit ae235b
  chunk = g_slice_new (FileChunk);
Packit ae235b
  chunk->offset = fb->offset;
Packit ae235b
  chunk->size = length;
Packit ae235b
  chunk->data = g_malloc (length);
Packit ae235b
  if (length != 0)
Packit ae235b
    memcpy (chunk->data, string, length);
Packit ae235b
Packit ae235b
  *start = guint32_to_le (fb->offset);
Packit ae235b
  *size = guint16_to_le (length);
Packit ae235b
  fb->offset += length;
Packit ae235b
Packit ae235b
  g_queue_push_tail (fb->chunks, chunk);
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
file_builder_allocate_for_hash (FileBuilder            *fb,
Packit ae235b
                                gsize                   n_buckets,
Packit ae235b
                                gsize                   n_items,
Packit ae235b
                                guint                   bloom_shift,
Packit ae235b
                                gsize                   n_bloom_words,
Packit ae235b
                                guint32_le            **bloom_filter,
Packit ae235b
                                guint32_le            **hash_buckets,
Packit ae235b
                                struct gvdb_hash_item **hash_items,
Packit ae235b
                                struct gvdb_pointer    *pointer)
Packit ae235b
{
Packit ae235b
  guint32_le bloom_hdr, table_hdr;
Packit ae235b
  guchar *data;
Packit ae235b
  gsize size;
Packit ae235b
Packit ae235b
  g_assert (n_bloom_words < (1u << 27));
Packit ae235b
Packit ae235b
  bloom_hdr = guint32_to_le (bloom_shift << 27 | n_bloom_words);
Packit ae235b
  table_hdr = guint32_to_le (n_buckets);
Packit ae235b
Packit ae235b
  size = sizeof bloom_hdr + sizeof table_hdr +
Packit ae235b
         n_bloom_words * sizeof (guint32_le) +
Packit ae235b
         n_buckets     * sizeof (guint32_le) +
Packit ae235b
         n_items       * sizeof (struct gvdb_hash_item);
Packit ae235b
Packit ae235b
  data = file_builder_allocate (fb, 4, size, pointer);
Packit ae235b
Packit ae235b
#define chunk(s) (size -= (s), data += (s), data - (s))
Packit ae235b
  memcpy (chunk (sizeof bloom_hdr), &bloom_hdr, sizeof bloom_hdr);
Packit ae235b
  memcpy (chunk (sizeof table_hdr), &table_hdr, sizeof table_hdr);
Packit ae235b
  *bloom_filter = (guint32_le *) chunk (n_bloom_words * sizeof (guint32_le));
Packit ae235b
  *hash_buckets = (guint32_le *) chunk (n_buckets * sizeof (guint32_le));
Packit ae235b
  *hash_items = (struct gvdb_hash_item *) chunk (n_items *
Packit ae235b
                  sizeof (struct gvdb_hash_item));
Packit ae235b
  g_assert (size == 0);
Packit ae235b
#undef chunk
Packit ae235b
Packit ae235b
  memset (*bloom_filter, 0, n_bloom_words * sizeof (guint32_le));
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
file_builder_add_hash (FileBuilder         *fb,
Packit ae235b
                       GHashTable          *table,
Packit ae235b
                       struct gvdb_pointer *pointer)
Packit ae235b
{
Packit ae235b
  guint32_le *buckets, *bloom_filter;
Packit ae235b
  struct gvdb_hash_item *items;
Packit ae235b
  HashTable *mytable;
Packit ae235b
  GvdbItem *item;
Packit ae235b
  guint32 index;
Packit ae235b
  gint bucket;
Packit ae235b
Packit ae235b
  mytable = hash_table_new (g_hash_table_size (table));
Packit ae235b
  g_hash_table_foreach (table, hash_table_insert, mytable);
Packit ae235b
  index = 0;
Packit ae235b
Packit ae235b
  for (bucket = 0; bucket < mytable->n_buckets; bucket++)
Packit ae235b
    for (item = mytable->buckets[bucket]; item; item = item->next)
Packit ae235b
      item->assigned_index = guint32_to_le (index++);
Packit ae235b
Packit ae235b
  file_builder_allocate_for_hash (fb, mytable->n_buckets, index, 5, 0,
Packit ae235b
                                  &bloom_filter, &buckets, &items, pointer);
Packit ae235b
Packit ae235b
  index = 0;
Packit ae235b
  for (bucket = 0; bucket < mytable->n_buckets; bucket++)
Packit ae235b
    {
Packit ae235b
      buckets[bucket] = guint32_to_le (index);
Packit ae235b
Packit ae235b
      for (item = mytable->buckets[bucket]; item; item = item->next)
Packit ae235b
        {
Packit ae235b
          struct gvdb_hash_item *entry = items++;
Packit ae235b
          const gchar *basename;
Packit ae235b
Packit ae235b
          g_assert (index == guint32_from_le (item->assigned_index));
Packit ae235b
          entry->hash_value = guint32_to_le (item->hash_value);
Packit ae235b
          entry->parent = item_to_index (item->parent);
Packit ae235b
          entry->unused = 0;
Packit ae235b
Packit ae235b
          if (item->parent != NULL)
Packit ae235b
            basename = item->key + strlen (item->parent->key);
Packit ae235b
          else
Packit ae235b
            basename = item->key;
Packit ae235b
Packit ae235b
          file_builder_add_string (fb, basename,
Packit ae235b
                                   &entry->key_start,
Packit ae235b
                                   &entry->key_size);
Packit ae235b
Packit ae235b
          if (item->value != NULL)
Packit ae235b
            {
Packit ae235b
              g_assert (item->child == NULL && item->table == NULL);
Packit ae235b
Packit ae235b
              file_builder_add_value (fb, item->value, &entry->value.pointer);
Packit ae235b
              entry->type = 'v';
Packit ae235b
            }
Packit ae235b
Packit ae235b
          if (item->child != NULL)
Packit ae235b
            {
Packit ae235b
              guint32 children = 0, i = 0;
Packit ae235b
              guint32_le *offsets;
Packit ae235b
              GvdbItem *child;
Packit ae235b
Packit ae235b
              g_assert (item->table == NULL);
Packit ae235b
Packit ae235b
              for (child = item->child; child; child = child->sibling)
Packit ae235b
                children++;
Packit ae235b
Packit ae235b
              offsets = file_builder_allocate (fb, 4, 4 * children,
Packit ae235b
                                               &entry->value.pointer);
Packit ae235b
              entry->type = 'L';
Packit ae235b
Packit ae235b
              for (child = item->child; child; child = child->sibling)
Packit ae235b
                offsets[i++] = child->assigned_index;
Packit ae235b
Packit ae235b
              g_assert (children == i);
Packit ae235b
            }
Packit ae235b
Packit ae235b
          if (item->table != NULL)
Packit ae235b
            {
Packit ae235b
              entry->type = 'H';
Packit ae235b
              file_builder_add_hash (fb, item->table, &entry->value.pointer);
Packit ae235b
            }
Packit ae235b
Packit ae235b
          index++;
Packit ae235b
        }
Packit ae235b
    }
Packit ae235b
Packit ae235b
  hash_table_free (mytable);
Packit ae235b
}
Packit ae235b
Packit ae235b
static FileBuilder *
Packit ae235b
file_builder_new (gboolean byteswap)
Packit ae235b
{
Packit ae235b
  FileBuilder *builder;
Packit ae235b
Packit ae235b
  builder = g_slice_new (FileBuilder);
Packit ae235b
  builder->chunks = g_queue_new ();
Packit ae235b
  builder->offset = sizeof (struct gvdb_header);
Packit ae235b
  builder->byteswap = byteswap;
Packit ae235b
Packit ae235b
  return builder;
Packit ae235b
}
Packit ae235b
Packit ae235b
static GString *
Packit ae235b
file_builder_serialise (FileBuilder          *fb,
Packit ae235b
                        struct gvdb_pointer   root)
Packit ae235b
{
Packit ae235b
  struct gvdb_header header = { { 0, }, };
Packit ae235b
  GString *result;
Packit ae235b
Packit ae235b
  if (fb->byteswap)
Packit ae235b
    {
Packit ae235b
      header.signature[0] = GVDB_SWAPPED_SIGNATURE0;
Packit ae235b
      header.signature[1] = GVDB_SWAPPED_SIGNATURE1;
Packit ae235b
    }
Packit ae235b
  else
Packit ae235b
    {
Packit ae235b
      header.signature[0] = GVDB_SIGNATURE0;
Packit ae235b
      header.signature[1] = GVDB_SIGNATURE1;
Packit ae235b
    }
Packit ae235b
Packit ae235b
  result = g_string_new (NULL);
Packit ae235b
Packit ae235b
  header.root = root;
Packit ae235b
  g_string_append_len (result, (gpointer) &header, sizeof header);
Packit ae235b
Packit ae235b
  while (!g_queue_is_empty (fb->chunks))
Packit ae235b
    {
Packit ae235b
      FileChunk *chunk = g_queue_pop_head (fb->chunks);
Packit ae235b
Packit ae235b
      if (result->len != chunk->offset)
Packit ae235b
        {
Packit ae235b
          gchar zero[8] = { 0, };
Packit ae235b
Packit ae235b
          g_assert (chunk->offset > result->len);
Packit ae235b
          g_assert (chunk->offset - result->len < 8);
Packit ae235b
Packit ae235b
          g_string_append_len (result, zero, chunk->offset - result->len);
Packit ae235b
          g_assert (result->len == chunk->offset);
Packit ae235b
        }
Packit ae235b
Packit ae235b
      g_string_append_len (result, chunk->data, chunk->size);
Packit ae235b
      g_free (chunk->data);
Packit ae235b
Packit ae235b
      g_slice_free (FileChunk, chunk);
Packit ae235b
    }
Packit ae235b
Packit ae235b
  g_queue_free (fb->chunks);
Packit ae235b
  g_slice_free (FileBuilder, fb);
Packit ae235b
Packit ae235b
  return result;
Packit ae235b
}
Packit ae235b
Packit ae235b
gboolean
Packit ae235b
gvdb_table_write_contents (GHashTable   *table,
Packit ae235b
                           const gchar  *filename,
Packit ae235b
                           gboolean      byteswap,
Packit ae235b
                           GError      **error)
Packit ae235b
{
Packit ae235b
  struct gvdb_pointer root;
Packit ae235b
  gboolean status;
Packit ae235b
  FileBuilder *fb;
Packit ae235b
  GString *str;
Packit ae235b
Packit ae235b
  fb = file_builder_new (byteswap);
Packit ae235b
  file_builder_add_hash (fb, table, &root);
Packit ae235b
  str = file_builder_serialise (fb, root);
Packit ae235b
Packit ae235b
  status = g_file_set_contents (filename, str->str, str->len, error);
Packit ae235b
  g_string_free (str, TRUE);
Packit ae235b
Packit ae235b
  return status;
Packit ae235b
}