Blame tests/unicode-encoding.c

Packit ae235b
#undef G_DISABLE_ASSERT
Packit ae235b
#undef G_LOG_DOMAIN
Packit ae235b
Packit ae235b
#include <stdarg.h>
Packit ae235b
#include <stdio.h>
Packit ae235b
#include <stdlib.h>
Packit ae235b
#include <string.h>
Packit ae235b
#include <glib.h>
Packit ae235b
Packit ae235b
static gint exit_status = 0;
Packit ae235b
Packit ae235b
static void
Packit ae235b
croak (char *format, ...)
Packit ae235b
{
Packit ae235b
  va_list va;
Packit ae235b
  
Packit ae235b
  va_start (va, format);
Packit ae235b
  vfprintf (stderr, format, va);
Packit ae235b
  va_end (va);
Packit ae235b
Packit ae235b
  exit (1);
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
fail (char *format, ...)
Packit ae235b
{
Packit ae235b
  va_list va;
Packit ae235b
  
Packit ae235b
  va_start (va, format);
Packit ae235b
  vfprintf (stderr, format, va);
Packit ae235b
  va_end (va);
Packit ae235b
Packit ae235b
  exit_status |= 1;
Packit ae235b
}
Packit ae235b
Packit ae235b
typedef enum
Packit ae235b
{
Packit ae235b
  VALID,
Packit ae235b
  INCOMPLETE,
Packit ae235b
  NOTUNICODE,
Packit ae235b
  OVERLONG,
Packit ae235b
  MALFORMED
Packit ae235b
} Status;
Packit ae235b
Packit ae235b
static gboolean
Packit ae235b
ucs4_equal (gunichar *a, gunichar *b)
Packit ae235b
{
Packit ae235b
  while (*a && *b && (*a == *b))
Packit ae235b
    {
Packit ae235b
      a++;
Packit ae235b
      b++;
Packit ae235b
    }
Packit ae235b
Packit ae235b
  return (*a == *b);
Packit ae235b
}
Packit ae235b
Packit ae235b
static gboolean
Packit ae235b
utf16_equal (gunichar2 *a, gunichar2 *b)
Packit ae235b
{
Packit ae235b
  while (*a && *b && (*a == *b))
Packit ae235b
    {
Packit ae235b
      a++;
Packit ae235b
      b++;
Packit ae235b
    }
Packit ae235b
Packit ae235b
  return (*a == *b);
Packit ae235b
}
Packit ae235b
Packit ae235b
static gint
Packit ae235b
utf16_count (gunichar2 *a)
Packit ae235b
{
Packit ae235b
  gint result = 0;
Packit ae235b
  
Packit ae235b
  while (a[result])
Packit ae235b
    result++;
Packit ae235b
Packit ae235b
  return result;
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
print_ucs4 (const gchar *prefix, gunichar *ucs4, gint ucs4_len)
Packit ae235b
{
Packit ae235b
  gint i;
Packit ae235b
  g_print ("%s ", prefix);
Packit ae235b
  for (i = 0; i < ucs4_len; i++)
Packit ae235b
    g_print ("%x ", ucs4[i]);
Packit ae235b
  g_print ("\n");
Packit ae235b
}
Packit ae235b
Packit ae235b
static void
Packit ae235b
process (gint      line,
Packit ae235b
	 gchar    *utf8,
Packit ae235b
	 Status    status,
Packit ae235b
	 gunichar *ucs4,
Packit ae235b
	 gint      ucs4_len)
Packit ae235b
{
Packit ae235b
  const gchar *end;
Packit ae235b
  gboolean is_valid = g_utf8_validate (utf8, -1, &end;;
Packit ae235b
  GError *error = NULL;
Packit ae235b
  glong items_read, items_written;
Packit ae235b
Packit ae235b
  switch (status)
Packit ae235b
    {
Packit ae235b
    case VALID:
Packit ae235b
      if (!is_valid)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: valid but g_utf8_validate returned FALSE\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
      break;
Packit ae235b
    case NOTUNICODE:
Packit ae235b
    case INCOMPLETE:
Packit ae235b
    case OVERLONG:
Packit ae235b
    case MALFORMED:
Packit ae235b
      if (is_valid)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: invalid but g_utf8_validate returned TRUE\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
      break;
Packit ae235b
    }
Packit ae235b
Packit ae235b
  if (status == INCOMPLETE)
Packit ae235b
    {
Packit ae235b
      gunichar *ucs4_result;
Packit ae235b
Packit ae235b
      ucs4_result = g_utf8_to_ucs4 (utf8, -1, NULL, NULL, &error);
Packit ae235b
Packit ae235b
      if (!error || !g_error_matches (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: incomplete input not properly detected\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
      g_clear_error (&error);
Packit ae235b
Packit ae235b
      ucs4_result = g_utf8_to_ucs4 (utf8, -1, &items_read, NULL, &error);
Packit ae235b
Packit ae235b
      if (!ucs4_result || items_read == strlen (utf8))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: incomplete input not properly detected\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      g_free (ucs4_result);
Packit ae235b
    }
Packit ae235b
Packit ae235b
  if (status == VALID || status == NOTUNICODE)
Packit ae235b
    {
Packit ae235b
      gunichar *ucs4_result;
Packit ae235b
Packit ae235b
      ucs4_result = g_utf8_to_ucs4 (utf8, -1, &items_read, &items_written, &error);
Packit ae235b
      if (!ucs4_result)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion with status %d to ucs4 failed: %s\n", line, status, error->message);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (!ucs4_equal (ucs4_result, ucs4) ||
Packit ae235b
	  items_read != strlen (utf8) ||
Packit ae235b
	  items_written != ucs4_len)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: results of conversion with status %d to ucs4 do not match expected.\n", line, status);
Packit ae235b
          print_ucs4 ("expected: ", ucs4, ucs4_len);
Packit ae235b
          print_ucs4 ("received: ", ucs4_result, items_written);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      g_free (ucs4_result);
Packit ae235b
    }
Packit ae235b
Packit ae235b
  if (status == VALID)
Packit ae235b
     {
Packit ae235b
      gunichar *ucs4_result;
Packit ae235b
      gchar *utf8_result;
Packit ae235b
Packit ae235b
      ucs4_result = g_utf8_to_ucs4_fast (utf8, -1, &items_written);
Packit ae235b
Packit ae235b
      if (!ucs4_equal (ucs4_result, ucs4) ||
Packit ae235b
	  items_written != ucs4_len)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: results of fast conversion with status %d to ucs4 do not match expected.\n", line, status);
Packit ae235b
          print_ucs4 ("expected: ", ucs4, ucs4_len);
Packit ae235b
          print_ucs4 ("received: ", ucs4_result, items_written);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      utf8_result = g_ucs4_to_utf8 (ucs4_result, -1, &items_read, &items_written, &error);
Packit ae235b
      if (!utf8_result)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion back to utf8 failed: %s", line, error->message);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (strcmp (utf8_result, utf8) != 0 ||
Packit ae235b
	  items_read != ucs4_len ||
Packit ae235b
	  items_written != strlen (utf8))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion back to utf8 did not match original\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      g_free (utf8_result);
Packit ae235b
      g_free (ucs4_result);
Packit ae235b
    }
Packit ae235b
Packit ae235b
  if (status == VALID)
Packit ae235b
    {
Packit ae235b
      gunichar2 *utf16_expected_tmp;
Packit ae235b
      gunichar2 *utf16_expected;
Packit ae235b
      gunichar2 *utf16_from_utf8;
Packit ae235b
      gunichar2 *utf16_from_ucs4;
Packit ae235b
      gunichar *ucs4_result;
Packit ae235b
      gsize bytes_written;
Packit ae235b
      gint n_chars;
Packit ae235b
      gchar *utf8_result;
Packit ae235b
Packit ae235b
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
Packit ae235b
#define TARGET "UTF-16LE"
Packit ae235b
#else
Packit ae235b
#define TARGET "UTF-16"
Packit ae235b
#endif
Packit ae235b
Packit ae235b
      if (!(utf16_expected_tmp = (gunichar2 *)g_convert (utf8, -1, TARGET, "UTF-8",
Packit ae235b
							 NULL, &bytes_written, NULL)))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: could not convert to UTF-16 via g_convert\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      /* zero-terminate and remove BOM
Packit ae235b
       */
Packit ae235b
      n_chars = bytes_written / 2;
Packit ae235b
      if (utf16_expected_tmp[0] == 0xfeff) /* BOM */
Packit ae235b
	{
Packit ae235b
	  n_chars--;
Packit ae235b
	  utf16_expected = g_new (gunichar2, n_chars + 1);
Packit ae235b
	  memcpy (utf16_expected, utf16_expected_tmp + 1, sizeof(gunichar2) * n_chars);
Packit ae235b
	}
Packit ae235b
      else if (utf16_expected_tmp[0] == 0xfffe) /* ANTI-BOM */
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion via iconv to \"UTF-16\" is not native-endian\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
      else
Packit ae235b
	{
Packit ae235b
	  utf16_expected = g_new (gunichar2, n_chars + 1);
Packit ae235b
	  memcpy (utf16_expected, utf16_expected_tmp, sizeof(gunichar2) * n_chars);
Packit ae235b
	}
Packit ae235b
Packit ae235b
      utf16_expected[n_chars] = '\0';
Packit ae235b
      
Packit ae235b
      if (!(utf16_from_utf8 = g_utf8_to_utf16 (utf8, -1, &items_read, &items_written, &error)))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion to ucs16 failed: %s\n", line, error->message);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (items_read != strlen (utf8) ||
Packit ae235b
	  utf16_count (utf16_from_utf8) != items_written)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: length error in conversion to ucs16\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (!(utf16_from_ucs4 = g_ucs4_to_utf16 (ucs4, -1, &items_read, &items_written, &error)))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion to ucs16 failed: %s\n", line, error->message);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (items_read != ucs4_len ||
Packit ae235b
	  utf16_count (utf16_from_ucs4) != items_written)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: length error in conversion to ucs16\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (!utf16_equal (utf16_from_utf8, utf16_expected) ||
Packit ae235b
	  !utf16_equal (utf16_from_ucs4, utf16_expected))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: results of conversion to ucs16 do not match\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (!(utf8_result = g_utf16_to_utf8 (utf16_from_utf8, -1, &items_read, &items_written, &error)))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion back to utf8 failed: %s\n", line, error->message);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (items_read != utf16_count (utf16_from_utf8) ||
Packit ae235b
	  items_written != strlen (utf8))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: length error in conversion from ucs16 to utf8\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (!(ucs4_result = g_utf16_to_ucs4 (utf16_from_ucs4, -1, &items_read, &items_written, &error)))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion back to utf8/ucs4 failed\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (items_read != utf16_count (utf16_from_utf8) ||
Packit ae235b
	  items_written != ucs4_len)
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: length error in conversion from ucs16 to ucs4\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      if (strcmp (utf8, utf8_result) != 0 ||
Packit ae235b
	  !ucs4_equal (ucs4, ucs4_result))
Packit ae235b
	{
Packit ae235b
	  fail ("line %d: conversion back to utf8/ucs4 did not match original\n", line);
Packit ae235b
	  return;
Packit ae235b
	}
Packit ae235b
      
Packit ae235b
      g_free (utf16_expected_tmp);
Packit ae235b
      g_free (utf16_expected);
Packit ae235b
      g_free (utf16_from_utf8);
Packit ae235b
      g_free (utf16_from_ucs4);
Packit ae235b
      g_free (utf8_result);
Packit ae235b
      g_free (ucs4_result);
Packit ae235b
    }
Packit ae235b
}
Packit ae235b
Packit ae235b
int
Packit ae235b
main (int argc, char **argv)
Packit ae235b
{
Packit ae235b
  gchar *testfile;
Packit ae235b
  gchar *contents;
Packit ae235b
  GError *error = NULL;
Packit ae235b
  gchar *p, *end;
Packit ae235b
  char *tmp;
Packit ae235b
  gint state = 0;
Packit ae235b
  gint line = 1;
Packit ae235b
  gint start_line = 0;		/* Quiet GCC */
Packit ae235b
  gchar *utf8 = NULL;		/* Quiet GCC */
Packit ae235b
  GArray *ucs4;
Packit ae235b
  Status status = VALID;	/* Quiet GCC */
Packit ae235b
Packit ae235b
  g_test_init (&argc, &argv, NULL);
Packit ae235b
Packit ae235b
  testfile = g_test_build_filename (G_TEST_DIST, "utf8.txt", NULL);
Packit ae235b
Packit ae235b
  g_file_get_contents (testfile, &contents, NULL, &error);
Packit ae235b
  if (error)
Packit ae235b
    croak ("Cannot open utf8.txt: %s", error->message);
Packit ae235b
Packit ae235b
  ucs4 = g_array_new (TRUE, FALSE, sizeof(gunichar));
Packit ae235b
Packit ae235b
  p = contents;
Packit ae235b
Packit ae235b
  /* Loop over lines */
Packit ae235b
  while (*p)
Packit ae235b
    {
Packit ae235b
      while (*p && (*p == ' ' || *p == '\t'))
Packit ae235b
	p++;
Packit ae235b
Packit ae235b
      end = p;
Packit ae235b
      while (*end && (*end != '\r' && *end != '\n'))
Packit ae235b
	end++;
Packit ae235b
      
Packit ae235b
      if (!*p || *p == '#' || *p == '\r' || *p == '\n')
Packit ae235b
	goto next_line;
Packit ae235b
Packit ae235b
      tmp = g_strstrip (g_strndup (p, end - p));
Packit ae235b
      
Packit ae235b
      switch (state)
Packit ae235b
	{
Packit ae235b
	case 0:
Packit ae235b
	  /* UTF-8 string */
Packit ae235b
	  start_line = line;
Packit ae235b
	  utf8 = tmp;
Packit ae235b
	  tmp = NULL;
Packit ae235b
	  break;
Packit ae235b
	  
Packit ae235b
	case 1:
Packit ae235b
	  /* Status */
Packit ae235b
	  if (!strcmp (tmp, "VALID"))
Packit ae235b
	    status = VALID;
Packit ae235b
	  else if (!strcmp (tmp, "INCOMPLETE"))
Packit ae235b
	    status = INCOMPLETE;
Packit ae235b
	  else if (!strcmp (tmp, "NOTUNICODE"))
Packit ae235b
	    status = NOTUNICODE;
Packit ae235b
	  else if (!strcmp (tmp, "OVERLONG"))
Packit ae235b
	    status = OVERLONG;
Packit ae235b
	  else if (!strcmp (tmp, "MALFORMED"))
Packit ae235b
	    status = MALFORMED;
Packit ae235b
	  else
Packit ae235b
	    croak ("Invalid status on line %d\n", line);
Packit ae235b
Packit ae235b
	  if (status != VALID && status != NOTUNICODE)
Packit ae235b
	    state++;		/* No UCS-4 data */
Packit ae235b
	  
Packit ae235b
	  break;
Packit ae235b
	  
Packit ae235b
	case 2:
Packit ae235b
	  /* UCS-4 version */
Packit ae235b
Packit ae235b
	  p = strtok (tmp, " \t");
Packit ae235b
	  while (p)
Packit ae235b
	    {
Packit ae235b
	      gchar *endptr;
Packit ae235b
	      
Packit ae235b
	      gunichar ch = strtoul (p, &endptr, 16);
Packit ae235b
	      if (*endptr != '\0')
Packit ae235b
		croak ("Invalid UCS-4 character on line %d\n", line);
Packit ae235b
Packit ae235b
	      g_array_append_val (ucs4, ch);
Packit ae235b
	      
Packit ae235b
	      p = strtok (NULL, " \t");
Packit ae235b
	    }
Packit ae235b
Packit ae235b
	  break;
Packit ae235b
	}
Packit ae235b
Packit ae235b
      g_free (tmp);
Packit ae235b
      state = (state + 1) % 3;
Packit ae235b
Packit ae235b
      if (state == 0)
Packit ae235b
	{
Packit ae235b
	  process (start_line, utf8, status, (gunichar *)ucs4->data, ucs4->len);
Packit ae235b
	  g_array_set_size (ucs4, 0);
Packit ae235b
	  g_free (utf8);
Packit ae235b
	}
Packit ae235b
      
Packit ae235b
    next_line:
Packit ae235b
      p = end;
Packit ae235b
      if (*p && *p == '\r')
Packit ae235b
	p++;
Packit ae235b
      if (*p && *p == '\n')
Packit ae235b
	p++;
Packit ae235b
      
Packit ae235b
      line++;
Packit ae235b
    }
Packit ae235b
Packit ae235b
  g_free (testfile);
Packit ae235b
  g_array_free (ucs4, TRUE);
Packit ae235b
  g_free (contents);
Packit ae235b
  return exit_status;
Packit ae235b
}