/* This Source Code Form is subject to the terms of the Mozilla Public

 * License, v. 2.0. If a copy of the MPL was not distributed with this

 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "seccomon.h"

#include "secport.h"

#ifdef TEST_UTF8

#include <assert.h>

#undef PORT_Assert

#define PORT_Assert assert

#endif

/*

 * From RFC 2044:

 *

 * UCS-4 range (hex.)           UTF-8 octet sequence (binary)

 * 0000 0000-0000 007F   0xxxxxxx

 * 0000 0080-0000 07FF   110xxxxx 10xxxxxx

 * 0000 0800-0000 FFFF   1110xxxx 10xxxxxx 10xxxxxx

 * 0001 0000-001F FFFF   11110xxx 10xxxxxx 10xxxxxx 10xxxxxx

 * 0020 0000-03FF FFFF   111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

 * 0400 0000-7FFF FFFF   1111110x 10xxxxxx ... 10xxxxxx

 */  

/*

 * From http://www.imc.org/draft-hoffman-utf16

 *

 * For U on [0x00010000,0x0010FFFF]:  Let U' = U - 0x00010000

 *

 * U' = yyyyyyyyyyxxxxxxxxxx

 * W1 = 110110yyyyyyyyyy

 * W2 = 110111xxxxxxxxxx

 */

/*

 * This code is assuming NETWORK BYTE ORDER for the 16- and 32-bit

 * character values.  If you wish to use this code for working with

 * host byte order values, define the following:

 *

 * #if IS_BIG_ENDIAN

 * #define L_0 0

 * #define L_1 1

 * #define L_2 2

 * #define L_3 3

 * #define H_0 0

 * #define H_1 1

 * #else / * not everyone has elif * /

 * #if IS_LITTLE_ENDIAN

 * #define L_0 3

 * #define L_1 2

 * #define L_2 1

 * #define L_3 0

 * #define H_0 1

 * #define H_1 0

 * #else

 * #error "PDP and NUXI support deferred"

 * #endif / * IS_LITTLE_ENDIAN * /

 * #endif / * IS_BIG_ENDIAN * /

 */

#define L_0 0

#define L_1 1

#define L_2 2

#define L_3 3

#define H_0 0

#define H_1 1

#define BAD_UTF8 ((PRUint32)-1)

/*

 * Parse a single UTF-8 character per the spec. in section 3.9 (D36)

 * of Unicode 4.0.0.

 *

 * Parameters:

 * index - Points to the byte offset in inBuf of character to read.  On success,

 *         updated to the offset of the following character.

 * inBuf - Input buffer, UTF-8 encoded

 * inbufLen - Length of input buffer, in bytes.

 *

 * Returns:

 * Success - The UCS4 encoded character

 * Failure - BAD_UTF8

 */

static PRUint32

sec_port_read_utf8(unsigned int *index, unsigned char *inBuf, unsigned int inBufLen)

{

  PRUint32 result;

  unsigned int i = *index;

  int bytes_left;

  PRUint32 min_value;

  PORT_Assert(i < inBufLen);

  if ( (inBuf[i] & 0x80) == 0x00 ) {

    result = inBuf[i++];

    bytes_left = 0;

    min_value = 0;

  } else if ( (inBuf[i] & 0xE0) == 0xC0 ) {

    result = inBuf[i++] & 0x1F;

    bytes_left = 1;

    min_value = 0x80;

  } else if ( (inBuf[i] & 0xF0) == 0xE0) {

    result = inBuf[i++] & 0x0F;

    bytes_left = 2;

    min_value = 0x800;

  } else if ( (inBuf[i] & 0xF8) == 0xF0) {

    result = inBuf[i++] & 0x07;

    bytes_left = 3;

    min_value = 0x10000;

  } else {

    return BAD_UTF8;

  }

  while (bytes_left--) {

    if (i >= inBufLen || (inBuf[i] & 0xC0) != 0x80) return BAD_UTF8;

    result = (result << 6) | (inBuf[i++] & 0x3F);

  }

  /* Check for overlong sequences, surrogates, and outside unicode range */

  if (result < min_value || (result & 0xFFFFF800) == 0xD800 || result > 0x10FFFF) {

    return BAD_UTF8;

  }

  *index = i;

  return result;

}

PRBool

sec_port_ucs4_utf8_conversion_function

(

  PRBool toUnicode,

  unsigned char *inBuf,

  unsigned int inBufLen,

  unsigned char *outBuf,

  unsigned int maxOutBufLen,

  unsigned int *outBufLen

)

{

  PORT_Assert((unsigned int *)NULL != outBufLen);

  if( toUnicode ) {

    unsigned int i, len = 0;

    for( i = 0; i < inBufLen; ) {

      if( (inBuf[i] & 0x80) == 0x00 ) i += 1;

      else if( (inBuf[i] & 0xE0) == 0xC0 ) i += 2;

      else if( (inBuf[i] & 0xF0) == 0xE0 ) i += 3;

      else if( (inBuf[i] & 0xF8) == 0xF0 ) i += 4;

      else return PR_FALSE;

      len += 4;

    }

    if( len > maxOutBufLen ) {

      *outBufLen = len;

      return PR_FALSE;

    }

    len = 0;

    for( i = 0; i < inBufLen; ) {

      PRUint32 ucs4 = sec_port_read_utf8(&i, inBuf, inBufLen);

      if (ucs4 == BAD_UTF8) return PR_FALSE;

      outBuf[len+L_0] = 0x00;

      outBuf[len+L_1] = (unsigned char)(ucs4 >> 16);

      outBuf[len+L_2] = (unsigned char)(ucs4 >> 8);

      outBuf[len+L_3] = (unsigned char)ucs4;

      len += 4;

    }

    *outBufLen = len;

    return PR_TRUE;

  } else {

    unsigned int i, len = 0;

    PORT_Assert((inBufLen % 4) == 0);

    if ((inBufLen % 4) != 0) {

      *outBufLen = 0;

      return PR_FALSE;

    }

    for( i = 0; i < inBufLen; i += 4 ) {

      if( (inBuf[i+L_0] > 0x00) || (inBuf[i+L_1] > 0x10) ) {

	*outBufLen = 0;

	return PR_FALSE;

      } else if( inBuf[i+L_1] >= 0x01 ) len += 4;

      else if( inBuf[i+L_2] >= 0x08 ) len += 3;

      else if( (inBuf[i+L_2] > 0x00) || (inBuf[i+L_3] >= 0x80) ) len += 2;

      else len += 1;

    }

    if( len > maxOutBufLen ) {

      *outBufLen = len;

      return PR_FALSE;

    }

    len = 0;

    for( i = 0; i < inBufLen; i += 4 ) {

      if( inBuf[i+L_1] >= 0x01 ) {

        /* 0001 0000-001F FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */

        /* 00000000 000abcde fghijklm nopqrstu ->

           11110abc 10defghi 10jklmno 10pqrstu */

        outBuf[len+0] = 0xF0 | ((inBuf[i+L_1] & 0x1C) >> 2);

        outBuf[len+1] = 0x80 | ((inBuf[i+L_1] & 0x03) << 4)

                             | ((inBuf[i+L_2] & 0xF0) >> 4);

        outBuf[len+2] = 0x80 | ((inBuf[i+L_2] & 0x0F) << 2)

                             | ((inBuf[i+L_3] & 0xC0) >> 6);

        outBuf[len+3] = 0x80 | ((inBuf[i+L_3] & 0x3F) >> 0);

        len += 4;

      } else if( inBuf[i+L_2] >= 0x08 ) {

        /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */

        /* 00000000 00000000 abcdefgh ijklmnop ->

           1110abcd 10efghij 10klmnop */

        outBuf[len+0] = 0xE0 | ((inBuf[i+L_2] & 0xF0) >> 4);

        outBuf[len+1] = 0x80 | ((inBuf[i+L_2] & 0x0F) << 2)

                             | ((inBuf[i+L_3] & 0xC0) >> 6);

        outBuf[len+2] = 0x80 | ((inBuf[i+L_3] & 0x3F) >> 0);

        len += 3;

      } else if( (inBuf[i+L_2] > 0x00) || (inBuf[i+L_3] >= 0x80) ) {

        /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */

        /* 00000000 00000000 00000abc defghijk ->

           110abcde 10fghijk */

        outBuf[len+0] = 0xC0 | ((inBuf[i+L_2] & 0x07) << 2)

                             | ((inBuf[i+L_3] & 0xC0) >> 6);

        outBuf[len+1] = 0x80 | ((inBuf[i+L_3] & 0x3F) >> 0);

        len += 2;

      } else {

        /* 0000 0000-0000 007F -> 0xxxxxx */

        /* 00000000 00000000 00000000 0abcdefg ->

           0abcdefg */

        outBuf[len+0] = (inBuf[i+L_3] & 0x7F);

        len += 1;

      }

    }

    *outBufLen = len;

    return PR_TRUE;

  }

}

PRBool

sec_port_ucs2_utf8_conversion_function

(

  PRBool toUnicode,

  unsigned char *inBuf,

  unsigned int inBufLen,

  unsigned char *outBuf,

  unsigned int maxOutBufLen,

  unsigned int *outBufLen

)

{

  PORT_Assert((unsigned int *)NULL != outBufLen);

  if( toUnicode ) {

    unsigned int i, len = 0;

    for( i = 0; i < inBufLen; ) {

      if( (inBuf[i] & 0x80) == 0x00 ) {

        i += 1;

        len += 2;

      } else if( (inBuf[i] & 0xE0) == 0xC0 ) {

        i += 2;

        len += 2;

      } else if( (inBuf[i] & 0xF0) == 0xE0 ) {

        i += 3;

        len += 2;

      } else if( (inBuf[i] & 0xF8) == 0xF0 ) { 

        i += 4;

        len += 4;

      } else return PR_FALSE;

    }

    if( len > maxOutBufLen ) {

      *outBufLen = len;

      return PR_FALSE;

    }

    len = 0;

    for( i = 0; i < inBufLen; ) {

      PRUint32 ucs4 = sec_port_read_utf8(&i, inBuf, inBufLen);

      if (ucs4 == BAD_UTF8) return PR_FALSE;

      if( ucs4 < 0x10000) {

        outBuf[len+H_0] = (unsigned char)(ucs4 >> 8);

        outBuf[len+H_1] = (unsigned char)ucs4;

        len += 2;

      } else {

	ucs4 -= 0x10000;

        outBuf[len+0+H_0] = (unsigned char)(0xD8 | ((ucs4 >> 18) & 0x3));

        outBuf[len+0+H_1] = (unsigned char)(ucs4 >> 10);

        outBuf[len+2+H_0] = (unsigned char)(0xDC | ((ucs4 >> 8) & 0x3));

        outBuf[len+2+H_1] = (unsigned char)ucs4;

	len += 4;

      }

    }

    *outBufLen = len;

    return PR_TRUE;

  } else {

    unsigned int i, len = 0;

    PORT_Assert((inBufLen % 2) == 0);

    if ((inBufLen % 2) != 0) {

      *outBufLen = 0;

      return PR_FALSE;

    }

    for( i = 0; i < inBufLen; i += 2 ) {

      if( (inBuf[i+H_0] == 0x00) && ((inBuf[i+H_0] & 0x80) == 0x00) ) len += 1;

      else if( inBuf[i+H_0] < 0x08 ) len += 2;

      else if( ((inBuf[i+0+H_0] & 0xDC) == 0xD8) ) {

        if( ((inBuf[i+2+H_0] & 0xDC) == 0xDC) && ((inBufLen - i) > 2) ) {

          i += 2;

          len += 4;

        } else {

          return PR_FALSE;

        }

      }

      else len += 3;

    }

    if( len > maxOutBufLen ) {

      *outBufLen = len;

      return PR_FALSE;

    }

    len = 0;

    for( i = 0; i < inBufLen; i += 2 ) {

      if( (inBuf[i+H_0] == 0x00) && ((inBuf[i+H_1] & 0x80) == 0x00) ) {

        /* 0000-007F -> 0xxxxxx */

        /* 00000000 0abcdefg -> 0abcdefg */

        outBuf[len] = inBuf[i+H_1] & 0x7F;

        len += 1;

      } else if( inBuf[i+H_0] < 0x08 ) {

        /* 0080-07FF -> 110xxxxx 10xxxxxx */

        /* 00000abc defghijk -> 110abcde 10fghijk */

        outBuf[len+0] = 0xC0 | ((inBuf[i+H_0] & 0x07) << 2) 

                             | ((inBuf[i+H_1] & 0xC0) >> 6);

        outBuf[len+1] = 0x80 | ((inBuf[i+H_1] & 0x3F) >> 0);

        len += 2;

      } else if( (inBuf[i+H_0] & 0xDC) == 0xD8 ) {

        int abcde, BCDE;

        PORT_Assert(((inBuf[i+2+H_0] & 0xDC) == 0xDC) && ((inBufLen - i) > 2));

        /* D800-DBFF DC00-DFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */

        /* 110110BC DEfghijk 110111lm nopqrstu ->

           { Let abcde = BCDE + 1 }

           11110abc 10defghi 10jklmno 10pqrstu */

        BCDE = ((inBuf[i+H_0] & 0x03) << 2) | ((inBuf[i+H_1] & 0xC0) >> 6);

        abcde = BCDE + 1;

        outBuf[len+0] = 0xF0 | ((abcde & 0x1C) >> 2);

        outBuf[len+1] = 0x80 | ((abcde & 0x03) << 4) 

                             | ((inBuf[i+0+H_1] & 0x3C) >> 2);

        outBuf[len+2] = 0x80 | ((inBuf[i+0+H_1] & 0x03) << 4)

                             | ((inBuf[i+2+H_0] & 0x03) << 2)

                             | ((inBuf[i+2+H_1] & 0xC0) >> 6);

        outBuf[len+3] = 0x80 | ((inBuf[i+2+H_1] & 0x3F) >> 0);

        i += 2;

        len += 4;

      } else {

        /* 0800-FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */

        /* abcdefgh ijklmnop -> 1110abcd 10efghij 10klmnop */

        outBuf[len+0] = 0xE0 | ((inBuf[i+H_0] & 0xF0) >> 4);

        outBuf[len+1] = 0x80 | ((inBuf[i+H_0] & 0x0F) << 2) 

                             | ((inBuf[i+H_1] & 0xC0) >> 6);

        outBuf[len+2] = 0x80 | ((inBuf[i+H_1] & 0x3F) >> 0);

        len += 3;

      }

    }

    *outBufLen = len;

    return PR_TRUE;

  }

}

PRBool

sec_port_iso88591_utf8_conversion_function

(

  const unsigned char *inBuf,

  unsigned int inBufLen,

  unsigned char *outBuf,

  unsigned int maxOutBufLen,

  unsigned int *outBufLen

)

{

  unsigned int i, len = 0;

  PORT_Assert((unsigned int *)NULL != outBufLen);

  for( i = 0; i < inBufLen; i++) {

    if( (inBuf[i] & 0x80) == 0x00 ) len += 1;

    else len += 2;

  }

  if( len > maxOutBufLen ) {

    *outBufLen = len;

    return PR_FALSE;

  }

  len = 0;

  for( i = 0; i < inBufLen; i++) {

    if( (inBuf[i] & 0x80) == 0x00 ) {

      /* 00-7F -> 0xxxxxxx */

      /* 0abcdefg -> 0abcdefg */

      outBuf[len] = inBuf[i];

      len += 1;

    } else {

      /* 80-FF <- 110xxxxx 10xxxxxx */

      /* 00000000 abcdefgh -> 110000ab 10cdefgh */

      outBuf[len+0] = 0xC0 | ((inBuf[i] & 0xC0) >> 6);

      outBuf[len+1] = 0x80 | ((inBuf[i] & 0x3F) >> 0);

      len += 2;

    }

  }

  *outBufLen = len;

  return PR_TRUE;

}

#ifdef TEST_UTF8

#include <stdio.h>

#include <string.h>

#include <stdlib.h>

#include <netinet/in.h> /* for htonl and htons */

/*

 * UCS-4 vectors

 */

struct ucs4 {

  PRUint32 c;

  char *utf8;

};

/*

 * UCS-2 vectors

 */

struct ucs2 {

  PRUint16 c;

  char *utf8;

};

/*

 * UTF-16 vectors

 */

struct utf16 {

  PRUint32 c;

  PRUint16 w[2];

};

/*

 * UCS-4 vectors

 */

struct ucs4 ucs4[] = {

  { 0x00000001, "\x01" },

  { 0x00000002, "\x02" },

  { 0x00000003, "\x03" },

  { 0x00000004, "\x04" },

  { 0x00000007, "\x07" },

  { 0x00000008, "\x08" },

  { 0x0000000F, "\x0F" },

  { 0x00000010, "\x10" },

  { 0x0000001F, "\x1F" },

  { 0x00000020, "\x20" },

  { 0x0000003F, "\x3F" },

  { 0x00000040, "\x40" },

  { 0x0000007F, "\x7F" },

  { 0x00000080, "\xC2\x80" },

  { 0x00000081, "\xC2\x81" },

  { 0x00000082, "\xC2\x82" },

  { 0x00000084, "\xC2\x84" },

  { 0x00000088, "\xC2\x88" },

  { 0x00000090, "\xC2\x90" },

  { 0x000000A0, "\xC2\xA0" },

  { 0x000000C0, "\xC3\x80" },

  { 0x000000FF, "\xC3\xBF" },

  { 0x00000100, "\xC4\x80" },

  { 0x00000101, "\xC4\x81" },

  { 0x00000102, "\xC4\x82" },

  { 0x00000104, "\xC4\x84" },

  { 0x00000108, "\xC4\x88" },

  { 0x00000110, "\xC4\x90" },

  { 0x00000120, "\xC4\xA0" },

  { 0x00000140, "\xC5\x80" },

  { 0x00000180, "\xC6\x80" },

  { 0x000001FF, "\xC7\xBF" },

  { 0x00000200, "\xC8\x80" },

  { 0x00000201, "\xC8\x81" },

  { 0x00000202, "\xC8\x82" },

  { 0x00000204, "\xC8\x84" },

  { 0x00000208, "\xC8\x88" },

  { 0x00000210, "\xC8\x90" },

  { 0x00000220, "\xC8\xA0" },

  { 0x00000240, "\xC9\x80" },

  { 0x00000280, "\xCA\x80" },

  { 0x00000300, "\xCC\x80" },

  { 0x000003FF, "\xCF\xBF" },

  { 0x00000400, "\xD0\x80" },

  { 0x00000401, "\xD0\x81" },

  { 0x00000402, "\xD0\x82" },

  { 0x00000404, "\xD0\x84" },

  { 0x00000408, "\xD0\x88" },

  { 0x00000410, "\xD0\x90" },

  { 0x00000420, "\xD0\xA0" },

  { 0x00000440, "\xD1\x80" },

  { 0x00000480, "\xD2\x80" },

  { 0x00000500, "\xD4\x80" },

  { 0x00000600, "\xD8\x80" },

  { 0x000007FF, "\xDF\xBF" },

  { 0x00000800, "\xE0\xA0\x80" },

  { 0x00000801, "\xE0\xA0\x81" },

  { 0x00000802, "\xE0\xA0\x82" },

  { 0x00000804, "\xE0\xA0\x84" },

  { 0x00000808, "\xE0\xA0\x88" },

  { 0x00000810, "\xE0\xA0\x90" },

  { 0x00000820, "\xE0\xA0\xA0" },

  { 0x00000840, "\xE0\xA1\x80" },

  { 0x00000880, "\xE0\xA2\x80" },

  { 0x00000900, "\xE0\xA4\x80" },

  { 0x00000A00, "\xE0\xA8\x80" },

  { 0x00000C00, "\xE0\xB0\x80" },

  { 0x00000FFF, "\xE0\xBF\xBF" },

  { 0x00001000, "\xE1\x80\x80" },

  { 0x00001001, "\xE1\x80\x81" },

  { 0x00001002, "\xE1\x80\x82" },

  { 0x00001004, "\xE1\x80\x84" },

  { 0x00001008, "\xE1\x80\x88" },

  { 0x00001010, "\xE1\x80\x90" },

  { 0x00001020, "\xE1\x80\xA0" },

  { 0x00001040, "\xE1\x81\x80" },

  { 0x00001080, "\xE1\x82\x80" },

  { 0x00001100, "\xE1\x84\x80" },

  { 0x00001200, "\xE1\x88\x80" },

  { 0x00001400, "\xE1\x90\x80" },

  { 0x00001800, "\xE1\xA0\x80" },

  { 0x00001FFF, "\xE1\xBF\xBF" },

  { 0x00002000, "\xE2\x80\x80" },

  { 0x00002001, "\xE2\x80\x81" },

  { 0x00002002, "\xE2\x80\x82" },

  { 0x00002004, "\xE2\x80\x84" },

  { 0x00002008, "\xE2\x80\x88" },

  { 0x00002010, "\xE2\x80\x90" },

  { 0x00002020, "\xE2\x80\xA0" },

  { 0x00002040, "\xE2\x81\x80" },

  { 0x00002080, "\xE2\x82\x80" },

  { 0x00002100, "\xE2\x84\x80" },

  { 0x00002200, "\xE2\x88\x80" },

  { 0x00002400, "\xE2\x90\x80" },

  { 0x00002800, "\xE2\xA0\x80" },

  { 0x00003000, "\xE3\x80\x80" },

  { 0x00003FFF, "\xE3\xBF\xBF" },

  { 0x00004000, "\xE4\x80\x80" },

  { 0x00004001, "\xE4\x80\x81" },

  { 0x00004002, "\xE4\x80\x82" },

  { 0x00004004, "\xE4\x80\x84" },

  { 0x00004008, "\xE4\x80\x88" },

  { 0x00004010, "\xE4\x80\x90" },

  { 0x00004020, "\xE4\x80\xA0" },

  { 0x00004040, "\xE4\x81\x80" },

  { 0x00004080, "\xE4\x82\x80" },

  { 0x00004100, "\xE4\x84\x80" },

  { 0x00004200, "\xE4\x88\x80" },

  { 0x00004400, "\xE4\x90\x80" },

  { 0x00004800, "\xE4\xA0\x80" },

  { 0x00005000, "\xE5\x80\x80" },

  { 0x00006000, "\xE6\x80\x80" },

  { 0x00007FFF, "\xE7\xBF\xBF" },

  { 0x00008000, "\xE8\x80\x80" },

  { 0x00008001, "\xE8\x80\x81" },

  { 0x00008002, "\xE8\x80\x82" },

  { 0x00008004, "\xE8\x80\x84" },

  { 0x00008008, "\xE8\x80\x88" },

  { 0x00008010, "\xE8\x80\x90" },

  { 0x00008020, "\xE8\x80\xA0" },

  { 0x00008040, "\xE8\x81\x80" },

  { 0x00008080, "\xE8\x82\x80" },

  { 0x00008100, "\xE8\x84\x80" },

  { 0x00008200, "\xE8\x88\x80" },

  { 0x00008400, "\xE8\x90\x80" },

  { 0x00008800, "\xE8\xA0\x80" },

  { 0x00009000, "\xE9\x80\x80" },

  { 0x0000A000, "\xEA\x80\x80" },

  { 0x0000C000, "\xEC\x80\x80" },

  { 0x0000FFFF, "\xEF\xBF\xBF" },

  { 0x00010000, "\xF0\x90\x80\x80" },

  { 0x00010001, "\xF0\x90\x80\x81" },

  { 0x00010002, "\xF0\x90\x80\x82" },

  { 0x00010004, "\xF0\x90\x80\x84" },

  { 0x00010008, "\xF0\x90\x80\x88" },

  { 0x00010010, "\xF0\x90\x80\x90" },

  { 0x00010020, "\xF0\x90\x80\xA0" },

  { 0x00010040, "\xF0\x90\x81\x80" },

  { 0x00010080, "\xF0\x90\x82\x80" },

  { 0x00010100, "\xF0\x90\x84\x80" },

  { 0x00010200, "\xF0\x90\x88\x80" },

  { 0x00010400, "\xF0\x90\x90\x80" },

  { 0x00010800, "\xF0\x90\xA0\x80" },

  { 0x00011000, "\xF0\x91\x80\x80" },

  { 0x00012000, "\xF0\x92\x80\x80" },

  { 0x00014000, "\xF0\x94\x80\x80" },

  { 0x00018000, "\xF0\x98\x80\x80" },

  { 0x0001FFFF, "\xF0\x9F\xBF\xBF" },

  { 0x00020000, "\xF0\xA0\x80\x80" },

  { 0x00020001, "\xF0\xA0\x80\x81" },

  { 0x00020002, "\xF0\xA0\x80\x82" },

  { 0x00020004, "\xF0\xA0\x80\x84" },

  { 0x00020008, "\xF0\xA0\x80\x88" },

  { 0x00020010, "\xF0\xA0\x80\x90" },

  { 0x00020020, "\xF0\xA0\x80\xA0" },

  { 0x00020040, "\xF0\xA0\x81\x80" },

  { 0x00020080, "\xF0\xA0\x82\x80" },

  { 0x00020100, "\xF0\xA0\x84\x80" },

  { 0x00020200, "\xF0\xA0\x88\x80" },

  { 0x00020400, "\xF0\xA0\x90\x80" },

  { 0x00020800, "\xF0\xA0\xA0\x80" },

  { 0x00021000, "\xF0\xA1\x80\x80" },

  { 0x00022000, "\xF0\xA2\x80\x80" },

  { 0x00024000, "\xF0\xA4\x80\x80" },

  { 0x00028000, "\xF0\xA8\x80\x80" },

  { 0x00030000, "\xF0\xB0\x80\x80" },

  { 0x0003FFFF, "\xF0\xBF\xBF\xBF" },

  { 0x00040000, "\xF1\x80\x80\x80" },

  { 0x00040001, "\xF1\x80\x80\x81" },

  { 0x00040002, "\xF1\x80\x80\x82" },

  { 0x00040004, "\xF1\x80\x80\x84" },

  { 0x00040008, "\xF1\x80\x80\x88" },

  { 0x00040010, "\xF1\x80\x80\x90" },

  { 0x00040020, "\xF1\x80\x80\xA0" },

  { 0x00040040, "\xF1\x80\x81\x80" },

  { 0x00040080, "\xF1\x80\x82\x80" },

  { 0x00040100, "\xF1\x80\x84\x80" },

  { 0x00040200, "\xF1\x80\x88\x80" },

  { 0x00040400, "\xF1\x80\x90\x80" },

  { 0x00040800, "\xF1\x80\xA0\x80" },

  { 0x00041000, "\xF1\x81\x80\x80" },

  { 0x00042000, "\xF1\x82\x80\x80" },

  { 0x00044000, "\xF1\x84\x80\x80" },

  { 0x00048000, "\xF1\x88\x80\x80" },

  { 0x00050000, "\xF1\x90\x80\x80" },

  { 0x00060000, "\xF1\xA0\x80\x80" },

  { 0x0007FFFF, "\xF1\xBF\xBF\xBF" },

  { 0x00080000, "\xF2\x80\x80\x80" },

  { 0x00080001, "\xF2\x80\x80\x81" },

  { 0x00080002, "\xF2\x80\x80\x82" },

  { 0x00080004, "\xF2\x80\x80\x84" },

  { 0x00080008, "\xF2\x80\x80\x88" },

  { 0x00080010, "\xF2\x80\x80\x90" },

  { 0x00080020, "\xF2\x80\x80\xA0" },

  { 0x00080040, "\xF2\x80\x81\x80" },

  { 0x00080080, "\xF2\x80\x82\x80" },

  { 0x00080100, "\xF2\x80\x84\x80" },

  { 0x00080200, "\xF2\x80\x88\x80" },

  { 0x00080400, "\xF2\x80\x90\x80" },

  { 0x00080800, "\xF2\x80\xA0\x80" },

  { 0x00081000, "\xF2\x81\x80\x80" },

  { 0x00082000, "\xF2\x82\x80\x80" },

  { 0x00084000, "\xF2\x84\x80\x80" },

  { 0x00088000, "\xF2\x88\x80\x80" },

  { 0x00090000, "\xF2\x90\x80\x80" },

  { 0x000A0000, "\xF2\xA0\x80\x80" },

  { 0x000C0000, "\xF3\x80\x80\x80" },

  { 0x000FFFFF, "\xF3\xBF\xBF\xBF" },

  { 0x00100000, "\xF4\x80\x80\x80" },

  { 0x00100001, "\xF4\x80\x80\x81" },

  { 0x00100002, "\xF4\x80\x80\x82" },

  { 0x00100004, "\xF4\x80\x80\x84" },

  { 0x00100008, "\xF4\x80\x80\x88" },

  { 0x00100010, "\xF4\x80\x80\x90" },

  { 0x00100020, "\xF4\x80\x80\xA0" },

  { 0x00100040, "\xF4\x80\x81\x80" },

  { 0x00100080, "\xF4\x80\x82\x80" },

  { 0x00100100, "\xF4\x80\x84\x80" },

  { 0x00100200, "\xF4\x80\x88\x80" },

  { 0x00100400, "\xF4\x80\x90\x80" },

  { 0x00100800, "\xF4\x80\xA0\x80" },

  { 0x00101000, "\xF4\x81\x80\x80" },

  { 0x00102000, "\xF4\x82\x80\x80" },

  { 0x00104000, "\xF4\x84\x80\x80" },

  { 0x00108000, "\xF4\x88\x80\x80" },

  { 0x0010FFFF, "\xF4\x8F\xBF\xBF" },

};

/*

 * UCS-2 vectors

 */

struct ucs2 ucs2[] = {

  { 0x0001, "\x01" },

  { 0x0002, "\x02" },

  { 0x0003, "\x03" },

  { 0x0004, "\x04" },

  { 0x0007, "\x07" },

  { 0x0008, "\x08" },

  { 0x000F, "\x0F" },

  { 0x0010, "\x10" },

  { 0x001F, "\x1F" },

  { 0x0020, "\x20" },

  { 0x003F, "\x3F" },

  { 0x0040, "\x40" },

  { 0x007F, "\x7F" },

  { 0x0080, "\xC2\x80" },

  { 0x0081, "\xC2\x81" },

  { 0x0082, "\xC2\x82" },

  { 0x0084, "\xC2\x84" },

  { 0x0088, "\xC2\x88" },

  { 0x0090, "\xC2\x90" },

  { 0x00A0, "\xC2\xA0" },

  { 0x00C0, "\xC3\x80" },

  { 0x00FF, "\xC3\xBF" },

  { 0x0100, "\xC4\x80" },

  { 0x0101, "\xC4\x81" },

  { 0x0102, "\xC4\x82" },

  { 0x0104, "\xC4\x84" },

  { 0x0108, "\xC4\x88" },

  { 0x0110, "\xC4\x90" },

  { 0x0120, "\xC4\xA0" },

  { 0x0140, "\xC5\x80" },

  { 0x0180, "\xC6\x80" },

  { 0x01FF, "\xC7\xBF" },

  { 0x0200, "\xC8\x80" },

  { 0x0201, "\xC8\x81" },

  { 0x0202, "\xC8\x82" },

  { 0x0204, "\xC8\x84" },

  { 0x0208, "\xC8\x88" },

  { 0x0210, "\xC8\x90" },

  { 0x0220, "\xC8\xA0" },

  { 0x0240, "\xC9\x80" },

  { 0x0280, "\xCA\x80" },

  { 0x0300, "\xCC\x80" },

  { 0x03FF, "\xCF\xBF" },

  { 0x0400, "\xD0\x80" },

  { 0x0401, "\xD0\x81" },

  { 0x0402, "\xD0\x82" },

  { 0x0404, "\xD0\x84" },

  { 0x0408, "\xD0\x88" },

  { 0x0410, "\xD0\x90" },

  { 0x0420, "\xD0\xA0" },

  { 0x0440, "\xD1\x80" },

  { 0x0480, "\xD2\x80" },

  { 0x0500, "\xD4\x80" },

  { 0x0600, "\xD8\x80" },

  { 0x07FF, "\xDF\xBF" },

  { 0x0800, "\xE0\xA0\x80" },

  { 0x0801, "\xE0\xA0\x81" },

  { 0x0802, "\xE0\xA0\x82" },

  { 0x0804, "\xE0\xA0\x84" },

  { 0x0808, "\xE0\xA0\x88" },

  { 0x0810, "\xE0\xA0\x90" },

  { 0x0820, "\xE0\xA0\xA0" },

  { 0x0840, "\xE0\xA1\x80" },

  { 0x0880, "\xE0\xA2\x80" },

  { 0x0900, "\xE0\xA4\x80" },

  { 0x0A00, "\xE0\xA8\x80" },

  { 0x0C00, "\xE0\xB0\x80" },

  { 0x0FFF, "\xE0\xBF\xBF" },

  { 0x1000, "\xE1\x80\x80" },

  { 0x1001, "\xE1\x80\x81" },

  { 0x1002, "\xE1\x80\x82" },

  { 0x1004, "\xE1\x80\x84" },

  { 0x1008, "\xE1\x80\x88" },

  { 0x1010, "\xE1\x80\x90" },

  { 0x1020, "\xE1\x80\xA0" },

  { 0x1040, "\xE1\x81\x80" },

  { 0x1080, "\xE1\x82\x80" },

  { 0x1100, "\xE1\x84\x80" },

  { 0x1200, "\xE1\x88\x80" },

  { 0x1400, "\xE1\x90\x80" },

  { 0x1800, "\xE1\xA0\x80" },

  { 0x1FFF, "\xE1\xBF\xBF" },

  { 0x2000, "\xE2\x80\x80" },

  { 0x2001, "\xE2\x80\x81" },

  { 0x2002, "\xE2\x80\x82" },

  { 0x2004, "\xE2\x80\x84" },

  { 0x2008, "\xE2\x80\x88" },

  { 0x2010, "\xE2\x80\x90" },

  { 0x2020, "\xE2\x80\xA0" },

  { 0x2040, "\xE2\x81\x80" },

  { 0x2080, "\xE2\x82\x80" },

  { 0x2100, "\xE2\x84\x80" },

  { 0x2200, "\xE2\x88\x80" },

  { 0x2400, "\xE2\x90\x80" },

  { 0x2800, "\xE2\xA0\x80" },

  { 0x3000, "\xE3\x80\x80" },

  { 0x3FFF, "\xE3\xBF\xBF" },

  { 0x4000, "\xE4\x80\x80" },

  { 0x4001, "\xE4\x80\x81" },

  { 0x4002, "\xE4\x80\x82" },

  { 0x4004, "\xE4\x80\x84" },

  { 0x4008, "\xE4\x80\x88" },

  { 0x4010, "\xE4\x80\x90" },

  { 0x4020, "\xE4\x80\xA0" },

  { 0x4040, "\xE4\x81\x80" },

  { 0x4080, "\xE4\x82\x80" },

  { 0x4100, "\xE4\x84\x80" },

  { 0x4200, "\xE4\x88\x80" },

  { 0x4400, "\xE4\x90\x80" },

  { 0x4800, "\xE4\xA0\x80" },

  { 0x5000, "\xE5\x80\x80" },

  { 0x6000, "\xE6\x80\x80" },

  { 0x7FFF, "\xE7\xBF\xBF" },

  { 0x8000, "\xE8\x80\x80" },

  { 0x8001, "\xE8\x80\x81" },

  { 0x8002, "\xE8\x80\x82" },

  { 0x8004, "\xE8\x80\x84" },

  { 0x8008, "\xE8\x80\x88" },

  { 0x8010, "\xE8\x80\x90" },

  { 0x8020, "\xE8\x80\xA0" },

  { 0x8040, "\xE8\x81\x80" },

  { 0x8080, "\xE8\x82\x80" },

  { 0x8100, "\xE8\x84\x80" },

  { 0x8200, "\xE8\x88\x80" },

  { 0x8400, "\xE8\x90\x80" },

  { 0x8800, "\xE8\xA0\x80" },

  { 0x9000, "\xE9\x80\x80" },

  { 0xA000, "\xEA\x80\x80" },

  { 0xC000, "\xEC\x80\x80" },

  { 0xFFFF, "\xEF\xBF\xBF" }

};

/*

 * UTF-16 vectors

 */

struct utf16 utf16[] = {

  { 0x00010000, { 0xD800, 0xDC00 } },

  { 0x00010001, { 0xD800, 0xDC01 } },

  { 0x00010002, { 0xD800, 0xDC02 } },

  { 0x00010003, { 0xD800, 0xDC03 } },

  { 0x00010004, { 0xD800, 0xDC04 } },

  { 0x00010007, { 0xD800, 0xDC07 } },

  { 0x00010008, { 0xD800, 0xDC08 } },

  { 0x0001000F, { 0xD800, 0xDC0F } },

  { 0x00010010, { 0xD800, 0xDC10 } },

  { 0x0001001F, { 0xD800, 0xDC1F } },

  { 0x00010020, { 0xD800, 0xDC20 } },

  { 0x0001003F, { 0xD800, 0xDC3F } },

  { 0x00010040, { 0xD800, 0xDC40 } },

  { 0x0001007F, { 0xD800, 0xDC7F } },

  { 0x00010080, { 0xD800, 0xDC80 } },

  { 0x00010081, { 0xD800, 0xDC81 } },