From 01422220ebf40f829c1f00418a96873b82f206ff Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <bonzini@gnu.org>
Date: Mon, 19 Apr 2010 14:50:23 +0200
Subject: [PATCH 1/2] dfa: optimize UTF-8 period

Backport of upstream commits 7a0ad00 and 42ac56a.

* src/dfa.h (struct dfa): Add utf8_anychar_classes.
* src/dfa.c (add_utf8_anychar): New.
(atom): Simplify if/else nesting.  Call add_utf8_anychar for ANYCHAR
in UTF-8 locales.
(dfaoptimize): Abort on ANYCHAR.
---
 src/dfa.c |   95 ++++++++++++++++++++++++++++++++++++++++++++++++++++---------
 src/dfa.h |    1 +
 2 files changed, 82 insertions(+), 14 deletions(-)

diff --git a/src/dfa.c b/src/dfa.c
index ba78b08..e13c361 100644
--- a/src/dfa.c
+++ b/src/dfa.c
@@ -1191,6 +1191,55 @@ addtok_wc (wint_t wc)
 }
 #endif
 
+static void
+add_utf8_anychar (void)
+{
+  static const charclass utf8_classes[5] = {
+      {  0,  0,  0,  0, ~0, ~0, 0, 0 },            /* 80-bf: non-lead bytes */
+      { ~0, ~0, ~0, ~0, ~0, ~0, 0, 0xff000000 },   /* 00-bf, f8-ff: 1-byte/invalid */
+      {  0,  0,  0,  0,  0,  0, ~0, 0 },           /* c0-df: 2-byte sequence */
+      {  0,  0,  0,  0,  0,  0,  0, 0xffff },      /* e0-ef: 3-byte sequence */
+      {  0,  0,  0,  0,  0,  0,  0, 0xff0000 }     /* f0-f7: 4-byte sequence */
+  };
+  const unsigned int n = sizeof (utf8_classes) / sizeof (utf8_classes[0]);
+  unsigned int i;
+
+  /* Define the five character classes that are needed below.  */
+  if (dfa->utf8_anychar_classes[0] == 0)
+    for (i = 0; i < n; i++)
+      {
+        charclass c;
+        memcpy (c, utf8_classes[i], sizeof c);
+        if (i == 1)
+          {
+            if (!(syntax_bits & RE_DOT_NEWLINE))
+              clrbit (eolbyte, c);
+            if (syntax_bits & RE_DOT_NOT_NULL)
+              clrbit ('\0', c);
+          }
+        dfa->utf8_anychar_classes[i] = CSET + charclass_index(c);
+      }
+
+  /* A valid UTF-8 character is
+
+          ([0x00-0x7f]
+           |[0xc2-0xdf][0x80-0xbf]
+           |[0xe0-0xef[0x80-0xbf][0x80-0xbf]
+           |[0xf0-f7][0x80-0xbf][0x80-0xbf][0x80-0xbf])
+
+     which I'll write more concisely "B|CA|DAA|EAAA".  Factor the [0x80-0xbf]
+     and you get "B|(C|(D|EA)A)A".  And since the token buffer is in reverse
+     Polish notation, you get "B C D E A CAT OR A CAT OR A CAT OR".  */
+  for (i = 1; i < n; i++)
+    addtok (dfa->utf8_anychar_classes[i]);
+  while (--i > 1)
+    {
+      addtok (dfa->utf8_anychar_classes[0]);
+      addtok (CAT);
+      addtok (OR);
+    }
+}
+
 /* The grammar understood by the parser is as follows.
 
    regexp:
@@ -1229,8 +1278,12 @@ addtok_wc (wint_t wc)
 static void
 atom (void)
 {
+  if (0)
+    {
+      /* empty */
+    }
 #ifdef MBS_SUPPORT
-  if (tok == WCHAR)
+  else if (tok == WCHAR)
     {
       addtok_wc (case_fold ? towlower(wctok) : wctok);
 #ifndef GREP
@@ -1242,16 +1295,28 @@ atom (void)
 #endif
 
       tok = lex();
-      return;
+    }
+
+  else if (tok == ANYCHAR && using_utf8())
+    {
+      /* For UTF-8 expand the period to a series of CSETs that define a valid
+	 UTF-8 character.  This avoids using the slow multibyte path.  I'm
+	 pretty sure it would be both profitable and correct to do it for
+	 any encoding; however, the optimization must be done manually as
+	 it is done above in add_utf8_anychar.	So, let's start with
+	 UTF-8: it is the most used, and the structure of the encoding
+	 makes the correctness more obvious.  */
+      add_utf8_anychar();
+      tok = lex();
     }
 #endif /* MBS_SUPPORT  */
 
-  if ((tok >= 0 && tok < NOTCHAR) || tok >= CSET || tok == BACKREF
-      || tok == BEGLINE || tok == ENDLINE || tok == BEGWORD
+  else if ((tok >= 0 && tok < NOTCHAR) || tok >= CSET || tok == BACKREF
+       	   || tok == BEGLINE || tok == ENDLINE || tok == BEGWORD
 #ifdef MBS_SUPPORT
-      || tok == ANYCHAR || tok == MBCSET /* MB_CUR_MAX > 1 */
+     	   || tok == ANYCHAR || tok == MBCSET
 #endif /* MBS_SUPPORT */
-      || tok == ENDWORD || tok == LIMWORD || tok == NOTLIMWORD)
+	   || tok == ENDWORD || tok == LIMWORD || tok == NOTLIMWORD)
     {
       addtok(tok);
       tok = lex();
@@ -3027,14 +3092,16 @@ dfaoptimize (struct dfa *d)
   for (i = 0; i < d->tindex; ++i)
     {
       switch(d->tokens[i])
-	{
-	case ANYCHAR:
-	case MBCSET:
-	  /* Requires multi-byte algorithm.  */
-	  return;
-	default:
-	  break;
-	}
+        {
+        case ANYCHAR:
+          /* Lowered.  */
+          abort ();
+        case MBCSET:
+          /* Requires multi-byte algorithm.  */
+          return;
+        default:
+          break;
+        }
     }
 
   free_mbdata (d);
diff --git a/src/dfa.h b/src/dfa.h
index 1c85207..42c177a 100644
--- a/src/dfa.h
+++ b/src/dfa.h
@@ -283,6 +283,7 @@ struct dfa
 				   with dfaparse(). */
 #ifdef MBS_SUPPORT
   unsigned int mb_cur_max;	/* Cached value of MB_CUR_MAX.  */
+  int utf8_anychar_classes[5];	/* To lower ANYCHAR in UTF-8 locales.  */
 
   /* The following are used only if MB_CUR_MAX > 1.  */
 
-- 
1.6.6.1