From 6f01f9784ae8c6af98bb8a6e848fcd2f1f58ddfe Mon Sep 17 00:00:00 2001
From: James Roseborough <james.roseborough@luaj.org>
Date: Fri, 23 Jul 2010 00:39:27 +0000
Subject: [PATCH] Switch to Unicode parsing.

---
 grammar/LuaParser.jj                          |  1 +
 .../vm2/parser/LuaParserTokenManager.java     | 33 +++++++++++++++++--
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/grammar/LuaParser.jj b/grammar/LuaParser.jj
index 2b1f9b64..3f654203 100644
--- a/grammar/LuaParser.jj
+++ b/grammar/LuaParser.jj
@@ -28,6 +28,7 @@ options {
   STATIC = false;
   JDK_VERSION = "1.3";
   ERROR_REPORTING = false;
+  UNICODE_INPUT = true;
   DEBUG_LOOKAHEAD = false;
   DEBUG_PARSER = false;
   DEBUG_TOKEN_MANAGER = false;
diff --git a/src/jse/org/luaj/vm2/parser/LuaParserTokenManager.java b/src/jse/org/luaj/vm2/parser/LuaParserTokenManager.java
index 17dc49eb..15379828 100644
--- a/src/jse/org/luaj/vm2/parser/LuaParserTokenManager.java
+++ b/src/jse/org/luaj/vm2/parser/LuaParserTokenManager.java
@@ -109,6 +109,9 @@ private int jjMoveNfa_11(int startState, int curPos)
       }
       else
       {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
          do
@@ -835,6 +838,9 @@ private int jjStartNfaWithStates_0(int pos, int kind, int state)
    return jjMoveNfa_0(state, pos + 1);
 }
 static final long[] jjbitVec0 = {
+   0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+static final long[] jjbitVec2 = {
    0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
 };
 private int jjMoveNfa_0(int startState, int curPos)
@@ -1221,6 +1227,9 @@ private int jjMoveNfa_0(int startState, int curPos)
       }
       else
       {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
          do
@@ -1229,12 +1238,12 @@ private int jjMoveNfa_0(int startState, int curPos)
             {
                case 27:
                case 30:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(7, 9);
                   break;
                case 40:
                case 43:
-                  if ((jjbitVec0[i2] & l2) != 0L)
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(4, 6);
                   break;
                default : break;
@@ -1331,6 +1340,9 @@ private int jjMoveNfa_1(int startState, int curPos)
       }
       else
       {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
          do
@@ -1339,7 +1351,7 @@ private int jjMoveNfa_1(int startState, int curPos)
             {
                case 4:
                case 0:
-                  if ((jjbitVec0[i2] & l2) == 0L)
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 17)
                      kind = 17;
@@ -1426,6 +1438,9 @@ private int jjMoveNfa_6(int startState, int curPos)
       }
       else
       {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
          do
@@ -1631,6 +1646,18 @@ static final int[] jjnextStates = {
    28, 29, 40, 41, 42, 50, 40, 51, 41, 42, 24, 25, 30, 31, 36, 43, 
    44, 49, 55, 56, 61, 62, 0, 1, 3, 
 };
+private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
+{
+   switch(hiByte)
+   {
+      case 0:
+         return ((jjbitVec2[i2] & l2) != 0L);
+      default :
+         if ((jjbitVec0[i1] & l1) != 0L)
+            return true;
+         return false;
+   }
+}
 
 /** Token literal values. */
 public static final String[] jjstrLiteralImages = {