From e0e2452d74b6994b4ffdad7bd46a4b274f653744 Mon Sep 17 00:00:00 2001 From: James Roseborough Date: Sat, 1 Sep 2012 04:59:14 +0000 Subject: [PATCH] Improve processing of first-line comments by LuaParser --- grammar/Lua52.jj | 3 +- grammar/LuaParser.jj | 3 +- src/jse/org/luaj/vm2/parser/LuaParser.java | 33 +++--- .../luaj/vm2/parser/LuaParserConstants.java | 11 +- .../vm2/parser/LuaParserTokenManager.java | 106 +++++------------- 5 files changed, 53 insertions(+), 103 deletions(-) diff --git a/grammar/Lua52.jj b/grammar/Lua52.jj index 4117a690..d84f4c6f 100644 --- a/grammar/Lua52.jj +++ b/grammar/Lua52.jj @@ -133,7 +133,6 @@ TOKEN : | < #QUOTED: | | > | < #DECIMAL: "\\" ["0"-"9"] (["0"-"9"])? (["0"-"9"])? > | < DBCOLON: "::" > -| < SHEBANG: "#" ["!"," "] (~["\n","\r"])* ()? > | < #UNICODE: "\\" "u" > | < #CHAR: "\\" (~[]) > | < #LF: ("\n" | "\r" | "\r\n") > @@ -143,7 +142,7 @@ TOKEN : void Chunk(): {} { - ( )? Block() + ( "#" { token_source.SwitchTo(IN_COMMENT); } )? Block() } void Block(): diff --git a/grammar/LuaParser.jj b/grammar/LuaParser.jj index f31fb6c2..0f710ab6 100644 --- a/grammar/LuaParser.jj +++ b/grammar/LuaParser.jj @@ -157,7 +157,6 @@ TOKEN : | < #QUOTED: | | > | < #DECIMAL: "\\" ["0"-"9"] (["0"-"9"])? (["0"-"9"])? > | < DBCOLON: "::" > -| < SHEBANG: "#" ["!"," "] (~["\n","\r"])* ()? > | < #UNICODE: "\\" "u" > | < #CHAR: "\\" (~[]) > | < #LF: ("\n" | "\r" | "\r\n") > @@ -169,7 +168,7 @@ Chunk Chunk(): Block b; } { - ( )? b=Block() { return new Chunk(b); } + ( "#" { token_source.SwitchTo(IN_COMMENT); } )? b=Block() { return new Chunk(b); } } Block Block(): diff --git a/src/jse/org/luaj/vm2/parser/LuaParser.java b/src/jse/org/luaj/vm2/parser/LuaParser.java index 5cc6b889..93d5a87b 100644 --- a/src/jse/org/luaj/vm2/parser/LuaParser.java +++ b/src/jse/org/luaj/vm2/parser/LuaParser.java @@ -27,15 +27,16 @@ public class LuaParser implements LuaParserConstants { final public Chunk Chunk() throws ParseException { Block b; switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { - case SHEBANG: - jj_consume_token(SHEBANG); + case 69: + jj_consume_token(69); + token_source.SwitchTo(IN_COMMENT); break; default: ; } b = Block(); jj_consume_token(0); - {if (true) return new Chunk(b);} + {if (true) return new Chunk(b);} throw new Error("Missing return statement in function"); } @@ -264,11 +265,11 @@ public class LuaParser implements LuaParserConstants { case NUMBER: case STRING: case CHARSTRING: + case 69: case 75: case 79: case 80: case 83: - case 95: el = ExpList(); break; default: @@ -472,11 +473,11 @@ public class LuaParser implements LuaParserConstants { case NUMBER: case STRING: case CHARSTRING: + case 69: case 75: case 79: case 80: case 83: - case 95: el = ExpList(); break; default: @@ -665,8 +666,8 @@ public class LuaParser implements LuaParserConstants { e = SimpleExp(); break; case NOT: + case 69: case 83: - case 95: op = Unop(); s = Exp(); e=Exp.unaryexp(op,s); @@ -764,12 +765,12 @@ public class LuaParser implements LuaParserConstants { case NUMBER: case STRING: case CHARSTRING: + case 69: case 75: case 77: case 79: case 80: case 83: - case 95: fl = FieldList(); tc.fields=fl; break; @@ -843,11 +844,11 @@ public class LuaParser implements LuaParserConstants { case NUMBER: case STRING: case CHARSTRING: + case 69: case 75: case 79: case 80: case 83: - case 95: rhs = Exp(); {if (true) return TableField.listField(rhs);} break; @@ -953,8 +954,8 @@ public class LuaParser implements LuaParserConstants { jj_consume_token(NOT); {if (true) return Lua.OP_NOT;} break; - case 95: - jj_consume_token(95); + case 69: + jj_consume_token(69); {if (true) return Lua.OP_LEN;} break; default: @@ -1006,6 +1007,11 @@ public class LuaParser implements LuaParserConstants { catch(LookaheadSuccess ls) { return true; } } + private boolean jj_3R_28() { + if (jj_scan_token(91)) return true; + return false; + } + private boolean jj_3R_27() { if (jj_scan_token(90)) return true; return false; @@ -1396,7 +1402,7 @@ public class LuaParser implements LuaParserConstants { } private boolean jj_3R_55() { - if (jj_scan_token(95)) return true; + if (jj_scan_token(69)) return true; return false; } @@ -1514,11 +1520,6 @@ public class LuaParser implements LuaParserConstants { return false; } - private boolean jj_3R_28() { - if (jj_scan_token(91)) return true; - return false; - } - /** Generated Token Manager. */ public LuaParserTokenManager token_source; SimpleCharStream jj_input_stream; diff --git a/src/jse/org/luaj/vm2/parser/LuaParserConstants.java b/src/jse/org/luaj/vm2/parser/LuaParserConstants.java index 20e50ad4..fbab92e7 100644 --- a/src/jse/org/luaj/vm2/parser/LuaParserConstants.java +++ b/src/jse/org/luaj/vm2/parser/LuaParserConstants.java @@ -107,13 +107,11 @@ public interface LuaParserConstants { /** RegularExpression Id. */ int DBCOLON = 65; /** RegularExpression Id. */ - int SHEBANG = 66; + int UNICODE = 66; /** RegularExpression Id. */ - int UNICODE = 67; + int CHAR = 67; /** RegularExpression Id. */ - int CHAR = 68; - /** RegularExpression Id. */ - int LF = 69; + int LF = 68; /** Lexical state. */ int DEFAULT = 0; @@ -208,10 +206,10 @@ public interface LuaParserConstants { "", "", "\"::\"", - "", "", "", "", + "\"#\"", "\";\"", "\"=\"", "\",\"", @@ -237,7 +235,6 @@ public interface LuaParserConstants { "\">=\"", "\"==\"", "\"~=\"", - "\"#\"", }; } diff --git a/src/jse/org/luaj/vm2/parser/LuaParserTokenManager.java b/src/jse/org/luaj/vm2/parser/LuaParserTokenManager.java index 2b4981fa..3b3dc0c2 100644 --- a/src/jse/org/luaj/vm2/parser/LuaParserTokenManager.java +++ b/src/jse/org/luaj/vm2/parser/LuaParserTokenManager.java @@ -353,8 +353,6 @@ private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1) jjmatchedKind = 51; return 17; } - if ((active1 & 0x80000000L) != 0L) - return 62; if ((active0 & 0x103c0L) != 0L || (active1 & 0x80000L) != 0L) return 7; return -1; @@ -451,7 +449,7 @@ private int jjMoveStringLiteralDfa0_0() switch(curChar) { case 35: - return jjStartNfaWithStates_0(0, 95, 62); + return jjStopAtPos(0, 69); case 37: return jjStopAtPos(0, 87); case 40: @@ -859,7 +857,7 @@ static final long[] jjbitVec2 = { private int jjMoveNfa_0(int startState, int curPos) { int startsAt = 0; - jjnewStateCnt = 72; + jjnewStateCnt = 66; int i = 1; jjstateSet[0] = startState; int kind = 0x7fffffff; @@ -881,8 +879,6 @@ private int jjMoveNfa_0(int startState, int curPos) kind = 52; jjCheckNAddStates(0, 3); } - else if (curChar == 35) - jjstateSet[jjnewStateCnt++] = 62; else if (curChar == 39) jjCheckNAddStates(4, 6); else if (curChar == 34) @@ -1100,66 +1096,36 @@ private int jjMoveNfa_0(int startState, int curPos) jjCheckNAddStates(26, 29); break; case 61: - if (curChar == 35) - jjstateSet[jjnewStateCnt++] = 62; - break; - case 62: - if ((0x300000000L & l) == 0L) - break; - if (kind > 66) - kind = 66; - jjCheckNAddStates(30, 32); - break; - case 63: - if ((0xffffffffffffdbffL & l) == 0L) - break; - if (kind > 66) - kind = 66; - jjCheckNAddStates(30, 32); - break; - case 64: - if ((0x2400L & l) != 0L && kind > 66) - kind = 66; - break; - case 65: - if (curChar == 10 && kind > 66) - kind = 66; - break; - case 66: - if (curChar == 13) - jjstateSet[jjnewStateCnt++] = 65; - break; - case 67: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 52) kind = 52; jjCheckNAddStates(0, 3); break; - case 68: + case 62: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(68, 69); + jjCheckNAddTwoStates(62, 63); break; - case 69: + case 63: if (curChar != 46) break; if (kind > 52) kind = 52; - jjCheckNAddTwoStates(70, 32); + jjCheckNAddTwoStates(64, 32); break; - case 70: + case 64: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 52) kind = 52; - jjCheckNAddTwoStates(70, 32); + jjCheckNAddTwoStates(64, 32); break; - case 71: + case 65: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 52) kind = 52; - jjCheckNAddTwoStates(71, 32); + jjCheckNAddTwoStates(65, 32); break; default : break; } @@ -1208,7 +1174,7 @@ private int jjMoveNfa_0(int startState, int curPos) break; case 19: if ((0x100000001000000L & l) != 0L) - jjAddStates(33, 34); + jjAddStates(30, 31); break; case 21: if ((0x7e0000007eL & l) == 0L) @@ -1219,7 +1185,7 @@ private int jjMoveNfa_0(int startState, int curPos) break; case 22: if ((0x1002000010020L & l) != 0L) - jjAddStates(35, 36); + jjAddStates(32, 33); break; case 25: if ((0x7e0000007eL & l) == 0L) @@ -1248,7 +1214,7 @@ private int jjMoveNfa_0(int startState, int curPos) break; case 32: if ((0x2000000020L & l) != 0L) - jjAddStates(37, 38); + jjAddStates(34, 35); break; case 36: if ((0xffffffffefffffffL & l) != 0L) @@ -1256,7 +1222,7 @@ private int jjMoveNfa_0(int startState, int curPos) break; case 38: if (curChar == 92) - jjAddStates(39, 41); + jjAddStates(36, 38); break; case 39: jjCheckNAddStates(7, 9); @@ -1287,7 +1253,7 @@ private int jjMoveNfa_0(int startState, int curPos) break; case 51: if (curChar == 92) - jjAddStates(42, 44); + jjAddStates(39, 41); break; case 52: jjCheckNAddStates(4, 6); @@ -1312,11 +1278,6 @@ private int jjMoveNfa_0(int startState, int curPos) if ((0x7e0000007eL & l) != 0L) jjCheckNAddStates(4, 6); break; - case 63: - if (kind > 66) - kind = 66; - jjAddStates(30, 32); - break; default : break; } } while(i != startsAt); @@ -1342,13 +1303,6 @@ private int jjMoveNfa_0(int startState, int curPos) if (jjCanMove_0(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(4, 6); break; - case 63: - if (!jjCanMove_0(hiByte, i1, i2, l1, l2)) - break; - if (kind > 66) - kind = 66; - jjAddStates(30, 32); - break; default : break; } } while(i != startsAt); @@ -1360,7 +1314,7 @@ private int jjMoveNfa_0(int startState, int curPos) kind = 0x7fffffff; } ++curPos; - if ((i = jjnewStateCnt) == (startsAt = 72 - (jjnewStateCnt = startsAt))) + if ((i = jjnewStateCnt) == (startsAt = 66 - (jjnewStateCnt = startsAt))) return curPos; try { curChar = input_stream.readChar(); } catch(java.io.IOException e) { return curPos; } @@ -1393,7 +1347,7 @@ private int jjMoveNfa_1(int startState, int curPos) { if (kind > 17) kind = 17; - jjCheckNAddStates(45, 47); + jjCheckNAddStates(42, 44); } else if ((0x2400L & l) != 0L) { @@ -1407,7 +1361,7 @@ private int jjMoveNfa_1(int startState, int curPos) if ((0xffffffffffffdbffL & l) == 0L) break; kind = 17; - jjCheckNAddStates(45, 47); + jjCheckNAddStates(42, 44); break; case 1: if ((0x2400L & l) != 0L && kind > 17) @@ -1435,7 +1389,7 @@ private int jjMoveNfa_1(int startState, int curPos) case 4: case 0: kind = 17; - jjCheckNAddStates(45, 47); + jjCheckNAddStates(42, 44); break; default : break; } @@ -1458,7 +1412,7 @@ private int jjMoveNfa_1(int startState, int curPos) break; if (kind > 17) kind = 17; - jjCheckNAddStates(45, 47); + jjCheckNAddStates(42, 44); break; default : break; } @@ -1745,9 +1699,9 @@ private int jjMoveStringLiteralDfa2_3(long old0, long active0) return 3; } static final int[] jjnextStates = { - 68, 69, 71, 32, 49, 50, 51, 36, 37, 38, 26, 27, 29, 22, 36, 37, - 38, 46, 36, 47, 37, 38, 49, 50, 51, 59, 49, 60, 50, 51, 63, 64, - 66, 20, 25, 23, 24, 33, 34, 39, 40, 45, 52, 53, 58, 0, 1, 3, + 62, 63, 65, 32, 49, 50, 51, 36, 37, 38, 26, 27, 29, 22, 36, 37, + 38, 46, 36, 47, 37, 38, 49, 50, 51, 59, 49, 60, 50, 51, 20, 25, + 23, 24, 33, 34, 39, 40, 45, 52, 53, 58, 0, 1, 3, }; private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2) { @@ -1772,9 +1726,9 @@ null, null, "\141\156\144", "\142\162\145\141\153", "\144\157", "\145\154\163\14 "\156\151\154", "\156\157\164", "\157\162", "\162\145\164\165\162\156", "\162\145\160\145\141\164", "\164\150\145\156", "\164\162\165\145", "\165\156\164\151\154", "\167\150\151\154\145", null, null, null, null, null, null, null, null, null, null, null, null, null, -null, "\72\72", null, null, null, null, "\73", "\75", "\54", "\56", "\72", "\50", +null, "\72\72", null, null, null, "\43", "\73", "\75", "\54", "\56", "\72", "\50", "\51", "\133", "\135", "\56\56\56", "\173", "\175", "\53", "\55", "\52", "\57", -"\136", "\45", "\56\56", "\74", "\74\75", "\76", "\76\75", "\75\75", "\176\75", "\43", }; +"\136", "\45", "\56\56", "\74", "\74\75", "\76", "\76\75", "\75\75", "\176\75", }; /** Lexer state names. */ public static final String[] lexStateNames = { @@ -1797,10 +1751,10 @@ public static final int[] jjnewLexState = { -1, -1, -1, -1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, }; static final long[] jjtoToken = { - 0x601fffffef800001L, 0xffffffc6L, + 0x601fffffef800001L, 0x7fffffe2L, }; static final long[] jjtoSkip = { 0x7e003eL, 0x0L, @@ -1812,8 +1766,8 @@ static final long[] jjtoMore = { 0x1001ffc0L, 0x0L, }; protected SimpleCharStream input_stream; -private final int[] jjrounds = new int[72]; -private final int[] jjstateSet = new int[144]; +private final int[] jjrounds = new int[66]; +private final int[] jjstateSet = new int[132]; private final StringBuffer jjimage = new StringBuffer(); private StringBuffer image = jjimage; private int jjimageLen; @@ -1844,7 +1798,7 @@ private void ReInitRounds() { int i; jjround = 0x80000001; - for (i = 72; i-- > 0;) + for (i = 66; i-- > 0;) jjrounds[i] = 0x80000000; }