Add line number tracking in elements parsed using LuaParser.

This commit is contained in:
James Roseborough
2012-09-01 15:56:09 +00:00
parent e0e2452d74
commit 2bf4c5767b
13 changed files with 637 additions and 472 deletions

View File

@@ -5,13 +5,15 @@
*/ */
import java.io.*; import java.io.*;
import org.luaj.vm2.*; import org.luaj.vm2.LuaValue;
import org.luaj.vm2.ast.*; import org.luaj.vm2.ast.*;
import org.luaj.vm2.ast.Exp.AnonFuncDef;
import org.luaj.vm2.ast.Stat.FuncDef;
import org.luaj.vm2.ast.Stat.LocalFuncDef;
import org.luaj.vm2.parser.*; import org.luaj.vm2.parser.*;
public class SampleParser { public class SampleParser {
static LuaValue NIL = LuaValue.NIL; // Workaround needed to ensure classes load in right order.
// Sample ParseException subclass that stores the file, line, and column info. // Sample ParseException subclass that stores the file, line, and column info.
static public class FileLineColumnParseException extends ParseException { static public class FileLineColumnParseException extends ParseException {
@@ -51,9 +53,29 @@ public class SampleParser {
// Perform the parsing. // Perform the parsing.
Chunk chunk = parser.Chunk(); Chunk chunk = parser.Chunk();
// Optionally recurse over the parse tree with a custom Visitor instance. // Print out line info for all function definitions.
chunk.accept( new Visitor() { chunk.accept( new Visitor() {
// TODO: override Visitor methods here to perform actions on the parse tree. public void visit(AnonFuncDef exp) {
System.out.println("Anonymous function definition at "
+ exp.beginLine + "." + exp.beginColumn + ","
+ exp.endLine + "." + exp.endColumn);
}
public void visit(FuncDef stat) {
System.out.println("Function definition '" + stat.name.name.name + "' at "
+ stat.beginLine + "." + stat.beginColumn + ","
+ stat.endLine + "." + stat.endColumn);
System.out.println("\tName location "
+ stat.name.beginLine + "." + stat.name.beginColumn + ","
+ stat.name.endLine + "." + stat.name.endColumn);
}
public void visit(LocalFuncDef stat) {
System.out.println("Local function definition '" + stat.name.name + "' at "
+ stat.beginLine + "." + stat.beginColumn + ","
+ stat.endLine + "." + stat.endColumn);
}
} ); } );
} catch ( FileLineColumnParseException e ) { } catch ( FileLineColumnParseException e ) {

View File

@@ -42,6 +42,9 @@ import org.luaj.vm2.ast.*;
import java.util.*; import java.util.*;
public class LuaParser { public class LuaParser {
static {
LuaValue.valueOf(true);
}
public static void main(String args[]) throws ParseException { public static void main(String args[]) throws ParseException {
LuaParser parser = new LuaParser(System.in); LuaParser parser = new LuaParser(System.in);
@@ -59,6 +62,29 @@ public class LuaParser {
throw new ParseException("expected function call"); throw new ParseException("expected function call");
return (Exp.FuncCall) pe; return (Exp.FuncCall) pe;
} }
public SimpleCharStream getCharStream() {
return jj_input_stream;
}
private long LineInfo() {
return ((long) jj_input_stream.getBeginLine() << 32) | jj_input_stream.getBeginColumn();
}
private void L(SyntaxElement e, long startinfo) {
e.beginLine = (int) (startinfo >> 32);
e.beginColumn = (short) startinfo;
e.endLine = token.endLine;
e.endColumn = (short) token.endColumn;
}
private void L(SyntaxElement e, Token starttoken) {
e.beginLine = starttoken.beginLine;
e.beginColumn = (short) starttoken.beginColumn;
e.endLine = token.endLine;
e.endColumn = (short) token.endColumn;
}
} }
PARSER_END(LuaParser) PARSER_END(LuaParser)
@@ -166,18 +192,21 @@ TOKEN :
Chunk Chunk(): Chunk Chunk():
{ {
Block b; Block b;
Chunk c;
long i = LineInfo();
} }
{ {
( "#" { token_source.SwitchTo(IN_COMMENT); } )? b=Block() <EOF> { return new Chunk(b); } ( "#" { token_source.SwitchTo(IN_COMMENT); } )? b=Block() <EOF> { c=new Chunk(b); L(c,i); return c; }
} }
Block Block(): Block Block():
{ {
Block b = new Block(); Block b = new Block();
Stat s; Stat s;
long i = LineInfo();
} }
{ {
(s=Stat() {b.add(s);} )* (s=ReturnStat() {b.add(s);} )? { return b; } (s=Stat() {b.add(s);} )* (s=ReturnStat() {b.add(s);} )? { L(b,i); return b; }
} }
Stat Stat(): Stat Stat():
@@ -190,22 +219,23 @@ Stat Stat():
Token n; Token n;
List<Name> nl; List<Name> nl;
List<Exp> el=null; List<Exp> el=null;
long i = LineInfo();
} }
{ {
";" { return null; } ";" { return null; }
| s=Label() { return s; } | s=Label() { L(s,i); return s; }
| <BREAK> { return Stat.breakstat(); } | <BREAK> { s=Stat.breakstat(); L(s,i); return s; }
| <GOTO> n=<NAME> { return Stat.gotostat(n.image); } | <GOTO> n=<NAME> { s=Stat.gotostat(n.image); L(s,i); return s; }
| <DO> b=Block() <END> { return Stat.block(b); } | <DO> b=Block() <END> { s=Stat.block(b); L(s,i); return s; }
| <WHILE> e=Exp() <DO> b=Block() <END> { return Stat.whiledo(e,b); } | <WHILE> e=Exp() <DO> b=Block() <END> { s=Stat.whiledo(e,b); L(s,i); return s; }
| <REPEAT> b=Block() <UNTIL> e=Exp() { return Stat.repeatuntil(b,e); } | <REPEAT> b=Block() <UNTIL> e=Exp() { s=Stat.repeatuntil(b,e); L(s,i); return s; }
| s=IfThenElse() { return s; } | s=IfThenElse() { L(s,i); return s; }
| LOOKAHEAD(3) <FOR> n=<NAME> "=" e=Exp() "," e2=Exp() ( "," e3=Exp() )? <DO> b=Block() <END> { return Stat.fornumeric(n.image,e,e2,e3,b); } | LOOKAHEAD(3) <FOR> n=<NAME> "=" e=Exp() "," e2=Exp() ( "," e3=Exp() )? <DO> b=Block() <END> { s=Stat.fornumeric(n.image,e,e2,e3,b); L(s,i); return s; }
| <FOR> nl=NameList() <IN> el=ExpList() <DO> b=Block() <END> { return Stat.forgeneric(nl,el,b); } | <FOR> nl=NameList() <IN> el=ExpList() <DO> b=Block() <END> { s=Stat.forgeneric(nl,el,b); L(s,i); return s; }
| <FUNCTION> fn=FuncName() fb=FuncBody() { return Stat.functiondef(fn,fb); } | <FUNCTION> fn=FuncName() fb=FuncBody() { s=Stat.functiondef(fn,fb); L(s,i); return s; }
| LOOKAHEAD(2) <LOCAL> <FUNCTION> n=<NAME> fb=FuncBody() { return Stat.localfunctiondef(n.image,fb); } | LOOKAHEAD(2) <LOCAL> <FUNCTION> n=<NAME> fb=FuncBody() { s=Stat.localfunctiondef(n.image,fb); L(s,i); return s; }
| <LOCAL> nl=NameList() ( "=" el=ExpList() )? { return Stat.localassignment(nl,el); } | <LOCAL> nl=NameList() ( "=" el=ExpList() )? { s=Stat.localassignment(nl,el); L(s,i); return s; }
| s=ExprStat() { return s; } | s=ExprStat() { L(s,i); return s; }
} }
Stat IfThenElse(): Stat IfThenElse():
@@ -231,9 +261,11 @@ Stat IfThenElse():
Stat ReturnStat(): Stat ReturnStat():
{ {
List<Exp> el=null; List<Exp> el=null;
Stat s;
long i = LineInfo();
} }
{ {
<RETURN> ( el=ExpList() )? ( ";" )? { return Stat.returnstat(el); } <RETURN> ( el=ExpList() )? ( ";" )? { s=Stat.returnstat(el); L(s,i); return s; }
} }
Stat Label(): Stat Label():
@@ -246,12 +278,13 @@ Stat Label():
Stat ExprStat(): Stat ExprStat():
{ {
Exp.PrimaryExp pe; Exp.PrimaryExp p;
Stat as=null; Stat s=null;
long i = LineInfo();
} }
{ {
pe=PrimaryExp() ( as=Assign(assertvarexp(pe)) )? p=PrimaryExp() ( s=Assign(assertvarexp(p)) )?
{ return as==null? Stat.functioncall(assertfunccall(pe)): as; } { if (s==null) { s=Stat.functioncall(assertfunccall(p)); } L(s,i); return s; }
} }
Stat Assign(Exp.VarExp v0): Stat Assign(Exp.VarExp v0):
@@ -260,47 +293,52 @@ Stat Assign(Exp.VarExp v0):
vl.add(v0); vl.add(v0);
Exp.VarExp ve; Exp.VarExp ve;
List<Exp> el; List<Exp> el;
Stat s;
long i = LineInfo();
} }
{ {
( "," ve=VarExp() { vl.add(ve); } )* "=" el=ExpList() { return Stat.assignment(vl,el); } ( "," ve=VarExp() { vl.add(ve); } )* "=" el=ExpList() { s=Stat.assignment(vl,el); L(s,i); return s; }
} }
Exp.VarExp VarExp(): Exp.VarExp VarExp():
{ {
Exp.PrimaryExp pe; Exp.PrimaryExp p;
} }
{ {
pe=PrimaryExp() { return assertvarexp(pe); } p=PrimaryExp() { return assertvarexp(p); }
} }
FuncName FuncName(): FuncName FuncName():
{ {
FuncName fn;
Token n; Token n;
FuncName f;
} }
{ {
n=<NAME> {fn=new FuncName(n.image);} n=<NAME> {f=new FuncName(n.image);}
( "." n=<NAME> {fn.adddot(n.image);} )* ( "." n=<NAME> {f.adddot(n.image);} )*
( ":" n=<NAME> {fn.method=n.image;} )? ( ":" n=<NAME> {f.method=n.image;} )?
{return fn;} {L(f,n); return f;}
} }
Exp.PrimaryExp PrefixExp(): Exp.PrimaryExp PrefixExp():
{ {
Token n; Token n;
Exp e; Exp e;
Exp.PrimaryExp p;
long i = LineInfo();
} }
{ {
n=<NAME> { return Exp.nameprefix(n.image); } n=<NAME> { p=Exp.nameprefix(n.image); L(p,i); return p; }
| "(" e=Exp() ")" { return Exp.parensprefix(e); } | "(" e=Exp() ")" { p=Exp.parensprefix(e); L(p,i); return p; }
} }
Exp.PrimaryExp PrimaryExp(): Exp.PrimaryExp PrimaryExp():
{ {
Exp.PrimaryExp pe; Exp.PrimaryExp p;
long i = LineInfo();
} }
{ {
pe=PrefixExp() ( LOOKAHEAD(2) pe=PostfixOp(pe) )* { return pe; } p=PrefixExp() ( LOOKAHEAD(2) p=PostfixOp(p) )* { L(p,i); return p; }
} }
Exp.PrimaryExp PostfixOp(Exp.PrimaryExp lhs): Exp.PrimaryExp PostfixOp(Exp.PrimaryExp lhs):
@@ -308,12 +346,14 @@ Exp.PrimaryExp PostfixOp(Exp.PrimaryExp lhs):
Token n; Token n;
Exp e; Exp e;
FuncArgs a; FuncArgs a;
Exp.PrimaryExp p;
long i = LineInfo();
} }
{ {
"." n=<NAME> { return Exp.fieldop(lhs, n.image); } "." n=<NAME> { p=Exp.fieldop(lhs, n.image); L(p,i); return p; }
| "[" e=Exp() "]" { return Exp.indexop(lhs, e); } | "[" e=Exp() "]" { p=Exp.indexop(lhs, e); L(p,i); return p; }
| ":" n=<NAME> a=FuncArgs() { return Exp.methodop(lhs, n.image,a); } | ":" n=<NAME> a=FuncArgs() { p=Exp.methodop(lhs, n.image,a); L(p,i); return p; }
| a=FuncArgs() { return Exp.functionop(lhs, a); } | a=FuncArgs() { p=Exp.functionop(lhs, a); L(p,i); return p; }
} }
FuncArgs FuncArgs(): FuncArgs FuncArgs():
@@ -321,29 +361,31 @@ FuncArgs FuncArgs():
List<Exp> el=null; List<Exp> el=null;
TableConstructor tc; TableConstructor tc;
LuaString s; LuaString s;
FuncArgs a;
long i = LineInfo();
} }
{ {
"(" ( el=ExpList() )? ")" { return FuncArgs.explist(el); } "(" ( el=ExpList() )? ")" { a=FuncArgs.explist(el); L(a,i); return a; }
| tc=TableConstructor() { return FuncArgs.tableconstructor(tc); } | tc=TableConstructor() { a=FuncArgs.tableconstructor(tc); L(a,i); return a; }
| s=Str() { return FuncArgs.string(s); } | s=Str() { a=FuncArgs.string(s); L(a,i); return a; }
} }
List<Name> NameList(): List<Name> NameList():
{ {
List<Name> nl = new ArrayList<Name>(); List<Name> l = new ArrayList<Name>();
Token name; Token name;
} }
{ {
name=<NAME> {nl.add(new Name(name.image));} ( LOOKAHEAD(2) "," name=<NAME> {nl.add(new Name(name.image));} )* {return nl;} name=<NAME> {l.add(new Name(name.image));} ( LOOKAHEAD(2) "," name=<NAME> {l.add(new Name(name.image));} )* {return l;}
} }
List<Exp> ExpList(): List<Exp> ExpList():
{ {
List<Exp> el = new ArrayList<Exp>(); List<Exp> l = new ArrayList<Exp>();
Exp e; Exp e;
} }
{ {
e=Exp() {el.add(e);} ( "," e=Exp() {el.add(e);} )* {return el;} e=Exp() {l.add(e);} ( "," e=Exp() {l.add(e);} )* {return l;}
} }
Exp SimpleExp(): Exp SimpleExp():
@@ -351,18 +393,19 @@ Exp SimpleExp():
Token n; Token n;
LuaString s; LuaString s;
Exp e; Exp e;
TableConstructor tc; TableConstructor c;
FuncBody fb; FuncBody b;
long i = LineInfo();
} }
{ {
<NIL> { return Exp.constant(LuaValue.NIL); } <NIL> { e=Exp.constant(LuaValue.NIL); L(e,i); return e; }
| <TRUE> { return Exp.constant(LuaValue.TRUE); } | <TRUE> { e=Exp.constant(LuaValue.TRUE); L(e,i); return e; }
| <FALSE> { return Exp.constant(LuaValue.FALSE); } | <FALSE> { e=Exp.constant(LuaValue.FALSE); L(e,i); return e; }
| n=<NUMBER> { return Exp.numberconstant(n.image); } | n=<NUMBER> { e=Exp.numberconstant(n.image); L(e,i); return e; }
| s=Str() { return Exp.constant(s); } | s=Str() { e=Exp.constant(s); L(e,i); return e; }
| "..." { return Exp.varargs(); } | "..." { e=Exp.varargs(); L(e,i); return e; }
| tc=TableConstructor() { return Exp.tableconstructor(tc); } | c=TableConstructor() { e=Exp.tableconstructor(c); L(e,i); return e; }
| fb=FunctionCall() { return Exp.anonymousfunction(fb); } | b=FunctionCall() { e=Exp.anonymousfunction(b); L(e,i); return e; }
| e=PrimaryExp() { return e; } | e=PrimaryExp() { return e; }
} }
@@ -382,66 +425,75 @@ Exp Exp():
{ {
Exp e,s; Exp e,s;
int op; int op;
long i = LineInfo();
} }
{ {
( e=SimpleExp() | op=Unop() s=Exp() {e=Exp.unaryexp(op,s);}) ( e=SimpleExp() | op=Unop() s=Exp() {e=Exp.unaryexp(op,s);})
(LOOKAHEAD(2) op=Binop() s=Exp() {e=Exp.binaryexp(e,op,s);} )* { return e; } (LOOKAHEAD(2) op=Binop() s=Exp() {e=Exp.binaryexp(e,op,s);} )* { L(e,i); return e; }
} }
FuncBody FunctionCall(): FuncBody FunctionCall():
{ {
FuncBody fb; FuncBody b;
long i = LineInfo();
} }
{ {
<FUNCTION> fb=FuncBody() { return fb; } <FUNCTION> b=FuncBody() { L(b,i); return b; }
} }
FuncBody FuncBody(): FuncBody FuncBody():
{ {
ParList pl=null; ParList pl=null;
Block b; Block b;
FuncBody f;
long i = LineInfo();
} }
{ {
"(" ( pl=ParList() )? ")" b=Block() <END> { return new FuncBody(pl,b); } "(" ( pl=ParList() )? ")" b=Block() <END> { f=new FuncBody(pl,b); L(f,i); return f; }
} }
ParList ParList(): ParList ParList():
{ {
List<Name> nl=null; List<Name> l=null;
boolean v=false; boolean v=false;
ParList p;
long i = LineInfo();
} }
{ {
nl=NameList() ( "," "..." { v=true; } )? { return new ParList(nl,v); } l=NameList() ( "," "..." { v=true; } )? { p=new ParList(l,v); L(p,i); return p; }
| "..." { return new ParList(null,true); ; } | "..." { p=new ParList(null,true); L(p,i); return p; }
} }
TableConstructor TableConstructor(): TableConstructor TableConstructor():
{ {
TableConstructor tc = new TableConstructor(); TableConstructor c = new TableConstructor();
List<TableField> fl = null; List<TableField> l = null;
long i = LineInfo();
} }
{ {
"{" ( fl=FieldList() {tc.fields=fl;} )? "}" { return tc; } "{" ( l=FieldList() {c.fields=l;} )? "}" { L(c,i); return c; }
} }
List<TableField> FieldList(): List<TableField> FieldList():
{ {
List<TableField> fl = new ArrayList<TableField>(); List<TableField> l = new ArrayList<TableField>();
TableField f; TableField f;
} }
{ {
f=Field() {fl.add(f);} (LOOKAHEAD(2) FieldSep() f=Field() {fl.add(f);})* (FieldSep())? { return fl; } f=Field() {l.add(f);} (LOOKAHEAD(2) FieldSep() f=Field() {l.add(f);})* (FieldSep())? { return l; }
} }
TableField Field(): TableField Field():
{ {
Token name; Token name;
Exp exp,rhs; Exp exp,rhs;
TableField f;
long i = LineInfo();
} }
{ {
"[" exp=Exp() "]" "=" rhs=Exp() { return TableField.keyedField(exp,rhs); } "[" exp=Exp() "]" "=" rhs=Exp() { f=TableField.keyedField(exp,rhs); L(f,i); return f; }
| LOOKAHEAD(2) name=<NAME> "=" rhs=Exp() { return TableField.namedField(name.image,rhs); } | LOOKAHEAD(2) name=<NAME> "=" rhs=Exp() { f=TableField.namedField(name.image,rhs); L(f,i); return f; }
| rhs=Exp() { return TableField.listField(rhs); } | rhs=Exp() { f=TableField.listField(rhs); L(f,i); return f; }
} }
void FieldSep(): void FieldSep():

View File

@@ -21,7 +21,7 @@
******************************************************************************/ ******************************************************************************/
package org.luaj.vm2.ast; package org.luaj.vm2.ast;
public class Chunk { public class Chunk extends SyntaxElement {
public final Block block; public final Block block;
public Chunk(Block b) { public Chunk(Block b) {

View File

@@ -25,7 +25,7 @@ import org.luaj.vm2.Lua;
import org.luaj.vm2.LuaValue; import org.luaj.vm2.LuaValue;
abstract abstract
public class Exp { public class Exp extends SyntaxElement {
abstract public void accept(Visitor visitor); abstract public void accept(Visitor visitor);
public static Exp constant(LuaValue value) { public static Exp constant(LuaValue value) {

View File

@@ -26,7 +26,7 @@ import java.util.List;
import org.luaj.vm2.LuaString; import org.luaj.vm2.LuaString;
public class FuncArgs { public class FuncArgs extends SyntaxElement {
public final List<Exp> exps; public final List<Exp> exps;

View File

@@ -21,7 +21,7 @@
******************************************************************************/ ******************************************************************************/
package org.luaj.vm2.ast; package org.luaj.vm2.ast;
public class FuncBody { public class FuncBody extends SyntaxElement {
public ParList parlist; public ParList parlist;
public Block block; public Block block;
public NameScope scope; public NameScope scope;

View File

@@ -24,7 +24,7 @@ package org.luaj.vm2.ast;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
public class FuncName { public class FuncName extends SyntaxElement {
// example: a.b.c.d:e // example: a.b.c.d:e
// initial base name: "a" // initial base name: "a"

View File

@@ -24,7 +24,7 @@ package org.luaj.vm2.ast;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
public class ParList { public class ParList extends SyntaxElement {
public static final List<Name> EMPTY_NAMELIST = new ArrayList<Name>(); public static final List<Name> EMPTY_NAMELIST = new ArrayList<Name>();
public static final ParList EMPTY_PARLIST = new ParList(EMPTY_NAMELIST,false); public static final ParList EMPTY_PARLIST = new ParList(EMPTY_NAMELIST,false);

View File

@@ -26,7 +26,7 @@ import java.util.List;
import org.luaj.vm2.ast.Exp.VarExp; import org.luaj.vm2.ast.Exp.VarExp;
abstract abstract
public class Stat { public class Stat extends SyntaxElement {
public abstract void accept(Visitor visitor); public abstract void accept(Visitor visitor);
public static Stat block(Block block) { public static Stat block(Block block) {

View File

@@ -0,0 +1,40 @@
/*******************************************************************************
* Copyright (c) 2012 Luaj.org. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
******************************************************************************/
package org.luaj.vm2.ast;
/** Base class for syntax elements of the parse tree that appear in source files.
* The LuaParser class will fill these values out during parsing for use in
* syntax highlighting, for example.
*/
public class SyntaxElement {
/** The line number on which the element begins. */
public int beginLine;
/** The column at which the element begins. */
public short beginColumn;
/** The line number on which the element ends. */
public int endLine;
/** The column at which the element ends. */
public short endColumn;
}

View File

@@ -21,7 +21,7 @@
******************************************************************************/ ******************************************************************************/
package org.luaj.vm2.ast; package org.luaj.vm2.ast;
public class TableField { public class TableField extends SyntaxElement {
public final Exp index; public final Exp index;
public final String name; public final String name;

File diff suppressed because it is too large Load Diff

View File

@@ -30,10 +30,10 @@ public class SimpleCharStream
protected char[] buffer; protected char[] buffer;
protected int maxNextCharInd = 0; protected int maxNextCharInd = 0;
protected int inBuf = 0; protected int inBuf = 0;
protected int tabSize = 8; protected int tabSize = 1;
protected void setTabSize(int i) { tabSize = i; } public void setTabSize(int i) { tabSize = i; }
protected int getTabSize(int i) { return tabSize; } public int getTabSize(int i) { return tabSize; }
protected void ExpandBuff(boolean wrapAround) protected void ExpandBuff(boolean wrapAround)