add compiler to vm as add-on

src/addon/java/lua/addon/compile/CheckCode.java

@@ -0,0 +1,12 @@
+package lua.addon.compile;
+
+import lua.io.Proto;
+
+public class CheckCode {
+
+	public static boolean checkcode(Proto f) {
+		// TODO: port logic from ldebug.c
+		return true;
+	}
+
+}

src/addon/java/lua/addon/compile/Compiler.java

@@ -0,0 +1,66 @@
+package lua.addon.compile;
+
+import java.io.Reader;
+import java.util.Hashtable;
+
+import lua.io.Proto;
+import lua.value.LString;
+
+public class Compiler {
+
+	public int nCcalls;
+	
+	public static Proto compile( Reader reader, String name ) {
+		Compiler compiler = new Compiler();
+		return compiler.luaY_parser(reader, name);
+	}
+
+			
+	Proto luaY_parser(Reader z, String name) {
+		LexState lexstate = new LexState(this, z);
+		FuncState funcstate = new FuncState();
+		// lexstate.buff = buff;
+		lexstate.setinput( this, z, new LString(name) );
+		lexstate.open_func(funcstate);
+		/* main func. is always vararg */
+		funcstate.varargflags = LuaC.VARARG_ISVARARG;
+		funcstate.f.is_vararg = true;
+		funcstate.f.source = new LString("@"+name);
+		lexstate.next(); /* read first token */
+		lexstate.chunk();
+		lexstate.check(LexState.TK_EOS);
+		lexstate.close_func();
+		LuaC._assert (funcstate.prev == null);
+		LuaC._assert (funcstate.f.nups == 0);
+		LuaC._assert (lexstate.fs == null);
+		return funcstate.f;
+	}
+	
+	
+	// these string utilities were originally 
+	// part of the Lua C State object, so we have 
+	// left them here for now until we deterimine 
+	// what exact purpose they serve.
+	
+	// table of all strings -> TString mapping. 
+	// this includes reserved words that must be identified 
+	// during lexical analysis for the compiler to work at all.
+	// TODO: consider moving this to LexState and simplifying 
+	// all the various "newstring()" like functions
+	Hashtable strings = new Hashtable();
+	
+	public LString newTString(String s) {
+		LString t = (LString) strings.get(s);
+		if ( t == null )
+			strings.put( s, t = new LString(s) );
+		return t;
+	}
+
+	public String pushfstring(String string) {
+		return string;
+	}
+
+	public LString newlstr(char[] chars, int offset, int len) {
+		return newTString( new String(chars,offset,len) );
+	}
+}

src/addon/java/lua/addon/compile/DumpState.java

@@ -0,0 +1,174 @@
+package lua.addon.compile;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import lua.io.Proto;
+import lua.value.LBoolean;
+import lua.value.LNil;
+import lua.value.LNumber;
+import lua.value.LString;
+import lua.value.LValue;
+
+public class DumpState {
+
+	/** mark for precompiled code (`<esc>Lua') */
+	public static final String LUA_SIGNATURE	= "\033Lua";
+
+	/** for header of binary files -- this is Lua 5.1 */
+	public static final int LUAC_VERSION		= 0x51;
+
+	/** for header of binary files -- this is the official format */
+	public static final int LUAC_FORMAT		= 0;
+
+	/** size of header of binary files */
+	public static final int LUAC_HEADERSIZE		= 12;
+
+	/** expected lua header bytes */
+	private static final byte[] LUAC_HEADER_SIGNATURE = { '\033', 'L', 'u', 'a' };
+
+	// header fields
+	private static final int IS_LITTLE_ENDIAN = 0;
+	private static final int SIZEOF_INT = 4;
+	private static final int SIZEOF_SIZET = 4;
+	private static final int SIZEOF_INSTRUCTION = 4;
+	private static final int SIZEOF_LUA_NUMBER = 8;
+	private static final int IS_NUMBER_INTEGRAL = 0;
+
+	// types of lua constants
+	private static final int LUA_TNIL		= 0;
+	private static final int LUA_TBOOLEAN		= 1;
+	private static final int LUA_TLIGHTUSERDATA	= 2;
+	private static final int LUA_TNUMBER		= 3;
+	private static final int LUA_TSTRING		= 4;
+	private static final int LUA_TTABLE		= 5;
+	private static final int LUA_TFUNCTION		= 6;
+	private static final int LUA_TUSERDATA		= 7;
+	private static final int LUA_TTHREAD		= 8;
+
+	DataOutputStream writer;
+	boolean strip;
+	int status;
+
+	public DumpState(OutputStream w, boolean strip) {
+		this.writer = new DataOutputStream( w );
+		this.strip = strip;
+		this.status = 0;
+	}
+
+	void dumpBlock(final byte[] b, int size) throws IOException {
+		writer.write(b, 0, size);
+	}
+
+	void dumpChar(int b) throws IOException {
+		writer.write( b );
+	}
+
+	void dumpInt(int x) throws IOException {
+		writer.writeInt(x);
+	}
+	
+	void dumpString(LString s) throws IOException {
+		final int len = s.length();
+		dumpInt( len+1 );
+		s.write( writer, 0, len );
+		writer.write( 0 );
+	}
+	
+	void dumpNumber(double d) throws IOException {
+		long l = Double.doubleToLongBits(d);
+		writer.writeLong(l);
+	}
+
+	void dumpCode( final Proto f ) throws IOException {
+		int n = f.code.length;
+		dumpInt( n );
+		for ( int i=0; i<n; i++ )
+			dumpInt( f.code[i] );
+	}
+	
+	void dumpConstants(final Proto f) throws IOException {
+		int i, n = f.k.length;
+		dumpInt(n);
+		for (i = 0; i < n; i++) {
+			final LValue o = f.k[i];
+			if (o == LNil.NIL) {
+				writer.write(LUA_TNIL);
+				// do nothing more
+			} else if (o instanceof LBoolean) {
+				writer.write(LUA_TBOOLEAN);
+				dumpChar(o.luaAsBoolean() ? 1 : 0);
+			} else if (o instanceof LNumber) {
+				writer.write(LUA_TNUMBER);
+				dumpNumber(o.luaAsDouble());
+			} else if (o instanceof LString) {
+				writer.write(LUA_TSTRING);
+				dumpString((LString) o);
+			} else {
+				throw new IllegalArgumentException("bad type for " + o);
+			}
+		}
+		n = f.p.length;
+		dumpInt(n);
+		for (i = 0; i < n; i++)
+			dumpFunction(f.p[i], f.source);
+	}
+	
+	void dumpDebug(final Proto f) throws IOException {
+		int i, n;
+		n = (strip) ? 0 : f.lineinfo.length;
+		dumpInt(n);
+		for (i = 0; i < n; i++)
+			dumpInt(f.lineinfo[i]);
+		n = (strip) ? 0 : f.locvars.length;
+		dumpInt(n);
+		for (i = 0; i < n; i++) {
+			dumpString(f.locvars[i].varname);
+			dumpInt(f.locvars[i].startpc);
+			dumpInt(f.locvars[i].endpc);
+		}
+		n = (strip) ? 0 : f.upvalues.length;
+		dumpInt(n);
+		for (i = 0; i < n; i++)
+			dumpString(f.upvalues[i]);
+	}
+	
+	void dumpFunction(final Proto f, final LString string) throws IOException {
+		if ( f.source == null || f.source.equals(string) || strip )
+			dumpInt(0);
+		else
+			dumpString(f.source);
+		dumpInt(f.linedefined);
+		dumpInt(f.lastlinedefined);
+		dumpChar(f.nups);
+		dumpChar(f.numparams);
+		dumpChar(f.is_vararg? 1: 0);
+		dumpChar(f.maxstacksize);
+		dumpCode(f);
+		dumpConstants(f);
+		dumpDebug(f);
+	}
+
+	void dumpHeader() throws IOException {
+		writer.write( LUAC_HEADER_SIGNATURE );
+		writer.write( LUAC_VERSION );
+		writer.write( LUAC_FORMAT );
+		writer.write( IS_LITTLE_ENDIAN );
+		writer.write( SIZEOF_INT );
+		writer.write( SIZEOF_SIZET );
+		writer.write( SIZEOF_INSTRUCTION );
+		writer.write( SIZEOF_LUA_NUMBER );
+		writer.write( IS_NUMBER_INTEGRAL );
+	}
+
+	/*
+	** dump Lua function as precompiled chunk
+	*/
+	public static int dump( Proto f, OutputStream w, boolean strip ) throws IOException {
+		DumpState D = new DumpState(w,strip);
+		D.dumpHeader();
+		D.dumpFunction(f,null);
+		return D.status;
+	}
+}

src/addon/java/lua/addon/compile/InstructionPtr.java

@@ -0,0 +1,17 @@
+
+package lua.addon.compile;
+
+class InstructionPtr {
+	final int[] code;
+	final int idx;
+	InstructionPtr(int[] code, int idx ) {
+		this.code = code;
+		this.idx = idx;
+	}
+	int get() {
+		return code[idx];
+	}
+	void set(int value) {
+		code[idx] = value;
+	}
+}

src/addon/java/lua/addon/compile/IntPtr.java

@@ -0,0 +1,10 @@
+package lua.addon.compile;
+
+public class IntPtr {
+	int i;
+	IntPtr() {
+	}
+	IntPtr(int value) {
+		this.i = value;
+	}
+}

src/addon/java/lua/addon/compile/LuaC.java

@@ -0,0 +1,125 @@
+package lua.addon.compile;
+
+import lua.Lua;
+import lua.io.LocVars;
+import lua.io.Proto;
+import lua.value.LString;
+import lua.value.LValue;
+
+/**
+ * Additional constants and utilities required for the compiler, 
+ * but not required for the interpreter.
+ * 
+ */
+public class LuaC extends Lua {
+	protected static void _assert(boolean b) {		
+		if (!b) throw new RuntimeException("assert failed");
+	}
+	
+	static final int LUAI_MAXUPVALUES = 60;
+	static final int LUAI_MAXVARS = 200;
+	static final int LFIELDS_PER_FLUSH = 50;
+	static final int NO_REG		 = MAXARG_A;
+	
+	/* masks for new-style vararg */
+	static final int     VARARG_HASARG		= 1;
+	static final int     VARARG_ISVARARG	= 2;
+	static final int     VARARG_NEEDSARG	= 4;
+
+
+	/* OpMode - basic instruction format */
+	static final int 
+		iABC = 0,
+		iABx = 1,
+		iAsBx = 2;
+
+	/* OpArgMask */
+	static final int 
+	  OpArgN = 0,  /* argument is not used */
+	  OpArgU = 1,  /* argument is used */
+	  OpArgR = 2,  /* argument is a register or a jump offset */
+	  OpArgK = 3;   /* argument is a constant or register/constant */
+
+
+	static void SET_OPCODE(InstructionPtr i,int o) {
+		i.set( ( i.get() & (MASK_NOT_OP)) | ((o << POS_OP) & MASK_OP) );
+	}
+	
+	static void SETARG_A(InstructionPtr i,int u) {
+		i.set( ( i.get() & (MASK_NOT_A)) | ((u << POS_A) & MASK_A) );
+	}
+
+	static void SETARG_B(InstructionPtr i,int u) {
+		i.set( ( i.get() & (MASK_NOT_B)) | ((u << POS_B) & MASK_B) );
+	}
+
+	static void SETARG_C(InstructionPtr i,int u) {
+		i.set( ( i.get() & (MASK_NOT_C)) | ((u << POS_C) & MASK_C) );
+	}
+	
+	static void SETARG_Bx(InstructionPtr i,int u) {
+		i.set( ( i.get() & (MASK_NOT_Bx)) | ((u << POS_Bx) & MASK_Bx) );
+	}
+	
+	static void SETARG_sBx(InstructionPtr i,int u) {
+		SETARG_Bx( i, u + MAXARG_sBx );
+	}
+
+	static int CREATE_ABC(int o, int a, int b, int c) {
+		return ((o << POS_OP) & MASK_OP) |
+				((a << POS_A) & MASK_A) |
+				((b << POS_B) & MASK_B) |
+				((c << POS_C) & MASK_C) ;
+	}
+	
+	static int CREATE_ABx(int o, int a, int bc) {
+		return ((o << POS_OP) & MASK_OP) |
+				((a << POS_A) & MASK_A) |
+				((bc << POS_Bx) & MASK_Bx) ;
+ 	}
+
+	// vector reallocation
+	
+	static LValue[] realloc(LValue[] v, int n) {
+		LValue[] a = new LValue[n];
+		if ( v != null )
+			System.arraycopy(v, 0, a, 0, Math.min(v.length,n));
+		return a;
+	}
+
+	static Proto[] realloc(Proto[] v, int n) {
+		Proto[] a = new Proto[n];
+		if ( v != null )
+			System.arraycopy(v, 0, a, 0, Math.min(v.length,n));
+		return a;
+	}
+
+	static LString[] realloc(LString[] v, int n) {
+		LString[] a = new LString[n];
+		if ( v != null )
+			System.arraycopy(v, 0, a, 0, Math.min(v.length,n));
+		return a;
+	}
+
+	static LocVars[] realloc(LocVars[] v, int n) {
+		LocVars[] a = new LocVars[n];
+		if ( v != null )
+			System.arraycopy(v, 0, a, 0, Math.min(v.length,n));
+		return a;
+	}
+
+	static int[] realloc(int[] v, int n) {
+		int[] a = new int[n];
+		if ( v != null )
+			System.arraycopy(v, 0, a, 0, Math.min(v.length,n));
+		return a;
+	}
+
+	static char[] realloc(char[] v, int n) {
+		char[] a = new char[n];
+		if ( v != null )
+			System.arraycopy(v, 0, a, 0, Math.min(v.length,n));
+		return a;
+	}
+
+}

src/test/compile/repack.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+LUA_HOME=/cygdrive/c/programs/lua5.1
+#DIRS="lua5.1-tests regressions"
+DIRS="regressions"
+for d in $DIRS; do
+
+	# clean out the old
+	rm -f $d/*.luac
+	
+	# compile the tests
+	TESTS=`echo $d/*.lua`
+	for x in $TESTS; do
+		echo compiling $x
+		luac -o ${x}c ${x}
+	done
+	
+	# rebuild the directory
+	rm -f ${d}.zip
+	jar -cvf ${d}.zip ${d}
+done

src/test/compile/unpack.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+LUA_HOME=/cygdrive/c/programs/lua5.1
+#DIRS="lua5.1-tests regressions"
+DIRS="regressions"
+for d in $DIRS; do
+
+	# unpack files into the directory
+	rm -rf $d
+	mkdir -p $d
+	jar -xvf $d.zip
+	
+done

src/test/java/lua/addon/compile/AbstractUnitTests.java

@@ -0,0 +1,96 @@
+package lua.addon.compile;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.io.Reader;
+import java.net.URL;
+
+import junit.framework.TestCase;
+
+import lua.Print;
+import lua.StackState;
+import lua.addon.compile.Compiler;
+import lua.addon.compile.DumpState;
+import lua.io.LoadState;
+import lua.io.Proto;
+
+abstract 
+public class AbstractUnitTests extends TestCase {
+	
+	private final String zipfile;
+	private final String dir;
+	
+	public AbstractUnitTests(String zipfile, String dir) {
+		this.zipfile = zipfile;
+		this.dir = dir;
+	}
+
+	protected void doTest( String file ) {
+		try {
+			// load source from jar
+			String path = "jar:file:" + zipfile + "!/" + dir + "/" + file;
+			byte[] lua = bytesFromJar( path );
+			
+			// compile in memory
+			InputStream is = new ByteArrayInputStream( lua );
+	    	Reader r = new InputStreamReader( is );
+	    	Proto p = Compiler.compile(r, dir+"/"+file);
+	    	String actual = protoToString( p );
+			
+			// load expected value from jar
+			byte[] luac = bytesFromJar( path + "c" );
+			Proto e = loadFromBytes( luac, file );
+	    	String expected = protoToString( e );
+
+			// compare results
+			assertEquals( expected, actual );
+			
+			// dump into memory
+			ByteArrayOutputStream baos = new ByteArrayOutputStream();
+			DumpState.dump(p, baos, false);
+			byte[] dumped = baos.toByteArray();
+			
+			// re-undump
+			Proto p2 = loadFromBytes( dumped, file );
+			String actual2 = protoToString( p2 );
+			
+			// compare again
+			assertEquals( actual, actual2 );
+			
+		} catch (IOException e) {
+			fail( e.toString() );
+		}
+	}
+	
+	protected byte[] bytesFromJar(String path) throws IOException {
+		URL url = new URL(path);
+		InputStream is = url.openStream();
+		ByteArrayOutputStream baos = new ByteArrayOutputStream();
+		byte[] buffer = new byte[2048];
+		int n;
+		while ( (n = is.read(buffer)) >= 0 )
+			baos.write( buffer, 0, n );
+		is.close();
+		return baos.toByteArray();
+	}
+	
+	protected Proto loadFromBytes(byte[] bytes, String script) throws IOException {
+		StackState state = new StackState();
+		InputStream is = new ByteArrayInputStream( bytes );
+		return LoadState.undump(state, is, script);
+	}
+	
+	protected String protoToString(Proto p) {
+		ByteArrayOutputStream baos = new ByteArrayOutputStream();
+		PrintStream ps = new PrintStream( baos );
+		Print.ps = ps;
+		new Print().printFunction(p, true);
+		return baos.toString();
+	}
+	
+
+}

src/test/java/lua/addon/compile/CompilerUnitTests.java

@@ -0,0 +1,35 @@
+package lua.addon.compile;
+
+
+public class CompilerUnitTests extends AbstractUnitTests {
+
+	public CompilerUnitTests() {
+		super( "src/test/compile/lua5.1-tests.zip", 
+				"lua5.1-tests" );
+	}
+	
+	public void testAll()        { doTest("all.lua"); }
+	public void testApi()        { doTest("api.lua"); }
+	public void testAttrib()     { doTest("attrib.lua"); }
+	public void testBig()        { doTest("big.lua"); }
+	public void testCalls()      { doTest("calls.lua"); }
+	public void testChecktable() { doTest("checktable.lua"); }
+	public void testClosure()    { doTest("closure.lua"); }
+	public void testCode()       { doTest("code.lua"); }
+	public void testConstruct()  { doTest("constructs.lua"); }
+	public void testDb()         { doTest("db.lua"); }
+	public void testErrors()     { doTest("errors.lua"); }
+	public void testEvents()     { doTest("events.lua"); }
+	public void testFiles()      { doTest("files.lua"); }
+	public void testGc()         { doTest("gc.lua"); }
+	public void testLiterals()   { doTest("literals.lua"); }
+	public void testLocals()     { doTest("locals.lua"); }
+	public void testMain()       { doTest("main.lua"); }
+	public void testMath()       { doTest("math.lua"); }
+	public void testNextvar()    { doTest("nextvar.lua"); }
+	public void testPm()         { doTest("pm.lua"); }
+	public void testSort()       { doTest("sort.lua"); }
+	public void testStrings()    { doTest("strings.lua"); }
+	public void testVararg()     { doTest("vararg.lua"); }
+	public void testVerybig()    { doTest("verybig.lua"); }
+}

src/test/java/lua/addon/compile/RegressionTests.java

@@ -0,0 +1,31 @@
+package lua.addon.compile;
+
+/**
+ * Framework to add regression tests as problem areas are found.
+ * 
+ * To add a new regression test:
+ *  1) run "unpack.sh" in the project root
+ *  2) add a new "lua" file in the "regressions" subdirectory
+ *  3) run "repack.sh" in the project root
+ *  4) add a line to the source file naming the new test
+ * 
+ * After adding a test, check in the zip file 
+ * rather than the individual regression test files.
+ * 
+ * @author jrosebor
+ */
+public class RegressionTests extends AbstractUnitTests {
+	
+	public RegressionTests() {
+		super( "src/test/compile/regressions.zip", 
+				"regressions" );
+	}
+	
+	public void testModulo()         { doTest("modulo.lua"); }
+	public void testConstruct()      { doTest("construct.lua"); }
+	public void testBigAttrs()       { doTest("bigattr.lua"); }
+	public void testControlChars()   { doTest("controlchars.lua"); }
+	public void testComparators()    { doTest("comparators.lua"); }
+	public void testMathRandomseed() { doTest("mathrandomseed.lua"); }
+
+}

src/test/java/lua/addon/compile/SimpleTests.java

@@ -0,0 +1,76 @@
+package lua.addon.compile;
+
+import java.io.Reader;
+import java.io.StringReader;
+
+import junit.framework.TestCase;
+import lua.Print;
+import lua.StackState;
+import lua.addon.compile.Compiler;
+import lua.io.Closure;
+import lua.io.Proto;
+import lua.value.LValue;
+
+public class SimpleTests extends TestCase {
+
+    private void doTest( String script ) {
+		Reader r = new StringReader( script );
+		Proto p = Compiler.compile( r, "script" );
+		assertNotNull( p );
+		Print.printCode( p );
+		
+		// try running the code!
+		StackState state = new StackState();
+		Closure c = new Closure( state, p );
+		state.doCall( c, new LValue[0] );
+	
+    }
+
+    public void testTrivial() {
+		String s = "print( 2 )\n";
+		doTest( s );
+	}
+	
+	public void testAlmostTrivial() {
+		String s = "print( 2 )\n" +
+				"print( 3 )\n";
+		doTest( s );
+	}
+	
+	public void testSimple() {
+		String s = "print( 'hello, world' )\n"+
+			"for i = 2,4 do\n" +
+			"	print( 'i', i )\n" +
+			"end\n";
+		doTest( s );
+	}
+	
+	public void testBreak() {
+		String s = "a=1\n"+
+			"while true do\n"+
+			"  if a>10 then\n"+
+			"     break\n"+
+			"  end\n"+
+			"  a=a+1\n"+
+			"  print( a )\n"+
+			"end\n";
+		doTest( s );
+	}
+	
+	public void testShebang() {
+		String s = "#!../lua\n"+
+			"print( 2 )\n";
+		doTest( s );
+	}
+	
+	public void testInlineTable() {
+		String s = "A = {g=10}\n"+
+			"print( A )\n";
+		doTest( s );
+	}
+
+	public void testEqualsAnd() {
+		String s = "print( 1 == b and b )\n";
+		doTest( s );
+	}
+}