Implement tail calls in generated bytecode classes.

src/core/org/luaj/vm2/LuaClosure.java

@@ -101,10 +101,10 @@ public class LuaClosure extends LuaFunction {
 	}
 
 	public final Varargs invoke(Varargs varargs) {
-		return oninvoke( varargs ).eval();
+		return onInvoke( varargs ).eval();
 	}
 	
-	final Varargs oninvoke(Varargs varargs) {
+	public Varargs onInvoke(Varargs varargs) {
 		LuaValue[] stack = new LuaValue[p.maxstacksize];
 		System.arraycopy(NILS, 0, stack, 0, p.maxstacksize);
 		for ( int i=0; i<p.numparams; i++ )

src/core/org/luaj/vm2/LuaValue.java

@@ -424,10 +424,18 @@ public class LuaValue extends Varargs {
 		default: return new ArrayVarargs(new LuaValue[] {v1,v2},v3); 
 		}
 	}
+	
+	// tail call support
 	public static Varargs tailcallOf(LuaValue func, Varargs args) { 
 		return new TailcallVarargs(func, args);
 	}
 	
+	// called by TailcallVarargs to invoke the function once.  
+	// may return TailcallVarargs to be evaluated by the caller. 
+	public Varargs onInvoke(Varargs args) {
+		return invoke(args);
+	}
+
 	// empty varargs
 	private static final class None extends LuaNil {
 		public LuaValue arg(int i) { return NIL; }

src/core/org/luaj/vm2/TailcallVarargs.java

@@ -32,22 +32,24 @@ public class TailcallVarargs extends Varargs {
 		this.args = args;
 	}
 	
+	public boolean isTailcall() {
+		return true;
+	}
+	
 	public Varargs eval() {
-		TailcallVarargs nextcall = this;
-		do {
-			LuaValue func = nextcall.func;
-			Varargs args = nextcall.args;
-			nextcall = null;
-			Varargs r = func.isclosure()? 
-					((LuaClosure) func).oninvoke(args):
-					func.invoke(args);
-			
-			if (r instanceof TailcallVarargs)
-				nextcall = (TailcallVarargs)r;
-			else
-				this.result = r;
-			
-		} while (nextcall != null);
+		while ( result == null ) {
+			Varargs r = func.onInvoke(args);
+			if (r.isTailcall()) {
+				TailcallVarargs t = (TailcallVarargs) r;
+				func = t.func;
+				args = t.args;
+			}
+			else {
+				result = r;			
+				func = null;
+				args = null;
+			}
+		}
 		return result;
 	}
 	

src/core/org/luaj/vm2/Varargs.java

@@ -52,6 +52,13 @@ public abstract class Varargs {
 	 */
 	public Varargs eval() { return this; }
 	
+	/**
+	 * Return true if this is a TailcallVarargs
+	 * @return true if a tail call, false otherwise
+	 */
+	public boolean isTailcall() {
+		return false;
+	}
 	// -----------------------------------------------------------------------
 	// utilities to get specific arguments and type-check them.
 	// -----------------------------------------------------------------------

src/core/org/luaj/vm2/lib/VarArgFunction.java

@@ -49,17 +49,31 @@ abstract public class VarArgFunction extends LibFunction {
 		return invoke(varargsOf(arg1,arg2,arg3)).arg1();
 	}
 
+	/** 
+	 * Override and implement for the best performance. 
+	 * May not have expected behavior for tail calls. 
+	 * Should not be used if either:
+	 * - function needs to be used as a module
+	 * - function has a possibility of returning a TailcallVarargs
+	 * @param args the arguments to the function call.
+	 */
 	public Varargs invoke(Varargs args) {
 		LuaThread.onCall(this);
 		try {
-			return onInvoke(args);
+			return this.onInvoke(args).eval();
 		} finally {
 			LuaThread.onReturn();
 		}
 	}
 
-	protected Varargs onInvoke(Varargs args) {
-		return NONE;
+	/**
+	 * Override to provide a call implementation that runs in an environment
+	 * that can participate in setfenv, and behaves as expected 
+	 * when returning TailcallVarargs. 
+	 * @param args the arguments to the function call.
+	 */
+	public Varargs onInvoke(Varargs args) {
+		return invoke(args);
 	}
 	
 } 

src/jse/org/luaj/vm2/luajc/JavaBuilder.java

@@ -108,7 +108,7 @@ public class JavaBuilder {
 	private static final ObjectType[] RETURN_TYPE_N  = { TYPE_LUAVALUE, TYPE_LUAVALUE, TYPE_LUAVALUE, TYPE_LUAVALUE, TYPE_VARARGS, };
 	private static final Type[][]     ARG_TYPES_N    = { {}, {TYPE_LUAVALUE}, {TYPE_LUAVALUE,TYPE_LUAVALUE}, {TYPE_LUAVALUE,TYPE_LUAVALUE,TYPE_LUAVALUE}, {TYPE_VARARGS},  };
 	private static final String[][]   ARG_NAMES_N    = { {}, {"arg"}, {"arg1","arg2"}, {"arg1","arg2","arg3"}, {"args"}, };
-	private static final String[]     METH_NAME_N    = { "call", "call", "call", "call", "invoke", };
+	private static final String[]     METH_NAME_N    = { "call", "call", "call", "call", "onInvoke", };
 
 	// varable naming
 	private static final String PREFIX_CONSTANT     = "k";
@@ -156,8 +156,12 @@ public class JavaBuilder {
 			superclassType = SUPERTYPE_VARARGS;
 		for ( int i=0, n=p.code.length; i<n; i++ ) {
 			int inst = p.code[i];
-			if ( Lua.GET_OPCODE(inst) == Lua.OP_RETURN && (Lua.GETARG_B(inst) < 1 || Lua.GETARG_B(inst) > 2) )
+			int o = Lua.GET_OPCODE(inst);
+			if ( (o == Lua.OP_TAILCALL) ||
+			     ((o == Lua.OP_RETURN) && (Lua.GETARG_B(inst) < 1 || Lua.GETARG_B(inst) > 2)) ) {
 				superclassType = SUPERTYPE_VARARGS;
+				break;
+			}
 		}
 		
 		// create class generator