Implement tail call optimization to avoid stack overflow.

src/core/org/luaj/vm2/LuaClosure.java

@@ -302,28 +302,17 @@ public class LuaClosure extends LuaFunction {
 					}
 					
 				case Lua.OP_TAILCALL: /*	A B C	return R(A)(R(A+1), ... ,R(A+B-1))		*/
-					switch ( i & (Lua.MASK_B | Lua.MASK_C) ) {
+					switch ( i & Lua.MASK_B ) {
-					case (1<<Lua.POS_B) | (0<<Lua.POS_C): return stack[a].invoke(NONE);
+					case (1<<Lua.POS_B): return new TailcallVarargs(stack[a], NONE);
-					case (2<<Lua.POS_B) | (0<<Lua.POS_C): return stack[a].invoke(stack[a+1]);
+					case (2<<Lua.POS_B): return new TailcallVarargs(stack[a], stack[a+1]);
-					case (1<<Lua.POS_B) | (1<<Lua.POS_C): stack[a].call(); return NONE;
+					case (3<<Lua.POS_B): return new TailcallVarargs(stack[a], varargsOf(stack[a+1],stack[a+2]));
-					case (2<<Lua.POS_B) | (1<<Lua.POS_C): stack[a].call(stack[a+1]); return NONE;
+					case (4<<Lua.POS_B): return new TailcallVarargs(stack[a], varargsOf(stack[a+1],stack[a+2],stack[a+3]));
-					case (3<<Lua.POS_B) | (1<<Lua.POS_C): stack[a].call(stack[a+1],stack[a+2]); return NONE;
-					case (4<<Lua.POS_B) | (1<<Lua.POS_C): stack[a].call(stack[a+1],stack[a+2],stack[a+3]); return NONE;
-					case (1<<Lua.POS_B) | (2<<Lua.POS_C): return stack[a].call();
-					case (2<<Lua.POS_B) | (2<<Lua.POS_C): return stack[a].call(stack[a+1]);
-					case (3<<Lua.POS_B) | (2<<Lua.POS_C): return stack[a].call(stack[a+1],stack[a+2]);
-					case (4<<Lua.POS_B) | (2<<Lua.POS_C): return stack[a].call(stack[a+1],stack[a+2],stack[a+3]);
 					default:
 						b = i>>>23;
-						c = (i>>14)&0x1ff;
 						v = b>0? 
 							varargsOf(stack,a+1,b-1): // exact arg count
 							varargsOf(stack, a+1, top-v.narg()-(a+1), v); // from prev top 
-						switch ( c ) {
+						return new TailcallVarargs( stack[a], v );
-						case 1: stack[a].invoke(v); return NONE;
-						case 2: return stack[a].invoke(v).arg1();
-						default: return stack[a].invoke(v);
-						}
 					}
 					
 				case Lua.OP_RETURN: /*	A B	return R(A), ... ,R(A+B-2)	(see note)	*/
@@ -461,6 +450,4 @@ public class LuaClosure extends LuaFunction {
 	protected void setUpvalue(int i, LuaValue v) {
 		upValues[i].setValue(v);
 	}
-	
-
 }

src/core/org/luaj/vm2/TailcallVarargs.java

@@ -0,0 +1,71 @@
+/*******************************************************************************
+* Copyright (c) 2010 Luaj.org. All rights reserved.
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in
+* all copies or substantial portions of the Software.
+* 
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+* THE SOFTWARE.
+******************************************************************************/
+package org.luaj.vm2;
+
+public class TailcallVarargs extends Varargs {
+
+	private LuaValue func;
+	private Varargs args;
+	private Varargs result;
+	
+	public TailcallVarargs(LuaValue f, Varargs args) {
+		this.func = f;
+		this.args = args;
+	}
+	
+	private void eval() {
+		TailcallVarargs nextcall = this;
+		do {
+			LuaValue func = nextcall.func;
+			Varargs args = nextcall.args;
+			nextcall = null;
+			Varargs r = func.invoke(args);
+			
+			if (r instanceof TailcallVarargs)
+				nextcall = (TailcallVarargs)r;
+			else
+				this.result = r;
+			
+		} while (nextcall != null);
+	}
+	
+	@Override
+	public LuaValue arg( int i ) {
+		if ( result == null )
+			eval();
+		return result.arg(i);
+	}
+	
+	@Override
+	public LuaValue arg1() {
+		if (result == null)
+			eval();
+		return result.arg1();
+	}
+	
+	@Override
+	public int narg() {
+		if (result == null)
+			eval();
+		return result.narg();
+	}
+}

test/lua/tailcalls.lua

@@ -98,3 +98,46 @@ print(factorial(5))
 
 local result2 = f(-1234)
 print( result2 )
+
+local function fib_bad(n)
+	local function helper(i, a, b)
+		if i >= n then
+			return a
+		else
+			-- not recognized by luac as a tailcall!
+			local result = helper(i + 1, b, a + b)
+			return result
+		end
+	end
+	return helper(1, 1, 1)
+end
+
+local function fib_good(n)
+	local function helper(i, a, b)
+		if i >= n then
+			return a
+		else
+			-- must be a tail call!
+			return helper(i + 1, b, a + b)
+		end
+	end
+	return helper(1, 1, 1)
+end
+
+print(pcall(fib_bad, 30))
+print(pcall(fib_bad, 25000))
+print(pcall(fib_good, 30))
+print(pcall(fib_good, 25000))
+
+local function fib_all(n, i, a, b)
+	i = i or 1
+	a = a or 1
+	b = b or 1
+	if i >= n then
+		return
+	else
+		return a, fib_all(n, i+1, b, a+b)
+	end
+end
+
+print(fib_all(10))