Fix gethook() return values and sethook() behavior when called from hook function.

README.html

@@ -340,6 +340,7 @@ and LuaForge:
 
 Main changes by version:
 <table cellspacing="10"><tr><td><table cellspacing="4">
-<tr><td>&nbsp;&nbsp;<b>1.0</b></td><td>Initial publicly supported release.</td></tr>
-<tr><td>&nbsp;&nbsp;<b>1.0.1</b></td><td>Fix arg check and behavior of xpcall() to leave stack intact.</td></tr>
+<tr valign="top"><td>&nbsp;&nbsp;<b>1.0</b></td><td>Initial publicly supported release.</td></tr>
+<tr valign="top"><td>&nbsp;&nbsp;<b>1.0.1</b></td><td>Fix arg check and behavior of xpcall() to leave stack intact.
+Fix debug.sethook() to when called from hook function.  Fix debug.gethook() return values.</td></tr>
 </table></td></tr></table>

src/core/org/luaj/lib/DebugLib.java

@@ -155,12 +155,16 @@ public class DebugLib extends LFunction {
 	}
 	
 	protected int gethook(LuaState vm) {
-		LuaState threadVm = vm;
-		if ( vm.gettop() >= 2 )
-			threadVm = vm.checkthread(1).vm;
-		vm.pushlvalue(threadVm.gethook());
-		vm.pushinteger(threadVm.gethookmask());
-		vm.pushinteger(threadVm.gethookcount());
+		LuaState threadVm = optthreadvm(vm, 1);
+		LValue hook = threadVm.gethook();
+		int mask = threadVm.gethookmask();
+		int count = threadVm.gethookcount();
+		vm.pushlvalue(hook!=null? hook: LNil.NIL);
+		vm.pushstring(""
+				+((mask&LuaState.LUA_MASKCALL)!=0? "c": "")
+				+((mask&LuaState.LUA_MASKRET) !=0? "r": "")
+				+((mask&LuaState.LUA_MASKLINE)!=0? "l": ""));
+		vm.pushinteger(count);
 		return 3;
 	}
 

src/core/org/luaj/vm/LuaState.java

@@ -105,7 +105,7 @@ public class LuaState extends Lua {
 
     // debug hooks - these MUST NOT be initialized, 
 	// so that a later obfuscation step can decide to remove them.
-	private boolean hooksenabled;
+	private boolean hooksenabled,inhook;
     private int hookmask;
     private int hookcount;
     private LFunction hookfunc;
@@ -2503,14 +2503,15 @@ public class LuaState extends Lua {
     }
 	
     private void debugCallHook(int mask, int line) {
-    	int oldmask = hookmask;
+    	if ( inhook )
+    		return;
         int oldtop  = top;
         int oldbase = base;
         int oldcc   = cc;
         int oldnresults = nresults;
         int beyond  = (cc>=0? base+calls[cc].closure.p.maxstacksize: top);
     	try {
-	    	hookmask = 0;
+    		inhook = true;
 
 	    	// adjust base and top to beyond call frame
 	    	top = base = beyond;
@@ -2541,7 +2542,7 @@ public class LuaState extends Lua {
     		base     = oldbase;
     		top      = oldtop;
     		nresults = oldnresults;
-    		hookmask = oldmask ;
+    		inhook   = false;
     	}
 	}
 }