luaj-core/src/main/java/org/luaj/vm2/lib/MathLib.java

@@ -231,6 +231,8 @@ public class MathLib extends TwoArgFunction {
 	static final class fmod extends TwoArgFunction {
 		@Override
 		public LuaValue call(LuaValue xv, LuaValue yv) {
+			if (yv.checkdouble() == 0.0d)
+				return LuaDouble.NAN;
 			if (xv.islong() && yv.islong()) {
 				return valueOf(xv.tolong()%yv.tolong());
 			}