Fix bug in Globals.UTF8Stream affecting loading from Readers. Add buffered input for compiling and loading of scripts.

README.html

@@ -968,6 +968,8 @@ Files are no longer hosted at LuaForge.
 <li>Pass user-supplied ScriptContext to script engine evaluation (fixes issue #21).</li>
 <li>Autoflush and encode written bytes in script contexts (fixes issue #20).</li>
 <li>Rename Globals.FINDER to Globals.finder.</li>
+<li>Fix bug in Globals.UTF8Stream affecting loading from Readers.</li>
+<li>Add buffered input for compiling and loading of scripts.</li>
 
 </ul></td></tr>
 </table></td></tr></table>

src/core/org/luaj/vm2/Globals.java

@@ -231,7 +231,7 @@ public class Globals extends LuaTable {
 			if (undumper == null)
 				error("No undumper.");
 			if (!is.markSupported())
-				is = new MarkStream(is);
+				is = new BufferedStream(is);
 			is.mark(4);
 			final Prototype p = undumper.undump(is, chunkname);
 			if (p != null)
@@ -277,7 +277,8 @@ public class Globals extends LuaTable {
 	/** Reader implementation to read chars from a String in JME or JSE. */
 	static class StrReader extends Reader {
 		final String s;
-		int i = 0, n;
+		int i = 0;
+		final int n;
 		StrReader(String s) {
 			this.s = s;
 			n = s.length();
@@ -285,6 +286,9 @@ public class Globals extends LuaTable {
 		public void close() throws IOException {
 			i = n;
 		}
+		public int read() throws IOException {
+			return i < n ? s.charAt(i++) : -1;
+		}
 		public int read(char[] cbuf, int off, int len) throws IOException {
 			int j = 0;
 			for (; j < len && i < n; ++j, ++i)
@@ -293,54 +297,115 @@ public class Globals extends LuaTable {
 		}
 	}
 
+	/* Abstract base class to provide basic buffered input storage and delivery.
+	 * This class may be moved to its own package in the future.
+	 */
+	public abstract static class AbstractBufferedStream extends InputStream {
+		protected byte[] b;
+		protected int i = 0, j = 0;
+		protected AbstractBufferedStream(int buflen) {
+			this.b = new byte[buflen];
+		}
+		abstract protected int avail() throws IOException;
+		public int read() throws IOException {
+			int a = avail();
+			return (a <= 0 ? -1 : 0xff & b[i++]);
+		}
+		public int read(byte[] b) throws IOException {
+			return read(b, 0, b.length);
+		}
+		public int read(byte[] b, int i0, int n) throws IOException {
+			int a = avail();
+			if (a <= 0) return -1;
+			final int n_read = Math.min(a, n);
+			System.arraycopy(this.b,  i,  b,  i0,  n_read);
+			i += n_read;
+			return n_read;
+		}
+		public long skip(long n) throws IOException {
+			final long k = Math.min(n, j - i);
+			i += k;
+			return k;
+		}		
+		public int available() throws IOException {
+			return j - i;
+		}
+	}
+
 	/**  Simple converter from Reader to InputStream using UTF8 encoding that will work
 	 * on both JME and JSE.
+	 * This class may be moved to its own package in the future.
 	 */
-	static class UTF8Stream extends InputStream {
+	static class UTF8Stream extends AbstractBufferedStream {
-		final char[] c = new char[32];
+		private final char[] c = new char[32];
-		final byte[] b = new byte[96];
+		private final Reader r;
-		int i = 0, j = 0;
-		final Reader r;
 		UTF8Stream(Reader r) {
+			super(96);
 			this.r = r;
 		}
-		public int read() throws IOException {
+		protected int avail() throws IOException {
-			if (i < j)
+			if (i < j) return j - i;
-				return c[i++];
 			int n = r.read(c);
 			if (n < 0)
 				return -1;
+			if (n == 0) {
+				int u = r.read();
+				if (u < 0)
+					return -1;
+				c[0] = (char) u;
+				n = 1;
+			}
 			j = LuaString.encodeToUtf8(c, n, b, i = 0);
-			return b[i++];
+			return j;
+		}
+		public void close() throws IOException {
+			r.close();
 		}
 	}
 	
-	/** Simple InputStream that supports mark.
+	/** Simple buffered InputStream that supports mark.
 	 * Used to examine an InputStream for a 4-byte binary lua signature, 
-	 * and fall back to text input when the signature is not found.
+	 * and fall back to text input when the signature is not found,
+	 * as well as speed up normal compilation and reading of lua scripts.
+	 * This class may be moved to its own package in the future.
 	 */
-	static class MarkStream extends InputStream {
+	public static class BufferedStream extends AbstractBufferedStream {
-		private int[] b;
-		private int i = 0, j = 0;
 		private final InputStream s;
-		MarkStream(InputStream s) {
+		public BufferedStream(InputStream s) {
+			this(128, s);
+		}
+		BufferedStream(int buflen, InputStream s) {
+			super(buflen);
 			this.s = s;
 		}
-		public int read() throws IOException {
+		protected int avail() throws IOException {
-			if (i < j)
+			if (i < j) return j - i;
-				return b[i++];
+			if (j >= b.length) i = j = 0;
-			final int c = s.read();
+			// leave previous bytes in place to implement mark()/reset().
-			if (c < 0)
+			int n = s.read(b, j, b.length - j);
+			if (n < 0)
 				return -1;
-			if (j < b.length) {
+			if (n == 0) {
-				b[j++] = c;
+				int u = s.read();
-				i = j;
+				if (u < 0)
+					return -1;
+				b[j] = (byte) u;
+				n = 1;
 			}
-			return c;
+			j += n;
+			return n;
+		}
+		public void close() throws IOException {
+			s.close();
 		}
 		public synchronized void mark(int n) {
-			b = new int[n];
+			if (i > 0 || n > b.length) {
-			i = j = 0;
+				byte[] dest = n > b.length ? new byte[n] : b;
+				System.arraycopy(b, i, dest, 0, j - i);
+				j -= i;
+				i = 0;
+				b = dest;
+			}
 		}
 		public boolean markSupported() {
 			return true;
@@ -349,5 +414,4 @@ public class Globals extends LuaTable {
 			i = 0;
 		}
 	}
-
 }

src/jse/lua.java

@@ -20,6 +20,7 @@
 * THE SOFTWARE.
 ******************************************************************************/
 
+import java.io.BufferedInputStream;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.FileInputStream;
@@ -195,6 +196,7 @@ public class lua {
 		try {
 			LuaValue c;
 			try {
+				script = new BufferedInputStream(script);
 				c = encoding != null? 
 						globals.load(new InputStreamReader(script, encoding), chunkname):
 						globals.load(script, chunkname, "bt", globals);

src/jse/luac.java

@@ -20,6 +20,7 @@
 * THE SOFTWARE.
 ******************************************************************************/
 
+import java.io.BufferedInputStream;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -170,6 +171,7 @@ public class luac {
 	private void processScript( Globals globals, InputStream script, String chunkname, OutputStream out ) throws IOException {
 		try {
 	        // create the chunk
+			script = new BufferedInputStream(script);
 	        Prototype chunk = encoding != null?
 	        		globals.compilePrototype(new InputStreamReader(script, encoding), chunkname):
 	        		globals.compilePrototype(script, chunkname);

test/junit/org/luaj/vm2/AllTests.java

@@ -67,6 +67,12 @@ public class AllTests {
 		TestSuite bytecodetests = FragmentsTest.suite();
 		suite.addTest(bytecodetests);
 		
+		// I/O tests
+		TestSuite io = new TestSuite("I/O Tests");
+		io.addTestSuite(BufferedStreamTest.class);
+		io.addTestSuite(UTF8StreamTest.class);
+		suite.addTest(io);
+		
 		// prototype compiler
 		TestSuite compiler = new TestSuite("Lua Compiler Tests");
 		compiler.addTestSuite(CompilerUnitTests.class);

test/junit/org/luaj/vm2/BufferedStreamTest.java

@@ -0,0 +1,115 @@
+/*******************************************************************************
+ * Copyright (c) 2014 Luaj.org. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+package org.luaj.vm2;
+
+import java.io.ByteArrayInputStream;
+
+import junit.framework.TestCase;
+
+import org.luaj.vm2.Globals.BufferedStream;
+
+
+public class BufferedStreamTest extends TestCase {
+
+	public BufferedStreamTest() {}
+	
+	private BufferedStream NewBufferedStream(int buflen, String contents) {
+		return new BufferedStream(buflen, new ByteArrayInputStream(contents.getBytes()));
+	}
+	
+	protected void setUp() throws Exception {
+		super.setUp();
+	}
+	
+	public void testReadEmptyStream() throws java.io.IOException {
+		BufferedStream bs = NewBufferedStream(4, "");
+		assertEquals(-1, bs.read());
+		assertEquals(-1, bs.read(new byte[10]));
+		assertEquals(-1, bs.read(new byte[10], 0, 10));
+	}
+	
+	public void testReadByte() throws java.io.IOException {
+		BufferedStream bs = NewBufferedStream(2, "abc");
+		assertEquals('a', bs.read());
+		assertEquals('b', bs.read());
+		assertEquals('c', bs.read());
+		assertEquals(-1, bs.read());
+	}
+	
+	public void testReadByteArray() throws java.io.IOException {
+		byte[] array = new byte[3];
+		BufferedStream bs = NewBufferedStream(4, "abcdef");
+		assertEquals(3, bs.read(array));
+		assertEquals("abc", new String(array));
+		assertEquals(1, bs.read(array));
+		assertEquals("d", new String(array, 0, 1));
+		assertEquals(2, bs.read(array));
+		assertEquals("ef", new String(array, 0, 2));
+		assertEquals(-1, bs.read());
+	}
+	
+	public void testReadByteArrayOffsetLength() throws java.io.IOException {
+		byte[] array = new byte[10];
+		BufferedStream bs = NewBufferedStream(8, "abcdefghijklmn");
+		assertEquals(4, bs.read(array, 0, 4));
+		assertEquals("abcd", new String(array, 0, 4));
+		assertEquals(4, bs.read(array, 2, 8));
+		assertEquals("efgh", new String(array, 2, 4));
+		assertEquals(6, bs.read(array, 0, 10));
+		assertEquals("ijklmn", new String(array, 0, 6));
+		assertEquals(-1, bs.read());
+	}
+	
+	public void testMarkOffsetBeginningOfStream() throws java.io.IOException {
+		byte[] array = new byte[4];
+		BufferedStream bs = NewBufferedStream(8, "abcdefghijkl");
+		assertEquals(true, bs.markSupported());
+		bs.mark(4);
+		assertEquals(4, bs.read(array));
+		assertEquals("abcd", new String(array));
+		bs.reset();
+		assertEquals(4, bs.read(array));
+		assertEquals("abcd", new String(array));
+		assertEquals(4, bs.read(array));
+		assertEquals("efgh", new String(array));
+		assertEquals(4, bs.read(array));
+		assertEquals("ijkl", new String(array));
+		assertEquals(-1, bs.read());
+	}
+
+	public void testMarkOffsetMiddleOfStream() throws java.io.IOException {
+		byte[] array = new byte[4];
+		BufferedStream bs = NewBufferedStream(8, "abcdefghijkl");
+		assertEquals(true, bs.markSupported());
+		assertEquals(4, bs.read(array));
+		assertEquals("abcd", new String(array));
+		bs.mark(4);
+		assertEquals(4, bs.read(array));
+		assertEquals("efgh", new String(array));
+		bs.reset();
+		assertEquals(4, bs.read(array));
+		assertEquals("efgh", new String(array));
+		assertEquals(4, bs.read(array));
+		assertEquals("ijkl", new String(array));
+		assertEquals(-1, bs.read());
+	}
+}

test/junit/org/luaj/vm2/UTF8StreamTest.java

@@ -0,0 +1,41 @@
+/*******************************************************************************
+ * Copyright (c) 2014 Luaj.org. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+package org.luaj.vm2;
+
+import junit.framework.TestCase;
+
+import org.luaj.vm2.lib.jse.JsePlatform;
+
+public class UTF8StreamTest  extends TestCase {
+
+	public void testUtf8CharsInStream() {
+		String script = "x = \"98\u00b0: today's temp!\"\n"
+				+ "print('x = ', x)\n"
+				+ "return x";
+		Globals globals = JsePlatform.standardGlobals();
+		LuaValue chunk = globals.load(script);
+		LuaValue result = chunk.call();
+		String str = result.tojstring();
+		assertEquals("98\u00b0: today's temp!", str);
+	}
+
+}