From 1672e73c403ead7d0c554e9e0fa617e5173bf8a0 Mon Sep 17 00:00:00 2001 From: Ian Farmer Date: Wed, 13 Feb 2008 08:02:17 +0000 Subject: [PATCH] Change LuaC to read bytes instead of chars. This fixes some of the test cases by avoiding a lossy round-trip conversion from bytes presumed to be, but might not be, UTF-8 encoded characters, and then back to bytes. All of the compiler test cases now pass. --- src/core/org/luaj/compiler/LexState.java | 34 ++++++++++++------------ src/core/org/luaj/compiler/LuaC.java | 21 +++++++-------- src/core/org/luaj/vm/LString.java | 14 ++++++++++ src/debug/org/luaj/debug/Print.java | 21 ++++++++------- src/test/java/org/luaj/vm/LuaJTest.java | 5 +++- version.properties | 2 +- 6 files changed, 57 insertions(+), 40 deletions(-) diff --git a/src/core/org/luaj/compiler/LexState.java b/src/core/org/luaj/compiler/LexState.java index f108b61c..c34d5697 100644 --- a/src/core/org/luaj/compiler/LexState.java +++ b/src/core/org/luaj/compiler/LexState.java @@ -22,7 +22,7 @@ package org.luaj.compiler; import java.io.IOException; -import java.io.Reader; +import java.io.InputStream; import java.util.Hashtable; import org.luaj.compiler.FuncState.BlockCnt; @@ -138,11 +138,11 @@ public class LexState { final Token lookahead = new Token(); /* look ahead token */ FuncState fs; /* `FuncState' is private to the parser */ LuaC L; - Reader z; /* input stream */ - char[] buff; /* buffer for tokens */ + InputStream z; /* input stream */ + byte[] buff; /* buffer for tokens */ int nbuff; /* length of buffer */ LString source; /* current source name */ - char decpoint; /* locale decimal point */ + byte decpoint; /* locale decimal point */ /* ORDER RESERVED */ final static String luaX_tokens [] = { @@ -197,9 +197,9 @@ public class LexState { } - public LexState(LuaC state, Reader reader) { - this.z = reader; - this.buff = new char[32]; + public LexState(LuaC state, InputStream stream) { + this.z = stream; + this.buff = new byte[32]; this.L = state; } @@ -221,10 +221,10 @@ public class LexState { nextChar(); } - void save( int c) { + void save(int c) { if ( buff == null || nbuff + 1 > buff.length ) buff = LuaC.realloc( buff, nbuff*2+1 ); - buff[nbuff++] = (char) c; + buff[nbuff++] = (byte) c; } @@ -281,12 +281,12 @@ public class LexState { lexerror( msg, t.token ); } - LString newstring( char[] chars, int offset, int len) { - return newstring( new String(chars, offset, len) ); + LString newstring( String s ) { + return L.newTString( LString.valueOf(s) ); } - LString newstring( String s ) { - return L.newTString( s ); + LString newstring( byte[] chars, int offset, int len ) { + return L.newTString( LString.newStringNoCopy(chars, offset, len) ); } void inclinenumber() { @@ -299,7 +299,7 @@ public class LexState { syntaxerror("chunk has too many lines"); } - void setinput( LuaC L, int firstByte, Reader z, LString source ) { + void setinput( LuaC L, int firstByte, InputStream z, LString source ) { this.decpoint = '.'; this.L = L; this.lookahead.token = TK_EOS; /* no look-ahead token */ @@ -335,9 +335,9 @@ public class LexState { return true; } - void buffreplace(char from, char to) { + void buffreplace(byte from, byte to) { int n = nbuff; - char[] p = buff; + byte[] p = buff; while ((--n) >= 0) if (p[n] == from) p[n] = to; @@ -394,7 +394,7 @@ public class LexState { while (isalnum(current) || current == '_') save_and_next(); save('\0'); - buffreplace('.', decpoint); /* follow locale for decimal point */ + buffreplace((byte)'.', decpoint); /* follow locale for decimal point */ String str = new String(buff, 0, nbuff); // if (!str2d(str, seminfo)) /* format error? */ // trydecpoint(str, seminfo); /* try to update decimal point separator */ diff --git a/src/core/org/luaj/compiler/LuaC.java b/src/core/org/luaj/compiler/LuaC.java index 17601c71..094499a1 100644 --- a/src/core/org/luaj/compiler/LuaC.java +++ b/src/core/org/luaj/compiler/LuaC.java @@ -23,7 +23,6 @@ package org.luaj.compiler; import java.io.IOException; import java.io.InputStream; -import java.io.Reader; import java.util.Hashtable; import org.luaj.vm.LPrototype; @@ -33,7 +32,6 @@ import org.luaj.vm.LoadState; import org.luaj.vm.LocVars; import org.luaj.vm.Lua; import org.luaj.vm.LuaErrorException; -import org.luaj.vm.Platform; import org.luaj.vm.LoadState.LuaCompiler; @@ -154,8 +152,8 @@ public class LuaC extends Lua implements LuaCompiler { return a; } - static char[] realloc(char[] v, int n) { - char[] a = new char[n]; + static byte[] realloc(byte[] v, int n) { + byte[] a = new byte[n]; if ( v != null ) System.arraycopy(v, 0, a, 0, Math.min(v.length,n)); return a; @@ -186,13 +184,12 @@ public class LuaC extends Lua implements LuaCompiler { * @throws LuaErrorException if there is a syntax error. */ public LPrototype compile(int firstByte, InputStream stream, String name) throws IOException { - Reader r = Platform.getInstance().createReader( stream ); LuaC compiler = new LuaC(); - return compiler.luaY_parser(firstByte, r, name); + return compiler.luaY_parser(firstByte, stream, name); } /** Parse the input */ - private LPrototype luaY_parser(int firstByte, Reader z, String name) { + private LPrototype luaY_parser(int firstByte, InputStream z, String name) { LexState lexstate = new LexState(this, z); FuncState funcstate = new FuncState(); // lexstate.buff = buff; @@ -213,13 +210,15 @@ public class LuaC extends Lua implements LuaCompiler { } public LString newlstr(char[] chars, int offset, int len) { - return newTString( new String(chars,offset,len) ); + return newTString( LString.valueOf( new String(chars,offset,len) ) ); } - public LString newTString(String s) { + public LString newTString(LString s) { LString t = (LString) strings.get(s); - if ( t == null ) - strings.put( s, t = new LString(s) ); + if ( t == null ) { + t = LString.newStringCopy(s); + strings.put( t, t ); + } return t; } diff --git a/src/core/org/luaj/vm/LString.java b/src/core/org/luaj/vm/LString.java index 8c3f0280..e70e433b 100644 --- a/src/core/org/luaj/vm/LString.java +++ b/src/core/org/luaj/vm/LString.java @@ -149,6 +149,20 @@ public class LString extends LValue { this.m_hash = hashBytes( bytes, off, len ); } + public static LString newStringCopy(LString src) { + return newStringCopy( src.m_bytes, src.m_offset, src.m_length ); + } + + public static LString newStringCopy(byte[] buf, int off, int len) { + byte[] b = new byte[len]; + System.arraycopy( buf, off, b, 0, len ); + return new LString( b, 0, len ); + } + + public static LString newStringNoCopy(byte[] buf, int off, int len) { + return new LString( buf, off, len ); + } + public boolean equals(Object o) { if ( o != null && o instanceof LString ) { LString s = (LString) o; diff --git a/src/debug/org/luaj/debug/Print.java b/src/debug/org/luaj/debug/Print.java index a56135a0..9818dbfb 100644 --- a/src/debug/org/luaj/debug/Print.java +++ b/src/debug/org/luaj/debug/Print.java @@ -65,13 +65,15 @@ public class Print extends Lua { }; - static void printString(String s) { - char[] chars = s.toCharArray(); + static void printString(final LString s) { + final byte[] bytes = s.m_bytes; + final int off = s.m_offset; + ps.print('"'); - for (int i = 0, n = chars.length; i < n; i++) { - char c = chars[i]; + for (int i = 0, n = s.m_length; i < n; i++) { + int c = bytes[i+off] & 0x0FF; if ( c >= ' ' && c <= '~' && c != '\"' && c != '\\' ) - ps.print(c); + ps.print((char) c); else { switch (c) { case '"': @@ -99,9 +101,8 @@ public class Print extends Lua { ps.print("\\v"); break; default: - ps.print("\\u"); - ps.print(Integer.toHexString(0x10000 + (int) c) - .substring(1)); + ps.print('\\'); + ps.print(Integer.toString(1000 + c).substring(1)); break; } } @@ -111,7 +112,7 @@ public class Print extends Lua { static void printValue( LValue v ) { if ( v instanceof LString ) - printString(v.toString()); + printString( v.luaAsString() ); else if ( v instanceof LInteger ) { ps.print( v.toJavaInt() ); } else if ( v instanceof LDouble ) { @@ -180,7 +181,7 @@ public class Print extends Lua { } switch (o) { case OP_LOADK: - printString(" ; "); + ps.print(" ; "); printConstant(f, bx); break; case OP_GETUPVAL: diff --git a/src/test/java/org/luaj/vm/LuaJTest.java b/src/test/java/org/luaj/vm/LuaJTest.java index f166dbe8..05763ef1 100644 --- a/src/test/java/org/luaj/vm/LuaJTest.java +++ b/src/test/java/org/luaj/vm/LuaJTest.java @@ -179,6 +179,7 @@ public class LuaJTest extends TestCase { // load the file LPrototype p = loadScriptResource( state, testName ); + p.source = LString.valueOf("stdin"); // Replace System.out with a ByteArrayOutputStream ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); @@ -208,7 +209,9 @@ public class LuaJTest extends TestCase { } try { - return LoadState.undump(state, script, name); + // Use "stdin" instead of resource name so that output matches + // standard Lua. + return LoadState.undump(state, script, "stdin"); } finally { script.close(); } diff --git a/version.properties b/version.properties index 0cb0ea60..e188da16 100644 --- a/version.properties +++ b/version.properties @@ -1 +1 @@ -version: 0.20 +version: 0.21