Handle source files as byte streams for lua2java conversions.

src/jse/lua2java.java

@@ -47,6 +47,7 @@ public class lua2java {
 		"  -s src	source directory\n" +
 		"  -d dir	destination directory\n" +
 		"  -p pkg	package prefix to apply to all classes\n" +
+		"  -e enc	override default character encoding\n" +
 		"  -r		recursively compile all\n" +
 		"  -v   	verbose\n";
 	
@@ -58,6 +59,7 @@ public class lua2java {
 	private String srcdir = null;
 	private String destdir = null;
 	private String pkgprefix = null;
+	private String encoding = "ISO8859-1";
 	private boolean recurse = false;
 	private boolean verbose = false;
 	private List<InputFile> files = new ArrayList<InputFile>();
@@ -93,6 +95,11 @@ public class lua2java {
 							usageExit();
 						pkgprefix = args[i];
 						break;
+					case 'e':
+						if ( ++i >= args.length )
+							usageExit();
+						encoding = args[i];
+						break;
 					case 'r':
 						recurse = true;
 						break;
@@ -112,6 +119,7 @@ public class lua2java {
 				System.out.println("srcdir: "+srcdir);
 				System.out.println("destdir: "+destdir);
 				System.out.println("files: "+files);
+				System.out.println("encoding: "+encoding);
 				System.out.println("recurse: "+recurse);
 			}
 
@@ -186,7 +194,7 @@ public class lua2java {
 			FileInputStream in = new FileInputStream(inf.infile);
 			FileOutputStream out = new FileOutputStream(inf.outfile);
 			PrintWriter pw = new PrintWriter(out);
-		    LuaParser parser = new LuaParser(in);
+		    LuaParser parser = new LuaParser(in,encoding);
 		    Chunk chunk = parser.Chunk();
 			new JavaCodeGen(chunk,pw,inf.javapackage,inf.javaclassname);
 			pw.close();

src/jse/org/luaj/vm2/ast/Str.java

@@ -22,35 +22,41 @@
 package org.luaj.vm2.ast;
 
 import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
 
 import org.luaj.vm2.LuaString;
 
 public class Str {
 	
-	public final byte[] bytes;
+	private Str() {}
-	public final boolean isutf8;
 	
-	public Str(byte[] bytes) {
-		this.bytes = bytes;
-		this.isutf8 = true; // TODO: scan to see
-	}
 	public static LuaString quoteString(String image) {
 		String s = image.substring(1, image.length()-1);
 		byte[] bytes = unquote(s);
-		// TODO: check for non-utf8
 		return LuaString.valueOf(bytes);
 	}
+	
 	public static LuaString charString(String image) {
 		String s = image.substring(1, image.length()-1);
 		byte[] bytes = unquote(s);
-		// TODO: check for non-utf8
 		return LuaString.valueOf(bytes);
 	}
+	
 	public static LuaString longString(String image) {
 		int i = image.indexOf('[', image.indexOf('[')+1);
 		String s = image.substring(i,image.length()-i);
-		return LuaString.valueOf(s);
+		byte[] b = iso88591bytes(s);
+		return LuaString.valueOf(b);
 	}
+	
+	public static byte[] iso88591bytes( String s ) {
+		try {
+			return s.getBytes("ISO8859-1");
+		} catch (UnsupportedEncodingException e) {
+			throw new IllegalStateException("ISO8859-1 not supported");
+		}
+	}
+	
 	public static byte[] unquote(String s) {
 		ByteArrayOutputStream baos = new ByteArrayOutputStream();
 		char[] c = s.toCharArray();
@@ -84,13 +90,4 @@ public class Str {
 		}
 		return baos.toByteArray();
 	}
-	/*
-	private static byte[] utf8decode(String s) {
-		try {
-			return s.getBytes("UTF8");
-		} catch ( Exception e ) {
-			throw new RuntimeException("utf8 not found: "+e);
-		}
-	}
-	*/
 }

src/jse/org/luaj/vm2/lua2java/JavaCodeGen.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.luaj.vm2.Lua;
+import org.luaj.vm2.LuaString;
 import org.luaj.vm2.LuaValue;
 import org.luaj.vm2.ast.Block;
 import org.luaj.vm2.ast.Chunk;
@@ -88,7 +89,7 @@ public class JavaCodeGen {
 
 		JavaScope javascope = null;
 		List<String> constantDeclarations = new ArrayList<String>();
-		Map<String,String> stringConstants = new HashMap<String,String>();
+		Map<LuaString,String> stringConstants = new HashMap<LuaString,String>();
 		Map<Double,String> numberConstants = new HashMap<Double,String>();
 		
 		
@@ -359,8 +360,7 @@ public class JavaCodeGen {
 		public void visit(Constant exp) {
 			switch ( exp.value.type() ) {
 			case LuaValue.TSTRING: {
-				// TODO: non-UTF8 data
+				out( evalLuaStringConstant(exp.value.checkstring()) );
-				out( evalStringConstant(exp.value.tojstring()) );
 				break;
 			}
 			case LuaValue.TNIL:
@@ -379,11 +379,14 @@ public class JavaCodeGen {
 		}
 
 		private String evalStringConstant(String str) {
-			// TODO: quoting, data pooling
+			return evalLuaStringConstant( LuaValue.valueOf(str) );
+		}
+		
+		private String evalLuaStringConstant(LuaString str) {
 			if ( stringConstants.containsKey(str) )
 				return stringConstants.get(str);
-			String declvalue = quotedStringInitializer(str.getBytes());
+			String declvalue = quotedStringInitializer(str);
-			String javaname = javascope.createConstantName(str);
+			String javaname = javascope.createConstantName(str.tojstring());
 			constantDeclarations.add( "static final LuaValue "+javaname+" = valueOf("+declvalue+");" );
 			stringConstants.put(str,javaname);
 			return javaname;
@@ -726,40 +729,41 @@ public class JavaCodeGen {
 		}		
 	}
 
-	private static String quotedStringInitializer(byte[] bytes) {
+	private static String quotedStringInitializer(LuaString s) {
-		int n = bytes.length;
+		byte[] bytes = s.m_bytes;
+		int o = s.m_offset;
+		int n = s.m_length;
 		StringBuffer sb = new StringBuffer(n+2);		
 		
-		// check for characters beyond ascii 128
+		// check for bytes not encodable as utf8
-		for ( int i=0; i<n; i++ )
+		if ( ! s.isValidUtf8() ) {
-			if (bytes[i]<0) {
+			sb.append( "new byte[]{" );
-				sb.append( "new byte[]{" );
+			for ( int j=0; j<n; j++ ) {
-				for ( int j=0; j<n; j++ ) {
+				if ( j>0 ) sb.append(",");
-					if ( j>0 ) sb.append(",");
+				byte b = bytes[o+j];
-					byte b = bytes[j];
+				switch ( b ) {
-					switch ( b ) {
+					case '\n': sb.append( "'\\n'" ); break; 
-						case '\n': sb.append( "'\\n'" ); break; 
+					case '\r': sb.append( "'\\r'" ); break; 
-						case '\r': sb.append( "'\\r'" ); break; 
+					case '\t': sb.append( "'\\t'" ); break; 
-						case '\t': sb.append( "'\\t'" ); break; 
+					case '\\': sb.append( "'\\\\'" ); break;
-						case '\\': sb.append( "'\\\\'" ); break;
+					default:
-						default:
+						if ( b >= ' ' ) {
-							if ( b >= ' ' ) {
+							sb.append( '\'');
-								sb.append( '\'');
+							sb.append( (char) b );
-								sb.append( (char) b );
+							sb.append( '\'');
-								sb.append( '\'');
+						} else {
-							} else {
+							sb.append( String.valueOf((int)b) );
-								sb.append( String.valueOf((int)b) );
+						}
-							}
+					break;
-						break;
+				}					
-					}					
-				}
-				sb.append( "}" );
-				return sb.toString();
 			}
+			sb.append( "}" );
+			return sb.toString();
+		}
 
 		sb.append('"');
 		for ( int i=0; i<n; i++ ) {
-			byte b = bytes[i];
+			byte b = bytes[o+i];
 			switch ( b ) {
 				case '\b': sb.append( "\\b" ); break; 
 				case '\f': sb.append( "\\f" ); break;