Add check for too complex patterns.

```
-- bug since 2.5 (C-stack overflow)
do
  local function f (size)
    local s = string.rep("a", size)
    local p = string.rep(".?", size)
    return pcall(string.match, s, p)
  end
  local r, m = f(80)
  assert(r and #m == 80)
  r, m = f(200000)
  assert(not r and string.find(m, "too complex"), tostring(r)..", "..tostring(m))
end
```

src/core/org/luaj/vm2/lib/StringLib.java

@@ -803,6 +803,8 @@ public class StringLib extends TwoArgFunction {
 	private static final LuaString SPECIALS = valueOf("^$*+?.([%-");
 	private static final int MAX_CAPTURES = 32;
 	
+	private static final int MAXCCALLS = 200;
+	
 	private static final int CAP_UNFINISHED = -1;
 	private static final int CAP_POSITION = -2;
 	
@@ -846,6 +848,7 @@ public class StringLib extends TwoArgFunction {
 	};
 	
 	static class MatchState {
+		int matchdepth;  /* control for recursive depth (to avoid C stack overflow) */
 		final LuaString s;
 		final LuaString p;
 		final Varargs args;
@@ -860,10 +863,12 @@ public class StringLib extends TwoArgFunction {
 			this.level = 0;
 			this.cinit = new int[ MAX_CAPTURES ];
 			this.clen = new int[ MAX_CAPTURES ];
+			this.matchdepth = MAXCCALLS;
 		}
 		
 		void reset() {
 			level = 0;
+			this.matchdepth = MAXCCALLS;
 		}
 		
 		private void add_s( Buffer lbuf, LuaString news, int soff, int e ) {
@@ -1052,81 +1057,86 @@ public class StringLib extends TwoArgFunction {
 		 * where match ends, otherwise returns -1.
 		 */
 		int match( int soffset, int poffset ) {
-			while ( true ) {
+			if (matchdepth-- == 0) error("pattern too complex");
-				// Check if we are at the end of the pattern -
+			try {
-				// equivalent to the '\0' case in the C version, but our pattern
+				while ( true ) {
-				// string is not NUL-terminated.
+					// Check if we are at the end of the pattern -
-				if ( poffset == p.length() )
+					// equivalent to the '\0' case in the C version, but our pattern
-					return soffset;
+					// string is not NUL-terminated.
-				switch ( p.luaByte( poffset ) ) {
+					if ( poffset == p.length() )
-				case '(':
+						return soffset;
-					if ( ++poffset < p.length() && p.luaByte( poffset ) == ')' )
+					switch ( p.luaByte( poffset ) ) {
-						return start_capture( soffset, poffset + 1, CAP_POSITION );
+					case '(':
-					else
+						if ( ++poffset < p.length() && p.luaByte( poffset ) == ')' )
-						return start_capture( soffset, poffset, CAP_UNFINISHED );
+							return start_capture( soffset, poffset + 1, CAP_POSITION );
-				case ')':
+						else
-					return end_capture( soffset, poffset + 1 );
+							return start_capture( soffset, poffset, CAP_UNFINISHED );
-				case L_ESC:
+					case ')':
-					if ( poffset + 1 == p.length() )
+						return end_capture( soffset, poffset + 1 );
-						error("malformed pattern (ends with '%')");
+					case L_ESC:
-					switch ( p.luaByte( poffset + 1 ) ) {
+						if ( poffset + 1 == p.length() )
-					case 'b':
+							error("malformed pattern (ends with '%')");
-						soffset = matchbalance( soffset, poffset + 2 );
+						switch ( p.luaByte( poffset + 1 ) ) {
-						if ( soffset == -1 ) return -1;
+						case 'b':
-						poffset += 4;
+							soffset = matchbalance( soffset, poffset + 2 );
-						continue;
+							if ( soffset == -1 ) return -1;
-					case 'f': {
+							poffset += 4;
-						poffset += 2;
+							continue;
-						if ( poffset == p.length() || p.luaByte( poffset ) != '[' ) {
+						case 'f': {
-							error("Missing '[' after '%f' in pattern");
+							poffset += 2;
+							if ( poffset == p.length() || p.luaByte( poffset ) != '[' ) {
+								error("Missing '[' after '%f' in pattern");
+							}
+							int ep = classend( poffset );
+							int previous = ( soffset == 0 ) ? '\0' : s.luaByte( soffset - 1 );
+							int next = ( soffset == s.length() ) ? '\0' : s.luaByte( soffset );
+							if ( matchbracketclass( previous, poffset, ep - 1 ) ||
+								 !matchbracketclass( next, poffset, ep - 1 ) )
+								return -1;
+							poffset = ep;
+							continue;
 						}
-						int ep = classend( poffset );
+						default: {
-						int previous = ( soffset == 0 ) ? '\0' : s.luaByte( soffset - 1 );
+							int c = p.luaByte( poffset + 1 );
-						int next = ( soffset == s.length() ) ? '\0' : s.luaByte( soffset );
+							if ( Character.isDigit( (char) c ) ) {
-						if ( matchbracketclass( previous, poffset, ep - 1 ) ||
+								soffset = match_capture( soffset, c );
-							 !matchbracketclass( next, poffset, ep - 1 ) )
+								if ( soffset == -1 )
+									return -1;
+								return match( soffset, poffset + 2 );
+							}
+						}
+						}
+					case '$':
+						if ( poffset + 1 == p.length() )
+							return ( soffset == s.length() ) ? soffset : -1;
+					}
+					int ep = classend( poffset );
+					boolean m = soffset < s.length() && singlematch( s.luaByte( soffset ), poffset, ep );
+					int pc = ( ep < p.length() ) ? p.luaByte( ep ) : '\0';
+					
+					switch ( pc ) {
+					case '?':
+						int res;
+						if ( m && ( ( res = match( soffset + 1, ep + 1 ) ) != -1 ) )
+							return res;
+						poffset = ep + 1;
+						continue;
+					case '*':
+						return max_expand( soffset, poffset, ep );
+					case '+':
+						return ( m ? max_expand( soffset + 1, poffset, ep ) : -1 );
+					case '-':
+						return min_expand( soffset, poffset, ep );
+					default:
+						if ( !m )
 							return -1;
+						soffset++;
 						poffset = ep;
 						continue;
 					}
-					default: {
-						int c = p.luaByte( poffset + 1 );
-						if ( Character.isDigit( (char) c ) ) {
-							soffset = match_capture( soffset, c );
-							if ( soffset == -1 )
-								return -1;
-							return match( soffset, poffset + 2 );
-						}
-					}
-					}
-				case '$':
-					if ( poffset + 1 == p.length() )
-						return ( soffset == s.length() ) ? soffset : -1;
-				}
-				int ep = classend( poffset );
-				boolean m = soffset < s.length() && singlematch( s.luaByte( soffset ), poffset, ep );
-				int pc = ( ep < p.length() ) ? p.luaByte( ep ) : '\0';
-				
-				switch ( pc ) {
-				case '?':
-					int res;
-					if ( m && ( ( res = match( soffset + 1, ep + 1 ) ) != -1 ) )
-						return res;
-					poffset = ep + 1;
-					continue;
-				case '*':
-					return max_expand( soffset, poffset, ep );
-				case '+':
-					return ( m ? max_expand( soffset + 1, poffset, ep ) : -1 );
-				case '-':
-					return min_expand( soffset, poffset, ep );
-				default:
-					if ( !m )
-						return -1;
-					soffset++;
-					poffset = ep;
-					continue;
 				}
+			} finally {
+				matchdepth++;
 			}
 		}