Skip to content

Commit

Permalink
fix: Complex Parsing Approach
Browse files Browse the repository at this point in the history 
- optionally provide a global Executor, instead spawning one for each parse
- run into Complex Parsing only, when Complex Parsing was allowed
- provide a Logger
- fixes JSQLParser#1792
  • Loading branch information
@manticore-projects
manticore-projects committed May 17, 2023
1 parent da32442 commit 4f0488c
Show file tree
Hide file tree
Showing 8 changed files with 232 additions and 88 deletions.
120 changes: 84 additions & 36 deletions src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;

import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.feature.Feature;
Expand All @@ -33,13 +36,25 @@

@SuppressWarnings("PMD.CyclomaticComplexity")
public final class CCJSqlParserUtil {
public final static Logger LOGGER = Logger.getLogger(CCJSqlParserUtil.class.getName());
static {
LOGGER.setLevel(Level.WARNING);
}

public final static int ALLOWED_NESTING_DEPTH = 10;

private CCJSqlParserUtil() {}

public static Statement parse(Reader statementReader) throws JSQLParserException {
ExecutorService executorService = Executors.newSingleThreadExecutor();
Statement statement = null;
CCJSqlParser parser = new CCJSqlParser(new StreamProvider(statementReader));
return parseStatement(parser);
try {
statement = parseStatement(parser, executorService);
} finally {
executorService.shutdown();
}
return statement;
}

public static Statement parse(String sql) throws JSQLParserException {
Expand All @@ -62,22 +77,41 @@ public static Statement parse(String sql) throws JSQLParserException {
*/
public static Statement parse(String sql, Consumer<CCJSqlParser> consumer)
throws JSQLParserException {

ExecutorService executorService = Executors.newSingleThreadExecutor();
Statement statement = null;
try {
statement = parse(sql, executorService, consumer);
} finally {
executorService.shutdown();
}
return statement;
}

public static Statement parse(String sql, ExecutorService executorService,
Consumer<CCJSqlParser> consumer)
throws JSQLParserException {
Statement statement = null;

// first, try to parse fast and simple
CCJSqlParser parser = newParser(sql);
if (consumer != null) {
consumer.accept(parser);
}
boolean allowComplex = parser.getConfiguration().getAsBoolean(Feature.allowComplexParsing);
LOGGER.info("Allowed Complex Parsing: " + allowComplex);
try {
CCJSqlParser parser = newParser(sql).withAllowComplexParsing(false);
if (consumer != null) {
consumer.accept(parser);
}
statement = parseStatement(parser);
LOGGER.info("Trying SIMPLE parsing " + (allowComplex ? "first" : "only"));
statement = parseStatement(parser.withAllowComplexParsing(false), executorService);
} catch (JSQLParserException ex) {
if (getNestingDepth(sql) <= ALLOWED_NESTING_DEPTH) {
CCJSqlParser parser = newParser(sql).withAllowComplexParsing(true);
if (allowComplex && getNestingDepth(sql) <= ALLOWED_NESTING_DEPTH) {
LOGGER.info("Trying COMPLEX parsing when SIMPLE parsing failed");
// beware: the parser must not be reused, but needs to be re-initiated
parser = newParser(sql);
if (consumer != null) {
consumer.accept(parser);
}
statement = parseStatement(parser);
statement = parseStatement(parser.withAllowComplexParsing(true), executorService);
} else {
throw ex;
}
Expand Down Expand Up @@ -252,24 +286,25 @@ public static Expression parseCondExpression(String conditionalExpressionStr,
}

/**
* @param parser
* @return the statement parsed
* @throws JSQLParserException
* @param parser the Parser armed with a Statement text
* @param executorService the Executor Service for parsing within a Thread
* @return the parsed Statement
* @throws JSQLParserException when either the Statement can't be parsed or the configured
* timeout is reached
*/
public static Statement parseStatement(CCJSqlParser parser) throws JSQLParserException {

public static Statement parseStatement(CCJSqlParser parser, ExecutorService executorService)
throws JSQLParserException {
Statement statement = null;
try {
ExecutorService executorService = Executors.newSingleThreadExecutor();
Future<Statement> future = executorService.submit(new Callable<Statement>() {
@Override
public Statement call() throws Exception {
return parser.Statement();
}
});
executorService.shutdown();

statement = future.get( parser.getConfiguration().getAsLong(Feature.timeOut), TimeUnit.MILLISECONDS);

statement = future.get(parser.getConfiguration().getAsLong(Feature.timeOut),
TimeUnit.MILLISECONDS);
} catch (TimeoutException ex) {
parser.interrupted = true;
throw new JSQLParserException("Time out occurred.", ex);
Expand All @@ -288,55 +323,68 @@ public static Statements parseStatements(String sqls) throws JSQLParserException
return parseStatements(sqls, null);
}

public static Statements parseStatements(String sqls, Consumer<CCJSqlParser> consumer)
throws JSQLParserException {
ExecutorService executorService = Executors.newSingleThreadExecutor();
final Statements statements = parseStatements(sqls, executorService, consumer);
executorService.shutdown();

return statements;
}

/**
* Parse a statement list.
*
* @return the statements parsed
*/
public static Statements parseStatements(String sqls, Consumer<CCJSqlParser> consumer)
public static Statements parseStatements(String sqls, ExecutorService executorService,
Consumer<CCJSqlParser> consumer)
throws JSQLParserException {
Statements statements = null;

CCJSqlParser parser = newParser(sqls);
if (consumer != null) {
consumer.accept(parser);
}
boolean allowComplex = parser.getConfiguration().getAsBoolean(Feature.allowComplexParsing);

// first, try to parse fast and simple
try {
CCJSqlParser parser = newParser(sqls).withAllowComplexParsing(false);
if (consumer != null) {
consumer.accept(parser);
}
statements = parseStatements(parser);
statements = parseStatements(parser.withAllowComplexParsing(false), executorService);
} catch (JSQLParserException ex) {
// when fast simple parsing fails, try complex parsing but only if it has a chance to
// succeed
if (getNestingDepth(sqls) <= ALLOWED_NESTING_DEPTH) {
CCJSqlParser parser = newParser(sqls).withAllowComplexParsing(true);
if (allowComplex && getNestingDepth(sqls) <= ALLOWED_NESTING_DEPTH) {
// beware: parser must not be re-used but needs to be re-initiated
parser = newParser(sqls);
if (consumer != null) {
consumer.accept(parser);
}
statements = parseStatements(parser);
statements = parseStatements(parser.withAllowComplexParsing(true), executorService);
}
}
return statements;
}

/**
* @param parser
* @return the statements parsed
* @throws JSQLParserException
* @param parser the Parser armed with a Statement text
* @param executorService the Executor Service for parsing within a Thread
* @return the Statements (representing a List of single statements)
* @throws JSQLParserException when either the Statement can't be parsed or the configured
* timeout is reached
*/
public static Statements parseStatements(CCJSqlParser parser) throws JSQLParserException {
public static Statements parseStatements(CCJSqlParser parser, ExecutorService executorService)
throws JSQLParserException {
Statements statements = null;
try {
ExecutorService executorService = Executors.newSingleThreadExecutor();
Future<Statements> future = executorService.submit(new Callable<Statements>() {
@Override
public Statements call() throws Exception {
return parser.Statements();
}
});
executorService.shutdown();

statements = future.get( parser.getConfiguration().getAsLong(Feature.timeOut) , TimeUnit.MILLISECONDS);

statements = future.get(parser.getConfiguration().getAsLong(Feature.timeOut),
TimeUnit.MILLISECONDS);
} catch (TimeoutException ex) {
parser.interrupted = true;
throw new JSQLParserException("Time out occurred.", ex);
Expand Down
19 changes: 13 additions & 6 deletions src/main/java/net/sf/jsqlparser/util/validation/ParseCapability.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,16 @@
*/
package net.sf.jsqlparser.util.validation;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statements;

/**
* package - private class for {@link Validation} to parse the statements
* within it's own {@link ValidationCapability}
* package - private class for {@link Validation} to parse the statements within it's own
* {@link ValidationCapability}
*
* @author gitmotte
*/
Expand All @@ -36,20 +38,25 @@ public String getStatements() {
}

/**
* @return <code>null</code> on parse error, otherwise the {@link Statements}
* parsed.
* @return <code>null</code> on parse error, otherwise the {@link Statements} parsed.
*/
public Statements getParsedStatements() {
return parsedStatement;
}

@Override
public void validate(ValidationContext context, Consumer<ValidationException> errorConsumer) {
ExecutorService executorService = Executors.newSingleThreadExecutor();
try {
this.parsedStatement = CCJSqlParserUtil.parseStatements(
CCJSqlParserUtil.newParser(statements).withConfiguration(context.getConfiguration()));
CCJSqlParserUtil.newParser(statements)
.withConfiguration(context.getConfiguration()),
executorService);
} catch (JSQLParserException e) {
errorConsumer.accept(new ParseException("Cannot parse statement: " + e.getMessage(), e));
errorConsumer
.accept(new ParseException("Cannot parse statement: " + e.getMessage(), e));
} finally {
executorService.shutdown();
}
}

Expand Down
45 changes: 26 additions & 19 deletions src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3081,7 +3081,7 @@ Expression AndExpression() :
}
{
(
LOOKAHEAD(Condition())
LOOKAHEAD(Condition(), {!interrupted})
left=Condition()
|
[ <K_NOT> { not=true; } | "!" { not=true; exclamationMarkNot=true; } ]
Expand All @@ -3093,7 +3093,7 @@ Expression AndExpression() :
{ boolean useOperator = false; }
(<K_AND> | <K_AND_OPERATOR> {useOperator=true;} )
(
LOOKAHEAD(Condition())
LOOKAHEAD(Condition(), {!interrupted})
right=Condition()
|
[ <K_NOT> { not=true; } | "!" { not=true; exclamationMarkNot=true; } ]
Expand Down Expand Up @@ -3216,8 +3216,8 @@ Expression SQLCondition():
{
(
result=ExistsExpression()
| LOOKAHEAD(InExpression()) result=InExpression()
| LOOKAHEAD(OverlapsCondition()) result=OverlapsCondition()
| LOOKAHEAD(InExpression() , {!interrupted}) result=InExpression()
| LOOKAHEAD(OverlapsCondition(), {!interrupted}) result=OverlapsCondition()
| left = SimpleExpression() { result = left; }
[
LOOKAHEAD(2) (
Expand Down Expand Up @@ -3258,7 +3258,7 @@ Expression InExpression() #InExpression :
(
LOOKAHEAD(2) token=<S_CHAR_LITERAL> { rightExpression = new StringValue(token.image); }
| LOOKAHEAD(3) rightExpression = Function()
| LOOKAHEAD(ParenthesedSelect()) rightExpression = ParenthesedSelect()
| LOOKAHEAD(ParenthesedSelect(), {!interrupted}) rightExpression = ParenthesedSelect()
| LOOKAHEAD(3) rightExpression = ParenthesedExpressionList()
| rightExpression = SimpleExpression()
)
Expand Down Expand Up @@ -4057,6 +4057,7 @@ JsonExpression JsonExpression() : {
CastExpression castExpr = null;
}
{
// LOOKAHEAD(3, {getAsBoolean(Feature.allowComplexParsing) && !interrupted})
(
LOOKAHEAD(3, {!interrupted}) expr=CaseWhenExpression()
|
Expand All @@ -4066,13 +4067,13 @@ JsonExpression JsonExpression() : {
|
expr=UserVariable()
|
LOOKAHEAD(JsonFunction(), {!interrupted}) expr = JsonFunction()
LOOKAHEAD(JsonFunction(), {getAsBoolean(Feature.allowComplexParsing) && !interrupted}) expr = JsonFunction()
|
LOOKAHEAD(JsonAggregateFunction(), {!interrupted}) expr = JsonAggregateFunction()
LOOKAHEAD(JsonAggregateFunction(), {getAsBoolean(Feature.allowComplexParsing) && !interrupted}) expr = JsonAggregateFunction()
|
LOOKAHEAD(FullTextSearch(), {!interrupted}) expr = FullTextSearch()
LOOKAHEAD(FullTextSearch(), {getAsBoolean(Feature.allowComplexParsing) && !interrupted}) expr = FullTextSearch()
|
LOOKAHEAD( 3 , {!interrupted && getAsBoolean(Feature.allowComplexParsing) } ) expr=Function()
LOOKAHEAD( 3 , {getAsBoolean(Feature.allowComplexParsing) && !interrupted} ) expr=Function()
|
LOOKAHEAD( 2, {!interrupted} ) expr=Column()
|
Expand Down Expand Up @@ -4551,9 +4552,14 @@ Expression CaseWhenExpression() #CaseWhenExpression:
<K_CASE> { caseCounter++; }
[ switchExp=Expression() ]
( clause=WhenThenSearchCondition() { whenClauses.add(clause); } )+
[<K_ELSE> (LOOKAHEAD( ["("] CaseWhenExpression() [")"] ( <K_WHEN> | <K_ELSE> | <K_END> ) ) ["("] elseExp=CaseWhenExpression() [")" { ((CaseExpression) elseExp).setUsingBrackets(true); } ]
| elseExp=Expression()
)
[
<K_ELSE>
(
LOOKAHEAD(3, {!interrupted}) "(" elseExp=CaseWhenExpression() ")" { elseExp = new Parenthesis( elseExp ); }
| LOOKAHEAD(3, {!interrupted}) elseExp=CaseWhenExpression()
| LOOKAHEAD(3, {getAsBoolean(Feature.allowComplexParsing) && !interrupted}) elseExp=Expression()
| elseExp=SimpleExpression()
)
]
<K_END> { caseCounter--; }
{
Expand All @@ -4567,16 +4573,17 @@ Expression CaseWhenExpression() #CaseWhenExpression:
WhenClause WhenThenSearchCondition():
{
WhenClause whenThen = new WhenClause();
Expression whenExp = null;
Expression thenExp = null;
Expression whenExp;
Expression thenExp;
}
{
<K_WHEN> whenExp=Expression()
<K_THEN> (
LOOKAHEAD( ["("] CaseWhenExpression() [")"] ( <K_WHEN> | <K_ELSE> | <K_END> ) ) ["("] thenExp=CaseWhenExpression() [")" { ((CaseExpression) thenExp).setUsingBrackets(true); }]
|
thenExp=Expression()
)
<K_THEN>
(
LOOKAHEAD({getAsBoolean(Feature.allowComplexParsing) && !interrupted}) thenExp=Expression()
|
thenExp=SimpleExpression()
)
{
whenThen.setWhenExpression(whenExp);
whenThen.setThenExpression(thenExp);
Expand Down
Loading

0 comments on commit 4f0488c

Please sign in to comment.