From a5aae93b65257276a5d8e43e28742ae145a41a3f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 12 Dec 2023 22:37:07 +0100 Subject: [PATCH 01/54] Small cleanup to IndicesRequestCache (#103348) Noticed this in reviewing a heap dump. We can simplify the removal listener logic here, this can just be a non-capturing lambda, no need to have the cache implement the listener interface. Also, removing 2 redundant fields. --- .../indices/IndicesRequestCache.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java index de7bfa91dda42..4042b21b865e4 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.cache.CacheLoader; -import org.elasticsearch.common.cache.RemovalListener; import org.elasticsearch.common.cache.RemovalNotification; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Setting; @@ -48,7 +47,7 @@ * There are still several TODOs left in this class, some easily addressable, some more complex, but the support * is functional. */ -public final class IndicesRequestCache implements RemovalListener, Closeable { +public final class IndicesRequestCache implements Closeable { /** * A setting to enable or disable request caching on an index level. Its dynamic by default @@ -73,18 +72,14 @@ public final class IndicesRequestCache implements RemovalListener registeredClosedListeners = ConcurrentCollections.newConcurrentMap(); private final Set keysToClean = ConcurrentCollections.newConcurrentSet(); - private final ByteSizeValue size; - private final TimeValue expire; private final Cache cache; IndicesRequestCache(Settings settings) { - this.size = INDICES_CACHE_QUERY_SIZE.get(settings); - this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null; - long sizeInBytes = size.getBytes(); + TimeValue expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null; CacheBuilder cacheBuilder = CacheBuilder.builder() - .setMaximumWeight(sizeInBytes) + .setMaximumWeight(INDICES_CACHE_QUERY_SIZE.get(settings).getBytes()) .weigher((k, v) -> k.ramBytesUsed() + v.ramBytesUsed()) - .removalListener(this); + .removalListener(notification -> notification.getKey().entity.onRemoval(notification)); if (expire != null) { cacheBuilder.setExpireAfterAccess(expire); } @@ -101,11 +96,6 @@ void clear(CacheEntity entity) { cleanCache(); } - @Override - public void onRemoval(RemovalNotification notification) { - notification.getKey().entity.onRemoval(notification); - } - BytesReference getOrCompute( CacheEntity cacheEntity, CheckedSupplier loader, From 1bc84770a09286aa9005a34708617fa4fe96e13c Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Tue, 12 Dec 2023 13:41:52 -0800 Subject: [PATCH 02/54] Mute ReactiveStorageDeciderServiceTests.testUnmovableSize --- .../autoscaling/storage/ReactiveStorageDeciderServiceTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java index a15ff51fc24ee..c5f7ea1622178 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java @@ -491,6 +491,7 @@ static void addNode(ClusterState.Builder stateBuilder, DiscoveryNodeRole role) { ); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103357") public void testUnmovableSize() { Settings.Builder settingsBuilder = Settings.builder(); if (randomBoolean()) { From a8a956f1e3bd46900d3af0872395f7a0515d7ad2 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 12 Dec 2023 14:04:16 -0800 Subject: [PATCH 03/54] ESQL: Improve grammar to allow identifiers with . (#100740) Extend the unquoted identifier to contain . not just numbers. Without it the lexer picks the characters as decimal literal which leads to errors Additionally fix a bug in quoting identifiers. Fix #100312 --- docs/changelog/100740.yaml | 6 + .../src/main/resources/date.csv-spec | 4 +- .../src/main/resources/stats.csv-spec | 9 + .../esql/src/main/antlr/EsqlBaseLexer.g4 | 284 ++- .../esql/src/main/antlr/EsqlBaseLexer.tokens | 137 -- .../esql/src/main/antlr/EsqlBaseParser.g4 | 32 +- .../esql/src/main/antlr/EsqlBaseParser.tokens | 137 -- .../xpack/esql/parser/EsqlBaseLexer.interp | 154 +- .../xpack/esql/parser/EsqlBaseLexer.java | 1236 ++++++++----- .../xpack/esql/parser/EsqlBaseParser.interp | 70 +- .../xpack/esql/parser/EsqlBaseParser.java | 1641 +++++++++-------- .../parser/EsqlBaseParserBaseListener.java | 28 +- .../parser/EsqlBaseParserBaseVisitor.java | 16 +- .../esql/parser/EsqlBaseParserListener.java | 28 +- .../esql/parser/EsqlBaseParserVisitor.java | 16 +- .../xpack/esql/parser/ExpressionBuilder.java | 49 +- .../xpack/esql/parser/IdentifierBuilder.java | 21 +- .../xpack/esql/parser/LogicalPlanBuilder.java | 58 +- .../elasticsearch/xpack/esql/CsvTests.java | 2 +- .../esql/parser/StatementParserTests.java | 36 +- .../session/IndexResolverFieldNamesTests.java | 2 +- 21 files changed, 2255 insertions(+), 1711 deletions(-) create mode 100644 docs/changelog/100740.yaml delete mode 100644 x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens delete mode 100644 x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens diff --git a/docs/changelog/100740.yaml b/docs/changelog/100740.yaml new file mode 100644 index 0000000000000..c93fbf676ef81 --- /dev/null +++ b/docs/changelog/100740.yaml @@ -0,0 +1,6 @@ +pr: 100740 +summary: "ESQL: Referencing expressions that contain backticks requires <>." +area: ES|QL +type: enhancement +issues: + - 100312 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index f6c0666c54ed8..591d395661afa 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -139,9 +139,9 @@ emp_no:integer | birth_date:date | x:date ; evalDateTruncGrouping -from employees | eval y = date_trunc(1 year, hire_date) | stats count(emp_no) by y | sort y | keep y, count(emp_no) | limit 5; +from employees | eval y = date_trunc(1 year, hire_date) | stats c = count(emp_no) by y | sort y | keep y, c | limit 5; -y:date | count(emp_no):long +y:date | c:long 1985-01-01T00:00:00.000Z | 11 1986-01-01T00:00:00.000Z | 11 1987-01-01T00:00:00.000Z | 15 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index dc96d1736858c..0ad759feeeea0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -782,3 +782,12 @@ FROM sample_data median_duration:double | client_ip:ip ; + +fieldEscaping#[skip:-8.12.99, reason:Fixed bug in 8.13 of removing the leading/trailing backquotes of an identifier] +FROM sample_data +| stats count(`event_duration`) | keep `count(``event_duration``)` +; + +count(`event_duration`):l +7 +; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 747c1fdcd1921..dbaefa2e5aebf 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -1,24 +1,24 @@ lexer grammar EsqlBaseLexer; -DISSECT : 'dissect' -> pushMode(EXPRESSION); -DROP : 'drop' -> pushMode(SOURCE_IDENTIFIERS); -ENRICH : 'enrich' -> pushMode(SOURCE_IDENTIFIERS); -EVAL : 'eval' -> pushMode(EXPRESSION); -EXPLAIN : 'explain' -> pushMode(EXPLAIN_MODE); -FROM : 'from' -> pushMode(SOURCE_IDENTIFIERS); -GROK : 'grok' -> pushMode(EXPRESSION); -INLINESTATS : 'inlinestats' -> pushMode(EXPRESSION); -KEEP : 'keep' -> pushMode(SOURCE_IDENTIFIERS); -LIMIT : 'limit' -> pushMode(EXPRESSION); -MV_EXPAND : 'mv_expand' -> pushMode(SOURCE_IDENTIFIERS); -PROJECT : 'project' -> pushMode(SOURCE_IDENTIFIERS); -RENAME : 'rename' -> pushMode(SOURCE_IDENTIFIERS); -ROW : 'row' -> pushMode(EXPRESSION); -SHOW : 'show' -> pushMode(EXPRESSION); -SORT : 'sort' -> pushMode(EXPRESSION); -STATS : 'stats' -> pushMode(EXPRESSION); -WHERE : 'where' -> pushMode(EXPRESSION); -UNKNOWN_CMD : ~[ \r\n\t[\]/]+ -> pushMode(EXPRESSION); +DISSECT : 'dissect' -> pushMode(EXPRESSION_MODE); +DROP : 'drop' -> pushMode(PROJECT_MODE); +ENRICH : 'enrich' -> pushMode(ENRICH_MODE); +EVAL : 'eval' -> pushMode(EXPRESSION_MODE); +EXPLAIN : 'explain' -> pushMode(EXPLAIN_MODE); +FROM : 'from' -> pushMode(FROM_MODE); +GROK : 'grok' -> pushMode(EXPRESSION_MODE); +INLINESTATS : 'inlinestats' -> pushMode(EXPRESSION_MODE); +KEEP : 'keep' -> pushMode(PROJECT_MODE); +LIMIT : 'limit' -> pushMode(EXPRESSION_MODE); +MV_EXPAND : 'mv_expand' -> pushMode(MVEXPAND_MODE); +PROJECT : 'project' -> pushMode(PROJECT_MODE); +RENAME : 'rename' -> pushMode(RENAME_MODE); +ROW : 'row' -> pushMode(EXPRESSION_MODE); +SHOW : 'show' -> pushMode(SHOW_MODE); +SORT : 'sort' -> pushMode(EXPRESSION_MODE); +STATS : 'stats' -> pushMode(EXPRESSION_MODE); +WHERE : 'where' -> pushMode(EXPRESSION_MODE); +UNKNOWN_CMD : ~[ \r\n\t[\]/]+ -> pushMode(EXPRESSION_MODE); LINE_COMMENT : '//' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN) @@ -31,16 +31,20 @@ MULTILINE_COMMENT WS : [ \r\n\t]+ -> channel(HIDDEN) ; - - +// +// Explain +// mode EXPLAIN_MODE; -EXPLAIN_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE); -EXPLAIN_PIPE : '|' -> type(PIPE), popMode; +EXPLAIN_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE); +EXPLAIN_PIPE : PIPE -> type(PIPE), popMode; EXPLAIN_WS : WS -> channel(HIDDEN); EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN); EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN); -mode EXPRESSION; +// +// Expression - used by most command +// +mode EXPRESSION_MODE; PIPE : '|' -> popMode; @@ -64,6 +68,27 @@ fragment EXPONENT : [Ee] [+-]? DIGIT+ ; +fragment ASPERAND + : '@' + ; + +fragment BACKQUOTE + : '`' + ; + +fragment BACKQUOTE_BLOCK + : ~'`' + | '``' + ; + +fragment UNDERSCORE + : '_' + ; + +fragment UNQUOTED_ID_BODY + : (LETTER | DIGIT | UNDERSCORE) + ; + STRING : '"' (ESCAPE_SEQUENCE | UNESCAPED_CHARS)* '"' | '"""' (~[\r\n])*? '"""' '"'? '"'? @@ -103,8 +128,6 @@ PARAM: '?'; RLIKE: 'rlike'; RP : ')'; TRUE : 'true'; -INFO : 'info'; -FUNCTIONS : 'functions'; EQ : '=='; NEQ : '!='; @@ -124,19 +147,18 @@ PERCENT : '%'; // mode. Thus, the two popModes on CLOSING_BRACKET. The other way could as // the start of a multivalued field constant. To line up with the double pop // the explain mode needs, we double push when we see that. -OPENING_BRACKET : '[' -> pushMode(EXPRESSION), pushMode(EXPRESSION); +OPENING_BRACKET : '[' -> pushMode(EXPRESSION_MODE), pushMode(EXPRESSION_MODE); CLOSING_BRACKET : ']' -> popMode, popMode; - UNQUOTED_IDENTIFIER - : LETTER (LETTER | DIGIT | '_')* + : LETTER UNQUOTED_ID_BODY* // only allow @ at beginning of identifier to keep the option to allow @ as infix operator in the future // also, single `_` and `@` characters are not valid identifiers - | ('_' | '@') (LETTER | DIGIT | '_')+ + | (UNDERSCORE | ASPERAND) UNQUOTED_ID_BODY+ ; QUOTED_IDENTIFIER - : '`' ( ~'`' | '``' )* '`' + : BACKQUOTE BACKQUOTE_BLOCK+ BACKQUOTE ; EXPR_LINE_COMMENT @@ -150,42 +172,204 @@ EXPR_MULTILINE_COMMENT EXPR_WS : WS -> channel(HIDDEN) ; +// +// FROM command +// +mode FROM_MODE; +FROM_PIPE : PIPE -> type(PIPE), popMode; +FROM_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET), pushMode(FROM_MODE), pushMode(FROM_MODE); +FROM_CLOSING_BRACKET : CLOSING_BRACKET -> type(CLOSING_BRACKET), popMode, popMode; +FROM_COMMA : COMMA -> type(COMMA); +FROM_ASSIGN : ASSIGN -> type(ASSIGN); +METADATA: 'metadata'; +fragment FROM_UNQUOTED_IDENTIFIER_PART + : ~[=`|,[\]/ \t\r\n] + | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment + ; -mode SOURCE_IDENTIFIERS; +FROM_UNQUOTED_IDENTIFIER + : FROM_UNQUOTED_IDENTIFIER_PART+ + ; + +FROM_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +FROM_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +FROM_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +FROM_WS + : WS -> channel(HIDDEN) + ; +// +// DROP, KEEP, PROJECT +// +mode PROJECT_MODE; +PROJECT_PIPE : PIPE -> type(PIPE), popMode; +PROJECT_DOT: DOT -> type(DOT); +PROJECT_COMMA : COMMA -> type(COMMA); + +fragment UNQUOTED_ID_BODY_WITH_PATTERN + : (LETTER | DIGIT | UNDERSCORE | ASTERISK) + ; + +PROJECT_UNQUOTED_IDENTIFIER + : (LETTER | ASTERISK) UNQUOTED_ID_BODY_WITH_PATTERN* + | (UNDERSCORE | ASPERAND) UNQUOTED_ID_BODY_WITH_PATTERN+ + ; + +PROJECT_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +PROJECT_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +PROJECT_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +PROJECT_WS + : WS -> channel(HIDDEN) + ; +// +// | RENAME a.b AS x, c AS y +// +mode RENAME_MODE; +RENAME_PIPE : PIPE -> type(PIPE), popMode; +RENAME_ASSIGN : ASSIGN -> type(ASSIGN); +RENAME_COMMA : COMMA -> type(COMMA); +RENAME_DOT: DOT -> type(DOT); -SRC_PIPE : '|' -> type(PIPE), popMode; -SRC_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(SOURCE_IDENTIFIERS), pushMode(SOURCE_IDENTIFIERS); -SRC_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET); -SRC_COMMA : ',' -> type(COMMA); -SRC_ASSIGN : '=' -> type(ASSIGN); AS : 'as'; -METADATA: 'metadata'; -ON : 'on'; -WITH : 'with'; -SRC_UNQUOTED_IDENTIFIER - : SRC_UNQUOTED_IDENTIFIER_PART+ +RENAME_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) ; -fragment SRC_UNQUOTED_IDENTIFIER_PART - : ~[=`|,[\]/ \t\r\n]+ - | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment +// use the unquoted pattern to let the parser invalidate fields with * +RENAME_UNQUOTED_IDENTIFIER + : PROJECT_UNQUOTED_IDENTIFIER -> type(PROJECT_UNQUOTED_IDENTIFIER) + ; + +RENAME_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) ; -SRC_QUOTED_IDENTIFIER - : QUOTED_IDENTIFIER +RENAME_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) ; -SRC_LINE_COMMENT +RENAME_WS + : WS -> channel(HIDDEN) + ; + +// | ENRICH ON key WITH fields +mode ENRICH_MODE; +ENRICH_PIPE : PIPE -> type(PIPE), popMode; + +ON : 'on' -> pushMode(ENRICH_FIELD_MODE); +WITH : 'with' -> pushMode(ENRICH_FIELD_MODE); + +// use the unquoted pattern to let the parser invalidate fields with * +ENRICH_POLICY_UNQUOTED_IDENTIFIER + : FROM_UNQUOTED_IDENTIFIER -> type(FROM_UNQUOTED_IDENTIFIER) + ; + +ENRICH_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +ENRICH_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +ENRICH_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +ENRICH_WS + : WS -> channel(HIDDEN) + ; + +// submode for Enrich to allow different lexing between policy identifier (loose) and field identifiers +mode ENRICH_FIELD_MODE; +ENRICH_FIELD_PIPE : PIPE -> type(PIPE), popMode, popMode; +ENRICH_FIELD_ASSIGN : ASSIGN -> type(ASSIGN); +ENRICH_FIELD_COMMA : COMMA -> type(COMMA); +ENRICH_FIELD_DOT: DOT -> type(DOT); + +ENRICH_FIELD_WITH : WITH -> type(WITH) ; + +ENRICH_FIELD_UNQUOTED_IDENTIFIER + : PROJECT_UNQUOTED_IDENTIFIER -> type(PROJECT_UNQUOTED_IDENTIFIER) + ; + +ENRICH_FIELD_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +ENRICH_FIELD_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +ENRICH_FIELD_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +ENRICH_FIELD_WS + : WS -> channel(HIDDEN) + ; + +mode MVEXPAND_MODE; +MVEXPAND_PIPE : PIPE -> type(PIPE), popMode; +MVEXPAND_DOT: DOT -> type(DOT); + +MVEXPAND_QUOTED_IDENTIFIER + : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) + ; + +MVEXPAND_UNQUOTED_IDENTIFIER + : UNQUOTED_IDENTIFIER -> type(UNQUOTED_IDENTIFIER) + ; + +MVEXPAND_LINE_COMMENT + : LINE_COMMENT -> channel(HIDDEN) + ; + +MVEXPAND_MULTILINE_COMMENT + : MULTILINE_COMMENT -> channel(HIDDEN) + ; + +MVEXPAND_WS + : WS -> channel(HIDDEN) + ; + +// +// SHOW INFO +// +mode SHOW_MODE; +SHOW_PIPE : PIPE -> type(PIPE), popMode; + +INFO : 'info'; +FUNCTIONS : 'functions'; + +SHOW_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN) ; -SRC_MULTILINE_COMMENT +SHOW_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN) ; -SRC_WS +SHOW_WS : WS -> channel(HIDDEN) ; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens deleted file mode 100644 index d8761f5eb0d73..0000000000000 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ /dev/null @@ -1,137 +0,0 @@ -DISSECT=1 -DROP=2 -ENRICH=3 -EVAL=4 -EXPLAIN=5 -FROM=6 -GROK=7 -INLINESTATS=8 -KEEP=9 -LIMIT=10 -MV_EXPAND=11 -PROJECT=12 -RENAME=13 -ROW=14 -SHOW=15 -SORT=16 -STATS=17 -WHERE=18 -UNKNOWN_CMD=19 -LINE_COMMENT=20 -MULTILINE_COMMENT=21 -WS=22 -EXPLAIN_WS=23 -EXPLAIN_LINE_COMMENT=24 -EXPLAIN_MULTILINE_COMMENT=25 -PIPE=26 -STRING=27 -INTEGER_LITERAL=28 -DECIMAL_LITERAL=29 -BY=30 -AND=31 -ASC=32 -ASSIGN=33 -COMMA=34 -DESC=35 -DOT=36 -FALSE=37 -FIRST=38 -LAST=39 -LP=40 -IN=41 -IS=42 -LIKE=43 -NOT=44 -NULL=45 -NULLS=46 -OR=47 -PARAM=48 -RLIKE=49 -RP=50 -TRUE=51 -INFO=52 -FUNCTIONS=53 -EQ=54 -NEQ=55 -LT=56 -LTE=57 -GT=58 -GTE=59 -PLUS=60 -MINUS=61 -ASTERISK=62 -SLASH=63 -PERCENT=64 -OPENING_BRACKET=65 -CLOSING_BRACKET=66 -UNQUOTED_IDENTIFIER=67 -QUOTED_IDENTIFIER=68 -EXPR_LINE_COMMENT=69 -EXPR_MULTILINE_COMMENT=70 -EXPR_WS=71 -AS=72 -METADATA=73 -ON=74 -WITH=75 -SRC_UNQUOTED_IDENTIFIER=76 -SRC_QUOTED_IDENTIFIER=77 -SRC_LINE_COMMENT=78 -SRC_MULTILINE_COMMENT=79 -SRC_WS=80 -EXPLAIN_PIPE=81 -'dissect'=1 -'drop'=2 -'enrich'=3 -'eval'=4 -'explain'=5 -'from'=6 -'grok'=7 -'inlinestats'=8 -'keep'=9 -'limit'=10 -'mv_expand'=11 -'project'=12 -'rename'=13 -'row'=14 -'show'=15 -'sort'=16 -'stats'=17 -'where'=18 -'by'=30 -'and'=31 -'asc'=32 -'desc'=35 -'.'=36 -'false'=37 -'first'=38 -'last'=39 -'('=40 -'in'=41 -'is'=42 -'like'=43 -'not'=44 -'null'=45 -'nulls'=46 -'or'=47 -'?'=48 -'rlike'=49 -')'=50 -'true'=51 -'info'=52 -'functions'=53 -'=='=54 -'!='=55 -'<'=56 -'<='=57 -'>'=58 -'>='=59 -'+'=60 -'-'=61 -'*'=62 -'/'=63 -'%'=64 -']'=66 -'as'=72 -'metadata'=73 -'on'=74 -'with'=75 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index 044e920744375..cdf0cea58b230 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -98,11 +98,11 @@ field ; fromCommand - : FROM sourceIdentifier (COMMA sourceIdentifier)* metadata? + : FROM fromIdentifier (COMMA fromIdentifier)* metadata? ; metadata - : OPENING_BRACKET METADATA sourceIdentifier (COMMA sourceIdentifier)* CLOSING_BRACKET + : OPENING_BRACKET METADATA fromIdentifier (COMMA fromIdentifier)* CLOSING_BRACKET ; @@ -122,21 +122,29 @@ grouping : qualifiedName (COMMA qualifiedName)* ; -sourceIdentifier - : SRC_UNQUOTED_IDENTIFIER - | SRC_QUOTED_IDENTIFIER +fromIdentifier + : FROM_UNQUOTED_IDENTIFIER + | QUOTED_IDENTIFIER ; qualifiedName : identifier (DOT identifier)* ; +qualifiedNamePattern + : identifierPattern (DOT identifierPattern)* + ; identifier : UNQUOTED_IDENTIFIER | QUOTED_IDENTIFIER ; +identifierPattern + : PROJECT_UNQUOTED_IDENTIFIER + | QUOTED_IDENTIFIER + ; + constant : NULL #nullLiteral | integerValue UNQUOTED_IDENTIFIER #qualifiedIntegerLiteral @@ -163,12 +171,12 @@ orderExpression ; keepCommand - : KEEP sourceIdentifier (COMMA sourceIdentifier)* - | PROJECT sourceIdentifier (COMMA sourceIdentifier)* + : KEEP qualifiedNamePattern (COMMA qualifiedNamePattern)* + | PROJECT qualifiedNamePattern (COMMA qualifiedNamePattern)* ; dropCommand - : DROP sourceIdentifier (COMMA sourceIdentifier)* + : DROP qualifiedNamePattern (COMMA qualifiedNamePattern)* ; renameCommand @@ -176,7 +184,7 @@ renameCommand ; renameClause: - oldName=sourceIdentifier AS newName=sourceIdentifier + oldName=qualifiedNamePattern AS newName=qualifiedNamePattern ; dissectCommand @@ -188,7 +196,7 @@ grokCommand ; mvExpandCommand - : MV_EXPAND sourceIdentifier + : MV_EXPAND qualifiedName ; commandOptions @@ -238,9 +246,9 @@ showCommand ; enrichCommand - : ENRICH policyName=sourceIdentifier (ON matchField=sourceIdentifier)? (WITH enrichWithClause (COMMA enrichWithClause)*)? + : ENRICH policyName=fromIdentifier (ON matchField=qualifiedNamePattern)? (WITH enrichWithClause (COMMA enrichWithClause)*)? ; enrichWithClause - : (newName=sourceIdentifier ASSIGN)? enrichField=sourceIdentifier + : (newName=qualifiedNamePattern ASSIGN)? enrichField=qualifiedNamePattern ; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens deleted file mode 100644 index d8761f5eb0d73..0000000000000 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ /dev/null @@ -1,137 +0,0 @@ -DISSECT=1 -DROP=2 -ENRICH=3 -EVAL=4 -EXPLAIN=5 -FROM=6 -GROK=7 -INLINESTATS=8 -KEEP=9 -LIMIT=10 -MV_EXPAND=11 -PROJECT=12 -RENAME=13 -ROW=14 -SHOW=15 -SORT=16 -STATS=17 -WHERE=18 -UNKNOWN_CMD=19 -LINE_COMMENT=20 -MULTILINE_COMMENT=21 -WS=22 -EXPLAIN_WS=23 -EXPLAIN_LINE_COMMENT=24 -EXPLAIN_MULTILINE_COMMENT=25 -PIPE=26 -STRING=27 -INTEGER_LITERAL=28 -DECIMAL_LITERAL=29 -BY=30 -AND=31 -ASC=32 -ASSIGN=33 -COMMA=34 -DESC=35 -DOT=36 -FALSE=37 -FIRST=38 -LAST=39 -LP=40 -IN=41 -IS=42 -LIKE=43 -NOT=44 -NULL=45 -NULLS=46 -OR=47 -PARAM=48 -RLIKE=49 -RP=50 -TRUE=51 -INFO=52 -FUNCTIONS=53 -EQ=54 -NEQ=55 -LT=56 -LTE=57 -GT=58 -GTE=59 -PLUS=60 -MINUS=61 -ASTERISK=62 -SLASH=63 -PERCENT=64 -OPENING_BRACKET=65 -CLOSING_BRACKET=66 -UNQUOTED_IDENTIFIER=67 -QUOTED_IDENTIFIER=68 -EXPR_LINE_COMMENT=69 -EXPR_MULTILINE_COMMENT=70 -EXPR_WS=71 -AS=72 -METADATA=73 -ON=74 -WITH=75 -SRC_UNQUOTED_IDENTIFIER=76 -SRC_QUOTED_IDENTIFIER=77 -SRC_LINE_COMMENT=78 -SRC_MULTILINE_COMMENT=79 -SRC_WS=80 -EXPLAIN_PIPE=81 -'dissect'=1 -'drop'=2 -'enrich'=3 -'eval'=4 -'explain'=5 -'from'=6 -'grok'=7 -'inlinestats'=8 -'keep'=9 -'limit'=10 -'mv_expand'=11 -'project'=12 -'rename'=13 -'row'=14 -'show'=15 -'sort'=16 -'stats'=17 -'where'=18 -'by'=30 -'and'=31 -'asc'=32 -'desc'=35 -'.'=36 -'false'=37 -'first'=38 -'last'=39 -'('=40 -'in'=41 -'is'=42 -'like'=43 -'not'=44 -'null'=45 -'nulls'=46 -'or'=47 -'?'=48 -'rlike'=49 -')'=50 -'true'=51 -'info'=52 -'functions'=53 -'=='=54 -'!='=55 -'<'=56 -'<='=57 -'>'=58 -'>='=59 -'+'=60 -'-'=61 -'*'=62 -'/'=63 -'%'=64 -']'=66 -'as'=72 -'metadata'=73 -'on'=74 -'with'=75 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index 12542878c3ed3..585f722065e6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -25,15 +25,15 @@ null null null null -null +'|' null null null 'by' 'and' 'asc' -null -null +'=' +',' 'desc' '.' 'false' @@ -51,8 +51,6 @@ null 'rlike' ')' 'true' -'info' -'functions' '==' '!=' '<' @@ -71,8 +69,19 @@ null null null null -'as' 'metadata' +null +null +null +null +null +null +null +null +'as' +null +null +null 'on' 'with' null @@ -81,6 +90,14 @@ null null null null +null +null +null +'info' +'functions' +null +null +null token symbolic names: null @@ -135,8 +152,6 @@ PARAM RLIKE RP TRUE -INFO -FUNCTIONS EQ NEQ LT @@ -155,16 +170,35 @@ QUOTED_IDENTIFIER EXPR_LINE_COMMENT EXPR_MULTILINE_COMMENT EXPR_WS -AS METADATA +FROM_UNQUOTED_IDENTIFIER +FROM_LINE_COMMENT +FROM_MULTILINE_COMMENT +FROM_WS +PROJECT_UNQUOTED_IDENTIFIER +PROJECT_LINE_COMMENT +PROJECT_MULTILINE_COMMENT +PROJECT_WS +AS +RENAME_LINE_COMMENT +RENAME_MULTILINE_COMMENT +RENAME_WS ON WITH -SRC_UNQUOTED_IDENTIFIER -SRC_QUOTED_IDENTIFIER -SRC_LINE_COMMENT -SRC_MULTILINE_COMMENT -SRC_WS -EXPLAIN_PIPE +ENRICH_LINE_COMMENT +ENRICH_MULTILINE_COMMENT +ENRICH_WS +ENRICH_FIELD_LINE_COMMENT +ENRICH_FIELD_MULTILINE_COMMENT +ENRICH_FIELD_WS +MVEXPAND_LINE_COMMENT +MVEXPAND_MULTILINE_COMMENT +MVEXPAND_WS +INFO +FUNCTIONS +SHOW_LINE_COMMENT +SHOW_MULTILINE_COMMENT +SHOW_WS rule names: DISSECT @@ -200,6 +234,11 @@ LETTER ESCAPE_SEQUENCE UNESCAPED_CHARS EXPONENT +ASPERAND +BACKQUOTE +BACKQUOTE_BLOCK +UNDERSCORE +UNQUOTED_ID_BODY STRING INTEGER_LITERAL DECIMAL_LITERAL @@ -225,8 +264,6 @@ PARAM RLIKE RP TRUE -INFO -FUNCTIONS EQ NEQ LT @@ -245,21 +282,68 @@ QUOTED_IDENTIFIER EXPR_LINE_COMMENT EXPR_MULTILINE_COMMENT EXPR_WS -SRC_PIPE -SRC_OPENING_BRACKET -SRC_CLOSING_BRACKET -SRC_COMMA -SRC_ASSIGN -AS +FROM_PIPE +FROM_OPENING_BRACKET +FROM_CLOSING_BRACKET +FROM_COMMA +FROM_ASSIGN METADATA +FROM_UNQUOTED_IDENTIFIER_PART +FROM_UNQUOTED_IDENTIFIER +FROM_QUOTED_IDENTIFIER +FROM_LINE_COMMENT +FROM_MULTILINE_COMMENT +FROM_WS +PROJECT_PIPE +PROJECT_DOT +PROJECT_COMMA +UNQUOTED_ID_BODY_WITH_PATTERN +PROJECT_UNQUOTED_IDENTIFIER +PROJECT_QUOTED_IDENTIFIER +PROJECT_LINE_COMMENT +PROJECT_MULTILINE_COMMENT +PROJECT_WS +RENAME_PIPE +RENAME_ASSIGN +RENAME_COMMA +RENAME_DOT +AS +RENAME_QUOTED_IDENTIFIER +RENAME_UNQUOTED_IDENTIFIER +RENAME_LINE_COMMENT +RENAME_MULTILINE_COMMENT +RENAME_WS +ENRICH_PIPE ON WITH -SRC_UNQUOTED_IDENTIFIER -SRC_UNQUOTED_IDENTIFIER_PART -SRC_QUOTED_IDENTIFIER -SRC_LINE_COMMENT -SRC_MULTILINE_COMMENT -SRC_WS +ENRICH_POLICY_UNQUOTED_IDENTIFIER +ENRICH_QUOTED_IDENTIFIER +ENRICH_LINE_COMMENT +ENRICH_MULTILINE_COMMENT +ENRICH_WS +ENRICH_FIELD_PIPE +ENRICH_FIELD_ASSIGN +ENRICH_FIELD_COMMA +ENRICH_FIELD_DOT +ENRICH_FIELD_WITH +ENRICH_FIELD_UNQUOTED_IDENTIFIER +ENRICH_FIELD_QUOTED_IDENTIFIER +ENRICH_FIELD_LINE_COMMENT +ENRICH_FIELD_MULTILINE_COMMENT +ENRICH_FIELD_WS +MVEXPAND_PIPE +MVEXPAND_DOT +MVEXPAND_QUOTED_IDENTIFIER +MVEXPAND_UNQUOTED_IDENTIFIER +MVEXPAND_LINE_COMMENT +MVEXPAND_MULTILINE_COMMENT +MVEXPAND_WS +SHOW_PIPE +INFO +FUNCTIONS +SHOW_LINE_COMMENT +SHOW_MULTILINE_COMMENT +SHOW_WS channel names: DEFAULT_TOKEN_CHANNEL @@ -268,8 +352,14 @@ HIDDEN mode names: DEFAULT_MODE EXPLAIN_MODE -EXPRESSION -SOURCE_IDENTIFIERS +EXPRESSION_MODE +FROM_MODE +PROJECT_MODE +RENAME_MODE +ENRICH_MODE +ENRICH_FIELD_MODE +MVEXPAND_MODE +SHOW_MODE atn: -[4, 0, 81, 764, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 4, 18, 345, 8, 18, 11, 18, 12, 18, 346, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 355, 8, 19, 10, 19, 12, 19, 358, 9, 19, 1, 19, 3, 19, 361, 8, 19, 1, 19, 3, 19, 364, 8, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 373, 8, 20, 10, 20, 12, 20, 376, 9, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 4, 21, 384, 8, 21, 11, 21, 12, 21, 385, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 3, 32, 427, 8, 32, 1, 32, 4, 32, 430, 8, 32, 11, 32, 12, 32, 431, 1, 33, 1, 33, 1, 33, 5, 33, 437, 8, 33, 10, 33, 12, 33, 440, 9, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 5, 33, 448, 8, 33, 10, 33, 12, 33, 451, 9, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 3, 33, 458, 8, 33, 1, 33, 3, 33, 461, 8, 33, 3, 33, 463, 8, 33, 1, 34, 4, 34, 466, 8, 34, 11, 34, 12, 34, 467, 1, 35, 4, 35, 471, 8, 35, 11, 35, 12, 35, 472, 1, 35, 1, 35, 5, 35, 477, 8, 35, 10, 35, 12, 35, 480, 9, 35, 1, 35, 1, 35, 4, 35, 484, 8, 35, 11, 35, 12, 35, 485, 1, 35, 4, 35, 489, 8, 35, 11, 35, 12, 35, 490, 1, 35, 1, 35, 5, 35, 495, 8, 35, 10, 35, 12, 35, 498, 9, 35, 3, 35, 500, 8, 35, 1, 35, 1, 35, 1, 35, 1, 35, 4, 35, 506, 8, 35, 11, 35, 12, 35, 507, 1, 35, 1, 35, 3, 35, 512, 8, 35, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 71, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 5, 73, 654, 8, 73, 10, 73, 12, 73, 657, 9, 73, 1, 73, 1, 73, 1, 73, 1, 73, 4, 73, 663, 8, 73, 11, 73, 12, 73, 664, 3, 73, 667, 8, 73, 1, 74, 1, 74, 1, 74, 1, 74, 5, 74, 673, 8, 74, 10, 74, 12, 74, 676, 9, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 1, 77, 1, 77, 1, 78, 1, 78, 1, 78, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 4, 87, 738, 8, 87, 11, 87, 12, 87, 739, 1, 88, 4, 88, 743, 8, 88, 11, 88, 12, 88, 744, 1, 88, 1, 88, 3, 88, 749, 8, 88, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 2, 374, 449, 0, 93, 4, 1, 6, 2, 8, 3, 10, 4, 12, 5, 14, 6, 16, 7, 18, 8, 20, 9, 22, 10, 24, 11, 26, 12, 28, 13, 30, 14, 32, 15, 34, 16, 36, 17, 38, 18, 40, 19, 42, 20, 44, 21, 46, 22, 48, 0, 50, 81, 52, 23, 54, 24, 56, 25, 58, 26, 60, 0, 62, 0, 64, 0, 66, 0, 68, 0, 70, 27, 72, 28, 74, 29, 76, 30, 78, 31, 80, 32, 82, 33, 84, 34, 86, 35, 88, 36, 90, 37, 92, 38, 94, 39, 96, 40, 98, 41, 100, 42, 102, 43, 104, 44, 106, 45, 108, 46, 110, 47, 112, 48, 114, 49, 116, 50, 118, 51, 120, 52, 122, 53, 124, 54, 126, 55, 128, 56, 130, 57, 132, 58, 134, 59, 136, 60, 138, 61, 140, 62, 142, 63, 144, 64, 146, 65, 148, 66, 150, 67, 152, 68, 154, 69, 156, 70, 158, 71, 160, 0, 162, 0, 164, 0, 166, 0, 168, 0, 170, 72, 172, 73, 174, 74, 176, 75, 178, 76, 180, 0, 182, 77, 184, 78, 186, 79, 188, 80, 4, 0, 1, 2, 3, 13, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 2, 0, 64, 64, 95, 95, 1, 0, 96, 96, 10, 0, 9, 10, 13, 13, 32, 32, 44, 44, 47, 47, 61, 61, 91, 91, 93, 93, 96, 96, 124, 124, 2, 0, 42, 42, 47, 47, 792, 0, 4, 1, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 8, 1, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 1, 48, 1, 0, 0, 0, 1, 50, 1, 0, 0, 0, 1, 52, 1, 0, 0, 0, 1, 54, 1, 0, 0, 0, 1, 56, 1, 0, 0, 0, 2, 58, 1, 0, 0, 0, 2, 70, 1, 0, 0, 0, 2, 72, 1, 0, 0, 0, 2, 74, 1, 0, 0, 0, 2, 76, 1, 0, 0, 0, 2, 78, 1, 0, 0, 0, 2, 80, 1, 0, 0, 0, 2, 82, 1, 0, 0, 0, 2, 84, 1, 0, 0, 0, 2, 86, 1, 0, 0, 0, 2, 88, 1, 0, 0, 0, 2, 90, 1, 0, 0, 0, 2, 92, 1, 0, 0, 0, 2, 94, 1, 0, 0, 0, 2, 96, 1, 0, 0, 0, 2, 98, 1, 0, 0, 0, 2, 100, 1, 0, 0, 0, 2, 102, 1, 0, 0, 0, 2, 104, 1, 0, 0, 0, 2, 106, 1, 0, 0, 0, 2, 108, 1, 0, 0, 0, 2, 110, 1, 0, 0, 0, 2, 112, 1, 0, 0, 0, 2, 114, 1, 0, 0, 0, 2, 116, 1, 0, 0, 0, 2, 118, 1, 0, 0, 0, 2, 120, 1, 0, 0, 0, 2, 122, 1, 0, 0, 0, 2, 124, 1, 0, 0, 0, 2, 126, 1, 0, 0, 0, 2, 128, 1, 0, 0, 0, 2, 130, 1, 0, 0, 0, 2, 132, 1, 0, 0, 0, 2, 134, 1, 0, 0, 0, 2, 136, 1, 0, 0, 0, 2, 138, 1, 0, 0, 0, 2, 140, 1, 0, 0, 0, 2, 142, 1, 0, 0, 0, 2, 144, 1, 0, 0, 0, 2, 146, 1, 0, 0, 0, 2, 148, 1, 0, 0, 0, 2, 150, 1, 0, 0, 0, 2, 152, 1, 0, 0, 0, 2, 154, 1, 0, 0, 0, 2, 156, 1, 0, 0, 0, 2, 158, 1, 0, 0, 0, 3, 160, 1, 0, 0, 0, 3, 162, 1, 0, 0, 0, 3, 164, 1, 0, 0, 0, 3, 166, 1, 0, 0, 0, 3, 168, 1, 0, 0, 0, 3, 170, 1, 0, 0, 0, 3, 172, 1, 0, 0, 0, 3, 174, 1, 0, 0, 0, 3, 176, 1, 0, 0, 0, 3, 178, 1, 0, 0, 0, 3, 182, 1, 0, 0, 0, 3, 184, 1, 0, 0, 0, 3, 186, 1, 0, 0, 0, 3, 188, 1, 0, 0, 0, 4, 190, 1, 0, 0, 0, 6, 200, 1, 0, 0, 0, 8, 207, 1, 0, 0, 0, 10, 216, 1, 0, 0, 0, 12, 223, 1, 0, 0, 0, 14, 233, 1, 0, 0, 0, 16, 240, 1, 0, 0, 0, 18, 247, 1, 0, 0, 0, 20, 261, 1, 0, 0, 0, 22, 268, 1, 0, 0, 0, 24, 276, 1, 0, 0, 0, 26, 288, 1, 0, 0, 0, 28, 298, 1, 0, 0, 0, 30, 307, 1, 0, 0, 0, 32, 313, 1, 0, 0, 0, 34, 320, 1, 0, 0, 0, 36, 327, 1, 0, 0, 0, 38, 335, 1, 0, 0, 0, 40, 344, 1, 0, 0, 0, 42, 350, 1, 0, 0, 0, 44, 367, 1, 0, 0, 0, 46, 383, 1, 0, 0, 0, 48, 389, 1, 0, 0, 0, 50, 394, 1, 0, 0, 0, 52, 399, 1, 0, 0, 0, 54, 403, 1, 0, 0, 0, 56, 407, 1, 0, 0, 0, 58, 411, 1, 0, 0, 0, 60, 415, 1, 0, 0, 0, 62, 417, 1, 0, 0, 0, 64, 419, 1, 0, 0, 0, 66, 422, 1, 0, 0, 0, 68, 424, 1, 0, 0, 0, 70, 462, 1, 0, 0, 0, 72, 465, 1, 0, 0, 0, 74, 511, 1, 0, 0, 0, 76, 513, 1, 0, 0, 0, 78, 516, 1, 0, 0, 0, 80, 520, 1, 0, 0, 0, 82, 524, 1, 0, 0, 0, 84, 526, 1, 0, 0, 0, 86, 528, 1, 0, 0, 0, 88, 533, 1, 0, 0, 0, 90, 535, 1, 0, 0, 0, 92, 541, 1, 0, 0, 0, 94, 547, 1, 0, 0, 0, 96, 552, 1, 0, 0, 0, 98, 554, 1, 0, 0, 0, 100, 557, 1, 0, 0, 0, 102, 560, 1, 0, 0, 0, 104, 565, 1, 0, 0, 0, 106, 569, 1, 0, 0, 0, 108, 574, 1, 0, 0, 0, 110, 580, 1, 0, 0, 0, 112, 583, 1, 0, 0, 0, 114, 585, 1, 0, 0, 0, 116, 591, 1, 0, 0, 0, 118, 593, 1, 0, 0, 0, 120, 598, 1, 0, 0, 0, 122, 603, 1, 0, 0, 0, 124, 613, 1, 0, 0, 0, 126, 616, 1, 0, 0, 0, 128, 619, 1, 0, 0, 0, 130, 621, 1, 0, 0, 0, 132, 624, 1, 0, 0, 0, 134, 626, 1, 0, 0, 0, 136, 629, 1, 0, 0, 0, 138, 631, 1, 0, 0, 0, 140, 633, 1, 0, 0, 0, 142, 635, 1, 0, 0, 0, 144, 637, 1, 0, 0, 0, 146, 639, 1, 0, 0, 0, 148, 644, 1, 0, 0, 0, 150, 666, 1, 0, 0, 0, 152, 668, 1, 0, 0, 0, 154, 679, 1, 0, 0, 0, 156, 683, 1, 0, 0, 0, 158, 687, 1, 0, 0, 0, 160, 691, 1, 0, 0, 0, 162, 696, 1, 0, 0, 0, 164, 702, 1, 0, 0, 0, 166, 708, 1, 0, 0, 0, 168, 712, 1, 0, 0, 0, 170, 716, 1, 0, 0, 0, 172, 719, 1, 0, 0, 0, 174, 728, 1, 0, 0, 0, 176, 731, 1, 0, 0, 0, 178, 737, 1, 0, 0, 0, 180, 748, 1, 0, 0, 0, 182, 750, 1, 0, 0, 0, 184, 752, 1, 0, 0, 0, 186, 756, 1, 0, 0, 0, 188, 760, 1, 0, 0, 0, 190, 191, 5, 100, 0, 0, 191, 192, 5, 105, 0, 0, 192, 193, 5, 115, 0, 0, 193, 194, 5, 115, 0, 0, 194, 195, 5, 101, 0, 0, 195, 196, 5, 99, 0, 0, 196, 197, 5, 116, 0, 0, 197, 198, 1, 0, 0, 0, 198, 199, 6, 0, 0, 0, 199, 5, 1, 0, 0, 0, 200, 201, 5, 100, 0, 0, 201, 202, 5, 114, 0, 0, 202, 203, 5, 111, 0, 0, 203, 204, 5, 112, 0, 0, 204, 205, 1, 0, 0, 0, 205, 206, 6, 1, 1, 0, 206, 7, 1, 0, 0, 0, 207, 208, 5, 101, 0, 0, 208, 209, 5, 110, 0, 0, 209, 210, 5, 114, 0, 0, 210, 211, 5, 105, 0, 0, 211, 212, 5, 99, 0, 0, 212, 213, 5, 104, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 6, 2, 1, 0, 215, 9, 1, 0, 0, 0, 216, 217, 5, 101, 0, 0, 217, 218, 5, 118, 0, 0, 218, 219, 5, 97, 0, 0, 219, 220, 5, 108, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 6, 3, 0, 0, 222, 11, 1, 0, 0, 0, 223, 224, 5, 101, 0, 0, 224, 225, 5, 120, 0, 0, 225, 226, 5, 112, 0, 0, 226, 227, 5, 108, 0, 0, 227, 228, 5, 97, 0, 0, 228, 229, 5, 105, 0, 0, 229, 230, 5, 110, 0, 0, 230, 231, 1, 0, 0, 0, 231, 232, 6, 4, 2, 0, 232, 13, 1, 0, 0, 0, 233, 234, 5, 102, 0, 0, 234, 235, 5, 114, 0, 0, 235, 236, 5, 111, 0, 0, 236, 237, 5, 109, 0, 0, 237, 238, 1, 0, 0, 0, 238, 239, 6, 5, 1, 0, 239, 15, 1, 0, 0, 0, 240, 241, 5, 103, 0, 0, 241, 242, 5, 114, 0, 0, 242, 243, 5, 111, 0, 0, 243, 244, 5, 107, 0, 0, 244, 245, 1, 0, 0, 0, 245, 246, 6, 6, 0, 0, 246, 17, 1, 0, 0, 0, 247, 248, 5, 105, 0, 0, 248, 249, 5, 110, 0, 0, 249, 250, 5, 108, 0, 0, 250, 251, 5, 105, 0, 0, 251, 252, 5, 110, 0, 0, 252, 253, 5, 101, 0, 0, 253, 254, 5, 115, 0, 0, 254, 255, 5, 116, 0, 0, 255, 256, 5, 97, 0, 0, 256, 257, 5, 116, 0, 0, 257, 258, 5, 115, 0, 0, 258, 259, 1, 0, 0, 0, 259, 260, 6, 7, 0, 0, 260, 19, 1, 0, 0, 0, 261, 262, 5, 107, 0, 0, 262, 263, 5, 101, 0, 0, 263, 264, 5, 101, 0, 0, 264, 265, 5, 112, 0, 0, 265, 266, 1, 0, 0, 0, 266, 267, 6, 8, 1, 0, 267, 21, 1, 0, 0, 0, 268, 269, 5, 108, 0, 0, 269, 270, 5, 105, 0, 0, 270, 271, 5, 109, 0, 0, 271, 272, 5, 105, 0, 0, 272, 273, 5, 116, 0, 0, 273, 274, 1, 0, 0, 0, 274, 275, 6, 9, 0, 0, 275, 23, 1, 0, 0, 0, 276, 277, 5, 109, 0, 0, 277, 278, 5, 118, 0, 0, 278, 279, 5, 95, 0, 0, 279, 280, 5, 101, 0, 0, 280, 281, 5, 120, 0, 0, 281, 282, 5, 112, 0, 0, 282, 283, 5, 97, 0, 0, 283, 284, 5, 110, 0, 0, 284, 285, 5, 100, 0, 0, 285, 286, 1, 0, 0, 0, 286, 287, 6, 10, 1, 0, 287, 25, 1, 0, 0, 0, 288, 289, 5, 112, 0, 0, 289, 290, 5, 114, 0, 0, 290, 291, 5, 111, 0, 0, 291, 292, 5, 106, 0, 0, 292, 293, 5, 101, 0, 0, 293, 294, 5, 99, 0, 0, 294, 295, 5, 116, 0, 0, 295, 296, 1, 0, 0, 0, 296, 297, 6, 11, 1, 0, 297, 27, 1, 0, 0, 0, 298, 299, 5, 114, 0, 0, 299, 300, 5, 101, 0, 0, 300, 301, 5, 110, 0, 0, 301, 302, 5, 97, 0, 0, 302, 303, 5, 109, 0, 0, 303, 304, 5, 101, 0, 0, 304, 305, 1, 0, 0, 0, 305, 306, 6, 12, 1, 0, 306, 29, 1, 0, 0, 0, 307, 308, 5, 114, 0, 0, 308, 309, 5, 111, 0, 0, 309, 310, 5, 119, 0, 0, 310, 311, 1, 0, 0, 0, 311, 312, 6, 13, 0, 0, 312, 31, 1, 0, 0, 0, 313, 314, 5, 115, 0, 0, 314, 315, 5, 104, 0, 0, 315, 316, 5, 111, 0, 0, 316, 317, 5, 119, 0, 0, 317, 318, 1, 0, 0, 0, 318, 319, 6, 14, 0, 0, 319, 33, 1, 0, 0, 0, 320, 321, 5, 115, 0, 0, 321, 322, 5, 111, 0, 0, 322, 323, 5, 114, 0, 0, 323, 324, 5, 116, 0, 0, 324, 325, 1, 0, 0, 0, 325, 326, 6, 15, 0, 0, 326, 35, 1, 0, 0, 0, 327, 328, 5, 115, 0, 0, 328, 329, 5, 116, 0, 0, 329, 330, 5, 97, 0, 0, 330, 331, 5, 116, 0, 0, 331, 332, 5, 115, 0, 0, 332, 333, 1, 0, 0, 0, 333, 334, 6, 16, 0, 0, 334, 37, 1, 0, 0, 0, 335, 336, 5, 119, 0, 0, 336, 337, 5, 104, 0, 0, 337, 338, 5, 101, 0, 0, 338, 339, 5, 114, 0, 0, 339, 340, 5, 101, 0, 0, 340, 341, 1, 0, 0, 0, 341, 342, 6, 17, 0, 0, 342, 39, 1, 0, 0, 0, 343, 345, 8, 0, 0, 0, 344, 343, 1, 0, 0, 0, 345, 346, 1, 0, 0, 0, 346, 344, 1, 0, 0, 0, 346, 347, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 349, 6, 18, 0, 0, 349, 41, 1, 0, 0, 0, 350, 351, 5, 47, 0, 0, 351, 352, 5, 47, 0, 0, 352, 356, 1, 0, 0, 0, 353, 355, 8, 1, 0, 0, 354, 353, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 360, 1, 0, 0, 0, 358, 356, 1, 0, 0, 0, 359, 361, 5, 13, 0, 0, 360, 359, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 363, 1, 0, 0, 0, 362, 364, 5, 10, 0, 0, 363, 362, 1, 0, 0, 0, 363, 364, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 366, 6, 19, 3, 0, 366, 43, 1, 0, 0, 0, 367, 368, 5, 47, 0, 0, 368, 369, 5, 42, 0, 0, 369, 374, 1, 0, 0, 0, 370, 373, 3, 44, 20, 0, 371, 373, 9, 0, 0, 0, 372, 370, 1, 0, 0, 0, 372, 371, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 1, 0, 0, 0, 374, 372, 1, 0, 0, 0, 375, 377, 1, 0, 0, 0, 376, 374, 1, 0, 0, 0, 377, 378, 5, 42, 0, 0, 378, 379, 5, 47, 0, 0, 379, 380, 1, 0, 0, 0, 380, 381, 6, 20, 3, 0, 381, 45, 1, 0, 0, 0, 382, 384, 7, 2, 0, 0, 383, 382, 1, 0, 0, 0, 384, 385, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 385, 386, 1, 0, 0, 0, 386, 387, 1, 0, 0, 0, 387, 388, 6, 21, 3, 0, 388, 47, 1, 0, 0, 0, 389, 390, 5, 91, 0, 0, 390, 391, 1, 0, 0, 0, 391, 392, 6, 22, 4, 0, 392, 393, 6, 22, 5, 0, 393, 49, 1, 0, 0, 0, 394, 395, 5, 124, 0, 0, 395, 396, 1, 0, 0, 0, 396, 397, 6, 23, 6, 0, 397, 398, 6, 23, 7, 0, 398, 51, 1, 0, 0, 0, 399, 400, 3, 46, 21, 0, 400, 401, 1, 0, 0, 0, 401, 402, 6, 24, 3, 0, 402, 53, 1, 0, 0, 0, 403, 404, 3, 42, 19, 0, 404, 405, 1, 0, 0, 0, 405, 406, 6, 25, 3, 0, 406, 55, 1, 0, 0, 0, 407, 408, 3, 44, 20, 0, 408, 409, 1, 0, 0, 0, 409, 410, 6, 26, 3, 0, 410, 57, 1, 0, 0, 0, 411, 412, 5, 124, 0, 0, 412, 413, 1, 0, 0, 0, 413, 414, 6, 27, 7, 0, 414, 59, 1, 0, 0, 0, 415, 416, 7, 3, 0, 0, 416, 61, 1, 0, 0, 0, 417, 418, 7, 4, 0, 0, 418, 63, 1, 0, 0, 0, 419, 420, 5, 92, 0, 0, 420, 421, 7, 5, 0, 0, 421, 65, 1, 0, 0, 0, 422, 423, 8, 6, 0, 0, 423, 67, 1, 0, 0, 0, 424, 426, 7, 7, 0, 0, 425, 427, 7, 8, 0, 0, 426, 425, 1, 0, 0, 0, 426, 427, 1, 0, 0, 0, 427, 429, 1, 0, 0, 0, 428, 430, 3, 60, 28, 0, 429, 428, 1, 0, 0, 0, 430, 431, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 431, 432, 1, 0, 0, 0, 432, 69, 1, 0, 0, 0, 433, 438, 5, 34, 0, 0, 434, 437, 3, 64, 30, 0, 435, 437, 3, 66, 31, 0, 436, 434, 1, 0, 0, 0, 436, 435, 1, 0, 0, 0, 437, 440, 1, 0, 0, 0, 438, 436, 1, 0, 0, 0, 438, 439, 1, 0, 0, 0, 439, 441, 1, 0, 0, 0, 440, 438, 1, 0, 0, 0, 441, 463, 5, 34, 0, 0, 442, 443, 5, 34, 0, 0, 443, 444, 5, 34, 0, 0, 444, 445, 5, 34, 0, 0, 445, 449, 1, 0, 0, 0, 446, 448, 8, 1, 0, 0, 447, 446, 1, 0, 0, 0, 448, 451, 1, 0, 0, 0, 449, 450, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 450, 452, 1, 0, 0, 0, 451, 449, 1, 0, 0, 0, 452, 453, 5, 34, 0, 0, 453, 454, 5, 34, 0, 0, 454, 455, 5, 34, 0, 0, 455, 457, 1, 0, 0, 0, 456, 458, 5, 34, 0, 0, 457, 456, 1, 0, 0, 0, 457, 458, 1, 0, 0, 0, 458, 460, 1, 0, 0, 0, 459, 461, 5, 34, 0, 0, 460, 459, 1, 0, 0, 0, 460, 461, 1, 0, 0, 0, 461, 463, 1, 0, 0, 0, 462, 433, 1, 0, 0, 0, 462, 442, 1, 0, 0, 0, 463, 71, 1, 0, 0, 0, 464, 466, 3, 60, 28, 0, 465, 464, 1, 0, 0, 0, 466, 467, 1, 0, 0, 0, 467, 465, 1, 0, 0, 0, 467, 468, 1, 0, 0, 0, 468, 73, 1, 0, 0, 0, 469, 471, 3, 60, 28, 0, 470, 469, 1, 0, 0, 0, 471, 472, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 1, 0, 0, 0, 474, 478, 3, 88, 42, 0, 475, 477, 3, 60, 28, 0, 476, 475, 1, 0, 0, 0, 477, 480, 1, 0, 0, 0, 478, 476, 1, 0, 0, 0, 478, 479, 1, 0, 0, 0, 479, 512, 1, 0, 0, 0, 480, 478, 1, 0, 0, 0, 481, 483, 3, 88, 42, 0, 482, 484, 3, 60, 28, 0, 483, 482, 1, 0, 0, 0, 484, 485, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 485, 486, 1, 0, 0, 0, 486, 512, 1, 0, 0, 0, 487, 489, 3, 60, 28, 0, 488, 487, 1, 0, 0, 0, 489, 490, 1, 0, 0, 0, 490, 488, 1, 0, 0, 0, 490, 491, 1, 0, 0, 0, 491, 499, 1, 0, 0, 0, 492, 496, 3, 88, 42, 0, 493, 495, 3, 60, 28, 0, 494, 493, 1, 0, 0, 0, 495, 498, 1, 0, 0, 0, 496, 494, 1, 0, 0, 0, 496, 497, 1, 0, 0, 0, 497, 500, 1, 0, 0, 0, 498, 496, 1, 0, 0, 0, 499, 492, 1, 0, 0, 0, 499, 500, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 502, 3, 68, 32, 0, 502, 512, 1, 0, 0, 0, 503, 505, 3, 88, 42, 0, 504, 506, 3, 60, 28, 0, 505, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 1, 0, 0, 0, 509, 510, 3, 68, 32, 0, 510, 512, 1, 0, 0, 0, 511, 470, 1, 0, 0, 0, 511, 481, 1, 0, 0, 0, 511, 488, 1, 0, 0, 0, 511, 503, 1, 0, 0, 0, 512, 75, 1, 0, 0, 0, 513, 514, 5, 98, 0, 0, 514, 515, 5, 121, 0, 0, 515, 77, 1, 0, 0, 0, 516, 517, 5, 97, 0, 0, 517, 518, 5, 110, 0, 0, 518, 519, 5, 100, 0, 0, 519, 79, 1, 0, 0, 0, 520, 521, 5, 97, 0, 0, 521, 522, 5, 115, 0, 0, 522, 523, 5, 99, 0, 0, 523, 81, 1, 0, 0, 0, 524, 525, 5, 61, 0, 0, 525, 83, 1, 0, 0, 0, 526, 527, 5, 44, 0, 0, 527, 85, 1, 0, 0, 0, 528, 529, 5, 100, 0, 0, 529, 530, 5, 101, 0, 0, 530, 531, 5, 115, 0, 0, 531, 532, 5, 99, 0, 0, 532, 87, 1, 0, 0, 0, 533, 534, 5, 46, 0, 0, 534, 89, 1, 0, 0, 0, 535, 536, 5, 102, 0, 0, 536, 537, 5, 97, 0, 0, 537, 538, 5, 108, 0, 0, 538, 539, 5, 115, 0, 0, 539, 540, 5, 101, 0, 0, 540, 91, 1, 0, 0, 0, 541, 542, 5, 102, 0, 0, 542, 543, 5, 105, 0, 0, 543, 544, 5, 114, 0, 0, 544, 545, 5, 115, 0, 0, 545, 546, 5, 116, 0, 0, 546, 93, 1, 0, 0, 0, 547, 548, 5, 108, 0, 0, 548, 549, 5, 97, 0, 0, 549, 550, 5, 115, 0, 0, 550, 551, 5, 116, 0, 0, 551, 95, 1, 0, 0, 0, 552, 553, 5, 40, 0, 0, 553, 97, 1, 0, 0, 0, 554, 555, 5, 105, 0, 0, 555, 556, 5, 110, 0, 0, 556, 99, 1, 0, 0, 0, 557, 558, 5, 105, 0, 0, 558, 559, 5, 115, 0, 0, 559, 101, 1, 0, 0, 0, 560, 561, 5, 108, 0, 0, 561, 562, 5, 105, 0, 0, 562, 563, 5, 107, 0, 0, 563, 564, 5, 101, 0, 0, 564, 103, 1, 0, 0, 0, 565, 566, 5, 110, 0, 0, 566, 567, 5, 111, 0, 0, 567, 568, 5, 116, 0, 0, 568, 105, 1, 0, 0, 0, 569, 570, 5, 110, 0, 0, 570, 571, 5, 117, 0, 0, 571, 572, 5, 108, 0, 0, 572, 573, 5, 108, 0, 0, 573, 107, 1, 0, 0, 0, 574, 575, 5, 110, 0, 0, 575, 576, 5, 117, 0, 0, 576, 577, 5, 108, 0, 0, 577, 578, 5, 108, 0, 0, 578, 579, 5, 115, 0, 0, 579, 109, 1, 0, 0, 0, 580, 581, 5, 111, 0, 0, 581, 582, 5, 114, 0, 0, 582, 111, 1, 0, 0, 0, 583, 584, 5, 63, 0, 0, 584, 113, 1, 0, 0, 0, 585, 586, 5, 114, 0, 0, 586, 587, 5, 108, 0, 0, 587, 588, 5, 105, 0, 0, 588, 589, 5, 107, 0, 0, 589, 590, 5, 101, 0, 0, 590, 115, 1, 0, 0, 0, 591, 592, 5, 41, 0, 0, 592, 117, 1, 0, 0, 0, 593, 594, 5, 116, 0, 0, 594, 595, 5, 114, 0, 0, 595, 596, 5, 117, 0, 0, 596, 597, 5, 101, 0, 0, 597, 119, 1, 0, 0, 0, 598, 599, 5, 105, 0, 0, 599, 600, 5, 110, 0, 0, 600, 601, 5, 102, 0, 0, 601, 602, 5, 111, 0, 0, 602, 121, 1, 0, 0, 0, 603, 604, 5, 102, 0, 0, 604, 605, 5, 117, 0, 0, 605, 606, 5, 110, 0, 0, 606, 607, 5, 99, 0, 0, 607, 608, 5, 116, 0, 0, 608, 609, 5, 105, 0, 0, 609, 610, 5, 111, 0, 0, 610, 611, 5, 110, 0, 0, 611, 612, 5, 115, 0, 0, 612, 123, 1, 0, 0, 0, 613, 614, 5, 61, 0, 0, 614, 615, 5, 61, 0, 0, 615, 125, 1, 0, 0, 0, 616, 617, 5, 33, 0, 0, 617, 618, 5, 61, 0, 0, 618, 127, 1, 0, 0, 0, 619, 620, 5, 60, 0, 0, 620, 129, 1, 0, 0, 0, 621, 622, 5, 60, 0, 0, 622, 623, 5, 61, 0, 0, 623, 131, 1, 0, 0, 0, 624, 625, 5, 62, 0, 0, 625, 133, 1, 0, 0, 0, 626, 627, 5, 62, 0, 0, 627, 628, 5, 61, 0, 0, 628, 135, 1, 0, 0, 0, 629, 630, 5, 43, 0, 0, 630, 137, 1, 0, 0, 0, 631, 632, 5, 45, 0, 0, 632, 139, 1, 0, 0, 0, 633, 634, 5, 42, 0, 0, 634, 141, 1, 0, 0, 0, 635, 636, 5, 47, 0, 0, 636, 143, 1, 0, 0, 0, 637, 638, 5, 37, 0, 0, 638, 145, 1, 0, 0, 0, 639, 640, 5, 91, 0, 0, 640, 641, 1, 0, 0, 0, 641, 642, 6, 71, 0, 0, 642, 643, 6, 71, 0, 0, 643, 147, 1, 0, 0, 0, 644, 645, 5, 93, 0, 0, 645, 646, 1, 0, 0, 0, 646, 647, 6, 72, 7, 0, 647, 648, 6, 72, 7, 0, 648, 149, 1, 0, 0, 0, 649, 655, 3, 62, 29, 0, 650, 654, 3, 62, 29, 0, 651, 654, 3, 60, 28, 0, 652, 654, 5, 95, 0, 0, 653, 650, 1, 0, 0, 0, 653, 651, 1, 0, 0, 0, 653, 652, 1, 0, 0, 0, 654, 657, 1, 0, 0, 0, 655, 653, 1, 0, 0, 0, 655, 656, 1, 0, 0, 0, 656, 667, 1, 0, 0, 0, 657, 655, 1, 0, 0, 0, 658, 662, 7, 9, 0, 0, 659, 663, 3, 62, 29, 0, 660, 663, 3, 60, 28, 0, 661, 663, 5, 95, 0, 0, 662, 659, 1, 0, 0, 0, 662, 660, 1, 0, 0, 0, 662, 661, 1, 0, 0, 0, 663, 664, 1, 0, 0, 0, 664, 662, 1, 0, 0, 0, 664, 665, 1, 0, 0, 0, 665, 667, 1, 0, 0, 0, 666, 649, 1, 0, 0, 0, 666, 658, 1, 0, 0, 0, 667, 151, 1, 0, 0, 0, 668, 674, 5, 96, 0, 0, 669, 673, 8, 10, 0, 0, 670, 671, 5, 96, 0, 0, 671, 673, 5, 96, 0, 0, 672, 669, 1, 0, 0, 0, 672, 670, 1, 0, 0, 0, 673, 676, 1, 0, 0, 0, 674, 672, 1, 0, 0, 0, 674, 675, 1, 0, 0, 0, 675, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 678, 5, 96, 0, 0, 678, 153, 1, 0, 0, 0, 679, 680, 3, 42, 19, 0, 680, 681, 1, 0, 0, 0, 681, 682, 6, 75, 3, 0, 682, 155, 1, 0, 0, 0, 683, 684, 3, 44, 20, 0, 684, 685, 1, 0, 0, 0, 685, 686, 6, 76, 3, 0, 686, 157, 1, 0, 0, 0, 687, 688, 3, 46, 21, 0, 688, 689, 1, 0, 0, 0, 689, 690, 6, 77, 3, 0, 690, 159, 1, 0, 0, 0, 691, 692, 5, 124, 0, 0, 692, 693, 1, 0, 0, 0, 693, 694, 6, 78, 6, 0, 694, 695, 6, 78, 7, 0, 695, 161, 1, 0, 0, 0, 696, 697, 5, 91, 0, 0, 697, 698, 1, 0, 0, 0, 698, 699, 6, 79, 4, 0, 699, 700, 6, 79, 1, 0, 700, 701, 6, 79, 1, 0, 701, 163, 1, 0, 0, 0, 702, 703, 5, 93, 0, 0, 703, 704, 1, 0, 0, 0, 704, 705, 6, 80, 7, 0, 705, 706, 6, 80, 7, 0, 706, 707, 6, 80, 8, 0, 707, 165, 1, 0, 0, 0, 708, 709, 5, 44, 0, 0, 709, 710, 1, 0, 0, 0, 710, 711, 6, 81, 9, 0, 711, 167, 1, 0, 0, 0, 712, 713, 5, 61, 0, 0, 713, 714, 1, 0, 0, 0, 714, 715, 6, 82, 10, 0, 715, 169, 1, 0, 0, 0, 716, 717, 5, 97, 0, 0, 717, 718, 5, 115, 0, 0, 718, 171, 1, 0, 0, 0, 719, 720, 5, 109, 0, 0, 720, 721, 5, 101, 0, 0, 721, 722, 5, 116, 0, 0, 722, 723, 5, 97, 0, 0, 723, 724, 5, 100, 0, 0, 724, 725, 5, 97, 0, 0, 725, 726, 5, 116, 0, 0, 726, 727, 5, 97, 0, 0, 727, 173, 1, 0, 0, 0, 728, 729, 5, 111, 0, 0, 729, 730, 5, 110, 0, 0, 730, 175, 1, 0, 0, 0, 731, 732, 5, 119, 0, 0, 732, 733, 5, 105, 0, 0, 733, 734, 5, 116, 0, 0, 734, 735, 5, 104, 0, 0, 735, 177, 1, 0, 0, 0, 736, 738, 3, 180, 88, 0, 737, 736, 1, 0, 0, 0, 738, 739, 1, 0, 0, 0, 739, 737, 1, 0, 0, 0, 739, 740, 1, 0, 0, 0, 740, 179, 1, 0, 0, 0, 741, 743, 8, 11, 0, 0, 742, 741, 1, 0, 0, 0, 743, 744, 1, 0, 0, 0, 744, 742, 1, 0, 0, 0, 744, 745, 1, 0, 0, 0, 745, 749, 1, 0, 0, 0, 746, 747, 5, 47, 0, 0, 747, 749, 8, 12, 0, 0, 748, 742, 1, 0, 0, 0, 748, 746, 1, 0, 0, 0, 749, 181, 1, 0, 0, 0, 750, 751, 3, 152, 74, 0, 751, 183, 1, 0, 0, 0, 752, 753, 3, 42, 19, 0, 753, 754, 1, 0, 0, 0, 754, 755, 6, 90, 3, 0, 755, 185, 1, 0, 0, 0, 756, 757, 3, 44, 20, 0, 757, 758, 1, 0, 0, 0, 758, 759, 6, 91, 3, 0, 759, 187, 1, 0, 0, 0, 760, 761, 3, 46, 21, 0, 761, 762, 1, 0, 0, 0, 762, 763, 6, 92, 3, 0, 763, 189, 1, 0, 0, 0, 38, 0, 1, 2, 3, 346, 356, 360, 363, 372, 374, 385, 426, 431, 436, 438, 449, 457, 460, 462, 467, 472, 478, 485, 490, 496, 499, 507, 511, 653, 655, 662, 664, 666, 672, 674, 739, 744, 748, 11, 5, 2, 0, 5, 3, 0, 5, 1, 0, 0, 1, 0, 7, 65, 0, 5, 0, 0, 7, 26, 0, 4, 0, 0, 7, 66, 0, 7, 34, 0, 7, 33, 0] \ No newline at end of file +[4, 0, 98, 1090, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 4, 18, 451, 8, 18, 11, 18, 12, 18, 452, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 461, 8, 19, 10, 19, 12, 19, 464, 9, 19, 1, 19, 3, 19, 467, 8, 19, 1, 19, 3, 19, 470, 8, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 479, 8, 20, 10, 20, 12, 20, 482, 9, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 4, 21, 490, 8, 21, 11, 21, 12, 21, 491, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 3, 32, 533, 8, 32, 1, 32, 4, 32, 536, 8, 32, 11, 32, 12, 32, 537, 1, 33, 1, 33, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 3, 35, 547, 8, 35, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 3, 37, 554, 8, 37, 1, 38, 1, 38, 1, 38, 5, 38, 559, 8, 38, 10, 38, 12, 38, 562, 9, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 570, 8, 38, 10, 38, 12, 38, 573, 9, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 580, 8, 38, 1, 38, 3, 38, 583, 8, 38, 3, 38, 585, 8, 38, 1, 39, 4, 39, 588, 8, 39, 11, 39, 12, 39, 589, 1, 40, 4, 40, 593, 8, 40, 11, 40, 12, 40, 594, 1, 40, 1, 40, 5, 40, 599, 8, 40, 10, 40, 12, 40, 602, 9, 40, 1, 40, 1, 40, 4, 40, 606, 8, 40, 11, 40, 12, 40, 607, 1, 40, 4, 40, 611, 8, 40, 11, 40, 12, 40, 612, 1, 40, 1, 40, 5, 40, 617, 8, 40, 10, 40, 12, 40, 620, 9, 40, 3, 40, 622, 8, 40, 1, 40, 1, 40, 1, 40, 1, 40, 4, 40, 628, 8, 40, 11, 40, 12, 40, 629, 1, 40, 1, 40, 3, 40, 634, 8, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 5, 76, 759, 8, 76, 10, 76, 12, 76, 762, 9, 76, 1, 76, 1, 76, 3, 76, 766, 8, 76, 1, 76, 4, 76, 769, 8, 76, 11, 76, 12, 76, 770, 3, 76, 773, 8, 76, 1, 77, 1, 77, 4, 77, 777, 8, 77, 11, 77, 12, 77, 778, 1, 77, 1, 77, 1, 78, 1, 78, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 3, 87, 832, 8, 87, 1, 88, 4, 88, 835, 8, 88, 11, 88, 12, 88, 836, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 3, 96, 872, 8, 96, 1, 97, 1, 97, 3, 97, 876, 8, 97, 1, 97, 5, 97, 879, 8, 97, 10, 97, 12, 97, 882, 9, 97, 1, 97, 1, 97, 3, 97, 886, 8, 97, 1, 97, 4, 97, 889, 8, 97, 11, 97, 12, 97, 890, 3, 97, 893, 8, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 2, 480, 571, 0, 143, 10, 1, 12, 2, 14, 3, 16, 4, 18, 5, 20, 6, 22, 7, 24, 8, 26, 9, 28, 10, 30, 11, 32, 12, 34, 13, 36, 14, 38, 15, 40, 16, 42, 17, 44, 18, 46, 19, 48, 20, 50, 21, 52, 22, 54, 0, 56, 0, 58, 23, 60, 24, 62, 25, 64, 26, 66, 0, 68, 0, 70, 0, 72, 0, 74, 0, 76, 0, 78, 0, 80, 0, 82, 0, 84, 0, 86, 27, 88, 28, 90, 29, 92, 30, 94, 31, 96, 32, 98, 33, 100, 34, 102, 35, 104, 36, 106, 37, 108, 38, 110, 39, 112, 40, 114, 41, 116, 42, 118, 43, 120, 44, 122, 45, 124, 46, 126, 47, 128, 48, 130, 49, 132, 50, 134, 51, 136, 52, 138, 53, 140, 54, 142, 55, 144, 56, 146, 57, 148, 58, 150, 59, 152, 60, 154, 61, 156, 62, 158, 63, 160, 64, 162, 65, 164, 66, 166, 67, 168, 68, 170, 69, 172, 0, 174, 0, 176, 0, 178, 0, 180, 0, 182, 70, 184, 0, 186, 71, 188, 0, 190, 72, 192, 73, 194, 74, 196, 0, 198, 0, 200, 0, 202, 0, 204, 75, 206, 0, 208, 76, 210, 77, 212, 78, 214, 0, 216, 0, 218, 0, 220, 0, 222, 79, 224, 0, 226, 0, 228, 80, 230, 81, 232, 82, 234, 0, 236, 83, 238, 84, 240, 0, 242, 0, 244, 85, 246, 86, 248, 87, 250, 0, 252, 0, 254, 0, 256, 0, 258, 0, 260, 0, 262, 0, 264, 88, 266, 89, 268, 90, 270, 0, 272, 0, 274, 0, 276, 0, 278, 91, 280, 92, 282, 93, 284, 0, 286, 94, 288, 95, 290, 96, 292, 97, 294, 98, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 10, 0, 9, 10, 13, 13, 32, 32, 44, 44, 47, 47, 61, 61, 91, 91, 93, 93, 96, 96, 124, 124, 2, 0, 42, 42, 47, 47, 1112, 0, 10, 1, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 1, 54, 1, 0, 0, 0, 1, 56, 1, 0, 0, 0, 1, 58, 1, 0, 0, 0, 1, 60, 1, 0, 0, 0, 1, 62, 1, 0, 0, 0, 2, 64, 1, 0, 0, 0, 2, 86, 1, 0, 0, 0, 2, 88, 1, 0, 0, 0, 2, 90, 1, 0, 0, 0, 2, 92, 1, 0, 0, 0, 2, 94, 1, 0, 0, 0, 2, 96, 1, 0, 0, 0, 2, 98, 1, 0, 0, 0, 2, 100, 1, 0, 0, 0, 2, 102, 1, 0, 0, 0, 2, 104, 1, 0, 0, 0, 2, 106, 1, 0, 0, 0, 2, 108, 1, 0, 0, 0, 2, 110, 1, 0, 0, 0, 2, 112, 1, 0, 0, 0, 2, 114, 1, 0, 0, 0, 2, 116, 1, 0, 0, 0, 2, 118, 1, 0, 0, 0, 2, 120, 1, 0, 0, 0, 2, 122, 1, 0, 0, 0, 2, 124, 1, 0, 0, 0, 2, 126, 1, 0, 0, 0, 2, 128, 1, 0, 0, 0, 2, 130, 1, 0, 0, 0, 2, 132, 1, 0, 0, 0, 2, 134, 1, 0, 0, 0, 2, 136, 1, 0, 0, 0, 2, 138, 1, 0, 0, 0, 2, 140, 1, 0, 0, 0, 2, 142, 1, 0, 0, 0, 2, 144, 1, 0, 0, 0, 2, 146, 1, 0, 0, 0, 2, 148, 1, 0, 0, 0, 2, 150, 1, 0, 0, 0, 2, 152, 1, 0, 0, 0, 2, 154, 1, 0, 0, 0, 2, 156, 1, 0, 0, 0, 2, 158, 1, 0, 0, 0, 2, 160, 1, 0, 0, 0, 2, 162, 1, 0, 0, 0, 2, 164, 1, 0, 0, 0, 2, 166, 1, 0, 0, 0, 2, 168, 1, 0, 0, 0, 2, 170, 1, 0, 0, 0, 3, 172, 1, 0, 0, 0, 3, 174, 1, 0, 0, 0, 3, 176, 1, 0, 0, 0, 3, 178, 1, 0, 0, 0, 3, 180, 1, 0, 0, 0, 3, 182, 1, 0, 0, 0, 3, 186, 1, 0, 0, 0, 3, 188, 1, 0, 0, 0, 3, 190, 1, 0, 0, 0, 3, 192, 1, 0, 0, 0, 3, 194, 1, 0, 0, 0, 4, 196, 1, 0, 0, 0, 4, 198, 1, 0, 0, 0, 4, 200, 1, 0, 0, 0, 4, 204, 1, 0, 0, 0, 4, 206, 1, 0, 0, 0, 4, 208, 1, 0, 0, 0, 4, 210, 1, 0, 0, 0, 4, 212, 1, 0, 0, 0, 5, 214, 1, 0, 0, 0, 5, 216, 1, 0, 0, 0, 5, 218, 1, 0, 0, 0, 5, 220, 1, 0, 0, 0, 5, 222, 1, 0, 0, 0, 5, 224, 1, 0, 0, 0, 5, 226, 1, 0, 0, 0, 5, 228, 1, 0, 0, 0, 5, 230, 1, 0, 0, 0, 5, 232, 1, 0, 0, 0, 6, 234, 1, 0, 0, 0, 6, 236, 1, 0, 0, 0, 6, 238, 1, 0, 0, 0, 6, 240, 1, 0, 0, 0, 6, 242, 1, 0, 0, 0, 6, 244, 1, 0, 0, 0, 6, 246, 1, 0, 0, 0, 6, 248, 1, 0, 0, 0, 7, 250, 1, 0, 0, 0, 7, 252, 1, 0, 0, 0, 7, 254, 1, 0, 0, 0, 7, 256, 1, 0, 0, 0, 7, 258, 1, 0, 0, 0, 7, 260, 1, 0, 0, 0, 7, 262, 1, 0, 0, 0, 7, 264, 1, 0, 0, 0, 7, 266, 1, 0, 0, 0, 7, 268, 1, 0, 0, 0, 8, 270, 1, 0, 0, 0, 8, 272, 1, 0, 0, 0, 8, 274, 1, 0, 0, 0, 8, 276, 1, 0, 0, 0, 8, 278, 1, 0, 0, 0, 8, 280, 1, 0, 0, 0, 8, 282, 1, 0, 0, 0, 9, 284, 1, 0, 0, 0, 9, 286, 1, 0, 0, 0, 9, 288, 1, 0, 0, 0, 9, 290, 1, 0, 0, 0, 9, 292, 1, 0, 0, 0, 9, 294, 1, 0, 0, 0, 10, 296, 1, 0, 0, 0, 12, 306, 1, 0, 0, 0, 14, 313, 1, 0, 0, 0, 16, 322, 1, 0, 0, 0, 18, 329, 1, 0, 0, 0, 20, 339, 1, 0, 0, 0, 22, 346, 1, 0, 0, 0, 24, 353, 1, 0, 0, 0, 26, 367, 1, 0, 0, 0, 28, 374, 1, 0, 0, 0, 30, 382, 1, 0, 0, 0, 32, 394, 1, 0, 0, 0, 34, 404, 1, 0, 0, 0, 36, 413, 1, 0, 0, 0, 38, 419, 1, 0, 0, 0, 40, 426, 1, 0, 0, 0, 42, 433, 1, 0, 0, 0, 44, 441, 1, 0, 0, 0, 46, 450, 1, 0, 0, 0, 48, 456, 1, 0, 0, 0, 50, 473, 1, 0, 0, 0, 52, 489, 1, 0, 0, 0, 54, 495, 1, 0, 0, 0, 56, 500, 1, 0, 0, 0, 58, 505, 1, 0, 0, 0, 60, 509, 1, 0, 0, 0, 62, 513, 1, 0, 0, 0, 64, 517, 1, 0, 0, 0, 66, 521, 1, 0, 0, 0, 68, 523, 1, 0, 0, 0, 70, 525, 1, 0, 0, 0, 72, 528, 1, 0, 0, 0, 74, 530, 1, 0, 0, 0, 76, 539, 1, 0, 0, 0, 78, 541, 1, 0, 0, 0, 80, 546, 1, 0, 0, 0, 82, 548, 1, 0, 0, 0, 84, 553, 1, 0, 0, 0, 86, 584, 1, 0, 0, 0, 88, 587, 1, 0, 0, 0, 90, 633, 1, 0, 0, 0, 92, 635, 1, 0, 0, 0, 94, 638, 1, 0, 0, 0, 96, 642, 1, 0, 0, 0, 98, 646, 1, 0, 0, 0, 100, 648, 1, 0, 0, 0, 102, 650, 1, 0, 0, 0, 104, 655, 1, 0, 0, 0, 106, 657, 1, 0, 0, 0, 108, 663, 1, 0, 0, 0, 110, 669, 1, 0, 0, 0, 112, 674, 1, 0, 0, 0, 114, 676, 1, 0, 0, 0, 116, 679, 1, 0, 0, 0, 118, 682, 1, 0, 0, 0, 120, 687, 1, 0, 0, 0, 122, 691, 1, 0, 0, 0, 124, 696, 1, 0, 0, 0, 126, 702, 1, 0, 0, 0, 128, 705, 1, 0, 0, 0, 130, 707, 1, 0, 0, 0, 132, 713, 1, 0, 0, 0, 134, 715, 1, 0, 0, 0, 136, 720, 1, 0, 0, 0, 138, 723, 1, 0, 0, 0, 140, 726, 1, 0, 0, 0, 142, 728, 1, 0, 0, 0, 144, 731, 1, 0, 0, 0, 146, 733, 1, 0, 0, 0, 148, 736, 1, 0, 0, 0, 150, 738, 1, 0, 0, 0, 152, 740, 1, 0, 0, 0, 154, 742, 1, 0, 0, 0, 156, 744, 1, 0, 0, 0, 158, 746, 1, 0, 0, 0, 160, 751, 1, 0, 0, 0, 162, 772, 1, 0, 0, 0, 164, 774, 1, 0, 0, 0, 166, 782, 1, 0, 0, 0, 168, 786, 1, 0, 0, 0, 170, 790, 1, 0, 0, 0, 172, 794, 1, 0, 0, 0, 174, 799, 1, 0, 0, 0, 176, 805, 1, 0, 0, 0, 178, 811, 1, 0, 0, 0, 180, 815, 1, 0, 0, 0, 182, 819, 1, 0, 0, 0, 184, 831, 1, 0, 0, 0, 186, 834, 1, 0, 0, 0, 188, 838, 1, 0, 0, 0, 190, 842, 1, 0, 0, 0, 192, 846, 1, 0, 0, 0, 194, 850, 1, 0, 0, 0, 196, 854, 1, 0, 0, 0, 198, 859, 1, 0, 0, 0, 200, 863, 1, 0, 0, 0, 202, 871, 1, 0, 0, 0, 204, 892, 1, 0, 0, 0, 206, 894, 1, 0, 0, 0, 208, 898, 1, 0, 0, 0, 210, 902, 1, 0, 0, 0, 212, 906, 1, 0, 0, 0, 214, 910, 1, 0, 0, 0, 216, 915, 1, 0, 0, 0, 218, 919, 1, 0, 0, 0, 220, 923, 1, 0, 0, 0, 222, 927, 1, 0, 0, 0, 224, 930, 1, 0, 0, 0, 226, 934, 1, 0, 0, 0, 228, 938, 1, 0, 0, 0, 230, 942, 1, 0, 0, 0, 232, 946, 1, 0, 0, 0, 234, 950, 1, 0, 0, 0, 236, 955, 1, 0, 0, 0, 238, 960, 1, 0, 0, 0, 240, 967, 1, 0, 0, 0, 242, 971, 1, 0, 0, 0, 244, 975, 1, 0, 0, 0, 246, 979, 1, 0, 0, 0, 248, 983, 1, 0, 0, 0, 250, 987, 1, 0, 0, 0, 252, 993, 1, 0, 0, 0, 254, 997, 1, 0, 0, 0, 256, 1001, 1, 0, 0, 0, 258, 1005, 1, 0, 0, 0, 260, 1009, 1, 0, 0, 0, 262, 1013, 1, 0, 0, 0, 264, 1017, 1, 0, 0, 0, 266, 1021, 1, 0, 0, 0, 268, 1025, 1, 0, 0, 0, 270, 1029, 1, 0, 0, 0, 272, 1034, 1, 0, 0, 0, 274, 1038, 1, 0, 0, 0, 276, 1042, 1, 0, 0, 0, 278, 1046, 1, 0, 0, 0, 280, 1050, 1, 0, 0, 0, 282, 1054, 1, 0, 0, 0, 284, 1058, 1, 0, 0, 0, 286, 1063, 1, 0, 0, 0, 288, 1068, 1, 0, 0, 0, 290, 1078, 1, 0, 0, 0, 292, 1082, 1, 0, 0, 0, 294, 1086, 1, 0, 0, 0, 296, 297, 5, 100, 0, 0, 297, 298, 5, 105, 0, 0, 298, 299, 5, 115, 0, 0, 299, 300, 5, 115, 0, 0, 300, 301, 5, 101, 0, 0, 301, 302, 5, 99, 0, 0, 302, 303, 5, 116, 0, 0, 303, 304, 1, 0, 0, 0, 304, 305, 6, 0, 0, 0, 305, 11, 1, 0, 0, 0, 306, 307, 5, 100, 0, 0, 307, 308, 5, 114, 0, 0, 308, 309, 5, 111, 0, 0, 309, 310, 5, 112, 0, 0, 310, 311, 1, 0, 0, 0, 311, 312, 6, 1, 1, 0, 312, 13, 1, 0, 0, 0, 313, 314, 5, 101, 0, 0, 314, 315, 5, 110, 0, 0, 315, 316, 5, 114, 0, 0, 316, 317, 5, 105, 0, 0, 317, 318, 5, 99, 0, 0, 318, 319, 5, 104, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 6, 2, 2, 0, 321, 15, 1, 0, 0, 0, 322, 323, 5, 101, 0, 0, 323, 324, 5, 118, 0, 0, 324, 325, 5, 97, 0, 0, 325, 326, 5, 108, 0, 0, 326, 327, 1, 0, 0, 0, 327, 328, 6, 3, 0, 0, 328, 17, 1, 0, 0, 0, 329, 330, 5, 101, 0, 0, 330, 331, 5, 120, 0, 0, 331, 332, 5, 112, 0, 0, 332, 333, 5, 108, 0, 0, 333, 334, 5, 97, 0, 0, 334, 335, 5, 105, 0, 0, 335, 336, 5, 110, 0, 0, 336, 337, 1, 0, 0, 0, 337, 338, 6, 4, 3, 0, 338, 19, 1, 0, 0, 0, 339, 340, 5, 102, 0, 0, 340, 341, 5, 114, 0, 0, 341, 342, 5, 111, 0, 0, 342, 343, 5, 109, 0, 0, 343, 344, 1, 0, 0, 0, 344, 345, 6, 5, 4, 0, 345, 21, 1, 0, 0, 0, 346, 347, 5, 103, 0, 0, 347, 348, 5, 114, 0, 0, 348, 349, 5, 111, 0, 0, 349, 350, 5, 107, 0, 0, 350, 351, 1, 0, 0, 0, 351, 352, 6, 6, 0, 0, 352, 23, 1, 0, 0, 0, 353, 354, 5, 105, 0, 0, 354, 355, 5, 110, 0, 0, 355, 356, 5, 108, 0, 0, 356, 357, 5, 105, 0, 0, 357, 358, 5, 110, 0, 0, 358, 359, 5, 101, 0, 0, 359, 360, 5, 115, 0, 0, 360, 361, 5, 116, 0, 0, 361, 362, 5, 97, 0, 0, 362, 363, 5, 116, 0, 0, 363, 364, 5, 115, 0, 0, 364, 365, 1, 0, 0, 0, 365, 366, 6, 7, 0, 0, 366, 25, 1, 0, 0, 0, 367, 368, 5, 107, 0, 0, 368, 369, 5, 101, 0, 0, 369, 370, 5, 101, 0, 0, 370, 371, 5, 112, 0, 0, 371, 372, 1, 0, 0, 0, 372, 373, 6, 8, 1, 0, 373, 27, 1, 0, 0, 0, 374, 375, 5, 108, 0, 0, 375, 376, 5, 105, 0, 0, 376, 377, 5, 109, 0, 0, 377, 378, 5, 105, 0, 0, 378, 379, 5, 116, 0, 0, 379, 380, 1, 0, 0, 0, 380, 381, 6, 9, 0, 0, 381, 29, 1, 0, 0, 0, 382, 383, 5, 109, 0, 0, 383, 384, 5, 118, 0, 0, 384, 385, 5, 95, 0, 0, 385, 386, 5, 101, 0, 0, 386, 387, 5, 120, 0, 0, 387, 388, 5, 112, 0, 0, 388, 389, 5, 97, 0, 0, 389, 390, 5, 110, 0, 0, 390, 391, 5, 100, 0, 0, 391, 392, 1, 0, 0, 0, 392, 393, 6, 10, 5, 0, 393, 31, 1, 0, 0, 0, 394, 395, 5, 112, 0, 0, 395, 396, 5, 114, 0, 0, 396, 397, 5, 111, 0, 0, 397, 398, 5, 106, 0, 0, 398, 399, 5, 101, 0, 0, 399, 400, 5, 99, 0, 0, 400, 401, 5, 116, 0, 0, 401, 402, 1, 0, 0, 0, 402, 403, 6, 11, 1, 0, 403, 33, 1, 0, 0, 0, 404, 405, 5, 114, 0, 0, 405, 406, 5, 101, 0, 0, 406, 407, 5, 110, 0, 0, 407, 408, 5, 97, 0, 0, 408, 409, 5, 109, 0, 0, 409, 410, 5, 101, 0, 0, 410, 411, 1, 0, 0, 0, 411, 412, 6, 12, 6, 0, 412, 35, 1, 0, 0, 0, 413, 414, 5, 114, 0, 0, 414, 415, 5, 111, 0, 0, 415, 416, 5, 119, 0, 0, 416, 417, 1, 0, 0, 0, 417, 418, 6, 13, 0, 0, 418, 37, 1, 0, 0, 0, 419, 420, 5, 115, 0, 0, 420, 421, 5, 104, 0, 0, 421, 422, 5, 111, 0, 0, 422, 423, 5, 119, 0, 0, 423, 424, 1, 0, 0, 0, 424, 425, 6, 14, 7, 0, 425, 39, 1, 0, 0, 0, 426, 427, 5, 115, 0, 0, 427, 428, 5, 111, 0, 0, 428, 429, 5, 114, 0, 0, 429, 430, 5, 116, 0, 0, 430, 431, 1, 0, 0, 0, 431, 432, 6, 15, 0, 0, 432, 41, 1, 0, 0, 0, 433, 434, 5, 115, 0, 0, 434, 435, 5, 116, 0, 0, 435, 436, 5, 97, 0, 0, 436, 437, 5, 116, 0, 0, 437, 438, 5, 115, 0, 0, 438, 439, 1, 0, 0, 0, 439, 440, 6, 16, 0, 0, 440, 43, 1, 0, 0, 0, 441, 442, 5, 119, 0, 0, 442, 443, 5, 104, 0, 0, 443, 444, 5, 101, 0, 0, 444, 445, 5, 114, 0, 0, 445, 446, 5, 101, 0, 0, 446, 447, 1, 0, 0, 0, 447, 448, 6, 17, 0, 0, 448, 45, 1, 0, 0, 0, 449, 451, 8, 0, 0, 0, 450, 449, 1, 0, 0, 0, 451, 452, 1, 0, 0, 0, 452, 450, 1, 0, 0, 0, 452, 453, 1, 0, 0, 0, 453, 454, 1, 0, 0, 0, 454, 455, 6, 18, 0, 0, 455, 47, 1, 0, 0, 0, 456, 457, 5, 47, 0, 0, 457, 458, 5, 47, 0, 0, 458, 462, 1, 0, 0, 0, 459, 461, 8, 1, 0, 0, 460, 459, 1, 0, 0, 0, 461, 464, 1, 0, 0, 0, 462, 460, 1, 0, 0, 0, 462, 463, 1, 0, 0, 0, 463, 466, 1, 0, 0, 0, 464, 462, 1, 0, 0, 0, 465, 467, 5, 13, 0, 0, 466, 465, 1, 0, 0, 0, 466, 467, 1, 0, 0, 0, 467, 469, 1, 0, 0, 0, 468, 470, 5, 10, 0, 0, 469, 468, 1, 0, 0, 0, 469, 470, 1, 0, 0, 0, 470, 471, 1, 0, 0, 0, 471, 472, 6, 19, 8, 0, 472, 49, 1, 0, 0, 0, 473, 474, 5, 47, 0, 0, 474, 475, 5, 42, 0, 0, 475, 480, 1, 0, 0, 0, 476, 479, 3, 50, 20, 0, 477, 479, 9, 0, 0, 0, 478, 476, 1, 0, 0, 0, 478, 477, 1, 0, 0, 0, 479, 482, 1, 0, 0, 0, 480, 481, 1, 0, 0, 0, 480, 478, 1, 0, 0, 0, 481, 483, 1, 0, 0, 0, 482, 480, 1, 0, 0, 0, 483, 484, 5, 42, 0, 0, 484, 485, 5, 47, 0, 0, 485, 486, 1, 0, 0, 0, 486, 487, 6, 20, 8, 0, 487, 51, 1, 0, 0, 0, 488, 490, 7, 2, 0, 0, 489, 488, 1, 0, 0, 0, 490, 491, 1, 0, 0, 0, 491, 489, 1, 0, 0, 0, 491, 492, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 21, 8, 0, 494, 53, 1, 0, 0, 0, 495, 496, 3, 158, 74, 0, 496, 497, 1, 0, 0, 0, 497, 498, 6, 22, 9, 0, 498, 499, 6, 22, 10, 0, 499, 55, 1, 0, 0, 0, 500, 501, 3, 64, 27, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 23, 11, 0, 503, 504, 6, 23, 12, 0, 504, 57, 1, 0, 0, 0, 505, 506, 3, 52, 21, 0, 506, 507, 1, 0, 0, 0, 507, 508, 6, 24, 8, 0, 508, 59, 1, 0, 0, 0, 509, 510, 3, 48, 19, 0, 510, 511, 1, 0, 0, 0, 511, 512, 6, 25, 8, 0, 512, 61, 1, 0, 0, 0, 513, 514, 3, 50, 20, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 26, 8, 0, 516, 63, 1, 0, 0, 0, 517, 518, 5, 124, 0, 0, 518, 519, 1, 0, 0, 0, 519, 520, 6, 27, 12, 0, 520, 65, 1, 0, 0, 0, 521, 522, 7, 3, 0, 0, 522, 67, 1, 0, 0, 0, 523, 524, 7, 4, 0, 0, 524, 69, 1, 0, 0, 0, 525, 526, 5, 92, 0, 0, 526, 527, 7, 5, 0, 0, 527, 71, 1, 0, 0, 0, 528, 529, 8, 6, 0, 0, 529, 73, 1, 0, 0, 0, 530, 532, 7, 7, 0, 0, 531, 533, 7, 8, 0, 0, 532, 531, 1, 0, 0, 0, 532, 533, 1, 0, 0, 0, 533, 535, 1, 0, 0, 0, 534, 536, 3, 66, 28, 0, 535, 534, 1, 0, 0, 0, 536, 537, 1, 0, 0, 0, 537, 535, 1, 0, 0, 0, 537, 538, 1, 0, 0, 0, 538, 75, 1, 0, 0, 0, 539, 540, 5, 64, 0, 0, 540, 77, 1, 0, 0, 0, 541, 542, 5, 96, 0, 0, 542, 79, 1, 0, 0, 0, 543, 547, 8, 9, 0, 0, 544, 545, 5, 96, 0, 0, 545, 547, 5, 96, 0, 0, 546, 543, 1, 0, 0, 0, 546, 544, 1, 0, 0, 0, 547, 81, 1, 0, 0, 0, 548, 549, 5, 95, 0, 0, 549, 83, 1, 0, 0, 0, 550, 554, 3, 68, 29, 0, 551, 554, 3, 66, 28, 0, 552, 554, 3, 82, 36, 0, 553, 550, 1, 0, 0, 0, 553, 551, 1, 0, 0, 0, 553, 552, 1, 0, 0, 0, 554, 85, 1, 0, 0, 0, 555, 560, 5, 34, 0, 0, 556, 559, 3, 70, 30, 0, 557, 559, 3, 72, 31, 0, 558, 556, 1, 0, 0, 0, 558, 557, 1, 0, 0, 0, 559, 562, 1, 0, 0, 0, 560, 558, 1, 0, 0, 0, 560, 561, 1, 0, 0, 0, 561, 563, 1, 0, 0, 0, 562, 560, 1, 0, 0, 0, 563, 585, 5, 34, 0, 0, 564, 565, 5, 34, 0, 0, 565, 566, 5, 34, 0, 0, 566, 567, 5, 34, 0, 0, 567, 571, 1, 0, 0, 0, 568, 570, 8, 1, 0, 0, 569, 568, 1, 0, 0, 0, 570, 573, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 571, 569, 1, 0, 0, 0, 572, 574, 1, 0, 0, 0, 573, 571, 1, 0, 0, 0, 574, 575, 5, 34, 0, 0, 575, 576, 5, 34, 0, 0, 576, 577, 5, 34, 0, 0, 577, 579, 1, 0, 0, 0, 578, 580, 5, 34, 0, 0, 579, 578, 1, 0, 0, 0, 579, 580, 1, 0, 0, 0, 580, 582, 1, 0, 0, 0, 581, 583, 5, 34, 0, 0, 582, 581, 1, 0, 0, 0, 582, 583, 1, 0, 0, 0, 583, 585, 1, 0, 0, 0, 584, 555, 1, 0, 0, 0, 584, 564, 1, 0, 0, 0, 585, 87, 1, 0, 0, 0, 586, 588, 3, 66, 28, 0, 587, 586, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 587, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 89, 1, 0, 0, 0, 591, 593, 3, 66, 28, 0, 592, 591, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 592, 1, 0, 0, 0, 594, 595, 1, 0, 0, 0, 595, 596, 1, 0, 0, 0, 596, 600, 3, 104, 47, 0, 597, 599, 3, 66, 28, 0, 598, 597, 1, 0, 0, 0, 599, 602, 1, 0, 0, 0, 600, 598, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 634, 1, 0, 0, 0, 602, 600, 1, 0, 0, 0, 603, 605, 3, 104, 47, 0, 604, 606, 3, 66, 28, 0, 605, 604, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 605, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 608, 634, 1, 0, 0, 0, 609, 611, 3, 66, 28, 0, 610, 609, 1, 0, 0, 0, 611, 612, 1, 0, 0, 0, 612, 610, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 621, 1, 0, 0, 0, 614, 618, 3, 104, 47, 0, 615, 617, 3, 66, 28, 0, 616, 615, 1, 0, 0, 0, 617, 620, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 622, 1, 0, 0, 0, 620, 618, 1, 0, 0, 0, 621, 614, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 1, 0, 0, 0, 623, 624, 3, 74, 32, 0, 624, 634, 1, 0, 0, 0, 625, 627, 3, 104, 47, 0, 626, 628, 3, 66, 28, 0, 627, 626, 1, 0, 0, 0, 628, 629, 1, 0, 0, 0, 629, 627, 1, 0, 0, 0, 629, 630, 1, 0, 0, 0, 630, 631, 1, 0, 0, 0, 631, 632, 3, 74, 32, 0, 632, 634, 1, 0, 0, 0, 633, 592, 1, 0, 0, 0, 633, 603, 1, 0, 0, 0, 633, 610, 1, 0, 0, 0, 633, 625, 1, 0, 0, 0, 634, 91, 1, 0, 0, 0, 635, 636, 5, 98, 0, 0, 636, 637, 5, 121, 0, 0, 637, 93, 1, 0, 0, 0, 638, 639, 5, 97, 0, 0, 639, 640, 5, 110, 0, 0, 640, 641, 5, 100, 0, 0, 641, 95, 1, 0, 0, 0, 642, 643, 5, 97, 0, 0, 643, 644, 5, 115, 0, 0, 644, 645, 5, 99, 0, 0, 645, 97, 1, 0, 0, 0, 646, 647, 5, 61, 0, 0, 647, 99, 1, 0, 0, 0, 648, 649, 5, 44, 0, 0, 649, 101, 1, 0, 0, 0, 650, 651, 5, 100, 0, 0, 651, 652, 5, 101, 0, 0, 652, 653, 5, 115, 0, 0, 653, 654, 5, 99, 0, 0, 654, 103, 1, 0, 0, 0, 655, 656, 5, 46, 0, 0, 656, 105, 1, 0, 0, 0, 657, 658, 5, 102, 0, 0, 658, 659, 5, 97, 0, 0, 659, 660, 5, 108, 0, 0, 660, 661, 5, 115, 0, 0, 661, 662, 5, 101, 0, 0, 662, 107, 1, 0, 0, 0, 663, 664, 5, 102, 0, 0, 664, 665, 5, 105, 0, 0, 665, 666, 5, 114, 0, 0, 666, 667, 5, 115, 0, 0, 667, 668, 5, 116, 0, 0, 668, 109, 1, 0, 0, 0, 669, 670, 5, 108, 0, 0, 670, 671, 5, 97, 0, 0, 671, 672, 5, 115, 0, 0, 672, 673, 5, 116, 0, 0, 673, 111, 1, 0, 0, 0, 674, 675, 5, 40, 0, 0, 675, 113, 1, 0, 0, 0, 676, 677, 5, 105, 0, 0, 677, 678, 5, 110, 0, 0, 678, 115, 1, 0, 0, 0, 679, 680, 5, 105, 0, 0, 680, 681, 5, 115, 0, 0, 681, 117, 1, 0, 0, 0, 682, 683, 5, 108, 0, 0, 683, 684, 5, 105, 0, 0, 684, 685, 5, 107, 0, 0, 685, 686, 5, 101, 0, 0, 686, 119, 1, 0, 0, 0, 687, 688, 5, 110, 0, 0, 688, 689, 5, 111, 0, 0, 689, 690, 5, 116, 0, 0, 690, 121, 1, 0, 0, 0, 691, 692, 5, 110, 0, 0, 692, 693, 5, 117, 0, 0, 693, 694, 5, 108, 0, 0, 694, 695, 5, 108, 0, 0, 695, 123, 1, 0, 0, 0, 696, 697, 5, 110, 0, 0, 697, 698, 5, 117, 0, 0, 698, 699, 5, 108, 0, 0, 699, 700, 5, 108, 0, 0, 700, 701, 5, 115, 0, 0, 701, 125, 1, 0, 0, 0, 702, 703, 5, 111, 0, 0, 703, 704, 5, 114, 0, 0, 704, 127, 1, 0, 0, 0, 705, 706, 5, 63, 0, 0, 706, 129, 1, 0, 0, 0, 707, 708, 5, 114, 0, 0, 708, 709, 5, 108, 0, 0, 709, 710, 5, 105, 0, 0, 710, 711, 5, 107, 0, 0, 711, 712, 5, 101, 0, 0, 712, 131, 1, 0, 0, 0, 713, 714, 5, 41, 0, 0, 714, 133, 1, 0, 0, 0, 715, 716, 5, 116, 0, 0, 716, 717, 5, 114, 0, 0, 717, 718, 5, 117, 0, 0, 718, 719, 5, 101, 0, 0, 719, 135, 1, 0, 0, 0, 720, 721, 5, 61, 0, 0, 721, 722, 5, 61, 0, 0, 722, 137, 1, 0, 0, 0, 723, 724, 5, 33, 0, 0, 724, 725, 5, 61, 0, 0, 725, 139, 1, 0, 0, 0, 726, 727, 5, 60, 0, 0, 727, 141, 1, 0, 0, 0, 728, 729, 5, 60, 0, 0, 729, 730, 5, 61, 0, 0, 730, 143, 1, 0, 0, 0, 731, 732, 5, 62, 0, 0, 732, 145, 1, 0, 0, 0, 733, 734, 5, 62, 0, 0, 734, 735, 5, 61, 0, 0, 735, 147, 1, 0, 0, 0, 736, 737, 5, 43, 0, 0, 737, 149, 1, 0, 0, 0, 738, 739, 5, 45, 0, 0, 739, 151, 1, 0, 0, 0, 740, 741, 5, 42, 0, 0, 741, 153, 1, 0, 0, 0, 742, 743, 5, 47, 0, 0, 743, 155, 1, 0, 0, 0, 744, 745, 5, 37, 0, 0, 745, 157, 1, 0, 0, 0, 746, 747, 5, 91, 0, 0, 747, 748, 1, 0, 0, 0, 748, 749, 6, 74, 0, 0, 749, 750, 6, 74, 0, 0, 750, 159, 1, 0, 0, 0, 751, 752, 5, 93, 0, 0, 752, 753, 1, 0, 0, 0, 753, 754, 6, 75, 12, 0, 754, 755, 6, 75, 12, 0, 755, 161, 1, 0, 0, 0, 756, 760, 3, 68, 29, 0, 757, 759, 3, 84, 37, 0, 758, 757, 1, 0, 0, 0, 759, 762, 1, 0, 0, 0, 760, 758, 1, 0, 0, 0, 760, 761, 1, 0, 0, 0, 761, 773, 1, 0, 0, 0, 762, 760, 1, 0, 0, 0, 763, 766, 3, 82, 36, 0, 764, 766, 3, 76, 33, 0, 765, 763, 1, 0, 0, 0, 765, 764, 1, 0, 0, 0, 766, 768, 1, 0, 0, 0, 767, 769, 3, 84, 37, 0, 768, 767, 1, 0, 0, 0, 769, 770, 1, 0, 0, 0, 770, 768, 1, 0, 0, 0, 770, 771, 1, 0, 0, 0, 771, 773, 1, 0, 0, 0, 772, 756, 1, 0, 0, 0, 772, 765, 1, 0, 0, 0, 773, 163, 1, 0, 0, 0, 774, 776, 3, 78, 34, 0, 775, 777, 3, 80, 35, 0, 776, 775, 1, 0, 0, 0, 777, 778, 1, 0, 0, 0, 778, 776, 1, 0, 0, 0, 778, 779, 1, 0, 0, 0, 779, 780, 1, 0, 0, 0, 780, 781, 3, 78, 34, 0, 781, 165, 1, 0, 0, 0, 782, 783, 3, 48, 19, 0, 783, 784, 1, 0, 0, 0, 784, 785, 6, 78, 8, 0, 785, 167, 1, 0, 0, 0, 786, 787, 3, 50, 20, 0, 787, 788, 1, 0, 0, 0, 788, 789, 6, 79, 8, 0, 789, 169, 1, 0, 0, 0, 790, 791, 3, 52, 21, 0, 791, 792, 1, 0, 0, 0, 792, 793, 6, 80, 8, 0, 793, 171, 1, 0, 0, 0, 794, 795, 3, 64, 27, 0, 795, 796, 1, 0, 0, 0, 796, 797, 6, 81, 11, 0, 797, 798, 6, 81, 12, 0, 798, 173, 1, 0, 0, 0, 799, 800, 3, 158, 74, 0, 800, 801, 1, 0, 0, 0, 801, 802, 6, 82, 9, 0, 802, 803, 6, 82, 4, 0, 803, 804, 6, 82, 4, 0, 804, 175, 1, 0, 0, 0, 805, 806, 3, 160, 75, 0, 806, 807, 1, 0, 0, 0, 807, 808, 6, 83, 13, 0, 808, 809, 6, 83, 12, 0, 809, 810, 6, 83, 12, 0, 810, 177, 1, 0, 0, 0, 811, 812, 3, 100, 45, 0, 812, 813, 1, 0, 0, 0, 813, 814, 6, 84, 14, 0, 814, 179, 1, 0, 0, 0, 815, 816, 3, 98, 44, 0, 816, 817, 1, 0, 0, 0, 817, 818, 6, 85, 15, 0, 818, 181, 1, 0, 0, 0, 819, 820, 5, 109, 0, 0, 820, 821, 5, 101, 0, 0, 821, 822, 5, 116, 0, 0, 822, 823, 5, 97, 0, 0, 823, 824, 5, 100, 0, 0, 824, 825, 5, 97, 0, 0, 825, 826, 5, 116, 0, 0, 826, 827, 5, 97, 0, 0, 827, 183, 1, 0, 0, 0, 828, 832, 8, 10, 0, 0, 829, 830, 5, 47, 0, 0, 830, 832, 8, 11, 0, 0, 831, 828, 1, 0, 0, 0, 831, 829, 1, 0, 0, 0, 832, 185, 1, 0, 0, 0, 833, 835, 3, 184, 87, 0, 834, 833, 1, 0, 0, 0, 835, 836, 1, 0, 0, 0, 836, 834, 1, 0, 0, 0, 836, 837, 1, 0, 0, 0, 837, 187, 1, 0, 0, 0, 838, 839, 3, 164, 77, 0, 839, 840, 1, 0, 0, 0, 840, 841, 6, 89, 16, 0, 841, 189, 1, 0, 0, 0, 842, 843, 3, 48, 19, 0, 843, 844, 1, 0, 0, 0, 844, 845, 6, 90, 8, 0, 845, 191, 1, 0, 0, 0, 846, 847, 3, 50, 20, 0, 847, 848, 1, 0, 0, 0, 848, 849, 6, 91, 8, 0, 849, 193, 1, 0, 0, 0, 850, 851, 3, 52, 21, 0, 851, 852, 1, 0, 0, 0, 852, 853, 6, 92, 8, 0, 853, 195, 1, 0, 0, 0, 854, 855, 3, 64, 27, 0, 855, 856, 1, 0, 0, 0, 856, 857, 6, 93, 11, 0, 857, 858, 6, 93, 12, 0, 858, 197, 1, 0, 0, 0, 859, 860, 3, 104, 47, 0, 860, 861, 1, 0, 0, 0, 861, 862, 6, 94, 17, 0, 862, 199, 1, 0, 0, 0, 863, 864, 3, 100, 45, 0, 864, 865, 1, 0, 0, 0, 865, 866, 6, 95, 14, 0, 866, 201, 1, 0, 0, 0, 867, 872, 3, 68, 29, 0, 868, 872, 3, 66, 28, 0, 869, 872, 3, 82, 36, 0, 870, 872, 3, 152, 71, 0, 871, 867, 1, 0, 0, 0, 871, 868, 1, 0, 0, 0, 871, 869, 1, 0, 0, 0, 871, 870, 1, 0, 0, 0, 872, 203, 1, 0, 0, 0, 873, 876, 3, 68, 29, 0, 874, 876, 3, 152, 71, 0, 875, 873, 1, 0, 0, 0, 875, 874, 1, 0, 0, 0, 876, 880, 1, 0, 0, 0, 877, 879, 3, 202, 96, 0, 878, 877, 1, 0, 0, 0, 879, 882, 1, 0, 0, 0, 880, 878, 1, 0, 0, 0, 880, 881, 1, 0, 0, 0, 881, 893, 1, 0, 0, 0, 882, 880, 1, 0, 0, 0, 883, 886, 3, 82, 36, 0, 884, 886, 3, 76, 33, 0, 885, 883, 1, 0, 0, 0, 885, 884, 1, 0, 0, 0, 886, 888, 1, 0, 0, 0, 887, 889, 3, 202, 96, 0, 888, 887, 1, 0, 0, 0, 889, 890, 1, 0, 0, 0, 890, 888, 1, 0, 0, 0, 890, 891, 1, 0, 0, 0, 891, 893, 1, 0, 0, 0, 892, 875, 1, 0, 0, 0, 892, 885, 1, 0, 0, 0, 893, 205, 1, 0, 0, 0, 894, 895, 3, 164, 77, 0, 895, 896, 1, 0, 0, 0, 896, 897, 6, 98, 16, 0, 897, 207, 1, 0, 0, 0, 898, 899, 3, 48, 19, 0, 899, 900, 1, 0, 0, 0, 900, 901, 6, 99, 8, 0, 901, 209, 1, 0, 0, 0, 902, 903, 3, 50, 20, 0, 903, 904, 1, 0, 0, 0, 904, 905, 6, 100, 8, 0, 905, 211, 1, 0, 0, 0, 906, 907, 3, 52, 21, 0, 907, 908, 1, 0, 0, 0, 908, 909, 6, 101, 8, 0, 909, 213, 1, 0, 0, 0, 910, 911, 3, 64, 27, 0, 911, 912, 1, 0, 0, 0, 912, 913, 6, 102, 11, 0, 913, 914, 6, 102, 12, 0, 914, 215, 1, 0, 0, 0, 915, 916, 3, 98, 44, 0, 916, 917, 1, 0, 0, 0, 917, 918, 6, 103, 15, 0, 918, 217, 1, 0, 0, 0, 919, 920, 3, 100, 45, 0, 920, 921, 1, 0, 0, 0, 921, 922, 6, 104, 14, 0, 922, 219, 1, 0, 0, 0, 923, 924, 3, 104, 47, 0, 924, 925, 1, 0, 0, 0, 925, 926, 6, 105, 17, 0, 926, 221, 1, 0, 0, 0, 927, 928, 5, 97, 0, 0, 928, 929, 5, 115, 0, 0, 929, 223, 1, 0, 0, 0, 930, 931, 3, 164, 77, 0, 931, 932, 1, 0, 0, 0, 932, 933, 6, 107, 16, 0, 933, 225, 1, 0, 0, 0, 934, 935, 3, 204, 97, 0, 935, 936, 1, 0, 0, 0, 936, 937, 6, 108, 18, 0, 937, 227, 1, 0, 0, 0, 938, 939, 3, 48, 19, 0, 939, 940, 1, 0, 0, 0, 940, 941, 6, 109, 8, 0, 941, 229, 1, 0, 0, 0, 942, 943, 3, 50, 20, 0, 943, 944, 1, 0, 0, 0, 944, 945, 6, 110, 8, 0, 945, 231, 1, 0, 0, 0, 946, 947, 3, 52, 21, 0, 947, 948, 1, 0, 0, 0, 948, 949, 6, 111, 8, 0, 949, 233, 1, 0, 0, 0, 950, 951, 3, 64, 27, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 112, 11, 0, 953, 954, 6, 112, 12, 0, 954, 235, 1, 0, 0, 0, 955, 956, 5, 111, 0, 0, 956, 957, 5, 110, 0, 0, 957, 958, 1, 0, 0, 0, 958, 959, 6, 113, 19, 0, 959, 237, 1, 0, 0, 0, 960, 961, 5, 119, 0, 0, 961, 962, 5, 105, 0, 0, 962, 963, 5, 116, 0, 0, 963, 964, 5, 104, 0, 0, 964, 965, 1, 0, 0, 0, 965, 966, 6, 114, 19, 0, 966, 239, 1, 0, 0, 0, 967, 968, 3, 186, 88, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 115, 20, 0, 970, 241, 1, 0, 0, 0, 971, 972, 3, 164, 77, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 116, 16, 0, 974, 243, 1, 0, 0, 0, 975, 976, 3, 48, 19, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 117, 8, 0, 978, 245, 1, 0, 0, 0, 979, 980, 3, 50, 20, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 118, 8, 0, 982, 247, 1, 0, 0, 0, 983, 984, 3, 52, 21, 0, 984, 985, 1, 0, 0, 0, 985, 986, 6, 119, 8, 0, 986, 249, 1, 0, 0, 0, 987, 988, 3, 64, 27, 0, 988, 989, 1, 0, 0, 0, 989, 990, 6, 120, 11, 0, 990, 991, 6, 120, 12, 0, 991, 992, 6, 120, 12, 0, 992, 251, 1, 0, 0, 0, 993, 994, 3, 98, 44, 0, 994, 995, 1, 0, 0, 0, 995, 996, 6, 121, 15, 0, 996, 253, 1, 0, 0, 0, 997, 998, 3, 100, 45, 0, 998, 999, 1, 0, 0, 0, 999, 1000, 6, 122, 14, 0, 1000, 255, 1, 0, 0, 0, 1001, 1002, 3, 104, 47, 0, 1002, 1003, 1, 0, 0, 0, 1003, 1004, 6, 123, 17, 0, 1004, 257, 1, 0, 0, 0, 1005, 1006, 3, 238, 114, 0, 1006, 1007, 1, 0, 0, 0, 1007, 1008, 6, 124, 21, 0, 1008, 259, 1, 0, 0, 0, 1009, 1010, 3, 204, 97, 0, 1010, 1011, 1, 0, 0, 0, 1011, 1012, 6, 125, 18, 0, 1012, 261, 1, 0, 0, 0, 1013, 1014, 3, 164, 77, 0, 1014, 1015, 1, 0, 0, 0, 1015, 1016, 6, 126, 16, 0, 1016, 263, 1, 0, 0, 0, 1017, 1018, 3, 48, 19, 0, 1018, 1019, 1, 0, 0, 0, 1019, 1020, 6, 127, 8, 0, 1020, 265, 1, 0, 0, 0, 1021, 1022, 3, 50, 20, 0, 1022, 1023, 1, 0, 0, 0, 1023, 1024, 6, 128, 8, 0, 1024, 267, 1, 0, 0, 0, 1025, 1026, 3, 52, 21, 0, 1026, 1027, 1, 0, 0, 0, 1027, 1028, 6, 129, 8, 0, 1028, 269, 1, 0, 0, 0, 1029, 1030, 3, 64, 27, 0, 1030, 1031, 1, 0, 0, 0, 1031, 1032, 6, 130, 11, 0, 1032, 1033, 6, 130, 12, 0, 1033, 271, 1, 0, 0, 0, 1034, 1035, 3, 104, 47, 0, 1035, 1036, 1, 0, 0, 0, 1036, 1037, 6, 131, 17, 0, 1037, 273, 1, 0, 0, 0, 1038, 1039, 3, 164, 77, 0, 1039, 1040, 1, 0, 0, 0, 1040, 1041, 6, 132, 16, 0, 1041, 275, 1, 0, 0, 0, 1042, 1043, 3, 162, 76, 0, 1043, 1044, 1, 0, 0, 0, 1044, 1045, 6, 133, 22, 0, 1045, 277, 1, 0, 0, 0, 1046, 1047, 3, 48, 19, 0, 1047, 1048, 1, 0, 0, 0, 1048, 1049, 6, 134, 8, 0, 1049, 279, 1, 0, 0, 0, 1050, 1051, 3, 50, 20, 0, 1051, 1052, 1, 0, 0, 0, 1052, 1053, 6, 135, 8, 0, 1053, 281, 1, 0, 0, 0, 1054, 1055, 3, 52, 21, 0, 1055, 1056, 1, 0, 0, 0, 1056, 1057, 6, 136, 8, 0, 1057, 283, 1, 0, 0, 0, 1058, 1059, 3, 64, 27, 0, 1059, 1060, 1, 0, 0, 0, 1060, 1061, 6, 137, 11, 0, 1061, 1062, 6, 137, 12, 0, 1062, 285, 1, 0, 0, 0, 1063, 1064, 5, 105, 0, 0, 1064, 1065, 5, 110, 0, 0, 1065, 1066, 5, 102, 0, 0, 1066, 1067, 5, 111, 0, 0, 1067, 287, 1, 0, 0, 0, 1068, 1069, 5, 102, 0, 0, 1069, 1070, 5, 117, 0, 0, 1070, 1071, 5, 110, 0, 0, 1071, 1072, 5, 99, 0, 0, 1072, 1073, 5, 116, 0, 0, 1073, 1074, 5, 105, 0, 0, 1074, 1075, 5, 111, 0, 0, 1075, 1076, 5, 110, 0, 0, 1076, 1077, 5, 115, 0, 0, 1077, 289, 1, 0, 0, 0, 1078, 1079, 3, 48, 19, 0, 1079, 1080, 1, 0, 0, 0, 1080, 1081, 6, 140, 8, 0, 1081, 291, 1, 0, 0, 0, 1082, 1083, 3, 50, 20, 0, 1083, 1084, 1, 0, 0, 0, 1084, 1085, 6, 141, 8, 0, 1085, 293, 1, 0, 0, 0, 1086, 1087, 3, 52, 21, 0, 1087, 1088, 1, 0, 0, 0, 1088, 1089, 6, 142, 8, 0, 1089, 295, 1, 0, 0, 0, 49, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 452, 462, 466, 469, 478, 480, 491, 532, 537, 546, 553, 558, 560, 571, 579, 582, 584, 589, 594, 600, 607, 612, 618, 621, 629, 633, 760, 765, 770, 772, 778, 831, 836, 871, 875, 880, 885, 890, 892, 23, 5, 2, 0, 5, 4, 0, 5, 6, 0, 5, 1, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 0, 1, 0, 7, 63, 0, 5, 0, 0, 7, 26, 0, 4, 0, 0, 7, 64, 0, 7, 34, 0, 7, 33, 0, 7, 66, 0, 7, 36, 0, 7, 75, 0, 5, 7, 0, 7, 71, 0, 7, 84, 0, 7, 65, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index 5a01cfa11b3fd..8946172327bcc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -24,20 +24,28 @@ public class EsqlBaseLexer extends Lexer { PIPE=26, STRING=27, INTEGER_LITERAL=28, DECIMAL_LITERAL=29, BY=30, AND=31, ASC=32, ASSIGN=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, LAST=39, LP=40, IN=41, IS=42, LIKE=43, NOT=44, NULL=45, NULLS=46, OR=47, PARAM=48, - RLIKE=49, RP=50, TRUE=51, INFO=52, FUNCTIONS=53, EQ=54, NEQ=55, LT=56, - LTE=57, GT=58, GTE=59, PLUS=60, MINUS=61, ASTERISK=62, SLASH=63, PERCENT=64, - OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, - EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, AS=72, METADATA=73, - ON=74, WITH=75, SRC_UNQUOTED_IDENTIFIER=76, SRC_QUOTED_IDENTIFIER=77, - SRC_LINE_COMMENT=78, SRC_MULTILINE_COMMENT=79, SRC_WS=80, EXPLAIN_PIPE=81; + RLIKE=49, RP=50, TRUE=51, EQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, + PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, OPENING_BRACKET=63, + CLOSING_BRACKET=64, UNQUOTED_IDENTIFIER=65, QUOTED_IDENTIFIER=66, EXPR_LINE_COMMENT=67, + EXPR_MULTILINE_COMMENT=68, EXPR_WS=69, METADATA=70, FROM_UNQUOTED_IDENTIFIER=71, + FROM_LINE_COMMENT=72, FROM_MULTILINE_COMMENT=73, FROM_WS=74, PROJECT_UNQUOTED_IDENTIFIER=75, + PROJECT_LINE_COMMENT=76, PROJECT_MULTILINE_COMMENT=77, PROJECT_WS=78, + AS=79, RENAME_LINE_COMMENT=80, RENAME_MULTILINE_COMMENT=81, RENAME_WS=82, + ON=83, WITH=84, ENRICH_LINE_COMMENT=85, ENRICH_MULTILINE_COMMENT=86, ENRICH_WS=87, + ENRICH_FIELD_LINE_COMMENT=88, ENRICH_FIELD_MULTILINE_COMMENT=89, ENRICH_FIELD_WS=90, + MVEXPAND_LINE_COMMENT=91, MVEXPAND_MULTILINE_COMMENT=92, MVEXPAND_WS=93, + INFO=94, FUNCTIONS=95, SHOW_LINE_COMMENT=96, SHOW_MULTILINE_COMMENT=97, + SHOW_WS=98; public static final int - EXPLAIN_MODE=1, EXPRESSION=2, SOURCE_IDENTIFIERS=3; + EXPLAIN_MODE=1, EXPRESSION_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, + ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9; public static String[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" }; public static String[] modeNames = { - "DEFAULT_MODE", "EXPLAIN_MODE", "EXPRESSION", "SOURCE_IDENTIFIERS" + "DEFAULT_MODE", "EXPLAIN_MODE", "EXPRESSION_MODE", "FROM_MODE", "PROJECT_MODE", + "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "MVEXPAND_MODE", "SHOW_MODE" }; private static String[] makeRuleNames() { @@ -47,16 +55,30 @@ private static String[] makeRuleNames() { "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "EXPLAIN_OPENING_BRACKET", "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", - "UNESCAPED_CHARS", "EXPONENT", "STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", + "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", + "UNDERSCORE", "UNQUOTED_ID_BODY", "STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", - "RLIKE", "RP", "TRUE", "INFO", "FUNCTIONS", "EQ", "NEQ", "LT", "LTE", - "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", - "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", - "EXPR_MULTILINE_COMMENT", "EXPR_WS", "SRC_PIPE", "SRC_OPENING_BRACKET", - "SRC_CLOSING_BRACKET", "SRC_COMMA", "SRC_ASSIGN", "AS", "METADATA", "ON", - "WITH", "SRC_UNQUOTED_IDENTIFIER", "SRC_UNQUOTED_IDENTIFIER_PART", "SRC_QUOTED_IDENTIFIER", - "SRC_LINE_COMMENT", "SRC_MULTILINE_COMMENT", "SRC_WS" + "RLIKE", "RP", "TRUE", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", + "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", + "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", + "EXPR_WS", "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", + "FROM_COMMA", "FROM_ASSIGN", "METADATA", "FROM_UNQUOTED_IDENTIFIER_PART", + "FROM_UNQUOTED_IDENTIFIER", "FROM_QUOTED_IDENTIFIER", "FROM_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", + "UNQUOTED_ID_BODY_WITH_PATTERN", "PROJECT_UNQUOTED_IDENTIFIER", "PROJECT_QUOTED_IDENTIFIER", + "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "RENAME_PIPE", + "RENAME_ASSIGN", "RENAME_COMMA", "RENAME_DOT", "AS", "RENAME_QUOTED_IDENTIFIER", + "RENAME_UNQUOTED_IDENTIFIER", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", + "RENAME_WS", "ENRICH_PIPE", "ON", "WITH", "ENRICH_POLICY_UNQUOTED_IDENTIFIER", + "ENRICH_QUOTED_IDENTIFIER", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", + "ENRICH_WS", "ENRICH_FIELD_PIPE", "ENRICH_FIELD_ASSIGN", "ENRICH_FIELD_COMMA", + "ENRICH_FIELD_DOT", "ENRICH_FIELD_WITH", "ENRICH_FIELD_UNQUOTED_IDENTIFIER", + "ENRICH_FIELD_QUOTED_IDENTIFIER", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", + "ENRICH_FIELD_WS", "MVEXPAND_PIPE", "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", + "MVEXPAND_UNQUOTED_IDENTIFIER", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", + "MVEXPAND_WS", "SHOW_PIPE", "INFO", "FUNCTIONS", "SHOW_LINE_COMMENT", + "SHOW_MULTILINE_COMMENT", "SHOW_WS" }; } public static final String[] ruleNames = makeRuleNames(); @@ -66,12 +88,14 @@ private static String[] makeLiteralNames() { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'inlinestats'", "'keep'", "'limit'", "'mv_expand'", "'project'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", null, - null, null, null, null, null, null, null, null, null, null, "'by'", "'and'", - "'asc'", null, null, "'desc'", "'.'", "'false'", "'first'", "'last'", - "'('", "'in'", "'is'", "'like'", "'not'", "'null'", "'nulls'", "'or'", - "'?'", "'rlike'", "')'", "'true'", "'info'", "'functions'", "'=='", "'!='", - "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, - "']'", null, null, null, null, null, "'as'", "'metadata'", "'on'", "'with'" + null, null, null, null, null, null, "'|'", null, null, null, "'by'", + "'and'", "'asc'", "'='", "','", "'desc'", "'.'", "'false'", "'first'", + "'last'", "'('", "'in'", "'is'", "'like'", "'not'", "'null'", "'nulls'", + "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", "'!='", "'<'", "'<='", + "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, "']'", null, + null, null, null, null, "'metadata'", null, null, null, null, null, null, + null, null, "'as'", null, null, null, "'on'", "'with'", null, null, null, + null, null, null, null, null, null, "'info'", "'functions'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); @@ -84,12 +108,17 @@ private static String[] makeSymbolicNames() { "PIPE", "STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", - "TRUE", "INFO", "FUNCTIONS", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", - "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", - "EXPR_WS", "AS", "METADATA", "ON", "WITH", "SRC_UNQUOTED_IDENTIFIER", - "SRC_QUOTED_IDENTIFIER", "SRC_LINE_COMMENT", "SRC_MULTILINE_COMMENT", - "SRC_WS", "EXPLAIN_PIPE" + "TRUE", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", + "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", + "METADATA", "FROM_UNQUOTED_IDENTIFIER", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", + "FROM_WS", "PROJECT_UNQUOTED_IDENTIFIER", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", + "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", + "RENAME_WS", "ON", "WITH", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", + "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", + "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", + "MVEXPAND_WS", "INFO", "FUNCTIONS", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", + "SHOW_WS" }; } private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); @@ -152,482 +181,683 @@ public EsqlBaseLexer(CharStream input) { public ATN getATN() { return _ATN; } public static final String _serializedATN = - "\u0004\u0000Q\u02fc\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ - "\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ - "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ - "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ - "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ - "\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007\u000f"+ - "\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002\u0012\u0007\u0012"+ - "\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002\u0015\u0007\u0015"+ - "\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002\u0018\u0007\u0018"+ - "\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002\u001b\u0007\u001b"+ - "\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d\u0002\u001e\u0007\u001e"+ - "\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002"+ - "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002"+ - "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ - "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+ - "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+ - "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0002"+ - "<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007@\u0002"+ - "A\u0007A\u0002B\u0007B\u0002C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002"+ - "F\u0007F\u0002G\u0007G\u0002H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002"+ - "K\u0007K\u0002L\u0007L\u0002M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002"+ - "P\u0007P\u0002Q\u0007Q\u0002R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002"+ - "U\u0007U\u0002V\u0007V\u0002W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002"+ - "Z\u0007Z\u0002[\u0007[\u0002\\\u0007\\\u0001\u0000\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006"+ - "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+ - "\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+ - "\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ - "\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f"+ - "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ + "\u0004\u0000b\u0442\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0002\u0000\u0007"+ + "\u0000\u0002\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007"+ + "\u0003\u0002\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007"+ + "\u0006\u0002\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n"+ + "\u0007\n\u0002\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002"+ + "\u000e\u0007\u000e\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002"+ + "\u0011\u0007\u0011\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002"+ + "\u0014\u0007\u0014\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002"+ + "\u0017\u0007\u0017\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002"+ + "\u001a\u0007\u001a\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002"+ + "\u001d\u0007\u001d\u0002\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002"+ + " \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002"+ + "%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002"+ + "*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002"+ + "/\u0007/\u00020\u00070\u00021\u00071\u00022\u00072\u00023\u00073\u0002"+ + "4\u00074\u00025\u00075\u00026\u00076\u00027\u00077\u00028\u00078\u0002"+ + "9\u00079\u0002:\u0007:\u0002;\u0007;\u0002<\u0007<\u0002=\u0007=\u0002"+ + ">\u0007>\u0002?\u0007?\u0002@\u0007@\u0002A\u0007A\u0002B\u0007B\u0002"+ + "C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002F\u0007F\u0002G\u0007G\u0002"+ + "H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002K\u0007K\u0002L\u0007L\u0002"+ + "M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002P\u0007P\u0002Q\u0007Q\u0002"+ + "R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002U\u0007U\u0002V\u0007V\u0002"+ + "W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002Z\u0007Z\u0002[\u0007[\u0002"+ + "\\\u0007\\\u0002]\u0007]\u0002^\u0007^\u0002_\u0007_\u0002`\u0007`\u0002"+ + "a\u0007a\u0002b\u0007b\u0002c\u0007c\u0002d\u0007d\u0002e\u0007e\u0002"+ + "f\u0007f\u0002g\u0007g\u0002h\u0007h\u0002i\u0007i\u0002j\u0007j\u0002"+ + "k\u0007k\u0002l\u0007l\u0002m\u0007m\u0002n\u0007n\u0002o\u0007o\u0002"+ + "p\u0007p\u0002q\u0007q\u0002r\u0007r\u0002s\u0007s\u0002t\u0007t\u0002"+ + "u\u0007u\u0002v\u0007v\u0002w\u0007w\u0002x\u0007x\u0002y\u0007y\u0002"+ + "z\u0007z\u0002{\u0007{\u0002|\u0007|\u0002}\u0007}\u0002~\u0007~\u0002"+ + "\u007f\u0007\u007f\u0002\u0080\u0007\u0080\u0002\u0081\u0007\u0081\u0002"+ + "\u0082\u0007\u0082\u0002\u0083\u0007\u0083\u0002\u0084\u0007\u0084\u0002"+ + "\u0085\u0007\u0085\u0002\u0086\u0007\u0086\u0002\u0087\u0007\u0087\u0002"+ + "\u0088\u0007\u0088\u0002\u0089\u0007\u0089\u0002\u008a\u0007\u008a\u0002"+ + "\u008b\u0007\u008b\u0002\u008c\u0007\u008c\u0002\u008d\u0007\u008d\u0002"+ + "\u008e\u0007\u008e\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ + "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007\u0001"+ + "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\t"+ + "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001"+ + "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001"+ + "\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001"+ + "\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e"+ + "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ + "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010"+ "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ - "\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ - "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0004\u0012"+ - "\u0159\b\u0012\u000b\u0012\f\u0012\u015a\u0001\u0012\u0001\u0012\u0001"+ - "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u0163\b\u0013\n"+ - "\u0013\f\u0013\u0166\t\u0013\u0001\u0013\u0003\u0013\u0169\b\u0013\u0001"+ - "\u0013\u0003\u0013\u016c\b\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001"+ - "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0005\u0014\u0175\b\u0014\n"+ - "\u0014\f\u0014\u0178\t\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+ - "\u0014\u0001\u0014\u0001\u0015\u0004\u0015\u0180\b\u0015\u000b\u0015\f"+ - "\u0015\u0181\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016"+ - "\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017"+ - "\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019"+ - "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a"+ - "\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001c"+ - "\u0001\u001c\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001e"+ - "\u0001\u001f\u0001\u001f\u0001 \u0001 \u0003 \u01ab\b \u0001 \u0004 \u01ae"+ - "\b \u000b \f \u01af\u0001!\u0001!\u0001!\u0005!\u01b5\b!\n!\f!\u01b8\t"+ - "!\u0001!\u0001!\u0001!\u0001!\u0001!\u0001!\u0005!\u01c0\b!\n!\f!\u01c3"+ - "\t!\u0001!\u0001!\u0001!\u0001!\u0001!\u0003!\u01ca\b!\u0001!\u0003!\u01cd"+ - "\b!\u0003!\u01cf\b!\u0001\"\u0004\"\u01d2\b\"\u000b\"\f\"\u01d3\u0001"+ - "#\u0004#\u01d7\b#\u000b#\f#\u01d8\u0001#\u0001#\u0005#\u01dd\b#\n#\f#"+ - "\u01e0\t#\u0001#\u0001#\u0004#\u01e4\b#\u000b#\f#\u01e5\u0001#\u0004#"+ - "\u01e9\b#\u000b#\f#\u01ea\u0001#\u0001#\u0005#\u01ef\b#\n#\f#\u01f2\t"+ - "#\u0003#\u01f4\b#\u0001#\u0001#\u0001#\u0001#\u0004#\u01fa\b#\u000b#\f"+ - "#\u01fb\u0001#\u0001#\u0003#\u0200\b#\u0001$\u0001$\u0001$\u0001%\u0001"+ - "%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001(\u0001"+ - "(\u0001)\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001+\u0001+\u0001"+ - "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0001,\u0001,\u0001"+ - "-\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0001/\u0001"+ - "0\u00010\u00010\u00011\u00011\u00011\u00011\u00011\u00012\u00012\u0001"+ - "2\u00012\u00013\u00013\u00013\u00013\u00013\u00014\u00014\u00014\u0001"+ - "4\u00014\u00014\u00015\u00015\u00015\u00016\u00016\u00017\u00017\u0001"+ - "7\u00017\u00017\u00017\u00018\u00018\u00019\u00019\u00019\u00019\u0001"+ - "9\u0001:\u0001:\u0001:\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001"+ - ";\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001=\u0001"+ - "=\u0001=\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001A\u0001"+ - "A\u0001A\u0001B\u0001B\u0001C\u0001C\u0001D\u0001D\u0001E\u0001E\u0001"+ - "F\u0001F\u0001G\u0001G\u0001G\u0001G\u0001G\u0001H\u0001H\u0001H\u0001"+ - "H\u0001H\u0001I\u0001I\u0001I\u0001I\u0005I\u028e\bI\nI\fI\u0291\tI\u0001"+ - "I\u0001I\u0001I\u0001I\u0004I\u0297\bI\u000bI\fI\u0298\u0003I\u029b\b"+ - "I\u0001J\u0001J\u0001J\u0001J\u0005J\u02a1\bJ\nJ\fJ\u02a4\tJ\u0001J\u0001"+ - "J\u0001K\u0001K\u0001K\u0001K\u0001L\u0001L\u0001L\u0001L\u0001M\u0001"+ - "M\u0001M\u0001M\u0001N\u0001N\u0001N\u0001N\u0001N\u0001O\u0001O\u0001"+ - "O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001P\u0001P\u0001"+ - "Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001S\u0001S\u0001"+ - "S\u0001T\u0001T\u0001T\u0001T\u0001T\u0001T\u0001T\u0001T\u0001T\u0001"+ - "U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001V\u0001W\u0004W\u02e2"+ - "\bW\u000bW\fW\u02e3\u0001X\u0004X\u02e7\bX\u000bX\fX\u02e8\u0001X\u0001"+ - "X\u0003X\u02ed\bX\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001"+ - "[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0002\u0176\u01c1\u0000"+ - "]\u0004\u0001\u0006\u0002\b\u0003\n\u0004\f\u0005\u000e\u0006\u0010\u0007"+ - "\u0012\b\u0014\t\u0016\n\u0018\u000b\u001a\f\u001c\r\u001e\u000e \u000f"+ - "\"\u0010$\u0011&\u0012(\u0013*\u0014,\u0015.\u00160\u00002Q4\u00176\u0018"+ - "8\u0019:\u001a<\u0000>\u0000@\u0000B\u0000D\u0000F\u001bH\u001cJ\u001d"+ - "L\u001eN\u001fP R!T\"V#X$Z%\\&^\'`(b)d*f+h,j-l.n/p0r1t2v3x4z5|6~7\u0080"+ - "8\u00829\u0084:\u0086;\u0088<\u008a=\u008c>\u008e?\u0090@\u0092A\u0094"+ - "B\u0096C\u0098D\u009aE\u009cF\u009eG\u00a0\u0000\u00a2\u0000\u00a4\u0000"+ - "\u00a6\u0000\u00a8\u0000\u00aaH\u00acI\u00aeJ\u00b0K\u00b2L\u00b4\u0000"+ - "\u00b6M\u00b8N\u00baO\u00bcP\u0004\u0000\u0001\u0002\u0003\r\u0006\u0000"+ - "\t\n\r\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u0000"+ - "09\u0002\u0000AZaz\u0005\u0000\"\"\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\"+ - "\\\u0002\u0000EEee\u0002\u0000++--\u0002\u0000@@__\u0001\u0000``\n\u0000"+ - "\t\n\r\r ,,//==[[]]``||\u0002\u0000**//\u0318\u0000\u0004\u0001\u0000"+ - "\u0000\u0000\u0000\u0006\u0001\u0000\u0000\u0000\u0000\b\u0001\u0000\u0000"+ - "\u0000\u0000\n\u0001\u0000\u0000\u0000\u0000\f\u0001\u0000\u0000\u0000"+ - "\u0000\u000e\u0001\u0000\u0000\u0000\u0000\u0010\u0001\u0000\u0000\u0000"+ - "\u0000\u0012\u0001\u0000\u0000\u0000\u0000\u0014\u0001\u0000\u0000\u0000"+ - "\u0000\u0016\u0001\u0000\u0000\u0000\u0000\u0018\u0001\u0000\u0000\u0000"+ - "\u0000\u001a\u0001\u0000\u0000\u0000\u0000\u001c\u0001\u0000\u0000\u0000"+ - "\u0000\u001e\u0001\u0000\u0000\u0000\u0000 \u0001\u0000\u0000\u0000\u0000"+ - "\"\u0001\u0000\u0000\u0000\u0000$\u0001\u0000\u0000\u0000\u0000&\u0001"+ - "\u0000\u0000\u0000\u0000(\u0001\u0000\u0000\u0000\u0000*\u0001\u0000\u0000"+ - "\u0000\u0000,\u0001\u0000\u0000\u0000\u0000.\u0001\u0000\u0000\u0000\u0001"+ - "0\u0001\u0000\u0000\u0000\u00012\u0001\u0000\u0000\u0000\u00014\u0001"+ - "\u0000\u0000\u0000\u00016\u0001\u0000\u0000\u0000\u00018\u0001\u0000\u0000"+ - "\u0000\u0002:\u0001\u0000\u0000\u0000\u0002F\u0001\u0000\u0000\u0000\u0002"+ - "H\u0001\u0000\u0000\u0000\u0002J\u0001\u0000\u0000\u0000\u0002L\u0001"+ - "\u0000\u0000\u0000\u0002N\u0001\u0000\u0000\u0000\u0002P\u0001\u0000\u0000"+ - "\u0000\u0002R\u0001\u0000\u0000\u0000\u0002T\u0001\u0000\u0000\u0000\u0002"+ - "V\u0001\u0000\u0000\u0000\u0002X\u0001\u0000\u0000\u0000\u0002Z\u0001"+ - "\u0000\u0000\u0000\u0002\\\u0001\u0000\u0000\u0000\u0002^\u0001\u0000"+ - "\u0000\u0000\u0002`\u0001\u0000\u0000\u0000\u0002b\u0001\u0000\u0000\u0000"+ - "\u0002d\u0001\u0000\u0000\u0000\u0002f\u0001\u0000\u0000\u0000\u0002h"+ - "\u0001\u0000\u0000\u0000\u0002j\u0001\u0000\u0000\u0000\u0002l\u0001\u0000"+ - "\u0000\u0000\u0002n\u0001\u0000\u0000\u0000\u0002p\u0001\u0000\u0000\u0000"+ - "\u0002r\u0001\u0000\u0000\u0000\u0002t\u0001\u0000\u0000\u0000\u0002v"+ - "\u0001\u0000\u0000\u0000\u0002x\u0001\u0000\u0000\u0000\u0002z\u0001\u0000"+ - "\u0000\u0000\u0002|\u0001\u0000\u0000\u0000\u0002~\u0001\u0000\u0000\u0000"+ - "\u0002\u0080\u0001\u0000\u0000\u0000\u0002\u0082\u0001\u0000\u0000\u0000"+ - "\u0002\u0084\u0001\u0000\u0000\u0000\u0002\u0086\u0001\u0000\u0000\u0000"+ - "\u0002\u0088\u0001\u0000\u0000\u0000\u0002\u008a\u0001\u0000\u0000\u0000"+ - "\u0002\u008c\u0001\u0000\u0000\u0000\u0002\u008e\u0001\u0000\u0000\u0000"+ - "\u0002\u0090\u0001\u0000\u0000\u0000\u0002\u0092\u0001\u0000\u0000\u0000"+ - "\u0002\u0094\u0001\u0000\u0000\u0000\u0002\u0096\u0001\u0000\u0000\u0000"+ - "\u0002\u0098\u0001\u0000\u0000\u0000\u0002\u009a\u0001\u0000\u0000\u0000"+ - "\u0002\u009c\u0001\u0000\u0000\u0000\u0002\u009e\u0001\u0000\u0000\u0000"+ - "\u0003\u00a0\u0001\u0000\u0000\u0000\u0003\u00a2\u0001\u0000\u0000\u0000"+ - "\u0003\u00a4\u0001\u0000\u0000\u0000\u0003\u00a6\u0001\u0000\u0000\u0000"+ - "\u0003\u00a8\u0001\u0000\u0000\u0000\u0003\u00aa\u0001\u0000\u0000\u0000"+ - "\u0003\u00ac\u0001\u0000\u0000\u0000\u0003\u00ae\u0001\u0000\u0000\u0000"+ - "\u0003\u00b0\u0001\u0000\u0000\u0000\u0003\u00b2\u0001\u0000\u0000\u0000"+ - "\u0003\u00b6\u0001\u0000\u0000\u0000\u0003\u00b8\u0001\u0000\u0000\u0000"+ - "\u0003\u00ba\u0001\u0000\u0000\u0000\u0003\u00bc\u0001\u0000\u0000\u0000"+ - "\u0004\u00be\u0001\u0000\u0000\u0000\u0006\u00c8\u0001\u0000\u0000\u0000"+ - "\b\u00cf\u0001\u0000\u0000\u0000\n\u00d8\u0001\u0000\u0000\u0000\f\u00df"+ - "\u0001\u0000\u0000\u0000\u000e\u00e9\u0001\u0000\u0000\u0000\u0010\u00f0"+ - "\u0001\u0000\u0000\u0000\u0012\u00f7\u0001\u0000\u0000\u0000\u0014\u0105"+ - "\u0001\u0000\u0000\u0000\u0016\u010c\u0001\u0000\u0000\u0000\u0018\u0114"+ - "\u0001\u0000\u0000\u0000\u001a\u0120\u0001\u0000\u0000\u0000\u001c\u012a"+ - "\u0001\u0000\u0000\u0000\u001e\u0133\u0001\u0000\u0000\u0000 \u0139\u0001"+ - "\u0000\u0000\u0000\"\u0140\u0001\u0000\u0000\u0000$\u0147\u0001\u0000"+ - "\u0000\u0000&\u014f\u0001\u0000\u0000\u0000(\u0158\u0001\u0000\u0000\u0000"+ - "*\u015e\u0001\u0000\u0000\u0000,\u016f\u0001\u0000\u0000\u0000.\u017f"+ - "\u0001\u0000\u0000\u00000\u0185\u0001\u0000\u0000\u00002\u018a\u0001\u0000"+ - "\u0000\u00004\u018f\u0001\u0000\u0000\u00006\u0193\u0001\u0000\u0000\u0000"+ - "8\u0197\u0001\u0000\u0000\u0000:\u019b\u0001\u0000\u0000\u0000<\u019f"+ - "\u0001\u0000\u0000\u0000>\u01a1\u0001\u0000\u0000\u0000@\u01a3\u0001\u0000"+ - "\u0000\u0000B\u01a6\u0001\u0000\u0000\u0000D\u01a8\u0001\u0000\u0000\u0000"+ - "F\u01ce\u0001\u0000\u0000\u0000H\u01d1\u0001\u0000\u0000\u0000J\u01ff"+ - "\u0001\u0000\u0000\u0000L\u0201\u0001\u0000\u0000\u0000N\u0204\u0001\u0000"+ - "\u0000\u0000P\u0208\u0001\u0000\u0000\u0000R\u020c\u0001\u0000\u0000\u0000"+ - "T\u020e\u0001\u0000\u0000\u0000V\u0210\u0001\u0000\u0000\u0000X\u0215"+ - "\u0001\u0000\u0000\u0000Z\u0217\u0001\u0000\u0000\u0000\\\u021d\u0001"+ - "\u0000\u0000\u0000^\u0223\u0001\u0000\u0000\u0000`\u0228\u0001\u0000\u0000"+ - "\u0000b\u022a\u0001\u0000\u0000\u0000d\u022d\u0001\u0000\u0000\u0000f"+ - "\u0230\u0001\u0000\u0000\u0000h\u0235\u0001\u0000\u0000\u0000j\u0239\u0001"+ - "\u0000\u0000\u0000l\u023e\u0001\u0000\u0000\u0000n\u0244\u0001\u0000\u0000"+ - "\u0000p\u0247\u0001\u0000\u0000\u0000r\u0249\u0001\u0000\u0000\u0000t"+ - "\u024f\u0001\u0000\u0000\u0000v\u0251\u0001\u0000\u0000\u0000x\u0256\u0001"+ - "\u0000\u0000\u0000z\u025b\u0001\u0000\u0000\u0000|\u0265\u0001\u0000\u0000"+ - "\u0000~\u0268\u0001\u0000\u0000\u0000\u0080\u026b\u0001\u0000\u0000\u0000"+ - "\u0082\u026d\u0001\u0000\u0000\u0000\u0084\u0270\u0001\u0000\u0000\u0000"+ - "\u0086\u0272\u0001\u0000\u0000\u0000\u0088\u0275\u0001\u0000\u0000\u0000"+ - "\u008a\u0277\u0001\u0000\u0000\u0000\u008c\u0279\u0001\u0000\u0000\u0000"+ - "\u008e\u027b\u0001\u0000\u0000\u0000\u0090\u027d\u0001\u0000\u0000\u0000"+ - "\u0092\u027f\u0001\u0000\u0000\u0000\u0094\u0284\u0001\u0000\u0000\u0000"+ - "\u0096\u029a\u0001\u0000\u0000\u0000\u0098\u029c\u0001\u0000\u0000\u0000"+ - "\u009a\u02a7\u0001\u0000\u0000\u0000\u009c\u02ab\u0001\u0000\u0000\u0000"+ - "\u009e\u02af\u0001\u0000\u0000\u0000\u00a0\u02b3\u0001\u0000\u0000\u0000"+ - "\u00a2\u02b8\u0001\u0000\u0000\u0000\u00a4\u02be\u0001\u0000\u0000\u0000"+ - "\u00a6\u02c4\u0001\u0000\u0000\u0000\u00a8\u02c8\u0001\u0000\u0000\u0000"+ - "\u00aa\u02cc\u0001\u0000\u0000\u0000\u00ac\u02cf\u0001\u0000\u0000\u0000"+ - "\u00ae\u02d8\u0001\u0000\u0000\u0000\u00b0\u02db\u0001\u0000\u0000\u0000"+ - "\u00b2\u02e1\u0001\u0000\u0000\u0000\u00b4\u02ec\u0001\u0000\u0000\u0000"+ - "\u00b6\u02ee\u0001\u0000\u0000\u0000\u00b8\u02f0\u0001\u0000\u0000\u0000"+ - "\u00ba\u02f4\u0001\u0000\u0000\u0000\u00bc\u02f8\u0001\u0000\u0000\u0000"+ - "\u00be\u00bf\u0005d\u0000\u0000\u00bf\u00c0\u0005i\u0000\u0000\u00c0\u00c1"+ - "\u0005s\u0000\u0000\u00c1\u00c2\u0005s\u0000\u0000\u00c2\u00c3\u0005e"+ - "\u0000\u0000\u00c3\u00c4\u0005c\u0000\u0000\u00c4\u00c5\u0005t\u0000\u0000"+ - "\u00c5\u00c6\u0001\u0000\u0000\u0000\u00c6\u00c7\u0006\u0000\u0000\u0000"+ - "\u00c7\u0005\u0001\u0000\u0000\u0000\u00c8\u00c9\u0005d\u0000\u0000\u00c9"+ - "\u00ca\u0005r\u0000\u0000\u00ca\u00cb\u0005o\u0000\u0000\u00cb\u00cc\u0005"+ - "p\u0000\u0000\u00cc\u00cd\u0001\u0000\u0000\u0000\u00cd\u00ce\u0006\u0001"+ - "\u0001\u0000\u00ce\u0007\u0001\u0000\u0000\u0000\u00cf\u00d0\u0005e\u0000"+ - "\u0000\u00d0\u00d1\u0005n\u0000\u0000\u00d1\u00d2\u0005r\u0000\u0000\u00d2"+ - "\u00d3\u0005i\u0000\u0000\u00d3\u00d4\u0005c\u0000\u0000\u00d4\u00d5\u0005"+ - "h\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7\u0006\u0002"+ - "\u0001\u0000\u00d7\t\u0001\u0000\u0000\u0000\u00d8\u00d9\u0005e\u0000"+ - "\u0000\u00d9\u00da\u0005v\u0000\u0000\u00da\u00db\u0005a\u0000\u0000\u00db"+ - "\u00dc\u0005l\u0000\u0000\u00dc\u00dd\u0001\u0000\u0000\u0000\u00dd\u00de"+ - "\u0006\u0003\u0000\u0000\u00de\u000b\u0001\u0000\u0000\u0000\u00df\u00e0"+ - "\u0005e\u0000\u0000\u00e0\u00e1\u0005x\u0000\u0000\u00e1\u00e2\u0005p"+ - "\u0000\u0000\u00e2\u00e3\u0005l\u0000\u0000\u00e3\u00e4\u0005a\u0000\u0000"+ - "\u00e4\u00e5\u0005i\u0000\u0000\u00e5\u00e6\u0005n\u0000\u0000\u00e6\u00e7"+ - "\u0001\u0000\u0000\u0000\u00e7\u00e8\u0006\u0004\u0002\u0000\u00e8\r\u0001"+ - "\u0000\u0000\u0000\u00e9\u00ea\u0005f\u0000\u0000\u00ea\u00eb\u0005r\u0000"+ - "\u0000\u00eb\u00ec\u0005o\u0000\u0000\u00ec\u00ed\u0005m\u0000\u0000\u00ed"+ - "\u00ee\u0001\u0000\u0000\u0000\u00ee\u00ef\u0006\u0005\u0001\u0000\u00ef"+ - "\u000f\u0001\u0000\u0000\u0000\u00f0\u00f1\u0005g\u0000\u0000\u00f1\u00f2"+ - "\u0005r\u0000\u0000\u00f2\u00f3\u0005o\u0000\u0000\u00f3\u00f4\u0005k"+ - "\u0000\u0000\u00f4\u00f5\u0001\u0000\u0000\u0000\u00f5\u00f6\u0006\u0006"+ - "\u0000\u0000\u00f6\u0011\u0001\u0000\u0000\u0000\u00f7\u00f8\u0005i\u0000"+ - "\u0000\u00f8\u00f9\u0005n\u0000\u0000\u00f9\u00fa\u0005l\u0000\u0000\u00fa"+ - "\u00fb\u0005i\u0000\u0000\u00fb\u00fc\u0005n\u0000\u0000\u00fc\u00fd\u0005"+ - "e\u0000\u0000\u00fd\u00fe\u0005s\u0000\u0000\u00fe\u00ff\u0005t\u0000"+ - "\u0000\u00ff\u0100\u0005a\u0000\u0000\u0100\u0101\u0005t\u0000\u0000\u0101"+ - "\u0102\u0005s\u0000\u0000\u0102\u0103\u0001\u0000\u0000\u0000\u0103\u0104"+ - "\u0006\u0007\u0000\u0000\u0104\u0013\u0001\u0000\u0000\u0000\u0105\u0106"+ - "\u0005k\u0000\u0000\u0106\u0107\u0005e\u0000\u0000\u0107\u0108\u0005e"+ - "\u0000\u0000\u0108\u0109\u0005p\u0000\u0000\u0109\u010a\u0001\u0000\u0000"+ - "\u0000\u010a\u010b\u0006\b\u0001\u0000\u010b\u0015\u0001\u0000\u0000\u0000"+ - "\u010c\u010d\u0005l\u0000\u0000\u010d\u010e\u0005i\u0000\u0000\u010e\u010f"+ - "\u0005m\u0000\u0000\u010f\u0110\u0005i\u0000\u0000\u0110\u0111\u0005t"+ - "\u0000\u0000\u0111\u0112\u0001\u0000\u0000\u0000\u0112\u0113\u0006\t\u0000"+ - "\u0000\u0113\u0017\u0001\u0000\u0000\u0000\u0114\u0115\u0005m\u0000\u0000"+ - "\u0115\u0116\u0005v\u0000\u0000\u0116\u0117\u0005_\u0000\u0000\u0117\u0118"+ - "\u0005e\u0000\u0000\u0118\u0119\u0005x\u0000\u0000\u0119\u011a\u0005p"+ - "\u0000\u0000\u011a\u011b\u0005a\u0000\u0000\u011b\u011c\u0005n\u0000\u0000"+ - "\u011c\u011d\u0005d\u0000\u0000\u011d\u011e\u0001\u0000\u0000\u0000\u011e"+ - "\u011f\u0006\n\u0001\u0000\u011f\u0019\u0001\u0000\u0000\u0000\u0120\u0121"+ - "\u0005p\u0000\u0000\u0121\u0122\u0005r\u0000\u0000\u0122\u0123\u0005o"+ - "\u0000\u0000\u0123\u0124\u0005j\u0000\u0000\u0124\u0125\u0005e\u0000\u0000"+ - "\u0125\u0126\u0005c\u0000\u0000\u0126\u0127\u0005t\u0000\u0000\u0127\u0128"+ - "\u0001\u0000\u0000\u0000\u0128\u0129\u0006\u000b\u0001\u0000\u0129\u001b"+ - "\u0001\u0000\u0000\u0000\u012a\u012b\u0005r\u0000\u0000\u012b\u012c\u0005"+ - "e\u0000\u0000\u012c\u012d\u0005n\u0000\u0000\u012d\u012e\u0005a\u0000"+ - "\u0000\u012e\u012f\u0005m\u0000\u0000\u012f\u0130\u0005e\u0000\u0000\u0130"+ - "\u0131\u0001\u0000\u0000\u0000\u0131\u0132\u0006\f\u0001\u0000\u0132\u001d"+ - "\u0001\u0000\u0000\u0000\u0133\u0134\u0005r\u0000\u0000\u0134\u0135\u0005"+ - "o\u0000\u0000\u0135\u0136\u0005w\u0000\u0000\u0136\u0137\u0001\u0000\u0000"+ - "\u0000\u0137\u0138\u0006\r\u0000\u0000\u0138\u001f\u0001\u0000\u0000\u0000"+ - "\u0139\u013a\u0005s\u0000\u0000\u013a\u013b\u0005h\u0000\u0000\u013b\u013c"+ - "\u0005o\u0000\u0000\u013c\u013d\u0005w\u0000\u0000\u013d\u013e\u0001\u0000"+ - "\u0000\u0000\u013e\u013f\u0006\u000e\u0000\u0000\u013f!\u0001\u0000\u0000"+ - "\u0000\u0140\u0141\u0005s\u0000\u0000\u0141\u0142\u0005o\u0000\u0000\u0142"+ - "\u0143\u0005r\u0000\u0000\u0143\u0144\u0005t\u0000\u0000\u0144\u0145\u0001"+ - "\u0000\u0000\u0000\u0145\u0146\u0006\u000f\u0000\u0000\u0146#\u0001\u0000"+ - "\u0000\u0000\u0147\u0148\u0005s\u0000\u0000\u0148\u0149\u0005t\u0000\u0000"+ - "\u0149\u014a\u0005a\u0000\u0000\u014a\u014b\u0005t\u0000\u0000\u014b\u014c"+ - "\u0005s\u0000\u0000\u014c\u014d\u0001\u0000\u0000\u0000\u014d\u014e\u0006"+ - "\u0010\u0000\u0000\u014e%\u0001\u0000\u0000\u0000\u014f\u0150\u0005w\u0000"+ - "\u0000\u0150\u0151\u0005h\u0000\u0000\u0151\u0152\u0005e\u0000\u0000\u0152"+ - "\u0153\u0005r\u0000\u0000\u0153\u0154\u0005e\u0000\u0000\u0154\u0155\u0001"+ - "\u0000\u0000\u0000\u0155\u0156\u0006\u0011\u0000\u0000\u0156\'\u0001\u0000"+ - "\u0000\u0000\u0157\u0159\b\u0000\u0000\u0000\u0158\u0157\u0001\u0000\u0000"+ - "\u0000\u0159\u015a\u0001\u0000\u0000\u0000\u015a\u0158\u0001\u0000\u0000"+ - "\u0000\u015a\u015b\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000\u0000"+ - "\u0000\u015c\u015d\u0006\u0012\u0000\u0000\u015d)\u0001\u0000\u0000\u0000"+ - "\u015e\u015f\u0005/\u0000\u0000\u015f\u0160\u0005/\u0000\u0000\u0160\u0164"+ - "\u0001\u0000\u0000\u0000\u0161\u0163\b\u0001\u0000\u0000\u0162\u0161\u0001"+ - "\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164\u0162\u0001"+ - "\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000\u0000\u0165\u0168\u0001"+ - "\u0000\u0000\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167\u0169\u0005"+ - "\r\u0000\u0000\u0168\u0167\u0001\u0000\u0000\u0000\u0168\u0169\u0001\u0000"+ - "\u0000\u0000\u0169\u016b\u0001\u0000\u0000\u0000\u016a\u016c\u0005\n\u0000"+ - "\u0000\u016b\u016a\u0001\u0000\u0000\u0000\u016b\u016c\u0001\u0000\u0000"+ - "\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d\u016e\u0006\u0013\u0003"+ - "\u0000\u016e+\u0001\u0000\u0000\u0000\u016f\u0170\u0005/\u0000\u0000\u0170"+ - "\u0171\u0005*\u0000\u0000\u0171\u0176\u0001\u0000\u0000\u0000\u0172\u0175"+ - "\u0003,\u0014\u0000\u0173\u0175\t\u0000\u0000\u0000\u0174\u0172\u0001"+ - "\u0000\u0000\u0000\u0174\u0173\u0001\u0000\u0000\u0000\u0175\u0178\u0001"+ - "\u0000\u0000\u0000\u0176\u0177\u0001\u0000\u0000\u0000\u0176\u0174\u0001"+ - "\u0000\u0000\u0000\u0177\u0179\u0001\u0000\u0000\u0000\u0178\u0176\u0001"+ - "\u0000\u0000\u0000\u0179\u017a\u0005*\u0000\u0000\u017a\u017b\u0005/\u0000"+ - "\u0000\u017b\u017c\u0001\u0000\u0000\u0000\u017c\u017d\u0006\u0014\u0003"+ - "\u0000\u017d-\u0001\u0000\u0000\u0000\u017e\u0180\u0007\u0002\u0000\u0000"+ - "\u017f\u017e\u0001\u0000\u0000\u0000\u0180\u0181\u0001\u0000\u0000\u0000"+ - "\u0181\u017f\u0001\u0000\u0000\u0000\u0181\u0182\u0001\u0000\u0000\u0000"+ - "\u0182\u0183\u0001\u0000\u0000\u0000\u0183\u0184\u0006\u0015\u0003\u0000"+ - "\u0184/\u0001\u0000\u0000\u0000\u0185\u0186\u0005[\u0000\u0000\u0186\u0187"+ - "\u0001\u0000\u0000\u0000\u0187\u0188\u0006\u0016\u0004\u0000\u0188\u0189"+ - "\u0006\u0016\u0005\u0000\u01891\u0001\u0000\u0000\u0000\u018a\u018b\u0005"+ - "|\u0000\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c\u018d\u0006\u0017"+ - "\u0006\u0000\u018d\u018e\u0006\u0017\u0007\u0000\u018e3\u0001\u0000\u0000"+ - "\u0000\u018f\u0190\u0003.\u0015\u0000\u0190\u0191\u0001\u0000\u0000\u0000"+ - "\u0191\u0192\u0006\u0018\u0003\u0000\u01925\u0001\u0000\u0000\u0000\u0193"+ - "\u0194\u0003*\u0013\u0000\u0194\u0195\u0001\u0000\u0000\u0000\u0195\u0196"+ - "\u0006\u0019\u0003\u0000\u01967\u0001\u0000\u0000\u0000\u0197\u0198\u0003"+ - ",\u0014\u0000\u0198\u0199\u0001\u0000\u0000\u0000\u0199\u019a\u0006\u001a"+ - "\u0003\u0000\u019a9\u0001\u0000\u0000\u0000\u019b\u019c\u0005|\u0000\u0000"+ - "\u019c\u019d\u0001\u0000\u0000\u0000\u019d\u019e\u0006\u001b\u0007\u0000"+ - "\u019e;\u0001\u0000\u0000\u0000\u019f\u01a0\u0007\u0003\u0000\u0000\u01a0"+ - "=\u0001\u0000\u0000\u0000\u01a1\u01a2\u0007\u0004\u0000\u0000\u01a2?\u0001"+ - "\u0000\u0000\u0000\u01a3\u01a4\u0005\\\u0000\u0000\u01a4\u01a5\u0007\u0005"+ - "\u0000\u0000\u01a5A\u0001\u0000\u0000\u0000\u01a6\u01a7\b\u0006\u0000"+ - "\u0000\u01a7C\u0001\u0000\u0000\u0000\u01a8\u01aa\u0007\u0007\u0000\u0000"+ - "\u01a9\u01ab\u0007\b\u0000\u0000\u01aa\u01a9\u0001\u0000\u0000\u0000\u01aa"+ - "\u01ab\u0001\u0000\u0000\u0000\u01ab\u01ad\u0001\u0000\u0000\u0000\u01ac"+ - "\u01ae\u0003<\u001c\u0000\u01ad\u01ac\u0001\u0000\u0000\u0000\u01ae\u01af"+ - "\u0001\u0000\u0000\u0000\u01af\u01ad\u0001\u0000\u0000\u0000\u01af\u01b0"+ - "\u0001\u0000\u0000\u0000\u01b0E\u0001\u0000\u0000\u0000\u01b1\u01b6\u0005"+ - "\"\u0000\u0000\u01b2\u01b5\u0003@\u001e\u0000\u01b3\u01b5\u0003B\u001f"+ - "\u0000\u01b4\u01b2\u0001\u0000\u0000\u0000\u01b4\u01b3\u0001\u0000\u0000"+ - "\u0000\u01b5\u01b8\u0001\u0000\u0000\u0000\u01b6\u01b4\u0001\u0000\u0000"+ - "\u0000\u01b6\u01b7\u0001\u0000\u0000\u0000\u01b7\u01b9\u0001\u0000\u0000"+ - "\u0000\u01b8\u01b6\u0001\u0000\u0000\u0000\u01b9\u01cf\u0005\"\u0000\u0000"+ - "\u01ba\u01bb\u0005\"\u0000\u0000\u01bb\u01bc\u0005\"\u0000\u0000\u01bc"+ - "\u01bd\u0005\"\u0000\u0000\u01bd\u01c1\u0001\u0000\u0000\u0000\u01be\u01c0"+ - "\b\u0001\u0000\u0000\u01bf\u01be\u0001\u0000\u0000\u0000\u01c0\u01c3\u0001"+ - "\u0000\u0000\u0000\u01c1\u01c2\u0001\u0000\u0000\u0000\u01c1\u01bf\u0001"+ - "\u0000\u0000\u0000\u01c2\u01c4\u0001\u0000\u0000\u0000\u01c3\u01c1\u0001"+ - "\u0000\u0000\u0000\u01c4\u01c5\u0005\"\u0000\u0000\u01c5\u01c6\u0005\""+ - "\u0000\u0000\u01c6\u01c7\u0005\"\u0000\u0000\u01c7\u01c9\u0001\u0000\u0000"+ - "\u0000\u01c8\u01ca\u0005\"\u0000\u0000\u01c9\u01c8\u0001\u0000\u0000\u0000"+ - "\u01c9\u01ca\u0001\u0000\u0000\u0000\u01ca\u01cc\u0001\u0000\u0000\u0000"+ - "\u01cb\u01cd\u0005\"\u0000\u0000\u01cc\u01cb\u0001\u0000\u0000\u0000\u01cc"+ - "\u01cd\u0001\u0000\u0000\u0000\u01cd\u01cf\u0001\u0000\u0000\u0000\u01ce"+ - "\u01b1\u0001\u0000\u0000\u0000\u01ce\u01ba\u0001\u0000\u0000\u0000\u01cf"+ - "G\u0001\u0000\u0000\u0000\u01d0\u01d2\u0003<\u001c\u0000\u01d1\u01d0\u0001"+ - "\u0000\u0000\u0000\u01d2\u01d3\u0001\u0000\u0000\u0000\u01d3\u01d1\u0001"+ - "\u0000\u0000\u0000\u01d3\u01d4\u0001\u0000\u0000\u0000\u01d4I\u0001\u0000"+ - "\u0000\u0000\u01d5\u01d7\u0003<\u001c\u0000\u01d6\u01d5\u0001\u0000\u0000"+ - "\u0000\u01d7\u01d8\u0001\u0000\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000"+ - "\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000\u01d9\u01da\u0001\u0000\u0000"+ - "\u0000\u01da\u01de\u0003X*\u0000\u01db\u01dd\u0003<\u001c\u0000\u01dc"+ - "\u01db\u0001\u0000\u0000\u0000\u01dd\u01e0\u0001\u0000\u0000\u0000\u01de"+ - "\u01dc\u0001\u0000\u0000\u0000\u01de\u01df\u0001\u0000\u0000\u0000\u01df"+ - "\u0200\u0001\u0000\u0000\u0000\u01e0\u01de\u0001\u0000\u0000\u0000\u01e1"+ - "\u01e3\u0003X*\u0000\u01e2\u01e4\u0003<\u001c\u0000\u01e3\u01e2\u0001"+ - "\u0000\u0000\u0000\u01e4\u01e5\u0001\u0000\u0000\u0000\u01e5\u01e3\u0001"+ - "\u0000\u0000\u0000\u01e5\u01e6\u0001\u0000\u0000\u0000\u01e6\u0200\u0001"+ - "\u0000\u0000\u0000\u01e7\u01e9\u0003<\u001c\u0000\u01e8\u01e7\u0001\u0000"+ - "\u0000\u0000\u01e9\u01ea\u0001\u0000\u0000\u0000\u01ea\u01e8\u0001\u0000"+ - "\u0000\u0000\u01ea\u01eb\u0001\u0000\u0000\u0000\u01eb\u01f3\u0001\u0000"+ - "\u0000\u0000\u01ec\u01f0\u0003X*\u0000\u01ed\u01ef\u0003<\u001c\u0000"+ - "\u01ee\u01ed\u0001\u0000\u0000\u0000\u01ef\u01f2\u0001\u0000\u0000\u0000"+ - "\u01f0\u01ee\u0001\u0000\u0000\u0000\u01f0\u01f1\u0001\u0000\u0000\u0000"+ - "\u01f1\u01f4\u0001\u0000\u0000\u0000\u01f2\u01f0\u0001\u0000\u0000\u0000"+ - "\u01f3\u01ec\u0001\u0000\u0000\u0000\u01f3\u01f4\u0001\u0000\u0000\u0000"+ - "\u01f4\u01f5\u0001\u0000\u0000\u0000\u01f5\u01f6\u0003D \u0000\u01f6\u0200"+ - "\u0001\u0000\u0000\u0000\u01f7\u01f9\u0003X*\u0000\u01f8\u01fa\u0003<"+ - "\u001c\u0000\u01f9\u01f8\u0001\u0000\u0000\u0000\u01fa\u01fb\u0001\u0000"+ - "\u0000\u0000\u01fb\u01f9\u0001\u0000\u0000\u0000\u01fb\u01fc\u0001\u0000"+ - "\u0000\u0000\u01fc\u01fd\u0001\u0000\u0000\u0000\u01fd\u01fe\u0003D \u0000"+ - "\u01fe\u0200\u0001\u0000\u0000\u0000\u01ff\u01d6\u0001\u0000\u0000\u0000"+ - "\u01ff\u01e1\u0001\u0000\u0000\u0000\u01ff\u01e8\u0001\u0000\u0000\u0000"+ - "\u01ff\u01f7\u0001\u0000\u0000\u0000\u0200K\u0001\u0000\u0000\u0000\u0201"+ - "\u0202\u0005b\u0000\u0000\u0202\u0203\u0005y\u0000\u0000\u0203M\u0001"+ - "\u0000\u0000\u0000\u0204\u0205\u0005a\u0000\u0000\u0205\u0206\u0005n\u0000"+ - "\u0000\u0206\u0207\u0005d\u0000\u0000\u0207O\u0001\u0000\u0000\u0000\u0208"+ - "\u0209\u0005a\u0000\u0000\u0209\u020a\u0005s\u0000\u0000\u020a\u020b\u0005"+ - "c\u0000\u0000\u020bQ\u0001\u0000\u0000\u0000\u020c\u020d\u0005=\u0000"+ - "\u0000\u020dS\u0001\u0000\u0000\u0000\u020e\u020f\u0005,\u0000\u0000\u020f"+ - "U\u0001\u0000\u0000\u0000\u0210\u0211\u0005d\u0000\u0000\u0211\u0212\u0005"+ - "e\u0000\u0000\u0212\u0213\u0005s\u0000\u0000\u0213\u0214\u0005c\u0000"+ - "\u0000\u0214W\u0001\u0000\u0000\u0000\u0215\u0216\u0005.\u0000\u0000\u0216"+ - "Y\u0001\u0000\u0000\u0000\u0217\u0218\u0005f\u0000\u0000\u0218\u0219\u0005"+ - "a\u0000\u0000\u0219\u021a\u0005l\u0000\u0000\u021a\u021b\u0005s\u0000"+ - "\u0000\u021b\u021c\u0005e\u0000\u0000\u021c[\u0001\u0000\u0000\u0000\u021d"+ - "\u021e\u0005f\u0000\u0000\u021e\u021f\u0005i\u0000\u0000\u021f\u0220\u0005"+ - "r\u0000\u0000\u0220\u0221\u0005s\u0000\u0000\u0221\u0222\u0005t\u0000"+ - "\u0000\u0222]\u0001\u0000\u0000\u0000\u0223\u0224\u0005l\u0000\u0000\u0224"+ - "\u0225\u0005a\u0000\u0000\u0225\u0226\u0005s\u0000\u0000\u0226\u0227\u0005"+ - "t\u0000\u0000\u0227_\u0001\u0000\u0000\u0000\u0228\u0229\u0005(\u0000"+ - "\u0000\u0229a\u0001\u0000\u0000\u0000\u022a\u022b\u0005i\u0000\u0000\u022b"+ - "\u022c\u0005n\u0000\u0000\u022cc\u0001\u0000\u0000\u0000\u022d\u022e\u0005"+ - "i\u0000\u0000\u022e\u022f\u0005s\u0000\u0000\u022fe\u0001\u0000\u0000"+ - "\u0000\u0230\u0231\u0005l\u0000\u0000\u0231\u0232\u0005i\u0000\u0000\u0232"+ - "\u0233\u0005k\u0000\u0000\u0233\u0234\u0005e\u0000\u0000\u0234g\u0001"+ - "\u0000\u0000\u0000\u0235\u0236\u0005n\u0000\u0000\u0236\u0237\u0005o\u0000"+ - "\u0000\u0237\u0238\u0005t\u0000\u0000\u0238i\u0001\u0000\u0000\u0000\u0239"+ - "\u023a\u0005n\u0000\u0000\u023a\u023b\u0005u\u0000\u0000\u023b\u023c\u0005"+ - "l\u0000\u0000\u023c\u023d\u0005l\u0000\u0000\u023dk\u0001\u0000\u0000"+ - "\u0000\u023e\u023f\u0005n\u0000\u0000\u023f\u0240\u0005u\u0000\u0000\u0240"+ - "\u0241\u0005l\u0000\u0000\u0241\u0242\u0005l\u0000\u0000\u0242\u0243\u0005"+ - "s\u0000\u0000\u0243m\u0001\u0000\u0000\u0000\u0244\u0245\u0005o\u0000"+ - "\u0000\u0245\u0246\u0005r\u0000\u0000\u0246o\u0001\u0000\u0000\u0000\u0247"+ - "\u0248\u0005?\u0000\u0000\u0248q\u0001\u0000\u0000\u0000\u0249\u024a\u0005"+ - "r\u0000\u0000\u024a\u024b\u0005l\u0000\u0000\u024b\u024c\u0005i\u0000"+ - "\u0000\u024c\u024d\u0005k\u0000\u0000\u024d\u024e\u0005e\u0000\u0000\u024e"+ - "s\u0001\u0000\u0000\u0000\u024f\u0250\u0005)\u0000\u0000\u0250u\u0001"+ - "\u0000\u0000\u0000\u0251\u0252\u0005t\u0000\u0000\u0252\u0253\u0005r\u0000"+ - "\u0000\u0253\u0254\u0005u\u0000\u0000\u0254\u0255\u0005e\u0000\u0000\u0255"+ - "w\u0001\u0000\u0000\u0000\u0256\u0257\u0005i\u0000\u0000\u0257\u0258\u0005"+ - "n\u0000\u0000\u0258\u0259\u0005f\u0000\u0000\u0259\u025a\u0005o\u0000"+ - "\u0000\u025ay\u0001\u0000\u0000\u0000\u025b\u025c\u0005f\u0000\u0000\u025c"+ - "\u025d\u0005u\u0000\u0000\u025d\u025e\u0005n\u0000\u0000\u025e\u025f\u0005"+ - "c\u0000\u0000\u025f\u0260\u0005t\u0000\u0000\u0260\u0261\u0005i\u0000"+ - "\u0000\u0261\u0262\u0005o\u0000\u0000\u0262\u0263\u0005n\u0000\u0000\u0263"+ - "\u0264\u0005s\u0000\u0000\u0264{\u0001\u0000\u0000\u0000\u0265\u0266\u0005"+ - "=\u0000\u0000\u0266\u0267\u0005=\u0000\u0000\u0267}\u0001\u0000\u0000"+ - "\u0000\u0268\u0269\u0005!\u0000\u0000\u0269\u026a\u0005=\u0000\u0000\u026a"+ - "\u007f\u0001\u0000\u0000\u0000\u026b\u026c\u0005<\u0000\u0000\u026c\u0081"+ - "\u0001\u0000\u0000\u0000\u026d\u026e\u0005<\u0000\u0000\u026e\u026f\u0005"+ - "=\u0000\u0000\u026f\u0083\u0001\u0000\u0000\u0000\u0270\u0271\u0005>\u0000"+ - "\u0000\u0271\u0085\u0001\u0000\u0000\u0000\u0272\u0273\u0005>\u0000\u0000"+ - "\u0273\u0274\u0005=\u0000\u0000\u0274\u0087\u0001\u0000\u0000\u0000\u0275"+ - "\u0276\u0005+\u0000\u0000\u0276\u0089\u0001\u0000\u0000\u0000\u0277\u0278"+ - "\u0005-\u0000\u0000\u0278\u008b\u0001\u0000\u0000\u0000\u0279\u027a\u0005"+ - "*\u0000\u0000\u027a\u008d\u0001\u0000\u0000\u0000\u027b\u027c\u0005/\u0000"+ - "\u0000\u027c\u008f\u0001\u0000\u0000\u0000\u027d\u027e\u0005%\u0000\u0000"+ - "\u027e\u0091\u0001\u0000\u0000\u0000\u027f\u0280\u0005[\u0000\u0000\u0280"+ - "\u0281\u0001\u0000\u0000\u0000\u0281\u0282\u0006G\u0000\u0000\u0282\u0283"+ - "\u0006G\u0000\u0000\u0283\u0093\u0001\u0000\u0000\u0000\u0284\u0285\u0005"+ - "]\u0000\u0000\u0285\u0286\u0001\u0000\u0000\u0000\u0286\u0287\u0006H\u0007"+ - "\u0000\u0287\u0288\u0006H\u0007\u0000\u0288\u0095\u0001\u0000\u0000\u0000"+ - "\u0289\u028f\u0003>\u001d\u0000\u028a\u028e\u0003>\u001d\u0000\u028b\u028e"+ - "\u0003<\u001c\u0000\u028c\u028e\u0005_\u0000\u0000\u028d\u028a\u0001\u0000"+ - "\u0000\u0000\u028d\u028b\u0001\u0000\u0000\u0000\u028d\u028c\u0001\u0000"+ - "\u0000\u0000\u028e\u0291\u0001\u0000\u0000\u0000\u028f\u028d\u0001\u0000"+ - "\u0000\u0000\u028f\u0290\u0001\u0000\u0000\u0000\u0290\u029b\u0001\u0000"+ - "\u0000\u0000\u0291\u028f\u0001\u0000\u0000\u0000\u0292\u0296\u0007\t\u0000"+ - "\u0000\u0293\u0297\u0003>\u001d\u0000\u0294\u0297\u0003<\u001c\u0000\u0295"+ - "\u0297\u0005_\u0000\u0000\u0296\u0293\u0001\u0000\u0000\u0000\u0296\u0294"+ - "\u0001\u0000\u0000\u0000\u0296\u0295\u0001\u0000\u0000\u0000\u0297\u0298"+ - "\u0001\u0000\u0000\u0000\u0298\u0296\u0001\u0000\u0000\u0000\u0298\u0299"+ - "\u0001\u0000\u0000\u0000\u0299\u029b\u0001\u0000\u0000\u0000\u029a\u0289"+ - "\u0001\u0000\u0000\u0000\u029a\u0292\u0001\u0000\u0000\u0000\u029b\u0097"+ - "\u0001\u0000\u0000\u0000\u029c\u02a2\u0005`\u0000\u0000\u029d\u02a1\b"+ - "\n\u0000\u0000\u029e\u029f\u0005`\u0000\u0000\u029f\u02a1\u0005`\u0000"+ - "\u0000\u02a0\u029d\u0001\u0000\u0000\u0000\u02a0\u029e\u0001\u0000\u0000"+ - "\u0000\u02a1\u02a4\u0001\u0000\u0000\u0000\u02a2\u02a0\u0001\u0000\u0000"+ - "\u0000\u02a2\u02a3\u0001\u0000\u0000\u0000\u02a3\u02a5\u0001\u0000\u0000"+ - "\u0000\u02a4\u02a2\u0001\u0000\u0000\u0000\u02a5\u02a6\u0005`\u0000\u0000"+ - "\u02a6\u0099\u0001\u0000\u0000\u0000\u02a7\u02a8\u0003*\u0013\u0000\u02a8"+ - "\u02a9\u0001\u0000\u0000\u0000\u02a9\u02aa\u0006K\u0003\u0000\u02aa\u009b"+ - "\u0001\u0000\u0000\u0000\u02ab\u02ac\u0003,\u0014\u0000\u02ac\u02ad\u0001"+ - "\u0000\u0000\u0000\u02ad\u02ae\u0006L\u0003\u0000\u02ae\u009d\u0001\u0000"+ - "\u0000\u0000\u02af\u02b0\u0003.\u0015\u0000\u02b0\u02b1\u0001\u0000\u0000"+ - "\u0000\u02b1\u02b2\u0006M\u0003\u0000\u02b2\u009f\u0001\u0000\u0000\u0000"+ - "\u02b3\u02b4\u0005|\u0000\u0000\u02b4\u02b5\u0001\u0000\u0000\u0000\u02b5"+ - "\u02b6\u0006N\u0006\u0000\u02b6\u02b7\u0006N\u0007\u0000\u02b7\u00a1\u0001"+ - "\u0000\u0000\u0000\u02b8\u02b9\u0005[\u0000\u0000\u02b9\u02ba\u0001\u0000"+ - "\u0000\u0000\u02ba\u02bb\u0006O\u0004\u0000\u02bb\u02bc\u0006O\u0001\u0000"+ - "\u02bc\u02bd\u0006O\u0001\u0000\u02bd\u00a3\u0001\u0000\u0000\u0000\u02be"+ - "\u02bf\u0005]\u0000\u0000\u02bf\u02c0\u0001\u0000\u0000\u0000\u02c0\u02c1"+ - "\u0006P\u0007\u0000\u02c1\u02c2\u0006P\u0007\u0000\u02c2\u02c3\u0006P"+ - "\b\u0000\u02c3\u00a5\u0001\u0000\u0000\u0000\u02c4\u02c5\u0005,\u0000"+ - "\u0000\u02c5\u02c6\u0001\u0000\u0000\u0000\u02c6\u02c7\u0006Q\t\u0000"+ - "\u02c7\u00a7\u0001\u0000\u0000\u0000\u02c8\u02c9\u0005=\u0000\u0000\u02c9"+ - "\u02ca\u0001\u0000\u0000\u0000\u02ca\u02cb\u0006R\n\u0000\u02cb\u00a9"+ - "\u0001\u0000\u0000\u0000\u02cc\u02cd\u0005a\u0000\u0000\u02cd\u02ce\u0005"+ - "s\u0000\u0000\u02ce\u00ab\u0001\u0000\u0000\u0000\u02cf\u02d0\u0005m\u0000"+ - "\u0000\u02d0\u02d1\u0005e\u0000\u0000\u02d1\u02d2\u0005t\u0000\u0000\u02d2"+ - "\u02d3\u0005a\u0000\u0000\u02d3\u02d4\u0005d\u0000\u0000\u02d4\u02d5\u0005"+ - "a\u0000\u0000\u02d5\u02d6\u0005t\u0000\u0000\u02d6\u02d7\u0005a\u0000"+ - "\u0000\u02d7\u00ad\u0001\u0000\u0000\u0000\u02d8\u02d9\u0005o\u0000\u0000"+ - "\u02d9\u02da\u0005n\u0000\u0000\u02da\u00af\u0001\u0000\u0000\u0000\u02db"+ - "\u02dc\u0005w\u0000\u0000\u02dc\u02dd\u0005i\u0000\u0000\u02dd\u02de\u0005"+ - "t\u0000\u0000\u02de\u02df\u0005h\u0000\u0000\u02df\u00b1\u0001\u0000\u0000"+ - "\u0000\u02e0\u02e2\u0003\u00b4X\u0000\u02e1\u02e0\u0001\u0000\u0000\u0000"+ - "\u02e2\u02e3\u0001\u0000\u0000\u0000\u02e3\u02e1\u0001\u0000\u0000\u0000"+ - "\u02e3\u02e4\u0001\u0000\u0000\u0000\u02e4\u00b3\u0001\u0000\u0000\u0000"+ - "\u02e5\u02e7\b\u000b\u0000\u0000\u02e6\u02e5\u0001\u0000\u0000\u0000\u02e7"+ - "\u02e8\u0001\u0000\u0000\u0000\u02e8\u02e6\u0001\u0000\u0000\u0000\u02e8"+ - "\u02e9\u0001\u0000\u0000\u0000\u02e9\u02ed\u0001\u0000\u0000\u0000\u02ea"+ - "\u02eb\u0005/\u0000\u0000\u02eb\u02ed\b\f\u0000\u0000\u02ec\u02e6\u0001"+ - "\u0000\u0000\u0000\u02ec\u02ea\u0001\u0000\u0000\u0000\u02ed\u00b5\u0001"+ - "\u0000\u0000\u0000\u02ee\u02ef\u0003\u0098J\u0000\u02ef\u00b7\u0001\u0000"+ - "\u0000\u0000\u02f0\u02f1\u0003*\u0013\u0000\u02f1\u02f2\u0001\u0000\u0000"+ - "\u0000\u02f2\u02f3\u0006Z\u0003\u0000\u02f3\u00b9\u0001\u0000\u0000\u0000"+ - "\u02f4\u02f5\u0003,\u0014\u0000\u02f5\u02f6\u0001\u0000\u0000\u0000\u02f6"+ - "\u02f7\u0006[\u0003\u0000\u02f7\u00bb\u0001\u0000\u0000\u0000\u02f8\u02f9"+ - "\u0003.\u0015\u0000\u02f9\u02fa\u0001\u0000\u0000\u0000\u02fa\u02fb\u0006"+ - "\\\u0003\u0000\u02fb\u00bd\u0001\u0000\u0000\u0000&\u0000\u0001\u0002"+ - "\u0003\u015a\u0164\u0168\u016b\u0174\u0176\u0181\u01aa\u01af\u01b4\u01b6"+ - "\u01c1\u01c9\u01cc\u01ce\u01d3\u01d8\u01de\u01e5\u01ea\u01f0\u01f3\u01fb"+ - "\u01ff\u028d\u028f\u0296\u0298\u029a\u02a0\u02a2\u02e3\u02e8\u02ec\u000b"+ - "\u0005\u0002\u0000\u0005\u0003\u0000\u0005\u0001\u0000\u0000\u0001\u0000"+ - "\u0007A\u0000\u0005\u0000\u0000\u0007\u001a\u0000\u0004\u0000\u0000\u0007"+ - "B\u0000\u0007\"\u0000\u0007!\u0000"; + "\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0004\u0012\u01c3\b\u0012"+ + "\u000b\u0012\f\u0012\u01c4\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+ + "\u0001\u0013\u0001\u0013\u0005\u0013\u01cd\b\u0013\n\u0013\f\u0013\u01d0"+ + "\t\u0013\u0001\u0013\u0003\u0013\u01d3\b\u0013\u0001\u0013\u0003\u0013"+ + "\u01d6\b\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014"+ + "\u0001\u0014\u0001\u0014\u0005\u0014\u01df\b\u0014\n\u0014\f\u0014\u01e2"+ + "\t\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+ + "\u0015\u0004\u0015\u01ea\b\u0015\u000b\u0015\f\u0015\u01eb\u0001\u0015"+ + "\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016"+ + "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018"+ + "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u0019"+ + "\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b"+ + "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001d"+ + "\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f"+ + "\u0001 \u0001 \u0003 \u0215\b \u0001 \u0004 \u0218\b \u000b \f \u0219"+ + "\u0001!\u0001!\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0003#\u0223\b#\u0001"+ + "$\u0001$\u0001%\u0001%\u0001%\u0003%\u022a\b%\u0001&\u0001&\u0001&\u0005"+ + "&\u022f\b&\n&\f&\u0232\t&\u0001&\u0001&\u0001&\u0001&\u0001&\u0001&\u0005"+ + "&\u023a\b&\n&\f&\u023d\t&\u0001&\u0001&\u0001&\u0001&\u0001&\u0003&\u0244"+ + "\b&\u0001&\u0003&\u0247\b&\u0003&\u0249\b&\u0001\'\u0004\'\u024c\b\'\u000b"+ + "\'\f\'\u024d\u0001(\u0004(\u0251\b(\u000b(\f(\u0252\u0001(\u0001(\u0005"+ + "(\u0257\b(\n(\f(\u025a\t(\u0001(\u0001(\u0004(\u025e\b(\u000b(\f(\u025f"+ + "\u0001(\u0004(\u0263\b(\u000b(\f(\u0264\u0001(\u0001(\u0005(\u0269\b("+ + "\n(\f(\u026c\t(\u0003(\u026e\b(\u0001(\u0001(\u0001(\u0001(\u0004(\u0274"+ + "\b(\u000b(\f(\u0275\u0001(\u0001(\u0003(\u027a\b(\u0001)\u0001)\u0001"+ + ")\u0001*\u0001*\u0001*\u0001*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001"+ + ",\u0001-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u0001"+ + "0\u00010\u00010\u00010\u00010\u00010\u00011\u00011\u00011\u00011\u0001"+ + "1\u00011\u00012\u00012\u00012\u00012\u00012\u00013\u00013\u00014\u0001"+ + "4\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00016\u0001"+ + "7\u00017\u00017\u00017\u00018\u00018\u00018\u00018\u00018\u00019\u0001"+ + "9\u00019\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001;\u0001;\u0001"+ + "<\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001>\u0001>\u0001"+ + ">\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001@\u0001A\u0001"+ + "A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001"+ + "E\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001J\u0001"+ + "J\u0001J\u0001J\u0001J\u0001K\u0001K\u0001K\u0001K\u0001K\u0001L\u0001"+ + "L\u0005L\u02f7\bL\nL\fL\u02fa\tL\u0001L\u0001L\u0003L\u02fe\bL\u0001L"+ + "\u0004L\u0301\bL\u000bL\fL\u0302\u0003L\u0305\bL\u0001M\u0001M\u0004M"+ + "\u0309\bM\u000bM\fM\u030a\u0001M\u0001M\u0001N\u0001N\u0001N\u0001N\u0001"+ + "O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001"+ + "Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001R\u0001R\u0001S\u0001"+ + "S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001T\u0001U\u0001"+ + "U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001V\u0001V\u0001V\u0001"+ + "V\u0001V\u0001W\u0001W\u0001W\u0003W\u0340\bW\u0001X\u0004X\u0343\bX\u000b"+ + "X\fX\u0344\u0001Y\u0001Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001"+ + "[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001"+ + "]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001"+ + "_\u0001`\u0001`\u0001`\u0001`\u0003`\u0368\b`\u0001a\u0001a\u0003a\u036c"+ + "\ba\u0001a\u0005a\u036f\ba\na\fa\u0372\ta\u0001a\u0001a\u0003a\u0376\b"+ + "a\u0001a\u0004a\u0379\ba\u000ba\fa\u037a\u0003a\u037d\ba\u0001b\u0001"+ + "b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001"+ + "d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001f\u0001"+ + "g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001"+ + "i\u0001i\u0001j\u0001j\u0001j\u0001k\u0001k\u0001k\u0001k\u0001l\u0001"+ + "l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001m\u0001n\u0001n\u0001n\u0001"+ + "n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001p\u0001p\u0001p\u0001"+ + "q\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001r\u0001r\u0001"+ + "r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001t\u0001"+ + "u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001w\u0001"+ + "w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001"+ + "y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001"+ + "|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001"+ + "~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080\u0001"+ + "\u0080\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081\u0001\u0081\u0001"+ + "\u0081\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001"+ + "\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001"+ + "\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001"+ + "\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087\u0001\u0087\u0001"+ + "\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0088\u0001"+ + "\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a\u0001"+ + "\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001"+ + "\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001"+ + "\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+ + "\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e\u0001"+ + "\u008e\u0001\u008e\u0002\u01e0\u023b\u0000\u008f\n\u0001\f\u0002\u000e"+ + "\u0003\u0010\u0004\u0012\u0005\u0014\u0006\u0016\u0007\u0018\b\u001a\t"+ + "\u001c\n\u001e\u000b \f\"\r$\u000e&\u000f(\u0010*\u0011,\u0012.\u0013"+ + "0\u00142\u00154\u00166\u00008\u0000:\u0017<\u0018>\u0019@\u001aB\u0000"+ + "D\u0000F\u0000H\u0000J\u0000L\u0000N\u0000P\u0000R\u0000T\u0000V\u001b"+ + "X\u001cZ\u001d\\\u001e^\u001f` b!d\"f#h$j%l&n\'p(r)t*v+x,z-|.~/\u0080"+ + "0\u00821\u00842\u00863\u00884\u008a5\u008c6\u008e7\u00908\u00929\u0094"+ + ":\u0096;\u0098<\u009a=\u009c>\u009e?\u00a0@\u00a2A\u00a4B\u00a6C\u00a8"+ + "D\u00aaE\u00ac\u0000\u00ae\u0000\u00b0\u0000\u00b2\u0000\u00b4\u0000\u00b6"+ + "F\u00b8\u0000\u00baG\u00bc\u0000\u00beH\u00c0I\u00c2J\u00c4\u0000\u00c6"+ + "\u0000\u00c8\u0000\u00ca\u0000\u00ccK\u00ce\u0000\u00d0L\u00d2M\u00d4"+ + "N\u00d6\u0000\u00d8\u0000\u00da\u0000\u00dc\u0000\u00deO\u00e0\u0000\u00e2"+ + "\u0000\u00e4P\u00e6Q\u00e8R\u00ea\u0000\u00ecS\u00eeT\u00f0\u0000\u00f2"+ + "\u0000\u00f4U\u00f6V\u00f8W\u00fa\u0000\u00fc\u0000\u00fe\u0000\u0100"+ + "\u0000\u0102\u0000\u0104\u0000\u0106\u0000\u0108X\u010aY\u010cZ\u010e"+ + "\u0000\u0110\u0000\u0112\u0000\u0114\u0000\u0116[\u0118\\\u011a]\u011c"+ + "\u0000\u011e^\u0120_\u0122`\u0124a\u0126b\n\u0000\u0001\u0002\u0003\u0004"+ + "\u0005\u0006\u0007\b\t\f\u0006\u0000\t\n\r\r //[[]]\u0002\u0000\n\n\r"+ + "\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002\u0000AZaz\u0005\u0000\"\""+ + "\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000EEee\u0002\u0000++-"+ + "-\u0001\u0000``\n\u0000\t\n\r\r ,,//==[[]]``||\u0002\u0000**//\u0458"+ + "\u0000\n\u0001\u0000\u0000\u0000\u0000\f\u0001\u0000\u0000\u0000\u0000"+ + "\u000e\u0001\u0000\u0000\u0000\u0000\u0010\u0001\u0000\u0000\u0000\u0000"+ + "\u0012\u0001\u0000\u0000\u0000\u0000\u0014\u0001\u0000\u0000\u0000\u0000"+ + "\u0016\u0001\u0000\u0000\u0000\u0000\u0018\u0001\u0000\u0000\u0000\u0000"+ + "\u001a\u0001\u0000\u0000\u0000\u0000\u001c\u0001\u0000\u0000\u0000\u0000"+ + "\u001e\u0001\u0000\u0000\u0000\u0000 \u0001\u0000\u0000\u0000\u0000\""+ + "\u0001\u0000\u0000\u0000\u0000$\u0001\u0000\u0000\u0000\u0000&\u0001\u0000"+ + "\u0000\u0000\u0000(\u0001\u0000\u0000\u0000\u0000*\u0001\u0000\u0000\u0000"+ + "\u0000,\u0001\u0000\u0000\u0000\u0000.\u0001\u0000\u0000\u0000\u00000"+ + "\u0001\u0000\u0000\u0000\u00002\u0001\u0000\u0000\u0000\u00004\u0001\u0000"+ + "\u0000\u0000\u00016\u0001\u0000\u0000\u0000\u00018\u0001\u0000\u0000\u0000"+ + "\u0001:\u0001\u0000\u0000\u0000\u0001<\u0001\u0000\u0000\u0000\u0001>"+ + "\u0001\u0000\u0000\u0000\u0002@\u0001\u0000\u0000\u0000\u0002V\u0001\u0000"+ + "\u0000\u0000\u0002X\u0001\u0000\u0000\u0000\u0002Z\u0001\u0000\u0000\u0000"+ + "\u0002\\\u0001\u0000\u0000\u0000\u0002^\u0001\u0000\u0000\u0000\u0002"+ + "`\u0001\u0000\u0000\u0000\u0002b\u0001\u0000\u0000\u0000\u0002d\u0001"+ + "\u0000\u0000\u0000\u0002f\u0001\u0000\u0000\u0000\u0002h\u0001\u0000\u0000"+ + "\u0000\u0002j\u0001\u0000\u0000\u0000\u0002l\u0001\u0000\u0000\u0000\u0002"+ + "n\u0001\u0000\u0000\u0000\u0002p\u0001\u0000\u0000\u0000\u0002r\u0001"+ + "\u0000\u0000\u0000\u0002t\u0001\u0000\u0000\u0000\u0002v\u0001\u0000\u0000"+ + "\u0000\u0002x\u0001\u0000\u0000\u0000\u0002z\u0001\u0000\u0000\u0000\u0002"+ + "|\u0001\u0000\u0000\u0000\u0002~\u0001\u0000\u0000\u0000\u0002\u0080\u0001"+ + "\u0000\u0000\u0000\u0002\u0082\u0001\u0000\u0000\u0000\u0002\u0084\u0001"+ + "\u0000\u0000\u0000\u0002\u0086\u0001\u0000\u0000\u0000\u0002\u0088\u0001"+ + "\u0000\u0000\u0000\u0002\u008a\u0001\u0000\u0000\u0000\u0002\u008c\u0001"+ + "\u0000\u0000\u0000\u0002\u008e\u0001\u0000\u0000\u0000\u0002\u0090\u0001"+ + "\u0000\u0000\u0000\u0002\u0092\u0001\u0000\u0000\u0000\u0002\u0094\u0001"+ + "\u0000\u0000\u0000\u0002\u0096\u0001\u0000\u0000\u0000\u0002\u0098\u0001"+ + "\u0000\u0000\u0000\u0002\u009a\u0001\u0000\u0000\u0000\u0002\u009c\u0001"+ + "\u0000\u0000\u0000\u0002\u009e\u0001\u0000\u0000\u0000\u0002\u00a0\u0001"+ + "\u0000\u0000\u0000\u0002\u00a2\u0001\u0000\u0000\u0000\u0002\u00a4\u0001"+ + "\u0000\u0000\u0000\u0002\u00a6\u0001\u0000\u0000\u0000\u0002\u00a8\u0001"+ + "\u0000\u0000\u0000\u0002\u00aa\u0001\u0000\u0000\u0000\u0003\u00ac\u0001"+ + "\u0000\u0000\u0000\u0003\u00ae\u0001\u0000\u0000\u0000\u0003\u00b0\u0001"+ + "\u0000\u0000\u0000\u0003\u00b2\u0001\u0000\u0000\u0000\u0003\u00b4\u0001"+ + "\u0000\u0000\u0000\u0003\u00b6\u0001\u0000\u0000\u0000\u0003\u00ba\u0001"+ + "\u0000\u0000\u0000\u0003\u00bc\u0001\u0000\u0000\u0000\u0003\u00be\u0001"+ + "\u0000\u0000\u0000\u0003\u00c0\u0001\u0000\u0000\u0000\u0003\u00c2\u0001"+ + "\u0000\u0000\u0000\u0004\u00c4\u0001\u0000\u0000\u0000\u0004\u00c6\u0001"+ + "\u0000\u0000\u0000\u0004\u00c8\u0001\u0000\u0000\u0000\u0004\u00cc\u0001"+ + "\u0000\u0000\u0000\u0004\u00ce\u0001\u0000\u0000\u0000\u0004\u00d0\u0001"+ + "\u0000\u0000\u0000\u0004\u00d2\u0001\u0000\u0000\u0000\u0004\u00d4\u0001"+ + "\u0000\u0000\u0000\u0005\u00d6\u0001\u0000\u0000\u0000\u0005\u00d8\u0001"+ + "\u0000\u0000\u0000\u0005\u00da\u0001\u0000\u0000\u0000\u0005\u00dc\u0001"+ + "\u0000\u0000\u0000\u0005\u00de\u0001\u0000\u0000\u0000\u0005\u00e0\u0001"+ + "\u0000\u0000\u0000\u0005\u00e2\u0001\u0000\u0000\u0000\u0005\u00e4\u0001"+ + "\u0000\u0000\u0000\u0005\u00e6\u0001\u0000\u0000\u0000\u0005\u00e8\u0001"+ + "\u0000\u0000\u0000\u0006\u00ea\u0001\u0000\u0000\u0000\u0006\u00ec\u0001"+ + "\u0000\u0000\u0000\u0006\u00ee\u0001\u0000\u0000\u0000\u0006\u00f0\u0001"+ + "\u0000\u0000\u0000\u0006\u00f2\u0001\u0000\u0000\u0000\u0006\u00f4\u0001"+ + "\u0000\u0000\u0000\u0006\u00f6\u0001\u0000\u0000\u0000\u0006\u00f8\u0001"+ + "\u0000\u0000\u0000\u0007\u00fa\u0001\u0000\u0000\u0000\u0007\u00fc\u0001"+ + "\u0000\u0000\u0000\u0007\u00fe\u0001\u0000\u0000\u0000\u0007\u0100\u0001"+ + "\u0000\u0000\u0000\u0007\u0102\u0001\u0000\u0000\u0000\u0007\u0104\u0001"+ + "\u0000\u0000\u0000\u0007\u0106\u0001\u0000\u0000\u0000\u0007\u0108\u0001"+ + "\u0000\u0000\u0000\u0007\u010a\u0001\u0000\u0000\u0000\u0007\u010c\u0001"+ + "\u0000\u0000\u0000\b\u010e\u0001\u0000\u0000\u0000\b\u0110\u0001\u0000"+ + "\u0000\u0000\b\u0112\u0001\u0000\u0000\u0000\b\u0114\u0001\u0000\u0000"+ + "\u0000\b\u0116\u0001\u0000\u0000\u0000\b\u0118\u0001\u0000\u0000\u0000"+ + "\b\u011a\u0001\u0000\u0000\u0000\t\u011c\u0001\u0000\u0000\u0000\t\u011e"+ + "\u0001\u0000\u0000\u0000\t\u0120\u0001\u0000\u0000\u0000\t\u0122\u0001"+ + "\u0000\u0000\u0000\t\u0124\u0001\u0000\u0000\u0000\t\u0126\u0001\u0000"+ + "\u0000\u0000\n\u0128\u0001\u0000\u0000\u0000\f\u0132\u0001\u0000\u0000"+ + "\u0000\u000e\u0139\u0001\u0000\u0000\u0000\u0010\u0142\u0001\u0000\u0000"+ + "\u0000\u0012\u0149\u0001\u0000\u0000\u0000\u0014\u0153\u0001\u0000\u0000"+ + "\u0000\u0016\u015a\u0001\u0000\u0000\u0000\u0018\u0161\u0001\u0000\u0000"+ + "\u0000\u001a\u016f\u0001\u0000\u0000\u0000\u001c\u0176\u0001\u0000\u0000"+ + "\u0000\u001e\u017e\u0001\u0000\u0000\u0000 \u018a\u0001\u0000\u0000\u0000"+ + "\"\u0194\u0001\u0000\u0000\u0000$\u019d\u0001\u0000\u0000\u0000&\u01a3"+ + "\u0001\u0000\u0000\u0000(\u01aa\u0001\u0000\u0000\u0000*\u01b1\u0001\u0000"+ + "\u0000\u0000,\u01b9\u0001\u0000\u0000\u0000.\u01c2\u0001\u0000\u0000\u0000"+ + "0\u01c8\u0001\u0000\u0000\u00002\u01d9\u0001\u0000\u0000\u00004\u01e9"+ + "\u0001\u0000\u0000\u00006\u01ef\u0001\u0000\u0000\u00008\u01f4\u0001\u0000"+ + "\u0000\u0000:\u01f9\u0001\u0000\u0000\u0000<\u01fd\u0001\u0000\u0000\u0000"+ + ">\u0201\u0001\u0000\u0000\u0000@\u0205\u0001\u0000\u0000\u0000B\u0209"+ + "\u0001\u0000\u0000\u0000D\u020b\u0001\u0000\u0000\u0000F\u020d\u0001\u0000"+ + "\u0000\u0000H\u0210\u0001\u0000\u0000\u0000J\u0212\u0001\u0000\u0000\u0000"+ + "L\u021b\u0001\u0000\u0000\u0000N\u021d\u0001\u0000\u0000\u0000P\u0222"+ + "\u0001\u0000\u0000\u0000R\u0224\u0001\u0000\u0000\u0000T\u0229\u0001\u0000"+ + "\u0000\u0000V\u0248\u0001\u0000\u0000\u0000X\u024b\u0001\u0000\u0000\u0000"+ + "Z\u0279\u0001\u0000\u0000\u0000\\\u027b\u0001\u0000\u0000\u0000^\u027e"+ + "\u0001\u0000\u0000\u0000`\u0282\u0001\u0000\u0000\u0000b\u0286\u0001\u0000"+ + "\u0000\u0000d\u0288\u0001\u0000\u0000\u0000f\u028a\u0001\u0000\u0000\u0000"+ + "h\u028f\u0001\u0000\u0000\u0000j\u0291\u0001\u0000\u0000\u0000l\u0297"+ + "\u0001\u0000\u0000\u0000n\u029d\u0001\u0000\u0000\u0000p\u02a2\u0001\u0000"+ + "\u0000\u0000r\u02a4\u0001\u0000\u0000\u0000t\u02a7\u0001\u0000\u0000\u0000"+ + "v\u02aa\u0001\u0000\u0000\u0000x\u02af\u0001\u0000\u0000\u0000z\u02b3"+ + "\u0001\u0000\u0000\u0000|\u02b8\u0001\u0000\u0000\u0000~\u02be\u0001\u0000"+ + "\u0000\u0000\u0080\u02c1\u0001\u0000\u0000\u0000\u0082\u02c3\u0001\u0000"+ + "\u0000\u0000\u0084\u02c9\u0001\u0000\u0000\u0000\u0086\u02cb\u0001\u0000"+ + "\u0000\u0000\u0088\u02d0\u0001\u0000\u0000\u0000\u008a\u02d3\u0001\u0000"+ + "\u0000\u0000\u008c\u02d6\u0001\u0000\u0000\u0000\u008e\u02d8\u0001\u0000"+ + "\u0000\u0000\u0090\u02db\u0001\u0000\u0000\u0000\u0092\u02dd\u0001\u0000"+ + "\u0000\u0000\u0094\u02e0\u0001\u0000\u0000\u0000\u0096\u02e2\u0001\u0000"+ + "\u0000\u0000\u0098\u02e4\u0001\u0000\u0000\u0000\u009a\u02e6\u0001\u0000"+ + "\u0000\u0000\u009c\u02e8\u0001\u0000\u0000\u0000\u009e\u02ea\u0001\u0000"+ + "\u0000\u0000\u00a0\u02ef\u0001\u0000\u0000\u0000\u00a2\u0304\u0001\u0000"+ + "\u0000\u0000\u00a4\u0306\u0001\u0000\u0000\u0000\u00a6\u030e\u0001\u0000"+ + "\u0000\u0000\u00a8\u0312\u0001\u0000\u0000\u0000\u00aa\u0316\u0001\u0000"+ + "\u0000\u0000\u00ac\u031a\u0001\u0000\u0000\u0000\u00ae\u031f\u0001\u0000"+ + "\u0000\u0000\u00b0\u0325\u0001\u0000\u0000\u0000\u00b2\u032b\u0001\u0000"+ + "\u0000\u0000\u00b4\u032f\u0001\u0000\u0000\u0000\u00b6\u0333\u0001\u0000"+ + "\u0000\u0000\u00b8\u033f\u0001\u0000\u0000\u0000\u00ba\u0342\u0001\u0000"+ + "\u0000\u0000\u00bc\u0346\u0001\u0000\u0000\u0000\u00be\u034a\u0001\u0000"+ + "\u0000\u0000\u00c0\u034e\u0001\u0000\u0000\u0000\u00c2\u0352\u0001\u0000"+ + "\u0000\u0000\u00c4\u0356\u0001\u0000\u0000\u0000\u00c6\u035b\u0001\u0000"+ + "\u0000\u0000\u00c8\u035f\u0001\u0000\u0000\u0000\u00ca\u0367\u0001\u0000"+ + "\u0000\u0000\u00cc\u037c\u0001\u0000\u0000\u0000\u00ce\u037e\u0001\u0000"+ + "\u0000\u0000\u00d0\u0382\u0001\u0000\u0000\u0000\u00d2\u0386\u0001\u0000"+ + "\u0000\u0000\u00d4\u038a\u0001\u0000\u0000\u0000\u00d6\u038e\u0001\u0000"+ + "\u0000\u0000\u00d8\u0393\u0001\u0000\u0000\u0000\u00da\u0397\u0001\u0000"+ + "\u0000\u0000\u00dc\u039b\u0001\u0000\u0000\u0000\u00de\u039f\u0001\u0000"+ + "\u0000\u0000\u00e0\u03a2\u0001\u0000\u0000\u0000\u00e2\u03a6\u0001\u0000"+ + "\u0000\u0000\u00e4\u03aa\u0001\u0000\u0000\u0000\u00e6\u03ae\u0001\u0000"+ + "\u0000\u0000\u00e8\u03b2\u0001\u0000\u0000\u0000\u00ea\u03b6\u0001\u0000"+ + "\u0000\u0000\u00ec\u03bb\u0001\u0000\u0000\u0000\u00ee\u03c0\u0001\u0000"+ + "\u0000\u0000\u00f0\u03c7\u0001\u0000\u0000\u0000\u00f2\u03cb\u0001\u0000"+ + "\u0000\u0000\u00f4\u03cf\u0001\u0000\u0000\u0000\u00f6\u03d3\u0001\u0000"+ + "\u0000\u0000\u00f8\u03d7\u0001\u0000\u0000\u0000\u00fa\u03db\u0001\u0000"+ + "\u0000\u0000\u00fc\u03e1\u0001\u0000\u0000\u0000\u00fe\u03e5\u0001\u0000"+ + "\u0000\u0000\u0100\u03e9\u0001\u0000\u0000\u0000\u0102\u03ed\u0001\u0000"+ + "\u0000\u0000\u0104\u03f1\u0001\u0000\u0000\u0000\u0106\u03f5\u0001\u0000"+ + "\u0000\u0000\u0108\u03f9\u0001\u0000\u0000\u0000\u010a\u03fd\u0001\u0000"+ + "\u0000\u0000\u010c\u0401\u0001\u0000\u0000\u0000\u010e\u0405\u0001\u0000"+ + "\u0000\u0000\u0110\u040a\u0001\u0000\u0000\u0000\u0112\u040e\u0001\u0000"+ + "\u0000\u0000\u0114\u0412\u0001\u0000\u0000\u0000\u0116\u0416\u0001\u0000"+ + "\u0000\u0000\u0118\u041a\u0001\u0000\u0000\u0000\u011a\u041e\u0001\u0000"+ + "\u0000\u0000\u011c\u0422\u0001\u0000\u0000\u0000\u011e\u0427\u0001\u0000"+ + "\u0000\u0000\u0120\u042c\u0001\u0000\u0000\u0000\u0122\u0436\u0001\u0000"+ + "\u0000\u0000\u0124\u043a\u0001\u0000\u0000\u0000\u0126\u043e\u0001\u0000"+ + "\u0000\u0000\u0128\u0129\u0005d\u0000\u0000\u0129\u012a\u0005i\u0000\u0000"+ + "\u012a\u012b\u0005s\u0000\u0000\u012b\u012c\u0005s\u0000\u0000\u012c\u012d"+ + "\u0005e\u0000\u0000\u012d\u012e\u0005c\u0000\u0000\u012e\u012f\u0005t"+ + "\u0000\u0000\u012f\u0130\u0001\u0000\u0000\u0000\u0130\u0131\u0006\u0000"+ + "\u0000\u0000\u0131\u000b\u0001\u0000\u0000\u0000\u0132\u0133\u0005d\u0000"+ + "\u0000\u0133\u0134\u0005r\u0000\u0000\u0134\u0135\u0005o\u0000\u0000\u0135"+ + "\u0136\u0005p\u0000\u0000\u0136\u0137\u0001\u0000\u0000\u0000\u0137\u0138"+ + "\u0006\u0001\u0001\u0000\u0138\r\u0001\u0000\u0000\u0000\u0139\u013a\u0005"+ + "e\u0000\u0000\u013a\u013b\u0005n\u0000\u0000\u013b\u013c\u0005r\u0000"+ + "\u0000\u013c\u013d\u0005i\u0000\u0000\u013d\u013e\u0005c\u0000\u0000\u013e"+ + "\u013f\u0005h\u0000\u0000\u013f\u0140\u0001\u0000\u0000\u0000\u0140\u0141"+ + "\u0006\u0002\u0002\u0000\u0141\u000f\u0001\u0000\u0000\u0000\u0142\u0143"+ + "\u0005e\u0000\u0000\u0143\u0144\u0005v\u0000\u0000\u0144\u0145\u0005a"+ + "\u0000\u0000\u0145\u0146\u0005l\u0000\u0000\u0146\u0147\u0001\u0000\u0000"+ + "\u0000\u0147\u0148\u0006\u0003\u0000\u0000\u0148\u0011\u0001\u0000\u0000"+ + "\u0000\u0149\u014a\u0005e\u0000\u0000\u014a\u014b\u0005x\u0000\u0000\u014b"+ + "\u014c\u0005p\u0000\u0000\u014c\u014d\u0005l\u0000\u0000\u014d\u014e\u0005"+ + "a\u0000\u0000\u014e\u014f\u0005i\u0000\u0000\u014f\u0150\u0005n\u0000"+ + "\u0000\u0150\u0151\u0001\u0000\u0000\u0000\u0151\u0152\u0006\u0004\u0003"+ + "\u0000\u0152\u0013\u0001\u0000\u0000\u0000\u0153\u0154\u0005f\u0000\u0000"+ + "\u0154\u0155\u0005r\u0000\u0000\u0155\u0156\u0005o\u0000\u0000\u0156\u0157"+ + "\u0005m\u0000\u0000\u0157\u0158\u0001\u0000\u0000\u0000\u0158\u0159\u0006"+ + "\u0005\u0004\u0000\u0159\u0015\u0001\u0000\u0000\u0000\u015a\u015b\u0005"+ + "g\u0000\u0000\u015b\u015c\u0005r\u0000\u0000\u015c\u015d\u0005o\u0000"+ + "\u0000\u015d\u015e\u0005k\u0000\u0000\u015e\u015f\u0001\u0000\u0000\u0000"+ + "\u015f\u0160\u0006\u0006\u0000\u0000\u0160\u0017\u0001\u0000\u0000\u0000"+ + "\u0161\u0162\u0005i\u0000\u0000\u0162\u0163\u0005n\u0000\u0000\u0163\u0164"+ + "\u0005l\u0000\u0000\u0164\u0165\u0005i\u0000\u0000\u0165\u0166\u0005n"+ + "\u0000\u0000\u0166\u0167\u0005e\u0000\u0000\u0167\u0168\u0005s\u0000\u0000"+ + "\u0168\u0169\u0005t\u0000\u0000\u0169\u016a\u0005a\u0000\u0000\u016a\u016b"+ + "\u0005t\u0000\u0000\u016b\u016c\u0005s\u0000\u0000\u016c\u016d\u0001\u0000"+ + "\u0000\u0000\u016d\u016e\u0006\u0007\u0000\u0000\u016e\u0019\u0001\u0000"+ + "\u0000\u0000\u016f\u0170\u0005k\u0000\u0000\u0170\u0171\u0005e\u0000\u0000"+ + "\u0171\u0172\u0005e\u0000\u0000\u0172\u0173\u0005p\u0000\u0000\u0173\u0174"+ + "\u0001\u0000\u0000\u0000\u0174\u0175\u0006\b\u0001\u0000\u0175\u001b\u0001"+ + "\u0000\u0000\u0000\u0176\u0177\u0005l\u0000\u0000\u0177\u0178\u0005i\u0000"+ + "\u0000\u0178\u0179\u0005m\u0000\u0000\u0179\u017a\u0005i\u0000\u0000\u017a"+ + "\u017b\u0005t\u0000\u0000\u017b\u017c\u0001\u0000\u0000\u0000\u017c\u017d"+ + "\u0006\t\u0000\u0000\u017d\u001d\u0001\u0000\u0000\u0000\u017e\u017f\u0005"+ + "m\u0000\u0000\u017f\u0180\u0005v\u0000\u0000\u0180\u0181\u0005_\u0000"+ + "\u0000\u0181\u0182\u0005e\u0000\u0000\u0182\u0183\u0005x\u0000\u0000\u0183"+ + "\u0184\u0005p\u0000\u0000\u0184\u0185\u0005a\u0000\u0000\u0185\u0186\u0005"+ + "n\u0000\u0000\u0186\u0187\u0005d\u0000\u0000\u0187\u0188\u0001\u0000\u0000"+ + "\u0000\u0188\u0189\u0006\n\u0005\u0000\u0189\u001f\u0001\u0000\u0000\u0000"+ + "\u018a\u018b\u0005p\u0000\u0000\u018b\u018c\u0005r\u0000\u0000\u018c\u018d"+ + "\u0005o\u0000\u0000\u018d\u018e\u0005j\u0000\u0000\u018e\u018f\u0005e"+ + "\u0000\u0000\u018f\u0190\u0005c\u0000\u0000\u0190\u0191\u0005t\u0000\u0000"+ + "\u0191\u0192\u0001\u0000\u0000\u0000\u0192\u0193\u0006\u000b\u0001\u0000"+ + "\u0193!\u0001\u0000\u0000\u0000\u0194\u0195\u0005r\u0000\u0000\u0195\u0196"+ + "\u0005e\u0000\u0000\u0196\u0197\u0005n\u0000\u0000\u0197\u0198\u0005a"+ + "\u0000\u0000\u0198\u0199\u0005m\u0000\u0000\u0199\u019a\u0005e\u0000\u0000"+ + "\u019a\u019b\u0001\u0000\u0000\u0000\u019b\u019c\u0006\f\u0006\u0000\u019c"+ + "#\u0001\u0000\u0000\u0000\u019d\u019e\u0005r\u0000\u0000\u019e\u019f\u0005"+ + "o\u0000\u0000\u019f\u01a0\u0005w\u0000\u0000\u01a0\u01a1\u0001\u0000\u0000"+ + "\u0000\u01a1\u01a2\u0006\r\u0000\u0000\u01a2%\u0001\u0000\u0000\u0000"+ + "\u01a3\u01a4\u0005s\u0000\u0000\u01a4\u01a5\u0005h\u0000\u0000\u01a5\u01a6"+ + "\u0005o\u0000\u0000\u01a6\u01a7\u0005w\u0000\u0000\u01a7\u01a8\u0001\u0000"+ + "\u0000\u0000\u01a8\u01a9\u0006\u000e\u0007\u0000\u01a9\'\u0001\u0000\u0000"+ + "\u0000\u01aa\u01ab\u0005s\u0000\u0000\u01ab\u01ac\u0005o\u0000\u0000\u01ac"+ + "\u01ad\u0005r\u0000\u0000\u01ad\u01ae\u0005t\u0000\u0000\u01ae\u01af\u0001"+ + "\u0000\u0000\u0000\u01af\u01b0\u0006\u000f\u0000\u0000\u01b0)\u0001\u0000"+ + "\u0000\u0000\u01b1\u01b2\u0005s\u0000\u0000\u01b2\u01b3\u0005t\u0000\u0000"+ + "\u01b3\u01b4\u0005a\u0000\u0000\u01b4\u01b5\u0005t\u0000\u0000\u01b5\u01b6"+ + "\u0005s\u0000\u0000\u01b6\u01b7\u0001\u0000\u0000\u0000\u01b7\u01b8\u0006"+ + "\u0010\u0000\u0000\u01b8+\u0001\u0000\u0000\u0000\u01b9\u01ba\u0005w\u0000"+ + "\u0000\u01ba\u01bb\u0005h\u0000\u0000\u01bb\u01bc\u0005e\u0000\u0000\u01bc"+ + "\u01bd\u0005r\u0000\u0000\u01bd\u01be\u0005e\u0000\u0000\u01be\u01bf\u0001"+ + "\u0000\u0000\u0000\u01bf\u01c0\u0006\u0011\u0000\u0000\u01c0-\u0001\u0000"+ + "\u0000\u0000\u01c1\u01c3\b\u0000\u0000\u0000\u01c2\u01c1\u0001\u0000\u0000"+ + "\u0000\u01c3\u01c4\u0001\u0000\u0000\u0000\u01c4\u01c2\u0001\u0000\u0000"+ + "\u0000\u01c4\u01c5\u0001\u0000\u0000\u0000\u01c5\u01c6\u0001\u0000\u0000"+ + "\u0000\u01c6\u01c7\u0006\u0012\u0000\u0000\u01c7/\u0001\u0000\u0000\u0000"+ + "\u01c8\u01c9\u0005/\u0000\u0000\u01c9\u01ca\u0005/\u0000\u0000\u01ca\u01ce"+ + "\u0001\u0000\u0000\u0000\u01cb\u01cd\b\u0001\u0000\u0000\u01cc\u01cb\u0001"+ + "\u0000\u0000\u0000\u01cd\u01d0\u0001\u0000\u0000\u0000\u01ce\u01cc\u0001"+ + "\u0000\u0000\u0000\u01ce\u01cf\u0001\u0000\u0000\u0000\u01cf\u01d2\u0001"+ + "\u0000\u0000\u0000\u01d0\u01ce\u0001\u0000\u0000\u0000\u01d1\u01d3\u0005"+ + "\r\u0000\u0000\u01d2\u01d1\u0001\u0000\u0000\u0000\u01d2\u01d3\u0001\u0000"+ + "\u0000\u0000\u01d3\u01d5\u0001\u0000\u0000\u0000\u01d4\u01d6\u0005\n\u0000"+ + "\u0000\u01d5\u01d4\u0001\u0000\u0000\u0000\u01d5\u01d6\u0001\u0000\u0000"+ + "\u0000\u01d6\u01d7\u0001\u0000\u0000\u0000\u01d7\u01d8\u0006\u0013\b\u0000"+ + "\u01d81\u0001\u0000\u0000\u0000\u01d9\u01da\u0005/\u0000\u0000\u01da\u01db"+ + "\u0005*\u0000\u0000\u01db\u01e0\u0001\u0000\u0000\u0000\u01dc\u01df\u0003"+ + "2\u0014\u0000\u01dd\u01df\t\u0000\u0000\u0000\u01de\u01dc\u0001\u0000"+ + "\u0000\u0000\u01de\u01dd\u0001\u0000\u0000\u0000\u01df\u01e2\u0001\u0000"+ + "\u0000\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e0\u01de\u0001\u0000"+ + "\u0000\u0000\u01e1\u01e3\u0001\u0000\u0000\u0000\u01e2\u01e0\u0001\u0000"+ + "\u0000\u0000\u01e3\u01e4\u0005*\u0000\u0000\u01e4\u01e5\u0005/\u0000\u0000"+ + "\u01e5\u01e6\u0001\u0000\u0000\u0000\u01e6\u01e7\u0006\u0014\b\u0000\u01e7"+ + "3\u0001\u0000\u0000\u0000\u01e8\u01ea\u0007\u0002\u0000\u0000\u01e9\u01e8"+ + "\u0001\u0000\u0000\u0000\u01ea\u01eb\u0001\u0000\u0000\u0000\u01eb\u01e9"+ + "\u0001\u0000\u0000\u0000\u01eb\u01ec\u0001\u0000\u0000\u0000\u01ec\u01ed"+ + "\u0001\u0000\u0000\u0000\u01ed\u01ee\u0006\u0015\b\u0000\u01ee5\u0001"+ + "\u0000\u0000\u0000\u01ef\u01f0\u0003\u009eJ\u0000\u01f0\u01f1\u0001\u0000"+ + "\u0000\u0000\u01f1\u01f2\u0006\u0016\t\u0000\u01f2\u01f3\u0006\u0016\n"+ + "\u0000\u01f37\u0001\u0000\u0000\u0000\u01f4\u01f5\u0003@\u001b\u0000\u01f5"+ + "\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0006\u0017\u000b\u0000\u01f7"+ + "\u01f8\u0006\u0017\f\u0000\u01f89\u0001\u0000\u0000\u0000\u01f9\u01fa"+ + "\u00034\u0015\u0000\u01fa\u01fb\u0001\u0000\u0000\u0000\u01fb\u01fc\u0006"+ + "\u0018\b\u0000\u01fc;\u0001\u0000\u0000\u0000\u01fd\u01fe\u00030\u0013"+ + "\u0000\u01fe\u01ff\u0001\u0000\u0000\u0000\u01ff\u0200\u0006\u0019\b\u0000"+ + "\u0200=\u0001\u0000\u0000\u0000\u0201\u0202\u00032\u0014\u0000\u0202\u0203"+ + "\u0001\u0000\u0000\u0000\u0203\u0204\u0006\u001a\b\u0000\u0204?\u0001"+ + "\u0000\u0000\u0000\u0205\u0206\u0005|\u0000\u0000\u0206\u0207\u0001\u0000"+ + "\u0000\u0000\u0207\u0208\u0006\u001b\f\u0000\u0208A\u0001\u0000\u0000"+ + "\u0000\u0209\u020a\u0007\u0003\u0000\u0000\u020aC\u0001\u0000\u0000\u0000"+ + "\u020b\u020c\u0007\u0004\u0000\u0000\u020cE\u0001\u0000\u0000\u0000\u020d"+ + "\u020e\u0005\\\u0000\u0000\u020e\u020f\u0007\u0005\u0000\u0000\u020fG"+ + "\u0001\u0000\u0000\u0000\u0210\u0211\b\u0006\u0000\u0000\u0211I\u0001"+ + "\u0000\u0000\u0000\u0212\u0214\u0007\u0007\u0000\u0000\u0213\u0215\u0007"+ + "\b\u0000\u0000\u0214\u0213\u0001\u0000\u0000\u0000\u0214\u0215\u0001\u0000"+ + "\u0000\u0000\u0215\u0217\u0001\u0000\u0000\u0000\u0216\u0218\u0003B\u001c"+ + "\u0000\u0217\u0216\u0001\u0000\u0000\u0000\u0218\u0219\u0001\u0000\u0000"+ + "\u0000\u0219\u0217\u0001\u0000\u0000\u0000\u0219\u021a\u0001\u0000\u0000"+ + "\u0000\u021aK\u0001\u0000\u0000\u0000\u021b\u021c\u0005@\u0000\u0000\u021c"+ + "M\u0001\u0000\u0000\u0000\u021d\u021e\u0005`\u0000\u0000\u021eO\u0001"+ + "\u0000\u0000\u0000\u021f\u0223\b\t\u0000\u0000\u0220\u0221\u0005`\u0000"+ + "\u0000\u0221\u0223\u0005`\u0000\u0000\u0222\u021f\u0001\u0000\u0000\u0000"+ + "\u0222\u0220\u0001\u0000\u0000\u0000\u0223Q\u0001\u0000\u0000\u0000\u0224"+ + "\u0225\u0005_\u0000\u0000\u0225S\u0001\u0000\u0000\u0000\u0226\u022a\u0003"+ + "D\u001d\u0000\u0227\u022a\u0003B\u001c\u0000\u0228\u022a\u0003R$\u0000"+ + "\u0229\u0226\u0001\u0000\u0000\u0000\u0229\u0227\u0001\u0000\u0000\u0000"+ + "\u0229\u0228\u0001\u0000\u0000\u0000\u022aU\u0001\u0000\u0000\u0000\u022b"+ + "\u0230\u0005\"\u0000\u0000\u022c\u022f\u0003F\u001e\u0000\u022d\u022f"+ + "\u0003H\u001f\u0000\u022e\u022c\u0001\u0000\u0000\u0000\u022e\u022d\u0001"+ + "\u0000\u0000\u0000\u022f\u0232\u0001\u0000\u0000\u0000\u0230\u022e\u0001"+ + "\u0000\u0000\u0000\u0230\u0231\u0001\u0000\u0000\u0000\u0231\u0233\u0001"+ + "\u0000\u0000\u0000\u0232\u0230\u0001\u0000\u0000\u0000\u0233\u0249\u0005"+ + "\"\u0000\u0000\u0234\u0235\u0005\"\u0000\u0000\u0235\u0236\u0005\"\u0000"+ + "\u0000\u0236\u0237\u0005\"\u0000\u0000\u0237\u023b\u0001\u0000\u0000\u0000"+ + "\u0238\u023a\b\u0001\u0000\u0000\u0239\u0238\u0001\u0000\u0000\u0000\u023a"+ + "\u023d\u0001\u0000\u0000\u0000\u023b\u023c\u0001\u0000\u0000\u0000\u023b"+ + "\u0239\u0001\u0000\u0000\u0000\u023c\u023e\u0001\u0000\u0000\u0000\u023d"+ + "\u023b\u0001\u0000\u0000\u0000\u023e\u023f\u0005\"\u0000\u0000\u023f\u0240"+ + "\u0005\"\u0000\u0000\u0240\u0241\u0005\"\u0000\u0000\u0241\u0243\u0001"+ + "\u0000\u0000\u0000\u0242\u0244\u0005\"\u0000\u0000\u0243\u0242\u0001\u0000"+ + "\u0000\u0000\u0243\u0244\u0001\u0000\u0000\u0000\u0244\u0246\u0001\u0000"+ + "\u0000\u0000\u0245\u0247\u0005\"\u0000\u0000\u0246\u0245\u0001\u0000\u0000"+ + "\u0000\u0246\u0247\u0001\u0000\u0000\u0000\u0247\u0249\u0001\u0000\u0000"+ + "\u0000\u0248\u022b\u0001\u0000\u0000\u0000\u0248\u0234\u0001\u0000\u0000"+ + "\u0000\u0249W\u0001\u0000\u0000\u0000\u024a\u024c\u0003B\u001c\u0000\u024b"+ + "\u024a\u0001\u0000\u0000\u0000\u024c\u024d\u0001\u0000\u0000\u0000\u024d"+ + "\u024b\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e"+ + "Y\u0001\u0000\u0000\u0000\u024f\u0251\u0003B\u001c\u0000\u0250\u024f\u0001"+ + "\u0000\u0000\u0000\u0251\u0252\u0001\u0000\u0000\u0000\u0252\u0250\u0001"+ + "\u0000\u0000\u0000\u0252\u0253\u0001\u0000\u0000\u0000\u0253\u0254\u0001"+ + "\u0000\u0000\u0000\u0254\u0258\u0003h/\u0000\u0255\u0257\u0003B\u001c"+ + "\u0000\u0256\u0255\u0001\u0000\u0000\u0000\u0257\u025a\u0001\u0000\u0000"+ + "\u0000\u0258\u0256\u0001\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000"+ + "\u0000\u0259\u027a\u0001\u0000\u0000\u0000\u025a\u0258\u0001\u0000\u0000"+ + "\u0000\u025b\u025d\u0003h/\u0000\u025c\u025e\u0003B\u001c\u0000\u025d"+ + "\u025c\u0001\u0000\u0000\u0000\u025e\u025f\u0001\u0000\u0000\u0000\u025f"+ + "\u025d\u0001\u0000\u0000\u0000\u025f\u0260\u0001\u0000\u0000\u0000\u0260"+ + "\u027a\u0001\u0000\u0000\u0000\u0261\u0263\u0003B\u001c\u0000\u0262\u0261"+ + "\u0001\u0000\u0000\u0000\u0263\u0264\u0001\u0000\u0000\u0000\u0264\u0262"+ + "\u0001\u0000\u0000\u0000\u0264\u0265\u0001\u0000\u0000\u0000\u0265\u026d"+ + "\u0001\u0000\u0000\u0000\u0266\u026a\u0003h/\u0000\u0267\u0269\u0003B"+ + "\u001c\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0269\u026c\u0001\u0000"+ + "\u0000\u0000\u026a\u0268\u0001\u0000\u0000\u0000\u026a\u026b\u0001\u0000"+ + "\u0000\u0000\u026b\u026e\u0001\u0000\u0000\u0000\u026c\u026a\u0001\u0000"+ + "\u0000\u0000\u026d\u0266\u0001\u0000\u0000\u0000\u026d\u026e\u0001\u0000"+ + "\u0000\u0000\u026e\u026f\u0001\u0000\u0000\u0000\u026f\u0270\u0003J \u0000"+ + "\u0270\u027a\u0001\u0000\u0000\u0000\u0271\u0273\u0003h/\u0000\u0272\u0274"+ + "\u0003B\u001c\u0000\u0273\u0272\u0001\u0000\u0000\u0000\u0274\u0275\u0001"+ + "\u0000\u0000\u0000\u0275\u0273\u0001\u0000\u0000\u0000\u0275\u0276\u0001"+ + "\u0000\u0000\u0000\u0276\u0277\u0001\u0000\u0000\u0000\u0277\u0278\u0003"+ + "J \u0000\u0278\u027a\u0001\u0000\u0000\u0000\u0279\u0250\u0001\u0000\u0000"+ + "\u0000\u0279\u025b\u0001\u0000\u0000\u0000\u0279\u0262\u0001\u0000\u0000"+ + "\u0000\u0279\u0271\u0001\u0000\u0000\u0000\u027a[\u0001\u0000\u0000\u0000"+ + "\u027b\u027c\u0005b\u0000\u0000\u027c\u027d\u0005y\u0000\u0000\u027d]"+ + "\u0001\u0000\u0000\u0000\u027e\u027f\u0005a\u0000\u0000\u027f\u0280\u0005"+ + "n\u0000\u0000\u0280\u0281\u0005d\u0000\u0000\u0281_\u0001\u0000\u0000"+ + "\u0000\u0282\u0283\u0005a\u0000\u0000\u0283\u0284\u0005s\u0000\u0000\u0284"+ + "\u0285\u0005c\u0000\u0000\u0285a\u0001\u0000\u0000\u0000\u0286\u0287\u0005"+ + "=\u0000\u0000\u0287c\u0001\u0000\u0000\u0000\u0288\u0289\u0005,\u0000"+ + "\u0000\u0289e\u0001\u0000\u0000\u0000\u028a\u028b\u0005d\u0000\u0000\u028b"+ + "\u028c\u0005e\u0000\u0000\u028c\u028d\u0005s\u0000\u0000\u028d\u028e\u0005"+ + "c\u0000\u0000\u028eg\u0001\u0000\u0000\u0000\u028f\u0290\u0005.\u0000"+ + "\u0000\u0290i\u0001\u0000\u0000\u0000\u0291\u0292\u0005f\u0000\u0000\u0292"+ + "\u0293\u0005a\u0000\u0000\u0293\u0294\u0005l\u0000\u0000\u0294\u0295\u0005"+ + "s\u0000\u0000\u0295\u0296\u0005e\u0000\u0000\u0296k\u0001\u0000\u0000"+ + "\u0000\u0297\u0298\u0005f\u0000\u0000\u0298\u0299\u0005i\u0000\u0000\u0299"+ + "\u029a\u0005r\u0000\u0000\u029a\u029b\u0005s\u0000\u0000\u029b\u029c\u0005"+ + "t\u0000\u0000\u029cm\u0001\u0000\u0000\u0000\u029d\u029e\u0005l\u0000"+ + "\u0000\u029e\u029f\u0005a\u0000\u0000\u029f\u02a0\u0005s\u0000\u0000\u02a0"+ + "\u02a1\u0005t\u0000\u0000\u02a1o\u0001\u0000\u0000\u0000\u02a2\u02a3\u0005"+ + "(\u0000\u0000\u02a3q\u0001\u0000\u0000\u0000\u02a4\u02a5\u0005i\u0000"+ + "\u0000\u02a5\u02a6\u0005n\u0000\u0000\u02a6s\u0001\u0000\u0000\u0000\u02a7"+ + "\u02a8\u0005i\u0000\u0000\u02a8\u02a9\u0005s\u0000\u0000\u02a9u\u0001"+ + "\u0000\u0000\u0000\u02aa\u02ab\u0005l\u0000\u0000\u02ab\u02ac\u0005i\u0000"+ + "\u0000\u02ac\u02ad\u0005k\u0000\u0000\u02ad\u02ae\u0005e\u0000\u0000\u02ae"+ + "w\u0001\u0000\u0000\u0000\u02af\u02b0\u0005n\u0000\u0000\u02b0\u02b1\u0005"+ + "o\u0000\u0000\u02b1\u02b2\u0005t\u0000\u0000\u02b2y\u0001\u0000\u0000"+ + "\u0000\u02b3\u02b4\u0005n\u0000\u0000\u02b4\u02b5\u0005u\u0000\u0000\u02b5"+ + "\u02b6\u0005l\u0000\u0000\u02b6\u02b7\u0005l\u0000\u0000\u02b7{\u0001"+ + "\u0000\u0000\u0000\u02b8\u02b9\u0005n\u0000\u0000\u02b9\u02ba\u0005u\u0000"+ + "\u0000\u02ba\u02bb\u0005l\u0000\u0000\u02bb\u02bc\u0005l\u0000\u0000\u02bc"+ + "\u02bd\u0005s\u0000\u0000\u02bd}\u0001\u0000\u0000\u0000\u02be\u02bf\u0005"+ + "o\u0000\u0000\u02bf\u02c0\u0005r\u0000\u0000\u02c0\u007f\u0001\u0000\u0000"+ + "\u0000\u02c1\u02c2\u0005?\u0000\u0000\u02c2\u0081\u0001\u0000\u0000\u0000"+ + "\u02c3\u02c4\u0005r\u0000\u0000\u02c4\u02c5\u0005l\u0000\u0000\u02c5\u02c6"+ + "\u0005i\u0000\u0000\u02c6\u02c7\u0005k\u0000\u0000\u02c7\u02c8\u0005e"+ + "\u0000\u0000\u02c8\u0083\u0001\u0000\u0000\u0000\u02c9\u02ca\u0005)\u0000"+ + "\u0000\u02ca\u0085\u0001\u0000\u0000\u0000\u02cb\u02cc\u0005t\u0000\u0000"+ + "\u02cc\u02cd\u0005r\u0000\u0000\u02cd\u02ce\u0005u\u0000\u0000\u02ce\u02cf"+ + "\u0005e\u0000\u0000\u02cf\u0087\u0001\u0000\u0000\u0000\u02d0\u02d1\u0005"+ + "=\u0000\u0000\u02d1\u02d2\u0005=\u0000\u0000\u02d2\u0089\u0001\u0000\u0000"+ + "\u0000\u02d3\u02d4\u0005!\u0000\u0000\u02d4\u02d5\u0005=\u0000\u0000\u02d5"+ + "\u008b\u0001\u0000\u0000\u0000\u02d6\u02d7\u0005<\u0000\u0000\u02d7\u008d"+ + "\u0001\u0000\u0000\u0000\u02d8\u02d9\u0005<\u0000\u0000\u02d9\u02da\u0005"+ + "=\u0000\u0000\u02da\u008f\u0001\u0000\u0000\u0000\u02db\u02dc\u0005>\u0000"+ + "\u0000\u02dc\u0091\u0001\u0000\u0000\u0000\u02dd\u02de\u0005>\u0000\u0000"+ + "\u02de\u02df\u0005=\u0000\u0000\u02df\u0093\u0001\u0000\u0000\u0000\u02e0"+ + "\u02e1\u0005+\u0000\u0000\u02e1\u0095\u0001\u0000\u0000\u0000\u02e2\u02e3"+ + "\u0005-\u0000\u0000\u02e3\u0097\u0001\u0000\u0000\u0000\u02e4\u02e5\u0005"+ + "*\u0000\u0000\u02e5\u0099\u0001\u0000\u0000\u0000\u02e6\u02e7\u0005/\u0000"+ + "\u0000\u02e7\u009b\u0001\u0000\u0000\u0000\u02e8\u02e9\u0005%\u0000\u0000"+ + "\u02e9\u009d\u0001\u0000\u0000\u0000\u02ea\u02eb\u0005[\u0000\u0000\u02eb"+ + "\u02ec\u0001\u0000\u0000\u0000\u02ec\u02ed\u0006J\u0000\u0000\u02ed\u02ee"+ + "\u0006J\u0000\u0000\u02ee\u009f\u0001\u0000\u0000\u0000\u02ef\u02f0\u0005"+ + "]\u0000\u0000\u02f0\u02f1\u0001\u0000\u0000\u0000\u02f1\u02f2\u0006K\f"+ + "\u0000\u02f2\u02f3\u0006K\f\u0000\u02f3\u00a1\u0001\u0000\u0000\u0000"+ + "\u02f4\u02f8\u0003D\u001d\u0000\u02f5\u02f7\u0003T%\u0000\u02f6\u02f5"+ + "\u0001\u0000\u0000\u0000\u02f7\u02fa\u0001\u0000\u0000\u0000\u02f8\u02f6"+ + "\u0001\u0000\u0000\u0000\u02f8\u02f9\u0001\u0000\u0000\u0000\u02f9\u0305"+ + "\u0001\u0000\u0000\u0000\u02fa\u02f8\u0001\u0000\u0000\u0000\u02fb\u02fe"+ + "\u0003R$\u0000\u02fc\u02fe\u0003L!\u0000\u02fd\u02fb\u0001\u0000\u0000"+ + "\u0000\u02fd\u02fc\u0001\u0000\u0000\u0000\u02fe\u0300\u0001\u0000\u0000"+ + "\u0000\u02ff\u0301\u0003T%\u0000\u0300\u02ff\u0001\u0000\u0000\u0000\u0301"+ + "\u0302\u0001\u0000\u0000\u0000\u0302\u0300\u0001\u0000\u0000\u0000\u0302"+ + "\u0303\u0001\u0000\u0000\u0000\u0303\u0305\u0001\u0000\u0000\u0000\u0304"+ + "\u02f4\u0001\u0000\u0000\u0000\u0304\u02fd\u0001\u0000\u0000\u0000\u0305"+ + "\u00a3\u0001\u0000\u0000\u0000\u0306\u0308\u0003N\"\u0000\u0307\u0309"+ + "\u0003P#\u0000\u0308\u0307\u0001\u0000\u0000\u0000\u0309\u030a\u0001\u0000"+ + "\u0000\u0000\u030a\u0308\u0001\u0000\u0000\u0000\u030a\u030b\u0001\u0000"+ + "\u0000\u0000\u030b\u030c\u0001\u0000\u0000\u0000\u030c\u030d\u0003N\""+ + "\u0000\u030d\u00a5\u0001\u0000\u0000\u0000\u030e\u030f\u00030\u0013\u0000"+ + "\u030f\u0310\u0001\u0000\u0000\u0000\u0310\u0311\u0006N\b\u0000\u0311"+ + "\u00a7\u0001\u0000\u0000\u0000\u0312\u0313\u00032\u0014\u0000\u0313\u0314"+ + "\u0001\u0000\u0000\u0000\u0314\u0315\u0006O\b\u0000\u0315\u00a9\u0001"+ + "\u0000\u0000\u0000\u0316\u0317\u00034\u0015\u0000\u0317\u0318\u0001\u0000"+ + "\u0000\u0000\u0318\u0319\u0006P\b\u0000\u0319\u00ab\u0001\u0000\u0000"+ + "\u0000\u031a\u031b\u0003@\u001b\u0000\u031b\u031c\u0001\u0000\u0000\u0000"+ + "\u031c\u031d\u0006Q\u000b\u0000\u031d\u031e\u0006Q\f\u0000\u031e\u00ad"+ + "\u0001\u0000\u0000\u0000\u031f\u0320\u0003\u009eJ\u0000\u0320\u0321\u0001"+ + "\u0000\u0000\u0000\u0321\u0322\u0006R\t\u0000\u0322\u0323\u0006R\u0004"+ + "\u0000\u0323\u0324\u0006R\u0004\u0000\u0324\u00af\u0001\u0000\u0000\u0000"+ + "\u0325\u0326\u0003\u00a0K\u0000\u0326\u0327\u0001\u0000\u0000\u0000\u0327"+ + "\u0328\u0006S\r\u0000\u0328\u0329\u0006S\f\u0000\u0329\u032a\u0006S\f"+ + "\u0000\u032a\u00b1\u0001\u0000\u0000\u0000\u032b\u032c\u0003d-\u0000\u032c"+ + "\u032d\u0001\u0000\u0000\u0000\u032d\u032e\u0006T\u000e\u0000\u032e\u00b3"+ + "\u0001\u0000\u0000\u0000\u032f\u0330\u0003b,\u0000\u0330\u0331\u0001\u0000"+ + "\u0000\u0000\u0331\u0332\u0006U\u000f\u0000\u0332\u00b5\u0001\u0000\u0000"+ + "\u0000\u0333\u0334\u0005m\u0000\u0000\u0334\u0335\u0005e\u0000\u0000\u0335"+ + "\u0336\u0005t\u0000\u0000\u0336\u0337\u0005a\u0000\u0000\u0337\u0338\u0005"+ + "d\u0000\u0000\u0338\u0339\u0005a\u0000\u0000\u0339\u033a\u0005t\u0000"+ + "\u0000\u033a\u033b\u0005a\u0000\u0000\u033b\u00b7\u0001\u0000\u0000\u0000"+ + "\u033c\u0340\b\n\u0000\u0000\u033d\u033e\u0005/\u0000\u0000\u033e\u0340"+ + "\b\u000b\u0000\u0000\u033f\u033c\u0001\u0000\u0000\u0000\u033f\u033d\u0001"+ + "\u0000\u0000\u0000\u0340\u00b9\u0001\u0000\u0000\u0000\u0341\u0343\u0003"+ + "\u00b8W\u0000\u0342\u0341\u0001\u0000\u0000\u0000\u0343\u0344\u0001\u0000"+ + "\u0000\u0000\u0344\u0342\u0001\u0000\u0000\u0000\u0344\u0345\u0001\u0000"+ + "\u0000\u0000\u0345\u00bb\u0001\u0000\u0000\u0000\u0346\u0347\u0003\u00a4"+ + "M\u0000\u0347\u0348\u0001\u0000\u0000\u0000\u0348\u0349\u0006Y\u0010\u0000"+ + "\u0349\u00bd\u0001\u0000\u0000\u0000\u034a\u034b\u00030\u0013\u0000\u034b"+ + "\u034c\u0001\u0000\u0000\u0000\u034c\u034d\u0006Z\b\u0000\u034d\u00bf"+ + "\u0001\u0000\u0000\u0000\u034e\u034f\u00032\u0014\u0000\u034f\u0350\u0001"+ + "\u0000\u0000\u0000\u0350\u0351\u0006[\b\u0000\u0351\u00c1\u0001\u0000"+ + "\u0000\u0000\u0352\u0353\u00034\u0015\u0000\u0353\u0354\u0001\u0000\u0000"+ + "\u0000\u0354\u0355\u0006\\\b\u0000\u0355\u00c3\u0001\u0000\u0000\u0000"+ + "\u0356\u0357\u0003@\u001b\u0000\u0357\u0358\u0001\u0000\u0000\u0000\u0358"+ + "\u0359\u0006]\u000b\u0000\u0359\u035a\u0006]\f\u0000\u035a\u00c5\u0001"+ + "\u0000\u0000\u0000\u035b\u035c\u0003h/\u0000\u035c\u035d\u0001\u0000\u0000"+ + "\u0000\u035d\u035e\u0006^\u0011\u0000\u035e\u00c7\u0001\u0000\u0000\u0000"+ + "\u035f\u0360\u0003d-\u0000\u0360\u0361\u0001\u0000\u0000\u0000\u0361\u0362"+ + "\u0006_\u000e\u0000\u0362\u00c9\u0001\u0000\u0000\u0000\u0363\u0368\u0003"+ + "D\u001d\u0000\u0364\u0368\u0003B\u001c\u0000\u0365\u0368\u0003R$\u0000"+ + "\u0366\u0368\u0003\u0098G\u0000\u0367\u0363\u0001\u0000\u0000\u0000\u0367"+ + "\u0364\u0001\u0000\u0000\u0000\u0367\u0365\u0001\u0000\u0000\u0000\u0367"+ + "\u0366\u0001\u0000\u0000\u0000\u0368\u00cb\u0001\u0000\u0000\u0000\u0369"+ + "\u036c\u0003D\u001d\u0000\u036a\u036c\u0003\u0098G\u0000\u036b\u0369\u0001"+ + "\u0000\u0000\u0000\u036b\u036a\u0001\u0000\u0000\u0000\u036c\u0370\u0001"+ + "\u0000\u0000\u0000\u036d\u036f\u0003\u00ca`\u0000\u036e\u036d\u0001\u0000"+ + "\u0000\u0000\u036f\u0372\u0001\u0000\u0000\u0000\u0370\u036e\u0001\u0000"+ + "\u0000\u0000\u0370\u0371\u0001\u0000\u0000\u0000\u0371\u037d\u0001\u0000"+ + "\u0000\u0000\u0372\u0370\u0001\u0000\u0000\u0000\u0373\u0376\u0003R$\u0000"+ + "\u0374\u0376\u0003L!\u0000\u0375\u0373\u0001\u0000\u0000\u0000\u0375\u0374"+ + "\u0001\u0000\u0000\u0000\u0376\u0378\u0001\u0000\u0000\u0000\u0377\u0379"+ + "\u0003\u00ca`\u0000\u0378\u0377\u0001\u0000\u0000\u0000\u0379\u037a\u0001"+ + "\u0000\u0000\u0000\u037a\u0378\u0001\u0000\u0000\u0000\u037a\u037b\u0001"+ + "\u0000\u0000\u0000\u037b\u037d\u0001\u0000\u0000\u0000\u037c\u036b\u0001"+ + "\u0000\u0000\u0000\u037c\u0375\u0001\u0000\u0000\u0000\u037d\u00cd\u0001"+ + "\u0000\u0000\u0000\u037e\u037f\u0003\u00a4M\u0000\u037f\u0380\u0001\u0000"+ + "\u0000\u0000\u0380\u0381\u0006b\u0010\u0000\u0381\u00cf\u0001\u0000\u0000"+ + "\u0000\u0382\u0383\u00030\u0013\u0000\u0383\u0384\u0001\u0000\u0000\u0000"+ + "\u0384\u0385\u0006c\b\u0000\u0385\u00d1\u0001\u0000\u0000\u0000\u0386"+ + "\u0387\u00032\u0014\u0000\u0387\u0388\u0001\u0000\u0000\u0000\u0388\u0389"+ + "\u0006d\b\u0000\u0389\u00d3\u0001\u0000\u0000\u0000\u038a\u038b\u0003"+ + "4\u0015\u0000\u038b\u038c\u0001\u0000\u0000\u0000\u038c\u038d\u0006e\b"+ + "\u0000\u038d\u00d5\u0001\u0000\u0000\u0000\u038e\u038f\u0003@\u001b\u0000"+ + "\u038f\u0390\u0001\u0000\u0000\u0000\u0390\u0391\u0006f\u000b\u0000\u0391"+ + "\u0392\u0006f\f\u0000\u0392\u00d7\u0001\u0000\u0000\u0000\u0393\u0394"+ + "\u0003b,\u0000\u0394\u0395\u0001\u0000\u0000\u0000\u0395\u0396\u0006g"+ + "\u000f\u0000\u0396\u00d9\u0001\u0000\u0000\u0000\u0397\u0398\u0003d-\u0000"+ + "\u0398\u0399\u0001\u0000\u0000\u0000\u0399\u039a\u0006h\u000e\u0000\u039a"+ + "\u00db\u0001\u0000\u0000\u0000\u039b\u039c\u0003h/\u0000\u039c\u039d\u0001"+ + "\u0000\u0000\u0000\u039d\u039e\u0006i\u0011\u0000\u039e\u00dd\u0001\u0000"+ + "\u0000\u0000\u039f\u03a0\u0005a\u0000\u0000\u03a0\u03a1\u0005s\u0000\u0000"+ + "\u03a1\u00df\u0001\u0000\u0000\u0000\u03a2\u03a3\u0003\u00a4M\u0000\u03a3"+ + "\u03a4\u0001\u0000\u0000\u0000\u03a4\u03a5\u0006k\u0010\u0000\u03a5\u00e1"+ + "\u0001\u0000\u0000\u0000\u03a6\u03a7\u0003\u00cca\u0000\u03a7\u03a8\u0001"+ + "\u0000\u0000\u0000\u03a8\u03a9\u0006l\u0012\u0000\u03a9\u00e3\u0001\u0000"+ + "\u0000\u0000\u03aa\u03ab\u00030\u0013\u0000\u03ab\u03ac\u0001\u0000\u0000"+ + "\u0000\u03ac\u03ad\u0006m\b\u0000\u03ad\u00e5\u0001\u0000\u0000\u0000"+ + "\u03ae\u03af\u00032\u0014\u0000\u03af\u03b0\u0001\u0000\u0000\u0000\u03b0"+ + "\u03b1\u0006n\b\u0000\u03b1\u00e7\u0001\u0000\u0000\u0000\u03b2\u03b3"+ + "\u00034\u0015\u0000\u03b3\u03b4\u0001\u0000\u0000\u0000\u03b4\u03b5\u0006"+ + "o\b\u0000\u03b5\u00e9\u0001\u0000\u0000\u0000\u03b6\u03b7\u0003@\u001b"+ + "\u0000\u03b7\u03b8\u0001\u0000\u0000\u0000\u03b8\u03b9\u0006p\u000b\u0000"+ + "\u03b9\u03ba\u0006p\f\u0000\u03ba\u00eb\u0001\u0000\u0000\u0000\u03bb"+ + "\u03bc\u0005o\u0000\u0000\u03bc\u03bd\u0005n\u0000\u0000\u03bd\u03be\u0001"+ + "\u0000\u0000\u0000\u03be\u03bf\u0006q\u0013\u0000\u03bf\u00ed\u0001\u0000"+ + "\u0000\u0000\u03c0\u03c1\u0005w\u0000\u0000\u03c1\u03c2\u0005i\u0000\u0000"+ + "\u03c2\u03c3\u0005t\u0000\u0000\u03c3\u03c4\u0005h\u0000\u0000\u03c4\u03c5"+ + "\u0001\u0000\u0000\u0000\u03c5\u03c6\u0006r\u0013\u0000\u03c6\u00ef\u0001"+ + "\u0000\u0000\u0000\u03c7\u03c8\u0003\u00baX\u0000\u03c8\u03c9\u0001\u0000"+ + "\u0000\u0000\u03c9\u03ca\u0006s\u0014\u0000\u03ca\u00f1\u0001\u0000\u0000"+ + "\u0000\u03cb\u03cc\u0003\u00a4M\u0000\u03cc\u03cd\u0001\u0000\u0000\u0000"+ + "\u03cd\u03ce\u0006t\u0010\u0000\u03ce\u00f3\u0001\u0000\u0000\u0000\u03cf"+ + "\u03d0\u00030\u0013\u0000\u03d0\u03d1\u0001\u0000\u0000\u0000\u03d1\u03d2"+ + "\u0006u\b\u0000\u03d2\u00f5\u0001\u0000\u0000\u0000\u03d3\u03d4\u0003"+ + "2\u0014\u0000\u03d4\u03d5\u0001\u0000\u0000\u0000\u03d5\u03d6\u0006v\b"+ + "\u0000\u03d6\u00f7\u0001\u0000\u0000\u0000\u03d7\u03d8\u00034\u0015\u0000"+ + "\u03d8\u03d9\u0001\u0000\u0000\u0000\u03d9\u03da\u0006w\b\u0000\u03da"+ + "\u00f9\u0001\u0000\u0000\u0000\u03db\u03dc\u0003@\u001b\u0000\u03dc\u03dd"+ + "\u0001\u0000\u0000\u0000\u03dd\u03de\u0006x\u000b\u0000\u03de\u03df\u0006"+ + "x\f\u0000\u03df\u03e0\u0006x\f\u0000\u03e0\u00fb\u0001\u0000\u0000\u0000"+ + "\u03e1\u03e2\u0003b,\u0000\u03e2\u03e3\u0001\u0000\u0000\u0000\u03e3\u03e4"+ + "\u0006y\u000f\u0000\u03e4\u00fd\u0001\u0000\u0000\u0000\u03e5\u03e6\u0003"+ + "d-\u0000\u03e6\u03e7\u0001\u0000\u0000\u0000\u03e7\u03e8\u0006z\u000e"+ + "\u0000\u03e8\u00ff\u0001\u0000\u0000\u0000\u03e9\u03ea\u0003h/\u0000\u03ea"+ + "\u03eb\u0001\u0000\u0000\u0000\u03eb\u03ec\u0006{\u0011\u0000\u03ec\u0101"+ + "\u0001\u0000\u0000\u0000\u03ed\u03ee\u0003\u00eer\u0000\u03ee\u03ef\u0001"+ + "\u0000\u0000\u0000\u03ef\u03f0\u0006|\u0015\u0000\u03f0\u0103\u0001\u0000"+ + "\u0000\u0000\u03f1\u03f2\u0003\u00cca\u0000\u03f2\u03f3\u0001\u0000\u0000"+ + "\u0000\u03f3\u03f4\u0006}\u0012\u0000\u03f4\u0105\u0001\u0000\u0000\u0000"+ + "\u03f5\u03f6\u0003\u00a4M\u0000\u03f6\u03f7\u0001\u0000\u0000\u0000\u03f7"+ + "\u03f8\u0006~\u0010\u0000\u03f8\u0107\u0001\u0000\u0000\u0000\u03f9\u03fa"+ + "\u00030\u0013\u0000\u03fa\u03fb\u0001\u0000\u0000\u0000\u03fb\u03fc\u0006"+ + "\u007f\b\u0000\u03fc\u0109\u0001\u0000\u0000\u0000\u03fd\u03fe\u00032"+ + "\u0014\u0000\u03fe\u03ff\u0001\u0000\u0000\u0000\u03ff\u0400\u0006\u0080"+ + "\b\u0000\u0400\u010b\u0001\u0000\u0000\u0000\u0401\u0402\u00034\u0015"+ + "\u0000\u0402\u0403\u0001\u0000\u0000\u0000\u0403\u0404\u0006\u0081\b\u0000"+ + "\u0404\u010d\u0001\u0000\u0000\u0000\u0405\u0406\u0003@\u001b\u0000\u0406"+ + "\u0407\u0001\u0000\u0000\u0000\u0407\u0408\u0006\u0082\u000b\u0000\u0408"+ + "\u0409\u0006\u0082\f\u0000\u0409\u010f\u0001\u0000\u0000\u0000\u040a\u040b"+ + "\u0003h/\u0000\u040b\u040c\u0001\u0000\u0000\u0000\u040c\u040d\u0006\u0083"+ + "\u0011\u0000\u040d\u0111\u0001\u0000\u0000\u0000\u040e\u040f\u0003\u00a4"+ + "M\u0000\u040f\u0410\u0001\u0000\u0000\u0000\u0410\u0411\u0006\u0084\u0010"+ + "\u0000\u0411\u0113\u0001\u0000\u0000\u0000\u0412\u0413\u0003\u00a2L\u0000"+ + "\u0413\u0414\u0001\u0000\u0000\u0000\u0414\u0415\u0006\u0085\u0016\u0000"+ + "\u0415\u0115\u0001\u0000\u0000\u0000\u0416\u0417\u00030\u0013\u0000\u0417"+ + "\u0418\u0001\u0000\u0000\u0000\u0418\u0419\u0006\u0086\b\u0000\u0419\u0117"+ + "\u0001\u0000\u0000\u0000\u041a\u041b\u00032\u0014\u0000\u041b\u041c\u0001"+ + "\u0000\u0000\u0000\u041c\u041d\u0006\u0087\b\u0000\u041d\u0119\u0001\u0000"+ + "\u0000\u0000\u041e\u041f\u00034\u0015\u0000\u041f\u0420\u0001\u0000\u0000"+ + "\u0000\u0420\u0421\u0006\u0088\b\u0000\u0421\u011b\u0001\u0000\u0000\u0000"+ + "\u0422\u0423\u0003@\u001b\u0000\u0423\u0424\u0001\u0000\u0000\u0000\u0424"+ + "\u0425\u0006\u0089\u000b\u0000\u0425\u0426\u0006\u0089\f\u0000\u0426\u011d"+ + "\u0001\u0000\u0000\u0000\u0427\u0428\u0005i\u0000\u0000\u0428\u0429\u0005"+ + "n\u0000\u0000\u0429\u042a\u0005f\u0000\u0000\u042a\u042b\u0005o\u0000"+ + "\u0000\u042b\u011f\u0001\u0000\u0000\u0000\u042c\u042d\u0005f\u0000\u0000"+ + "\u042d\u042e\u0005u\u0000\u0000\u042e\u042f\u0005n\u0000\u0000\u042f\u0430"+ + "\u0005c\u0000\u0000\u0430\u0431\u0005t\u0000\u0000\u0431\u0432\u0005i"+ + "\u0000\u0000\u0432\u0433\u0005o\u0000\u0000\u0433\u0434\u0005n\u0000\u0000"+ + "\u0434\u0435\u0005s\u0000\u0000\u0435\u0121\u0001\u0000\u0000\u0000\u0436"+ + "\u0437\u00030\u0013\u0000\u0437\u0438\u0001\u0000\u0000\u0000\u0438\u0439"+ + "\u0006\u008c\b\u0000\u0439\u0123\u0001\u0000\u0000\u0000\u043a\u043b\u0003"+ + "2\u0014\u0000\u043b\u043c\u0001\u0000\u0000\u0000\u043c\u043d\u0006\u008d"+ + "\b\u0000\u043d\u0125\u0001\u0000\u0000\u0000\u043e\u043f\u00034\u0015"+ + "\u0000\u043f\u0440\u0001\u0000\u0000\u0000\u0440\u0441\u0006\u008e\b\u0000"+ + "\u0441\u0127\u0001\u0000\u0000\u00001\u0000\u0001\u0002\u0003\u0004\u0005"+ + "\u0006\u0007\b\t\u01c4\u01ce\u01d2\u01d5\u01de\u01e0\u01eb\u0214\u0219"+ + "\u0222\u0229\u022e\u0230\u023b\u0243\u0246\u0248\u024d\u0252\u0258\u025f"+ + "\u0264\u026a\u026d\u0275\u0279\u02f8\u02fd\u0302\u0304\u030a\u033f\u0344"+ + "\u0367\u036b\u0370\u0375\u037a\u037c\u0017\u0005\u0002\u0000\u0005\u0004"+ + "\u0000\u0005\u0006\u0000\u0005\u0001\u0000\u0005\u0003\u0000\u0005\b\u0000"+ + "\u0005\u0005\u0000\u0005\t\u0000\u0000\u0001\u0000\u0007?\u0000\u0005"+ + "\u0000\u0000\u0007\u001a\u0000\u0004\u0000\u0000\u0007@\u0000\u0007\""+ + "\u0000\u0007!\u0000\u0007B\u0000\u0007$\u0000\u0007K\u0000\u0005\u0007"+ + "\u0000\u0007G\u0000\u0007T\u0000\u0007A\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index 658e09ca4b190..3acc73b1b592c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -25,15 +25,15 @@ null null null null -null +'|' null null null 'by' 'and' 'asc' -null -null +'=' +',' 'desc' '.' 'false' @@ -51,8 +51,6 @@ null 'rlike' ')' 'true' -'info' -'functions' '==' '!=' '<' @@ -71,8 +69,19 @@ null null null null -'as' 'metadata' +null +null +null +null +null +null +null +null +'as' +null +null +null 'on' 'with' null @@ -81,6 +90,14 @@ null null null null +null +null +null +'info' +'functions' +null +null +null token symbolic names: null @@ -135,8 +152,6 @@ PARAM RLIKE RP TRUE -INFO -FUNCTIONS EQ NEQ LT @@ -155,16 +170,35 @@ QUOTED_IDENTIFIER EXPR_LINE_COMMENT EXPR_MULTILINE_COMMENT EXPR_WS -AS METADATA +FROM_UNQUOTED_IDENTIFIER +FROM_LINE_COMMENT +FROM_MULTILINE_COMMENT +FROM_WS +PROJECT_UNQUOTED_IDENTIFIER +PROJECT_LINE_COMMENT +PROJECT_MULTILINE_COMMENT +PROJECT_WS +AS +RENAME_LINE_COMMENT +RENAME_MULTILINE_COMMENT +RENAME_WS ON WITH -SRC_UNQUOTED_IDENTIFIER -SRC_QUOTED_IDENTIFIER -SRC_LINE_COMMENT -SRC_MULTILINE_COMMENT -SRC_WS -EXPLAIN_PIPE +ENRICH_LINE_COMMENT +ENRICH_MULTILINE_COMMENT +ENRICH_WS +ENRICH_FIELD_LINE_COMMENT +ENRICH_FIELD_MULTILINE_COMMENT +ENRICH_FIELD_WS +MVEXPAND_LINE_COMMENT +MVEXPAND_MULTILINE_COMMENT +MVEXPAND_WS +INFO +FUNCTIONS +SHOW_LINE_COMMENT +SHOW_MULTILINE_COMMENT +SHOW_WS rule names: singleStatement @@ -187,9 +221,11 @@ evalCommand statsCommand inlinestatsCommand grouping -sourceIdentifier +fromIdentifier qualifiedName +qualifiedNamePattern identifier +identifierPattern constant limitCommand sortCommand @@ -217,4 +253,4 @@ enrichWithClause atn: -[4, 1, 81, 505, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 104, 8, 1, 10, 1, 12, 1, 107, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 113, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 128, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 140, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 147, 8, 5, 10, 5, 12, 5, 150, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 157, 8, 5, 1, 5, 1, 5, 3, 5, 161, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 169, 8, 5, 10, 5, 12, 5, 172, 9, 5, 1, 6, 1, 6, 3, 6, 176, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 183, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 188, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 195, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 201, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 209, 8, 8, 10, 8, 12, 8, 212, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 221, 8, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 229, 8, 10, 10, 10, 12, 10, 232, 9, 10, 3, 10, 234, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 5, 12, 244, 8, 12, 10, 12, 12, 12, 247, 9, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 254, 8, 13, 1, 14, 1, 14, 1, 14, 1, 14, 5, 14, 260, 8, 14, 10, 14, 12, 14, 263, 9, 14, 1, 14, 3, 14, 266, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 5, 15, 273, 8, 15, 10, 15, 12, 15, 276, 9, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 3, 17, 285, 8, 17, 1, 17, 1, 17, 3, 17, 289, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 295, 8, 18, 1, 19, 1, 19, 1, 19, 5, 19, 300, 8, 19, 10, 19, 12, 19, 303, 9, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 5, 21, 310, 8, 21, 10, 21, 12, 21, 313, 9, 21, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 330, 8, 23, 10, 23, 12, 23, 333, 9, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 341, 8, 23, 10, 23, 12, 23, 344, 9, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 352, 8, 23, 10, 23, 12, 23, 355, 9, 23, 1, 23, 1, 23, 3, 23, 359, 8, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 368, 8, 25, 10, 25, 12, 25, 371, 9, 25, 1, 26, 1, 26, 3, 26, 375, 8, 26, 1, 26, 1, 26, 3, 26, 379, 8, 26, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 385, 8, 27, 10, 27, 12, 27, 388, 9, 27, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 394, 8, 27, 10, 27, 12, 27, 397, 9, 27, 3, 27, 399, 8, 27, 1, 28, 1, 28, 1, 28, 1, 28, 5, 28, 405, 8, 28, 10, 28, 12, 28, 408, 9, 28, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 414, 8, 29, 10, 29, 12, 29, 417, 9, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 3, 31, 427, 8, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 5, 34, 439, 8, 34, 10, 34, 12, 34, 442, 9, 34, 1, 35, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 37, 1, 37, 3, 37, 452, 8, 37, 1, 38, 3, 38, 455, 8, 38, 1, 38, 1, 38, 1, 39, 3, 39, 460, 8, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 479, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 3, 45, 485, 8, 45, 1, 45, 1, 45, 1, 45, 1, 45, 5, 45, 491, 8, 45, 10, 45, 12, 45, 494, 9, 45, 3, 45, 496, 8, 45, 1, 46, 1, 46, 1, 46, 3, 46, 501, 8, 46, 1, 46, 1, 46, 1, 46, 0, 3, 2, 10, 16, 47, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 0, 8, 1, 0, 60, 61, 1, 0, 62, 64, 1, 0, 76, 77, 1, 0, 67, 68, 2, 0, 32, 32, 35, 35, 1, 0, 38, 39, 2, 0, 37, 37, 51, 51, 1, 0, 54, 59, 535, 0, 94, 1, 0, 0, 0, 2, 97, 1, 0, 0, 0, 4, 112, 1, 0, 0, 0, 6, 127, 1, 0, 0, 0, 8, 129, 1, 0, 0, 0, 10, 160, 1, 0, 0, 0, 12, 187, 1, 0, 0, 0, 14, 194, 1, 0, 0, 0, 16, 200, 1, 0, 0, 0, 18, 220, 1, 0, 0, 0, 20, 222, 1, 0, 0, 0, 22, 237, 1, 0, 0, 0, 24, 240, 1, 0, 0, 0, 26, 253, 1, 0, 0, 0, 28, 255, 1, 0, 0, 0, 30, 267, 1, 0, 0, 0, 32, 279, 1, 0, 0, 0, 34, 282, 1, 0, 0, 0, 36, 290, 1, 0, 0, 0, 38, 296, 1, 0, 0, 0, 40, 304, 1, 0, 0, 0, 42, 306, 1, 0, 0, 0, 44, 314, 1, 0, 0, 0, 46, 358, 1, 0, 0, 0, 48, 360, 1, 0, 0, 0, 50, 363, 1, 0, 0, 0, 52, 372, 1, 0, 0, 0, 54, 398, 1, 0, 0, 0, 56, 400, 1, 0, 0, 0, 58, 409, 1, 0, 0, 0, 60, 418, 1, 0, 0, 0, 62, 422, 1, 0, 0, 0, 64, 428, 1, 0, 0, 0, 66, 432, 1, 0, 0, 0, 68, 435, 1, 0, 0, 0, 70, 443, 1, 0, 0, 0, 72, 447, 1, 0, 0, 0, 74, 451, 1, 0, 0, 0, 76, 454, 1, 0, 0, 0, 78, 459, 1, 0, 0, 0, 80, 463, 1, 0, 0, 0, 82, 465, 1, 0, 0, 0, 84, 467, 1, 0, 0, 0, 86, 470, 1, 0, 0, 0, 88, 478, 1, 0, 0, 0, 90, 480, 1, 0, 0, 0, 92, 500, 1, 0, 0, 0, 94, 95, 3, 2, 1, 0, 95, 96, 5, 0, 0, 1, 96, 1, 1, 0, 0, 0, 97, 98, 6, 1, -1, 0, 98, 99, 3, 4, 2, 0, 99, 105, 1, 0, 0, 0, 100, 101, 10, 1, 0, 0, 101, 102, 5, 26, 0, 0, 102, 104, 3, 6, 3, 0, 103, 100, 1, 0, 0, 0, 104, 107, 1, 0, 0, 0, 105, 103, 1, 0, 0, 0, 105, 106, 1, 0, 0, 0, 106, 3, 1, 0, 0, 0, 107, 105, 1, 0, 0, 0, 108, 113, 3, 84, 42, 0, 109, 113, 3, 28, 14, 0, 110, 113, 3, 22, 11, 0, 111, 113, 3, 88, 44, 0, 112, 108, 1, 0, 0, 0, 112, 109, 1, 0, 0, 0, 112, 110, 1, 0, 0, 0, 112, 111, 1, 0, 0, 0, 113, 5, 1, 0, 0, 0, 114, 128, 3, 32, 16, 0, 115, 128, 3, 36, 18, 0, 116, 128, 3, 48, 24, 0, 117, 128, 3, 54, 27, 0, 118, 128, 3, 50, 25, 0, 119, 128, 3, 34, 17, 0, 120, 128, 3, 8, 4, 0, 121, 128, 3, 56, 28, 0, 122, 128, 3, 58, 29, 0, 123, 128, 3, 62, 31, 0, 124, 128, 3, 64, 32, 0, 125, 128, 3, 90, 45, 0, 126, 128, 3, 66, 33, 0, 127, 114, 1, 0, 0, 0, 127, 115, 1, 0, 0, 0, 127, 116, 1, 0, 0, 0, 127, 117, 1, 0, 0, 0, 127, 118, 1, 0, 0, 0, 127, 119, 1, 0, 0, 0, 127, 120, 1, 0, 0, 0, 127, 121, 1, 0, 0, 0, 127, 122, 1, 0, 0, 0, 127, 123, 1, 0, 0, 0, 127, 124, 1, 0, 0, 0, 127, 125, 1, 0, 0, 0, 127, 126, 1, 0, 0, 0, 128, 7, 1, 0, 0, 0, 129, 130, 5, 18, 0, 0, 130, 131, 3, 10, 5, 0, 131, 9, 1, 0, 0, 0, 132, 133, 6, 5, -1, 0, 133, 134, 5, 44, 0, 0, 134, 161, 3, 10, 5, 7, 135, 161, 3, 14, 7, 0, 136, 161, 3, 12, 6, 0, 137, 139, 3, 14, 7, 0, 138, 140, 5, 44, 0, 0, 139, 138, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 141, 1, 0, 0, 0, 141, 142, 5, 41, 0, 0, 142, 143, 5, 40, 0, 0, 143, 148, 3, 14, 7, 0, 144, 145, 5, 34, 0, 0, 145, 147, 3, 14, 7, 0, 146, 144, 1, 0, 0, 0, 147, 150, 1, 0, 0, 0, 148, 146, 1, 0, 0, 0, 148, 149, 1, 0, 0, 0, 149, 151, 1, 0, 0, 0, 150, 148, 1, 0, 0, 0, 151, 152, 5, 50, 0, 0, 152, 161, 1, 0, 0, 0, 153, 154, 3, 14, 7, 0, 154, 156, 5, 42, 0, 0, 155, 157, 5, 44, 0, 0, 156, 155, 1, 0, 0, 0, 156, 157, 1, 0, 0, 0, 157, 158, 1, 0, 0, 0, 158, 159, 5, 45, 0, 0, 159, 161, 1, 0, 0, 0, 160, 132, 1, 0, 0, 0, 160, 135, 1, 0, 0, 0, 160, 136, 1, 0, 0, 0, 160, 137, 1, 0, 0, 0, 160, 153, 1, 0, 0, 0, 161, 170, 1, 0, 0, 0, 162, 163, 10, 4, 0, 0, 163, 164, 5, 31, 0, 0, 164, 169, 3, 10, 5, 5, 165, 166, 10, 3, 0, 0, 166, 167, 5, 47, 0, 0, 167, 169, 3, 10, 5, 4, 168, 162, 1, 0, 0, 0, 168, 165, 1, 0, 0, 0, 169, 172, 1, 0, 0, 0, 170, 168, 1, 0, 0, 0, 170, 171, 1, 0, 0, 0, 171, 11, 1, 0, 0, 0, 172, 170, 1, 0, 0, 0, 173, 175, 3, 14, 7, 0, 174, 176, 5, 44, 0, 0, 175, 174, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 177, 1, 0, 0, 0, 177, 178, 5, 43, 0, 0, 178, 179, 3, 80, 40, 0, 179, 188, 1, 0, 0, 0, 180, 182, 3, 14, 7, 0, 181, 183, 5, 44, 0, 0, 182, 181, 1, 0, 0, 0, 182, 183, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 185, 5, 49, 0, 0, 185, 186, 3, 80, 40, 0, 186, 188, 1, 0, 0, 0, 187, 173, 1, 0, 0, 0, 187, 180, 1, 0, 0, 0, 188, 13, 1, 0, 0, 0, 189, 195, 3, 16, 8, 0, 190, 191, 3, 16, 8, 0, 191, 192, 3, 82, 41, 0, 192, 193, 3, 16, 8, 0, 193, 195, 1, 0, 0, 0, 194, 189, 1, 0, 0, 0, 194, 190, 1, 0, 0, 0, 195, 15, 1, 0, 0, 0, 196, 197, 6, 8, -1, 0, 197, 201, 3, 18, 9, 0, 198, 199, 7, 0, 0, 0, 199, 201, 3, 16, 8, 3, 200, 196, 1, 0, 0, 0, 200, 198, 1, 0, 0, 0, 201, 210, 1, 0, 0, 0, 202, 203, 10, 2, 0, 0, 203, 204, 7, 1, 0, 0, 204, 209, 3, 16, 8, 3, 205, 206, 10, 1, 0, 0, 206, 207, 7, 0, 0, 0, 207, 209, 3, 16, 8, 2, 208, 202, 1, 0, 0, 0, 208, 205, 1, 0, 0, 0, 209, 212, 1, 0, 0, 0, 210, 208, 1, 0, 0, 0, 210, 211, 1, 0, 0, 0, 211, 17, 1, 0, 0, 0, 212, 210, 1, 0, 0, 0, 213, 221, 3, 46, 23, 0, 214, 221, 3, 42, 21, 0, 215, 221, 3, 20, 10, 0, 216, 217, 5, 40, 0, 0, 217, 218, 3, 10, 5, 0, 218, 219, 5, 50, 0, 0, 219, 221, 1, 0, 0, 0, 220, 213, 1, 0, 0, 0, 220, 214, 1, 0, 0, 0, 220, 215, 1, 0, 0, 0, 220, 216, 1, 0, 0, 0, 221, 19, 1, 0, 0, 0, 222, 223, 3, 44, 22, 0, 223, 233, 5, 40, 0, 0, 224, 234, 5, 62, 0, 0, 225, 230, 3, 10, 5, 0, 226, 227, 5, 34, 0, 0, 227, 229, 3, 10, 5, 0, 228, 226, 1, 0, 0, 0, 229, 232, 1, 0, 0, 0, 230, 228, 1, 0, 0, 0, 230, 231, 1, 0, 0, 0, 231, 234, 1, 0, 0, 0, 232, 230, 1, 0, 0, 0, 233, 224, 1, 0, 0, 0, 233, 225, 1, 0, 0, 0, 233, 234, 1, 0, 0, 0, 234, 235, 1, 0, 0, 0, 235, 236, 5, 50, 0, 0, 236, 21, 1, 0, 0, 0, 237, 238, 5, 14, 0, 0, 238, 239, 3, 24, 12, 0, 239, 23, 1, 0, 0, 0, 240, 245, 3, 26, 13, 0, 241, 242, 5, 34, 0, 0, 242, 244, 3, 26, 13, 0, 243, 241, 1, 0, 0, 0, 244, 247, 1, 0, 0, 0, 245, 243, 1, 0, 0, 0, 245, 246, 1, 0, 0, 0, 246, 25, 1, 0, 0, 0, 247, 245, 1, 0, 0, 0, 248, 254, 3, 10, 5, 0, 249, 250, 3, 42, 21, 0, 250, 251, 5, 33, 0, 0, 251, 252, 3, 10, 5, 0, 252, 254, 1, 0, 0, 0, 253, 248, 1, 0, 0, 0, 253, 249, 1, 0, 0, 0, 254, 27, 1, 0, 0, 0, 255, 256, 5, 6, 0, 0, 256, 261, 3, 40, 20, 0, 257, 258, 5, 34, 0, 0, 258, 260, 3, 40, 20, 0, 259, 257, 1, 0, 0, 0, 260, 263, 1, 0, 0, 0, 261, 259, 1, 0, 0, 0, 261, 262, 1, 0, 0, 0, 262, 265, 1, 0, 0, 0, 263, 261, 1, 0, 0, 0, 264, 266, 3, 30, 15, 0, 265, 264, 1, 0, 0, 0, 265, 266, 1, 0, 0, 0, 266, 29, 1, 0, 0, 0, 267, 268, 5, 65, 0, 0, 268, 269, 5, 73, 0, 0, 269, 274, 3, 40, 20, 0, 270, 271, 5, 34, 0, 0, 271, 273, 3, 40, 20, 0, 272, 270, 1, 0, 0, 0, 273, 276, 1, 0, 0, 0, 274, 272, 1, 0, 0, 0, 274, 275, 1, 0, 0, 0, 275, 277, 1, 0, 0, 0, 276, 274, 1, 0, 0, 0, 277, 278, 5, 66, 0, 0, 278, 31, 1, 0, 0, 0, 279, 280, 5, 4, 0, 0, 280, 281, 3, 24, 12, 0, 281, 33, 1, 0, 0, 0, 282, 284, 5, 17, 0, 0, 283, 285, 3, 24, 12, 0, 284, 283, 1, 0, 0, 0, 284, 285, 1, 0, 0, 0, 285, 288, 1, 0, 0, 0, 286, 287, 5, 30, 0, 0, 287, 289, 3, 38, 19, 0, 288, 286, 1, 0, 0, 0, 288, 289, 1, 0, 0, 0, 289, 35, 1, 0, 0, 0, 290, 291, 5, 8, 0, 0, 291, 294, 3, 24, 12, 0, 292, 293, 5, 30, 0, 0, 293, 295, 3, 38, 19, 0, 294, 292, 1, 0, 0, 0, 294, 295, 1, 0, 0, 0, 295, 37, 1, 0, 0, 0, 296, 301, 3, 42, 21, 0, 297, 298, 5, 34, 0, 0, 298, 300, 3, 42, 21, 0, 299, 297, 1, 0, 0, 0, 300, 303, 1, 0, 0, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 39, 1, 0, 0, 0, 303, 301, 1, 0, 0, 0, 304, 305, 7, 2, 0, 0, 305, 41, 1, 0, 0, 0, 306, 311, 3, 44, 22, 0, 307, 308, 5, 36, 0, 0, 308, 310, 3, 44, 22, 0, 309, 307, 1, 0, 0, 0, 310, 313, 1, 0, 0, 0, 311, 309, 1, 0, 0, 0, 311, 312, 1, 0, 0, 0, 312, 43, 1, 0, 0, 0, 313, 311, 1, 0, 0, 0, 314, 315, 7, 3, 0, 0, 315, 45, 1, 0, 0, 0, 316, 359, 5, 45, 0, 0, 317, 318, 3, 78, 39, 0, 318, 319, 5, 67, 0, 0, 319, 359, 1, 0, 0, 0, 320, 359, 3, 76, 38, 0, 321, 359, 3, 78, 39, 0, 322, 359, 3, 72, 36, 0, 323, 359, 5, 48, 0, 0, 324, 359, 3, 80, 40, 0, 325, 326, 5, 65, 0, 0, 326, 331, 3, 74, 37, 0, 327, 328, 5, 34, 0, 0, 328, 330, 3, 74, 37, 0, 329, 327, 1, 0, 0, 0, 330, 333, 1, 0, 0, 0, 331, 329, 1, 0, 0, 0, 331, 332, 1, 0, 0, 0, 332, 334, 1, 0, 0, 0, 333, 331, 1, 0, 0, 0, 334, 335, 5, 66, 0, 0, 335, 359, 1, 0, 0, 0, 336, 337, 5, 65, 0, 0, 337, 342, 3, 72, 36, 0, 338, 339, 5, 34, 0, 0, 339, 341, 3, 72, 36, 0, 340, 338, 1, 0, 0, 0, 341, 344, 1, 0, 0, 0, 342, 340, 1, 0, 0, 0, 342, 343, 1, 0, 0, 0, 343, 345, 1, 0, 0, 0, 344, 342, 1, 0, 0, 0, 345, 346, 5, 66, 0, 0, 346, 359, 1, 0, 0, 0, 347, 348, 5, 65, 0, 0, 348, 353, 3, 80, 40, 0, 349, 350, 5, 34, 0, 0, 350, 352, 3, 80, 40, 0, 351, 349, 1, 0, 0, 0, 352, 355, 1, 0, 0, 0, 353, 351, 1, 0, 0, 0, 353, 354, 1, 0, 0, 0, 354, 356, 1, 0, 0, 0, 355, 353, 1, 0, 0, 0, 356, 357, 5, 66, 0, 0, 357, 359, 1, 0, 0, 0, 358, 316, 1, 0, 0, 0, 358, 317, 1, 0, 0, 0, 358, 320, 1, 0, 0, 0, 358, 321, 1, 0, 0, 0, 358, 322, 1, 0, 0, 0, 358, 323, 1, 0, 0, 0, 358, 324, 1, 0, 0, 0, 358, 325, 1, 0, 0, 0, 358, 336, 1, 0, 0, 0, 358, 347, 1, 0, 0, 0, 359, 47, 1, 0, 0, 0, 360, 361, 5, 10, 0, 0, 361, 362, 5, 28, 0, 0, 362, 49, 1, 0, 0, 0, 363, 364, 5, 16, 0, 0, 364, 369, 3, 52, 26, 0, 365, 366, 5, 34, 0, 0, 366, 368, 3, 52, 26, 0, 367, 365, 1, 0, 0, 0, 368, 371, 1, 0, 0, 0, 369, 367, 1, 0, 0, 0, 369, 370, 1, 0, 0, 0, 370, 51, 1, 0, 0, 0, 371, 369, 1, 0, 0, 0, 372, 374, 3, 10, 5, 0, 373, 375, 7, 4, 0, 0, 374, 373, 1, 0, 0, 0, 374, 375, 1, 0, 0, 0, 375, 378, 1, 0, 0, 0, 376, 377, 5, 46, 0, 0, 377, 379, 7, 5, 0, 0, 378, 376, 1, 0, 0, 0, 378, 379, 1, 0, 0, 0, 379, 53, 1, 0, 0, 0, 380, 381, 5, 9, 0, 0, 381, 386, 3, 40, 20, 0, 382, 383, 5, 34, 0, 0, 383, 385, 3, 40, 20, 0, 384, 382, 1, 0, 0, 0, 385, 388, 1, 0, 0, 0, 386, 384, 1, 0, 0, 0, 386, 387, 1, 0, 0, 0, 387, 399, 1, 0, 0, 0, 388, 386, 1, 0, 0, 0, 389, 390, 5, 12, 0, 0, 390, 395, 3, 40, 20, 0, 391, 392, 5, 34, 0, 0, 392, 394, 3, 40, 20, 0, 393, 391, 1, 0, 0, 0, 394, 397, 1, 0, 0, 0, 395, 393, 1, 0, 0, 0, 395, 396, 1, 0, 0, 0, 396, 399, 1, 0, 0, 0, 397, 395, 1, 0, 0, 0, 398, 380, 1, 0, 0, 0, 398, 389, 1, 0, 0, 0, 399, 55, 1, 0, 0, 0, 400, 401, 5, 2, 0, 0, 401, 406, 3, 40, 20, 0, 402, 403, 5, 34, 0, 0, 403, 405, 3, 40, 20, 0, 404, 402, 1, 0, 0, 0, 405, 408, 1, 0, 0, 0, 406, 404, 1, 0, 0, 0, 406, 407, 1, 0, 0, 0, 407, 57, 1, 0, 0, 0, 408, 406, 1, 0, 0, 0, 409, 410, 5, 13, 0, 0, 410, 415, 3, 60, 30, 0, 411, 412, 5, 34, 0, 0, 412, 414, 3, 60, 30, 0, 413, 411, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 415, 416, 1, 0, 0, 0, 416, 59, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 418, 419, 3, 40, 20, 0, 419, 420, 5, 72, 0, 0, 420, 421, 3, 40, 20, 0, 421, 61, 1, 0, 0, 0, 422, 423, 5, 1, 0, 0, 423, 424, 3, 18, 9, 0, 424, 426, 3, 80, 40, 0, 425, 427, 3, 68, 34, 0, 426, 425, 1, 0, 0, 0, 426, 427, 1, 0, 0, 0, 427, 63, 1, 0, 0, 0, 428, 429, 5, 7, 0, 0, 429, 430, 3, 18, 9, 0, 430, 431, 3, 80, 40, 0, 431, 65, 1, 0, 0, 0, 432, 433, 5, 11, 0, 0, 433, 434, 3, 40, 20, 0, 434, 67, 1, 0, 0, 0, 435, 440, 3, 70, 35, 0, 436, 437, 5, 34, 0, 0, 437, 439, 3, 70, 35, 0, 438, 436, 1, 0, 0, 0, 439, 442, 1, 0, 0, 0, 440, 438, 1, 0, 0, 0, 440, 441, 1, 0, 0, 0, 441, 69, 1, 0, 0, 0, 442, 440, 1, 0, 0, 0, 443, 444, 3, 44, 22, 0, 444, 445, 5, 33, 0, 0, 445, 446, 3, 46, 23, 0, 446, 71, 1, 0, 0, 0, 447, 448, 7, 6, 0, 0, 448, 73, 1, 0, 0, 0, 449, 452, 3, 76, 38, 0, 450, 452, 3, 78, 39, 0, 451, 449, 1, 0, 0, 0, 451, 450, 1, 0, 0, 0, 452, 75, 1, 0, 0, 0, 453, 455, 7, 0, 0, 0, 454, 453, 1, 0, 0, 0, 454, 455, 1, 0, 0, 0, 455, 456, 1, 0, 0, 0, 456, 457, 5, 29, 0, 0, 457, 77, 1, 0, 0, 0, 458, 460, 7, 0, 0, 0, 459, 458, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 461, 1, 0, 0, 0, 461, 462, 5, 28, 0, 0, 462, 79, 1, 0, 0, 0, 463, 464, 5, 27, 0, 0, 464, 81, 1, 0, 0, 0, 465, 466, 7, 7, 0, 0, 466, 83, 1, 0, 0, 0, 467, 468, 5, 5, 0, 0, 468, 469, 3, 86, 43, 0, 469, 85, 1, 0, 0, 0, 470, 471, 5, 65, 0, 0, 471, 472, 3, 2, 1, 0, 472, 473, 5, 66, 0, 0, 473, 87, 1, 0, 0, 0, 474, 475, 5, 15, 0, 0, 475, 479, 5, 52, 0, 0, 476, 477, 5, 15, 0, 0, 477, 479, 5, 53, 0, 0, 478, 474, 1, 0, 0, 0, 478, 476, 1, 0, 0, 0, 479, 89, 1, 0, 0, 0, 480, 481, 5, 3, 0, 0, 481, 484, 3, 40, 20, 0, 482, 483, 5, 74, 0, 0, 483, 485, 3, 40, 20, 0, 484, 482, 1, 0, 0, 0, 484, 485, 1, 0, 0, 0, 485, 495, 1, 0, 0, 0, 486, 487, 5, 75, 0, 0, 487, 492, 3, 92, 46, 0, 488, 489, 5, 34, 0, 0, 489, 491, 3, 92, 46, 0, 490, 488, 1, 0, 0, 0, 491, 494, 1, 0, 0, 0, 492, 490, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 496, 1, 0, 0, 0, 494, 492, 1, 0, 0, 0, 495, 486, 1, 0, 0, 0, 495, 496, 1, 0, 0, 0, 496, 91, 1, 0, 0, 0, 497, 498, 3, 40, 20, 0, 498, 499, 5, 33, 0, 0, 499, 501, 1, 0, 0, 0, 500, 497, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 3, 40, 20, 0, 503, 93, 1, 0, 0, 0, 51, 105, 112, 127, 139, 148, 156, 160, 168, 170, 175, 182, 187, 194, 200, 208, 210, 220, 230, 233, 245, 253, 261, 265, 274, 284, 288, 294, 301, 311, 331, 342, 353, 358, 369, 374, 378, 386, 395, 398, 406, 415, 426, 440, 451, 454, 459, 478, 484, 492, 495, 500] \ No newline at end of file +[4, 1, 98, 519, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 108, 8, 1, 10, 1, 12, 1, 111, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 117, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 132, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 144, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 151, 8, 5, 10, 5, 12, 5, 154, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 161, 8, 5, 1, 5, 1, 5, 3, 5, 165, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 173, 8, 5, 10, 5, 12, 5, 176, 9, 5, 1, 6, 1, 6, 3, 6, 180, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 187, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 192, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 199, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 205, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 213, 8, 8, 10, 8, 12, 8, 216, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 225, 8, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 233, 8, 10, 10, 10, 12, 10, 236, 9, 10, 3, 10, 238, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 5, 12, 248, 8, 12, 10, 12, 12, 12, 251, 9, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 258, 8, 13, 1, 14, 1, 14, 1, 14, 1, 14, 5, 14, 264, 8, 14, 10, 14, 12, 14, 267, 9, 14, 1, 14, 3, 14, 270, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 5, 15, 277, 8, 15, 10, 15, 12, 15, 280, 9, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 3, 17, 289, 8, 17, 1, 17, 1, 17, 3, 17, 293, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 299, 8, 18, 1, 19, 1, 19, 1, 19, 5, 19, 304, 8, 19, 10, 19, 12, 19, 307, 9, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 5, 21, 314, 8, 21, 10, 21, 12, 21, 317, 9, 21, 1, 22, 1, 22, 1, 22, 5, 22, 322, 8, 22, 10, 22, 12, 22, 325, 9, 22, 1, 23, 1, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 344, 8, 25, 10, 25, 12, 25, 347, 9, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 355, 8, 25, 10, 25, 12, 25, 358, 9, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 5, 25, 366, 8, 25, 10, 25, 12, 25, 369, 9, 25, 1, 25, 1, 25, 3, 25, 373, 8, 25, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 382, 8, 27, 10, 27, 12, 27, 385, 9, 27, 1, 28, 1, 28, 3, 28, 389, 8, 28, 1, 28, 1, 28, 3, 28, 393, 8, 28, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 399, 8, 29, 10, 29, 12, 29, 402, 9, 29, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 408, 8, 29, 10, 29, 12, 29, 411, 9, 29, 3, 29, 413, 8, 29, 1, 30, 1, 30, 1, 30, 1, 30, 5, 30, 419, 8, 30, 10, 30, 12, 30, 422, 9, 30, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 428, 8, 31, 10, 31, 12, 31, 431, 9, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 33, 3, 33, 441, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 5, 36, 453, 8, 36, 10, 36, 12, 36, 456, 9, 36, 1, 37, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 39, 1, 39, 3, 39, 466, 8, 39, 1, 40, 3, 40, 469, 8, 40, 1, 40, 1, 40, 1, 41, 3, 41, 474, 8, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 3, 46, 493, 8, 46, 1, 47, 1, 47, 1, 47, 1, 47, 3, 47, 499, 8, 47, 1, 47, 1, 47, 1, 47, 1, 47, 5, 47, 505, 8, 47, 10, 47, 12, 47, 508, 9, 47, 3, 47, 510, 8, 47, 1, 48, 1, 48, 1, 48, 3, 48, 515, 8, 48, 1, 48, 1, 48, 1, 48, 0, 3, 2, 10, 16, 49, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 0, 9, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 66, 66, 71, 71, 1, 0, 65, 66, 2, 0, 66, 66, 75, 75, 2, 0, 32, 32, 35, 35, 1, 0, 38, 39, 2, 0, 37, 37, 51, 51, 1, 0, 52, 57, 548, 0, 98, 1, 0, 0, 0, 2, 101, 1, 0, 0, 0, 4, 116, 1, 0, 0, 0, 6, 131, 1, 0, 0, 0, 8, 133, 1, 0, 0, 0, 10, 164, 1, 0, 0, 0, 12, 191, 1, 0, 0, 0, 14, 198, 1, 0, 0, 0, 16, 204, 1, 0, 0, 0, 18, 224, 1, 0, 0, 0, 20, 226, 1, 0, 0, 0, 22, 241, 1, 0, 0, 0, 24, 244, 1, 0, 0, 0, 26, 257, 1, 0, 0, 0, 28, 259, 1, 0, 0, 0, 30, 271, 1, 0, 0, 0, 32, 283, 1, 0, 0, 0, 34, 286, 1, 0, 0, 0, 36, 294, 1, 0, 0, 0, 38, 300, 1, 0, 0, 0, 40, 308, 1, 0, 0, 0, 42, 310, 1, 0, 0, 0, 44, 318, 1, 0, 0, 0, 46, 326, 1, 0, 0, 0, 48, 328, 1, 0, 0, 0, 50, 372, 1, 0, 0, 0, 52, 374, 1, 0, 0, 0, 54, 377, 1, 0, 0, 0, 56, 386, 1, 0, 0, 0, 58, 412, 1, 0, 0, 0, 60, 414, 1, 0, 0, 0, 62, 423, 1, 0, 0, 0, 64, 432, 1, 0, 0, 0, 66, 436, 1, 0, 0, 0, 68, 442, 1, 0, 0, 0, 70, 446, 1, 0, 0, 0, 72, 449, 1, 0, 0, 0, 74, 457, 1, 0, 0, 0, 76, 461, 1, 0, 0, 0, 78, 465, 1, 0, 0, 0, 80, 468, 1, 0, 0, 0, 82, 473, 1, 0, 0, 0, 84, 477, 1, 0, 0, 0, 86, 479, 1, 0, 0, 0, 88, 481, 1, 0, 0, 0, 90, 484, 1, 0, 0, 0, 92, 492, 1, 0, 0, 0, 94, 494, 1, 0, 0, 0, 96, 514, 1, 0, 0, 0, 98, 99, 3, 2, 1, 0, 99, 100, 5, 0, 0, 1, 100, 1, 1, 0, 0, 0, 101, 102, 6, 1, -1, 0, 102, 103, 3, 4, 2, 0, 103, 109, 1, 0, 0, 0, 104, 105, 10, 1, 0, 0, 105, 106, 5, 26, 0, 0, 106, 108, 3, 6, 3, 0, 107, 104, 1, 0, 0, 0, 108, 111, 1, 0, 0, 0, 109, 107, 1, 0, 0, 0, 109, 110, 1, 0, 0, 0, 110, 3, 1, 0, 0, 0, 111, 109, 1, 0, 0, 0, 112, 117, 3, 88, 44, 0, 113, 117, 3, 28, 14, 0, 114, 117, 3, 22, 11, 0, 115, 117, 3, 92, 46, 0, 116, 112, 1, 0, 0, 0, 116, 113, 1, 0, 0, 0, 116, 114, 1, 0, 0, 0, 116, 115, 1, 0, 0, 0, 117, 5, 1, 0, 0, 0, 118, 132, 3, 32, 16, 0, 119, 132, 3, 36, 18, 0, 120, 132, 3, 52, 26, 0, 121, 132, 3, 58, 29, 0, 122, 132, 3, 54, 27, 0, 123, 132, 3, 34, 17, 0, 124, 132, 3, 8, 4, 0, 125, 132, 3, 60, 30, 0, 126, 132, 3, 62, 31, 0, 127, 132, 3, 66, 33, 0, 128, 132, 3, 68, 34, 0, 129, 132, 3, 94, 47, 0, 130, 132, 3, 70, 35, 0, 131, 118, 1, 0, 0, 0, 131, 119, 1, 0, 0, 0, 131, 120, 1, 0, 0, 0, 131, 121, 1, 0, 0, 0, 131, 122, 1, 0, 0, 0, 131, 123, 1, 0, 0, 0, 131, 124, 1, 0, 0, 0, 131, 125, 1, 0, 0, 0, 131, 126, 1, 0, 0, 0, 131, 127, 1, 0, 0, 0, 131, 128, 1, 0, 0, 0, 131, 129, 1, 0, 0, 0, 131, 130, 1, 0, 0, 0, 132, 7, 1, 0, 0, 0, 133, 134, 5, 18, 0, 0, 134, 135, 3, 10, 5, 0, 135, 9, 1, 0, 0, 0, 136, 137, 6, 5, -1, 0, 137, 138, 5, 44, 0, 0, 138, 165, 3, 10, 5, 7, 139, 165, 3, 14, 7, 0, 140, 165, 3, 12, 6, 0, 141, 143, 3, 14, 7, 0, 142, 144, 5, 44, 0, 0, 143, 142, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 145, 1, 0, 0, 0, 145, 146, 5, 41, 0, 0, 146, 147, 5, 40, 0, 0, 147, 152, 3, 14, 7, 0, 148, 149, 5, 34, 0, 0, 149, 151, 3, 14, 7, 0, 150, 148, 1, 0, 0, 0, 151, 154, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 152, 153, 1, 0, 0, 0, 153, 155, 1, 0, 0, 0, 154, 152, 1, 0, 0, 0, 155, 156, 5, 50, 0, 0, 156, 165, 1, 0, 0, 0, 157, 158, 3, 14, 7, 0, 158, 160, 5, 42, 0, 0, 159, 161, 5, 44, 0, 0, 160, 159, 1, 0, 0, 0, 160, 161, 1, 0, 0, 0, 161, 162, 1, 0, 0, 0, 162, 163, 5, 45, 0, 0, 163, 165, 1, 0, 0, 0, 164, 136, 1, 0, 0, 0, 164, 139, 1, 0, 0, 0, 164, 140, 1, 0, 0, 0, 164, 141, 1, 0, 0, 0, 164, 157, 1, 0, 0, 0, 165, 174, 1, 0, 0, 0, 166, 167, 10, 4, 0, 0, 167, 168, 5, 31, 0, 0, 168, 173, 3, 10, 5, 5, 169, 170, 10, 3, 0, 0, 170, 171, 5, 47, 0, 0, 171, 173, 3, 10, 5, 4, 172, 166, 1, 0, 0, 0, 172, 169, 1, 0, 0, 0, 173, 176, 1, 0, 0, 0, 174, 172, 1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 11, 1, 0, 0, 0, 176, 174, 1, 0, 0, 0, 177, 179, 3, 14, 7, 0, 178, 180, 5, 44, 0, 0, 179, 178, 1, 0, 0, 0, 179, 180, 1, 0, 0, 0, 180, 181, 1, 0, 0, 0, 181, 182, 5, 43, 0, 0, 182, 183, 3, 84, 42, 0, 183, 192, 1, 0, 0, 0, 184, 186, 3, 14, 7, 0, 185, 187, 5, 44, 0, 0, 186, 185, 1, 0, 0, 0, 186, 187, 1, 0, 0, 0, 187, 188, 1, 0, 0, 0, 188, 189, 5, 49, 0, 0, 189, 190, 3, 84, 42, 0, 190, 192, 1, 0, 0, 0, 191, 177, 1, 0, 0, 0, 191, 184, 1, 0, 0, 0, 192, 13, 1, 0, 0, 0, 193, 199, 3, 16, 8, 0, 194, 195, 3, 16, 8, 0, 195, 196, 3, 86, 43, 0, 196, 197, 3, 16, 8, 0, 197, 199, 1, 0, 0, 0, 198, 193, 1, 0, 0, 0, 198, 194, 1, 0, 0, 0, 199, 15, 1, 0, 0, 0, 200, 201, 6, 8, -1, 0, 201, 205, 3, 18, 9, 0, 202, 203, 7, 0, 0, 0, 203, 205, 3, 16, 8, 3, 204, 200, 1, 0, 0, 0, 204, 202, 1, 0, 0, 0, 205, 214, 1, 0, 0, 0, 206, 207, 10, 2, 0, 0, 207, 208, 7, 1, 0, 0, 208, 213, 3, 16, 8, 3, 209, 210, 10, 1, 0, 0, 210, 211, 7, 0, 0, 0, 211, 213, 3, 16, 8, 2, 212, 206, 1, 0, 0, 0, 212, 209, 1, 0, 0, 0, 213, 216, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 214, 215, 1, 0, 0, 0, 215, 17, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 217, 225, 3, 50, 25, 0, 218, 225, 3, 42, 21, 0, 219, 225, 3, 20, 10, 0, 220, 221, 5, 40, 0, 0, 221, 222, 3, 10, 5, 0, 222, 223, 5, 50, 0, 0, 223, 225, 1, 0, 0, 0, 224, 217, 1, 0, 0, 0, 224, 218, 1, 0, 0, 0, 224, 219, 1, 0, 0, 0, 224, 220, 1, 0, 0, 0, 225, 19, 1, 0, 0, 0, 226, 227, 3, 46, 23, 0, 227, 237, 5, 40, 0, 0, 228, 238, 5, 60, 0, 0, 229, 234, 3, 10, 5, 0, 230, 231, 5, 34, 0, 0, 231, 233, 3, 10, 5, 0, 232, 230, 1, 0, 0, 0, 233, 236, 1, 0, 0, 0, 234, 232, 1, 0, 0, 0, 234, 235, 1, 0, 0, 0, 235, 238, 1, 0, 0, 0, 236, 234, 1, 0, 0, 0, 237, 228, 1, 0, 0, 0, 237, 229, 1, 0, 0, 0, 237, 238, 1, 0, 0, 0, 238, 239, 1, 0, 0, 0, 239, 240, 5, 50, 0, 0, 240, 21, 1, 0, 0, 0, 241, 242, 5, 14, 0, 0, 242, 243, 3, 24, 12, 0, 243, 23, 1, 0, 0, 0, 244, 249, 3, 26, 13, 0, 245, 246, 5, 34, 0, 0, 246, 248, 3, 26, 13, 0, 247, 245, 1, 0, 0, 0, 248, 251, 1, 0, 0, 0, 249, 247, 1, 0, 0, 0, 249, 250, 1, 0, 0, 0, 250, 25, 1, 0, 0, 0, 251, 249, 1, 0, 0, 0, 252, 258, 3, 10, 5, 0, 253, 254, 3, 42, 21, 0, 254, 255, 5, 33, 0, 0, 255, 256, 3, 10, 5, 0, 256, 258, 1, 0, 0, 0, 257, 252, 1, 0, 0, 0, 257, 253, 1, 0, 0, 0, 258, 27, 1, 0, 0, 0, 259, 260, 5, 6, 0, 0, 260, 265, 3, 40, 20, 0, 261, 262, 5, 34, 0, 0, 262, 264, 3, 40, 20, 0, 263, 261, 1, 0, 0, 0, 264, 267, 1, 0, 0, 0, 265, 263, 1, 0, 0, 0, 265, 266, 1, 0, 0, 0, 266, 269, 1, 0, 0, 0, 267, 265, 1, 0, 0, 0, 268, 270, 3, 30, 15, 0, 269, 268, 1, 0, 0, 0, 269, 270, 1, 0, 0, 0, 270, 29, 1, 0, 0, 0, 271, 272, 5, 63, 0, 0, 272, 273, 5, 70, 0, 0, 273, 278, 3, 40, 20, 0, 274, 275, 5, 34, 0, 0, 275, 277, 3, 40, 20, 0, 276, 274, 1, 0, 0, 0, 277, 280, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 281, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 281, 282, 5, 64, 0, 0, 282, 31, 1, 0, 0, 0, 283, 284, 5, 4, 0, 0, 284, 285, 3, 24, 12, 0, 285, 33, 1, 0, 0, 0, 286, 288, 5, 17, 0, 0, 287, 289, 3, 24, 12, 0, 288, 287, 1, 0, 0, 0, 288, 289, 1, 0, 0, 0, 289, 292, 1, 0, 0, 0, 290, 291, 5, 30, 0, 0, 291, 293, 3, 38, 19, 0, 292, 290, 1, 0, 0, 0, 292, 293, 1, 0, 0, 0, 293, 35, 1, 0, 0, 0, 294, 295, 5, 8, 0, 0, 295, 298, 3, 24, 12, 0, 296, 297, 5, 30, 0, 0, 297, 299, 3, 38, 19, 0, 298, 296, 1, 0, 0, 0, 298, 299, 1, 0, 0, 0, 299, 37, 1, 0, 0, 0, 300, 305, 3, 42, 21, 0, 301, 302, 5, 34, 0, 0, 302, 304, 3, 42, 21, 0, 303, 301, 1, 0, 0, 0, 304, 307, 1, 0, 0, 0, 305, 303, 1, 0, 0, 0, 305, 306, 1, 0, 0, 0, 306, 39, 1, 0, 0, 0, 307, 305, 1, 0, 0, 0, 308, 309, 7, 2, 0, 0, 309, 41, 1, 0, 0, 0, 310, 315, 3, 46, 23, 0, 311, 312, 5, 36, 0, 0, 312, 314, 3, 46, 23, 0, 313, 311, 1, 0, 0, 0, 314, 317, 1, 0, 0, 0, 315, 313, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 43, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 318, 323, 3, 48, 24, 0, 319, 320, 5, 36, 0, 0, 320, 322, 3, 48, 24, 0, 321, 319, 1, 0, 0, 0, 322, 325, 1, 0, 0, 0, 323, 321, 1, 0, 0, 0, 323, 324, 1, 0, 0, 0, 324, 45, 1, 0, 0, 0, 325, 323, 1, 0, 0, 0, 326, 327, 7, 3, 0, 0, 327, 47, 1, 0, 0, 0, 328, 329, 7, 4, 0, 0, 329, 49, 1, 0, 0, 0, 330, 373, 5, 45, 0, 0, 331, 332, 3, 82, 41, 0, 332, 333, 5, 65, 0, 0, 333, 373, 1, 0, 0, 0, 334, 373, 3, 80, 40, 0, 335, 373, 3, 82, 41, 0, 336, 373, 3, 76, 38, 0, 337, 373, 5, 48, 0, 0, 338, 373, 3, 84, 42, 0, 339, 340, 5, 63, 0, 0, 340, 345, 3, 78, 39, 0, 341, 342, 5, 34, 0, 0, 342, 344, 3, 78, 39, 0, 343, 341, 1, 0, 0, 0, 344, 347, 1, 0, 0, 0, 345, 343, 1, 0, 0, 0, 345, 346, 1, 0, 0, 0, 346, 348, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 348, 349, 5, 64, 0, 0, 349, 373, 1, 0, 0, 0, 350, 351, 5, 63, 0, 0, 351, 356, 3, 76, 38, 0, 352, 353, 5, 34, 0, 0, 353, 355, 3, 76, 38, 0, 354, 352, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 359, 1, 0, 0, 0, 358, 356, 1, 0, 0, 0, 359, 360, 5, 64, 0, 0, 360, 373, 1, 0, 0, 0, 361, 362, 5, 63, 0, 0, 362, 367, 3, 84, 42, 0, 363, 364, 5, 34, 0, 0, 364, 366, 3, 84, 42, 0, 365, 363, 1, 0, 0, 0, 366, 369, 1, 0, 0, 0, 367, 365, 1, 0, 0, 0, 367, 368, 1, 0, 0, 0, 368, 370, 1, 0, 0, 0, 369, 367, 1, 0, 0, 0, 370, 371, 5, 64, 0, 0, 371, 373, 1, 0, 0, 0, 372, 330, 1, 0, 0, 0, 372, 331, 1, 0, 0, 0, 372, 334, 1, 0, 0, 0, 372, 335, 1, 0, 0, 0, 372, 336, 1, 0, 0, 0, 372, 337, 1, 0, 0, 0, 372, 338, 1, 0, 0, 0, 372, 339, 1, 0, 0, 0, 372, 350, 1, 0, 0, 0, 372, 361, 1, 0, 0, 0, 373, 51, 1, 0, 0, 0, 374, 375, 5, 10, 0, 0, 375, 376, 5, 28, 0, 0, 376, 53, 1, 0, 0, 0, 377, 378, 5, 16, 0, 0, 378, 383, 3, 56, 28, 0, 379, 380, 5, 34, 0, 0, 380, 382, 3, 56, 28, 0, 381, 379, 1, 0, 0, 0, 382, 385, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 55, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 386, 388, 3, 10, 5, 0, 387, 389, 7, 5, 0, 0, 388, 387, 1, 0, 0, 0, 388, 389, 1, 0, 0, 0, 389, 392, 1, 0, 0, 0, 390, 391, 5, 46, 0, 0, 391, 393, 7, 6, 0, 0, 392, 390, 1, 0, 0, 0, 392, 393, 1, 0, 0, 0, 393, 57, 1, 0, 0, 0, 394, 395, 5, 9, 0, 0, 395, 400, 3, 44, 22, 0, 396, 397, 5, 34, 0, 0, 397, 399, 3, 44, 22, 0, 398, 396, 1, 0, 0, 0, 399, 402, 1, 0, 0, 0, 400, 398, 1, 0, 0, 0, 400, 401, 1, 0, 0, 0, 401, 413, 1, 0, 0, 0, 402, 400, 1, 0, 0, 0, 403, 404, 5, 12, 0, 0, 404, 409, 3, 44, 22, 0, 405, 406, 5, 34, 0, 0, 406, 408, 3, 44, 22, 0, 407, 405, 1, 0, 0, 0, 408, 411, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 409, 410, 1, 0, 0, 0, 410, 413, 1, 0, 0, 0, 411, 409, 1, 0, 0, 0, 412, 394, 1, 0, 0, 0, 412, 403, 1, 0, 0, 0, 413, 59, 1, 0, 0, 0, 414, 415, 5, 2, 0, 0, 415, 420, 3, 44, 22, 0, 416, 417, 5, 34, 0, 0, 417, 419, 3, 44, 22, 0, 418, 416, 1, 0, 0, 0, 419, 422, 1, 0, 0, 0, 420, 418, 1, 0, 0, 0, 420, 421, 1, 0, 0, 0, 421, 61, 1, 0, 0, 0, 422, 420, 1, 0, 0, 0, 423, 424, 5, 13, 0, 0, 424, 429, 3, 64, 32, 0, 425, 426, 5, 34, 0, 0, 426, 428, 3, 64, 32, 0, 427, 425, 1, 0, 0, 0, 428, 431, 1, 0, 0, 0, 429, 427, 1, 0, 0, 0, 429, 430, 1, 0, 0, 0, 430, 63, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 433, 3, 44, 22, 0, 433, 434, 5, 79, 0, 0, 434, 435, 3, 44, 22, 0, 435, 65, 1, 0, 0, 0, 436, 437, 5, 1, 0, 0, 437, 438, 3, 18, 9, 0, 438, 440, 3, 84, 42, 0, 439, 441, 3, 72, 36, 0, 440, 439, 1, 0, 0, 0, 440, 441, 1, 0, 0, 0, 441, 67, 1, 0, 0, 0, 442, 443, 5, 7, 0, 0, 443, 444, 3, 18, 9, 0, 444, 445, 3, 84, 42, 0, 445, 69, 1, 0, 0, 0, 446, 447, 5, 11, 0, 0, 447, 448, 3, 42, 21, 0, 448, 71, 1, 0, 0, 0, 449, 454, 3, 74, 37, 0, 450, 451, 5, 34, 0, 0, 451, 453, 3, 74, 37, 0, 452, 450, 1, 0, 0, 0, 453, 456, 1, 0, 0, 0, 454, 452, 1, 0, 0, 0, 454, 455, 1, 0, 0, 0, 455, 73, 1, 0, 0, 0, 456, 454, 1, 0, 0, 0, 457, 458, 3, 46, 23, 0, 458, 459, 5, 33, 0, 0, 459, 460, 3, 50, 25, 0, 460, 75, 1, 0, 0, 0, 461, 462, 7, 7, 0, 0, 462, 77, 1, 0, 0, 0, 463, 466, 3, 80, 40, 0, 464, 466, 3, 82, 41, 0, 465, 463, 1, 0, 0, 0, 465, 464, 1, 0, 0, 0, 466, 79, 1, 0, 0, 0, 467, 469, 7, 0, 0, 0, 468, 467, 1, 0, 0, 0, 468, 469, 1, 0, 0, 0, 469, 470, 1, 0, 0, 0, 470, 471, 5, 29, 0, 0, 471, 81, 1, 0, 0, 0, 472, 474, 7, 0, 0, 0, 473, 472, 1, 0, 0, 0, 473, 474, 1, 0, 0, 0, 474, 475, 1, 0, 0, 0, 475, 476, 5, 28, 0, 0, 476, 83, 1, 0, 0, 0, 477, 478, 5, 27, 0, 0, 478, 85, 1, 0, 0, 0, 479, 480, 7, 8, 0, 0, 480, 87, 1, 0, 0, 0, 481, 482, 5, 5, 0, 0, 482, 483, 3, 90, 45, 0, 483, 89, 1, 0, 0, 0, 484, 485, 5, 63, 0, 0, 485, 486, 3, 2, 1, 0, 486, 487, 5, 64, 0, 0, 487, 91, 1, 0, 0, 0, 488, 489, 5, 15, 0, 0, 489, 493, 5, 94, 0, 0, 490, 491, 5, 15, 0, 0, 491, 493, 5, 95, 0, 0, 492, 488, 1, 0, 0, 0, 492, 490, 1, 0, 0, 0, 493, 93, 1, 0, 0, 0, 494, 495, 5, 3, 0, 0, 495, 498, 3, 40, 20, 0, 496, 497, 5, 83, 0, 0, 497, 499, 3, 44, 22, 0, 498, 496, 1, 0, 0, 0, 498, 499, 1, 0, 0, 0, 499, 509, 1, 0, 0, 0, 500, 501, 5, 84, 0, 0, 501, 506, 3, 96, 48, 0, 502, 503, 5, 34, 0, 0, 503, 505, 3, 96, 48, 0, 504, 502, 1, 0, 0, 0, 505, 508, 1, 0, 0, 0, 506, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 510, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 509, 500, 1, 0, 0, 0, 509, 510, 1, 0, 0, 0, 510, 95, 1, 0, 0, 0, 511, 512, 3, 44, 22, 0, 512, 513, 5, 33, 0, 0, 513, 515, 1, 0, 0, 0, 514, 511, 1, 0, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 1, 0, 0, 0, 516, 517, 3, 44, 22, 0, 517, 97, 1, 0, 0, 0, 52, 109, 116, 131, 143, 152, 160, 164, 172, 174, 179, 186, 191, 198, 204, 212, 214, 224, 234, 237, 249, 257, 265, 269, 278, 288, 292, 298, 305, 315, 323, 345, 356, 367, 372, 383, 388, 392, 400, 409, 412, 420, 429, 440, 454, 465, 468, 473, 492, 498, 506, 509, 514] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index d136c346927e6..54ec466de9623 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -24,12 +24,18 @@ public class EsqlBaseParser extends Parser { PIPE=26, STRING=27, INTEGER_LITERAL=28, DECIMAL_LITERAL=29, BY=30, AND=31, ASC=32, ASSIGN=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, LAST=39, LP=40, IN=41, IS=42, LIKE=43, NOT=44, NULL=45, NULLS=46, OR=47, PARAM=48, - RLIKE=49, RP=50, TRUE=51, INFO=52, FUNCTIONS=53, EQ=54, NEQ=55, LT=56, - LTE=57, GT=58, GTE=59, PLUS=60, MINUS=61, ASTERISK=62, SLASH=63, PERCENT=64, - OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, - EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, AS=72, METADATA=73, - ON=74, WITH=75, SRC_UNQUOTED_IDENTIFIER=76, SRC_QUOTED_IDENTIFIER=77, - SRC_LINE_COMMENT=78, SRC_MULTILINE_COMMENT=79, SRC_WS=80, EXPLAIN_PIPE=81; + RLIKE=49, RP=50, TRUE=51, EQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, + PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, OPENING_BRACKET=63, + CLOSING_BRACKET=64, UNQUOTED_IDENTIFIER=65, QUOTED_IDENTIFIER=66, EXPR_LINE_COMMENT=67, + EXPR_MULTILINE_COMMENT=68, EXPR_WS=69, METADATA=70, FROM_UNQUOTED_IDENTIFIER=71, + FROM_LINE_COMMENT=72, FROM_MULTILINE_COMMENT=73, FROM_WS=74, PROJECT_UNQUOTED_IDENTIFIER=75, + PROJECT_LINE_COMMENT=76, PROJECT_MULTILINE_COMMENT=77, PROJECT_WS=78, + AS=79, RENAME_LINE_COMMENT=80, RENAME_MULTILINE_COMMENT=81, RENAME_WS=82, + ON=83, WITH=84, ENRICH_LINE_COMMENT=85, ENRICH_MULTILINE_COMMENT=86, ENRICH_WS=87, + ENRICH_FIELD_LINE_COMMENT=88, ENRICH_FIELD_MULTILINE_COMMENT=89, ENRICH_FIELD_WS=90, + MVEXPAND_LINE_COMMENT=91, MVEXPAND_MULTILINE_COMMENT=92, MVEXPAND_WS=93, + INFO=94, FUNCTIONS=95, SHOW_LINE_COMMENT=96, SHOW_MULTILINE_COMMENT=97, + SHOW_WS=98; public static final int RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, @@ -37,26 +43,28 @@ public class EsqlBaseParser extends Parser { RULE_functionExpression = 10, RULE_rowCommand = 11, RULE_fields = 12, RULE_field = 13, RULE_fromCommand = 14, RULE_metadata = 15, RULE_evalCommand = 16, RULE_statsCommand = 17, RULE_inlinestatsCommand = 18, RULE_grouping = 19, - RULE_sourceIdentifier = 20, RULE_qualifiedName = 21, RULE_identifier = 22, - RULE_constant = 23, RULE_limitCommand = 24, RULE_sortCommand = 25, RULE_orderExpression = 26, - RULE_keepCommand = 27, RULE_dropCommand = 28, RULE_renameCommand = 29, - RULE_renameClause = 30, RULE_dissectCommand = 31, RULE_grokCommand = 32, - RULE_mvExpandCommand = 33, RULE_commandOptions = 34, RULE_commandOption = 35, - RULE_booleanValue = 36, RULE_numericValue = 37, RULE_decimalValue = 38, - RULE_integerValue = 39, RULE_string = 40, RULE_comparisonOperator = 41, - RULE_explainCommand = 42, RULE_subqueryExpression = 43, RULE_showCommand = 44, - RULE_enrichCommand = 45, RULE_enrichWithClause = 46; + RULE_fromIdentifier = 20, RULE_qualifiedName = 21, RULE_qualifiedNamePattern = 22, + RULE_identifier = 23, RULE_identifierPattern = 24, RULE_constant = 25, + RULE_limitCommand = 26, RULE_sortCommand = 27, RULE_orderExpression = 28, + RULE_keepCommand = 29, RULE_dropCommand = 30, RULE_renameCommand = 31, + RULE_renameClause = 32, RULE_dissectCommand = 33, RULE_grokCommand = 34, + RULE_mvExpandCommand = 35, RULE_commandOptions = 36, RULE_commandOption = 37, + RULE_booleanValue = 38, RULE_numericValue = 39, RULE_decimalValue = 40, + RULE_integerValue = 41, RULE_string = 42, RULE_comparisonOperator = 43, + RULE_explainCommand = 44, RULE_subqueryExpression = 45, RULE_showCommand = 46, + RULE_enrichCommand = 47, RULE_enrichWithClause = 48; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", "booleanExpression", "regexBooleanExpression", "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", "rowCommand", "fields", "field", "fromCommand", "metadata", "evalCommand", "statsCommand", "inlinestatsCommand", - "grouping", "sourceIdentifier", "qualifiedName", "identifier", "constant", - "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", - "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", - "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", - "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", + "grouping", "fromIdentifier", "qualifiedName", "qualifiedNamePattern", + "identifier", "identifierPattern", "constant", "limitCommand", "sortCommand", + "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", + "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", + "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", + "string", "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", "enrichWithClause" }; } @@ -67,12 +75,14 @@ private static String[] makeLiteralNames() { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", "'grok'", "'inlinestats'", "'keep'", "'limit'", "'mv_expand'", "'project'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", null, - null, null, null, null, null, null, null, null, null, null, "'by'", "'and'", - "'asc'", null, null, "'desc'", "'.'", "'false'", "'first'", "'last'", - "'('", "'in'", "'is'", "'like'", "'not'", "'null'", "'nulls'", "'or'", - "'?'", "'rlike'", "')'", "'true'", "'info'", "'functions'", "'=='", "'!='", - "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, - "']'", null, null, null, null, null, "'as'", "'metadata'", "'on'", "'with'" + null, null, null, null, null, null, "'|'", null, null, null, "'by'", + "'and'", "'asc'", "'='", "','", "'desc'", "'.'", "'false'", "'first'", + "'last'", "'('", "'in'", "'is'", "'like'", "'not'", "'null'", "'nulls'", + "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", "'!='", "'<'", "'<='", + "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, "']'", null, + null, null, null, null, "'metadata'", null, null, null, null, null, null, + null, null, "'as'", null, null, null, "'on'", "'with'", null, null, null, + null, null, null, null, null, null, "'info'", "'functions'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); @@ -85,12 +95,17 @@ private static String[] makeSymbolicNames() { "PIPE", "STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", - "TRUE", "INFO", "FUNCTIONS", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", - "MINUS", "ASTERISK", "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", - "EXPR_WS", "AS", "METADATA", "ON", "WITH", "SRC_UNQUOTED_IDENTIFIER", - "SRC_QUOTED_IDENTIFIER", "SRC_LINE_COMMENT", "SRC_MULTILINE_COMMENT", - "SRC_WS", "EXPLAIN_PIPE" + "TRUE", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "SLASH", "PERCENT", "OPENING_BRACKET", "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", + "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", + "METADATA", "FROM_UNQUOTED_IDENTIFIER", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", + "FROM_WS", "PROJECT_UNQUOTED_IDENTIFIER", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", + "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", + "RENAME_WS", "ON", "WITH", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", + "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", + "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", + "MVEXPAND_WS", "INFO", "FUNCTIONS", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", + "SHOW_WS" }; } private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); @@ -177,9 +192,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(94); + setState(98); query(0); - setState(95); + setState(99); match(EOF); } } @@ -275,11 +290,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(98); + setState(102); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(105); + setState(109); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -290,16 +305,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(100); + setState(104); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(101); + setState(105); match(PIPE); - setState(102); + setState(106); processingCommand(); } } } - setState(107); + setState(111); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -354,34 +369,34 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(112); + setState(116); _errHandler.sync(this); switch (_input.LA(1)) { case EXPLAIN: enterOuterAlt(_localctx, 1); { - setState(108); + setState(112); explainCommand(); } break; case FROM: enterOuterAlt(_localctx, 2); { - setState(109); + setState(113); fromCommand(); } break; case ROW: enterOuterAlt(_localctx, 3); { - setState(110); + setState(114); rowCommand(); } break; case SHOW: enterOuterAlt(_localctx, 4); { - setState(111); + setState(115); showCommand(); } break; @@ -465,27 +480,27 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(127); + setState(131); _errHandler.sync(this); switch (_input.LA(1)) { case EVAL: enterOuterAlt(_localctx, 1); { - setState(114); + setState(118); evalCommand(); } break; case INLINESTATS: enterOuterAlt(_localctx, 2); { - setState(115); + setState(119); inlinestatsCommand(); } break; case LIMIT: enterOuterAlt(_localctx, 3); { - setState(116); + setState(120); limitCommand(); } break; @@ -493,70 +508,70 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce case PROJECT: enterOuterAlt(_localctx, 4); { - setState(117); + setState(121); keepCommand(); } break; case SORT: enterOuterAlt(_localctx, 5); { - setState(118); + setState(122); sortCommand(); } break; case STATS: enterOuterAlt(_localctx, 6); { - setState(119); + setState(123); statsCommand(); } break; case WHERE: enterOuterAlt(_localctx, 7); { - setState(120); + setState(124); whereCommand(); } break; case DROP: enterOuterAlt(_localctx, 8); { - setState(121); + setState(125); dropCommand(); } break; case RENAME: enterOuterAlt(_localctx, 9); { - setState(122); + setState(126); renameCommand(); } break; case DISSECT: enterOuterAlt(_localctx, 10); { - setState(123); + setState(127); dissectCommand(); } break; case GROK: enterOuterAlt(_localctx, 11); { - setState(124); + setState(128); grokCommand(); } break; case ENRICH: enterOuterAlt(_localctx, 12); { - setState(125); + setState(129); enrichCommand(); } break; case MV_EXPAND: enterOuterAlt(_localctx, 13); { - setState(126); + setState(130); mvExpandCommand(); } break; @@ -607,9 +622,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(129); + setState(133); match(WHERE); - setState(130); + setState(134); booleanExpression(0); } } @@ -804,7 +819,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(160); + setState(164); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -813,9 +828,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(133); + setState(137); match(NOT); - setState(134); + setState(138); booleanExpression(7); } break; @@ -824,7 +839,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(135); + setState(139); valueExpression(); } break; @@ -833,7 +848,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(136); + setState(140); regexBooleanExpression(); } break; @@ -842,41 +857,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(137); + setState(141); valueExpression(); - setState(139); + setState(143); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(138); + setState(142); match(NOT); } } - setState(141); + setState(145); match(IN); - setState(142); + setState(146); match(LP); - setState(143); + setState(147); valueExpression(); - setState(148); + setState(152); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(144); + setState(148); match(COMMA); - setState(145); + setState(149); valueExpression(); } } - setState(150); + setState(154); _errHandler.sync(this); _la = _input.LA(1); } - setState(151); + setState(155); match(RP); } break; @@ -885,27 +900,27 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(153); + setState(157); valueExpression(); - setState(154); + setState(158); match(IS); - setState(156); + setState(160); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(155); + setState(159); match(NOT); } } - setState(158); + setState(162); match(NULL); } break; } _ctx.stop = _input.LT(-1); - setState(170); + setState(174); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -913,7 +928,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(168); + setState(172); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -921,11 +936,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(162); + setState(166); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(163); + setState(167); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(164); + setState(168); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; @@ -934,18 +949,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(165); + setState(169); if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)"); - setState(166); + setState(170); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(167); + setState(171); ((LogicalBinaryContext)_localctx).right = booleanExpression(4); } break; } } } - setState(172); + setState(176); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1000,48 +1015,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(187); + setState(191); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(173); + setState(177); valueExpression(); - setState(175); + setState(179); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(174); + setState(178); match(NOT); } } - setState(177); + setState(181); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(178); + setState(182); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(180); + setState(184); valueExpression(); - setState(182); + setState(186); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(181); + setState(185); match(NOT); } } - setState(184); + setState(188); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(185); + setState(189); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1127,14 +1142,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 14, RULE_valueExpression); try { - setState(194); + setState(198); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(189); + setState(193); operatorExpression(0); } break; @@ -1142,11 +1157,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(190); + setState(194); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(191); + setState(195); comparisonOperator(); - setState(192); + setState(196); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1271,7 +1286,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(200); + setState(204); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: @@ -1280,7 +1295,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(197); + setState(201); primaryExpression(); } break; @@ -1289,7 +1304,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(198); + setState(202); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1300,13 +1315,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(199); + setState(203); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(210); + setState(214); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1314,7 +1329,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(208); + setState(212); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1322,12 +1337,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(202); + setState(206); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(203); + setState(207); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !((((_la - 62)) & ~0x3f) == 0 && ((1L << (_la - 62)) & 7L) != 0) ) { + if ( !(((_la) & ~0x3f) == 0 && ((1L << _la) & 8070450532247928832L) != 0) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { @@ -1335,7 +1350,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(204); + setState(208); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1344,9 +1359,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(205); + setState(209); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(206); + setState(210); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1357,14 +1372,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(207); + setState(211); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(212); + setState(216); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); } @@ -1486,14 +1501,14 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce PrimaryExpressionContext _localctx = new PrimaryExpressionContext(_ctx, getState()); enterRule(_localctx, 18, RULE_primaryExpression); try { - setState(220); + setState(224); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) { case 1: _localctx = new ConstantDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(213); + setState(217); constant(); } break; @@ -1501,7 +1516,7 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new DereferenceContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(214); + setState(218); qualifiedName(); } break; @@ -1509,7 +1524,7 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new FunctionContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(215); + setState(219); functionExpression(); } break; @@ -1517,11 +1532,11 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce _localctx = new ParenthesizedExpressionContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(216); + setState(220); match(LP); - setState(217); + setState(221); booleanExpression(0); - setState(218); + setState(222); match(RP); } break; @@ -1583,16 +1598,16 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(222); + setState(226); identifier(); - setState(223); + setState(227); match(LP); - setState(233); + setState(237); _errHandler.sync(this); switch (_input.LA(1)) { case ASTERISK: { - setState(224); + setState(228); match(ASTERISK); } break; @@ -1612,21 +1627,21 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx case QUOTED_IDENTIFIER: { { - setState(225); + setState(229); booleanExpression(0); - setState(230); + setState(234); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(226); + setState(230); match(COMMA); - setState(227); + setState(231); booleanExpression(0); } } - setState(232); + setState(236); _errHandler.sync(this); _la = _input.LA(1); } @@ -1638,7 +1653,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx default: break; } - setState(235); + setState(239); match(RP); } } @@ -1685,9 +1700,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(237); + setState(241); match(ROW); - setState(238); + setState(242); fields(); } } @@ -1741,23 +1756,23 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(240); + setState(244); field(); - setState(245); + setState(249); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(241); + setState(245); match(COMMA); - setState(242); + setState(246); field(); } } } - setState(247); + setState(251); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,19,_ctx); } @@ -1807,24 +1822,24 @@ public final FieldContext field() throws RecognitionException { FieldContext _localctx = new FieldContext(_ctx, getState()); enterRule(_localctx, 26, RULE_field); try { - setState(253); + setState(257); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(248); + setState(252); booleanExpression(0); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(249); + setState(253); qualifiedName(); - setState(250); + setState(254); match(ASSIGN); - setState(251); + setState(255); booleanExpression(0); } break; @@ -1844,11 +1859,11 @@ public final FieldContext field() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class FromCommandContext extends ParserRuleContext { public TerminalNode FROM() { return getToken(EsqlBaseParser.FROM, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List fromIdentifier() { + return getRuleContexts(FromIdentifierContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public FromIdentifierContext fromIdentifier(int i) { + return getRuleContext(FromIdentifierContext.class,i); } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } public TerminalNode COMMA(int i) { @@ -1884,34 +1899,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(255); + setState(259); match(FROM); - setState(256); - sourceIdentifier(); - setState(261); + setState(260); + fromIdentifier(); + setState(265); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(257); + setState(261); match(COMMA); - setState(258); - sourceIdentifier(); + setState(262); + fromIdentifier(); } } } - setState(263); + setState(267); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } - setState(265); + setState(269); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: { - setState(264); + setState(268); metadata(); } break; @@ -1933,11 +1948,11 @@ public final FromCommandContext fromCommand() throws RecognitionException { public static class MetadataContext extends ParserRuleContext { public TerminalNode OPENING_BRACKET() { return getToken(EsqlBaseParser.OPENING_BRACKET, 0); } public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List fromIdentifier() { + return getRuleContexts(FromIdentifierContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public FromIdentifierContext fromIdentifier(int i) { + return getRuleContext(FromIdentifierContext.class,i); } public TerminalNode CLOSING_BRACKET() { return getToken(EsqlBaseParser.CLOSING_BRACKET, 0); } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } @@ -1971,29 +1986,29 @@ public final MetadataContext metadata() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(267); + setState(271); match(OPENING_BRACKET); - setState(268); + setState(272); match(METADATA); - setState(269); - sourceIdentifier(); - setState(274); + setState(273); + fromIdentifier(); + setState(278); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(270); + setState(274); match(COMMA); - setState(271); - sourceIdentifier(); + setState(275); + fromIdentifier(); } } - setState(276); + setState(280); _errHandler.sync(this); _la = _input.LA(1); } - setState(277); + setState(281); match(CLOSING_BRACKET); } } @@ -2040,9 +2055,9 @@ public final EvalCommandContext evalCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(279); + setState(283); match(EVAL); - setState(280); + setState(284); fields(); } } @@ -2093,26 +2108,26 @@ public final StatsCommandContext statsCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(282); + setState(286); match(STATS); - setState(284); + setState(288); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(283); + setState(287); fields(); } break; } - setState(288); + setState(292); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: { - setState(286); + setState(290); match(BY); - setState(287); + setState(291); grouping(); } break; @@ -2166,18 +2181,18 @@ public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(290); + setState(294); match(INLINESTATS); - setState(291); + setState(295); fields(); - setState(294); + setState(298); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) { case 1: { - setState(292); + setState(296); match(BY); - setState(293); + setState(297); grouping(); } break; @@ -2234,23 +2249,23 @@ public final GroupingContext grouping() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(296); + setState(300); qualifiedName(); - setState(301); + setState(305); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(297); + setState(301); match(COMMA); - setState(298); + setState(302); qualifiedName(); } } } - setState(303); + setState(307); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } @@ -2268,39 +2283,39 @@ public final GroupingContext grouping() throws RecognitionException { } @SuppressWarnings("CheckReturnValue") - public static class SourceIdentifierContext extends ParserRuleContext { - public TerminalNode SRC_UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.SRC_UNQUOTED_IDENTIFIER, 0); } - public TerminalNode SRC_QUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.SRC_QUOTED_IDENTIFIER, 0); } + public static class FromIdentifierContext extends ParserRuleContext { + public TerminalNode FROM_UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.FROM_UNQUOTED_IDENTIFIER, 0); } + public TerminalNode QUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.QUOTED_IDENTIFIER, 0); } @SuppressWarnings("this-escape") - public SourceIdentifierContext(ParserRuleContext parent, int invokingState) { + public FromIdentifierContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } - @Override public int getRuleIndex() { return RULE_sourceIdentifier; } + @Override public int getRuleIndex() { return RULE_fromIdentifier; } @Override public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterSourceIdentifier(this); + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterFromIdentifier(this); } @Override public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitSourceIdentifier(this); + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitFromIdentifier(this); } @Override public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitSourceIdentifier(this); + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitFromIdentifier(this); else return visitor.visitChildren(this); } } - public final SourceIdentifierContext sourceIdentifier() throws RecognitionException { - SourceIdentifierContext _localctx = new SourceIdentifierContext(_ctx, getState()); - enterRule(_localctx, 40, RULE_sourceIdentifier); + public final FromIdentifierContext fromIdentifier() throws RecognitionException { + FromIdentifierContext _localctx = new FromIdentifierContext(_ctx, getState()); + enterRule(_localctx, 40, RULE_fromIdentifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(304); + setState(308); _la = _input.LA(1); - if ( !(_la==SRC_UNQUOTED_IDENTIFIER || _la==SRC_QUOTED_IDENTIFIER) ) { + if ( !(_la==QUOTED_IDENTIFIER || _la==FROM_UNQUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); } else { @@ -2360,23 +2375,23 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(306); + setState(310); identifier(); - setState(311); + setState(315); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,28,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(307); + setState(311); match(DOT); - setState(308); + setState(312); identifier(); } } } - setState(313); + setState(317); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,28,_ctx); } @@ -2393,6 +2408,78 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class QualifiedNamePatternContext extends ParserRuleContext { + public List identifierPattern() { + return getRuleContexts(IdentifierPatternContext.class); + } + public IdentifierPatternContext identifierPattern(int i) { + return getRuleContext(IdentifierPatternContext.class,i); + } + public List DOT() { return getTokens(EsqlBaseParser.DOT); } + public TerminalNode DOT(int i) { + return getToken(EsqlBaseParser.DOT, i); + } + @SuppressWarnings("this-escape") + public QualifiedNamePatternContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_qualifiedNamePattern; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterQualifiedNamePattern(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitQualifiedNamePattern(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitQualifiedNamePattern(this); + else return visitor.visitChildren(this); + } + } + + public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException { + QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState()); + enterRule(_localctx, 44, RULE_qualifiedNamePattern); + try { + int _alt; + enterOuterAlt(_localctx, 1); + { + setState(318); + identifierPattern(); + setState(323); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,29,_ctx); + while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { + if ( _alt==1 ) { + { + { + setState(319); + match(DOT); + setState(320); + identifierPattern(); + } + } + } + setState(325); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,29,_ctx); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + @SuppressWarnings("CheckReturnValue") public static class IdentifierContext extends ParserRuleContext { public TerminalNode UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.UNQUOTED_IDENTIFIER, 0); } @@ -2419,12 +2506,12 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); - enterRule(_localctx, 44, RULE_identifier); + enterRule(_localctx, 46, RULE_identifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(314); + setState(326); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -2447,6 +2534,60 @@ public final IdentifierContext identifier() throws RecognitionException { return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class IdentifierPatternContext extends ParserRuleContext { + public TerminalNode PROJECT_UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.PROJECT_UNQUOTED_IDENTIFIER, 0); } + public TerminalNode QUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.QUOTED_IDENTIFIER, 0); } + @SuppressWarnings("this-escape") + public IdentifierPatternContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_identifierPattern; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterIdentifierPattern(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitIdentifierPattern(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitIdentifierPattern(this); + else return visitor.visitChildren(this); + } + } + + public final IdentifierPatternContext identifierPattern() throws RecognitionException { + IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); + enterRule(_localctx, 48, RULE_identifierPattern); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(328); + _la = _input.LA(1); + if ( !(_la==QUOTED_IDENTIFIER || _la==PROJECT_UNQUOTED_IDENTIFIER) ) { + _errHandler.recoverInline(this); + } + else { + if ( _input.LA(1)==Token.EOF ) matchedEOF = true; + _errHandler.reportMatch(this); + consume(); + } + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + @SuppressWarnings("CheckReturnValue") public static class ConstantContext extends ParserRuleContext { @SuppressWarnings("this-escape") @@ -2698,17 +2839,17 @@ public T accept(ParseTreeVisitor visitor) { public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 46, RULE_constant); + enterRule(_localctx, 50, RULE_constant); int _la; try { - setState(358); + setState(372); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(316); + setState(330); match(NULL); } break; @@ -2716,9 +2857,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(317); + setState(331); integerValue(); - setState(318); + setState(332); match(UNQUOTED_IDENTIFIER); } break; @@ -2726,7 +2867,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(320); + setState(334); decimalValue(); } break; @@ -2734,7 +2875,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(321); + setState(335); integerValue(); } break; @@ -2742,7 +2883,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(322); + setState(336); booleanValue(); } break; @@ -2750,7 +2891,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(323); + setState(337); match(PARAM); } break; @@ -2758,7 +2899,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(324); + setState(338); string(); } break; @@ -2766,27 +2907,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(325); + setState(339); match(OPENING_BRACKET); - setState(326); + setState(340); numericValue(); - setState(331); + setState(345); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(327); + setState(341); match(COMMA); - setState(328); + setState(342); numericValue(); } } - setState(333); + setState(347); _errHandler.sync(this); _la = _input.LA(1); } - setState(334); + setState(348); match(CLOSING_BRACKET); } break; @@ -2794,27 +2935,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(336); + setState(350); match(OPENING_BRACKET); - setState(337); + setState(351); booleanValue(); - setState(342); + setState(356); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(338); + setState(352); match(COMMA); - setState(339); + setState(353); booleanValue(); } } - setState(344); + setState(358); _errHandler.sync(this); _la = _input.LA(1); } - setState(345); + setState(359); match(CLOSING_BRACKET); } break; @@ -2822,27 +2963,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(347); + setState(361); match(OPENING_BRACKET); - setState(348); + setState(362); string(); - setState(353); + setState(367); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(349); + setState(363); match(COMMA); - setState(350); + setState(364); string(); } } - setState(355); + setState(369); _errHandler.sync(this); _la = _input.LA(1); } - setState(356); + setState(370); match(CLOSING_BRACKET); } break; @@ -2885,13 +3026,13 @@ public T accept(ParseTreeVisitor visitor) { public final LimitCommandContext limitCommand() throws RecognitionException { LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState()); - enterRule(_localctx, 48, RULE_limitCommand); + enterRule(_localctx, 52, RULE_limitCommand); try { enterOuterAlt(_localctx, 1); { - setState(360); + setState(374); match(LIMIT); - setState(361); + setState(375); match(INTEGER_LITERAL); } } @@ -2941,32 +3082,32 @@ public T accept(ParseTreeVisitor visitor) { public final SortCommandContext sortCommand() throws RecognitionException { SortCommandContext _localctx = new SortCommandContext(_ctx, getState()); - enterRule(_localctx, 50, RULE_sortCommand); + enterRule(_localctx, 54, RULE_sortCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(363); + setState(377); match(SORT); - setState(364); + setState(378); orderExpression(); - setState(369); + setState(383); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(365); + setState(379); match(COMMA); - setState(366); + setState(380); orderExpression(); } } } - setState(371); + setState(385); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } } } @@ -3015,19 +3156,19 @@ public T accept(ParseTreeVisitor visitor) { public final OrderExpressionContext orderExpression() throws RecognitionException { OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_orderExpression); + enterRule(_localctx, 56, RULE_orderExpression); int _la; try { enterOuterAlt(_localctx, 1); { - setState(372); + setState(386); booleanExpression(0); - setState(374); + setState(388); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) { case 1: { - setState(373); + setState(387); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -3041,14 +3182,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(378); + setState(392); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) { case 1: { - setState(376); + setState(390); match(NULLS); - setState(377); + setState(391); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -3078,11 +3219,11 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio @SuppressWarnings("CheckReturnValue") public static class KeepCommandContext extends ParserRuleContext { public TerminalNode KEEP() { return getToken(EsqlBaseParser.KEEP, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List qualifiedNamePattern() { + return getRuleContexts(QualifiedNamePatternContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public QualifiedNamePatternContext qualifiedNamePattern(int i) { + return getRuleContext(QualifiedNamePatternContext.class,i); } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } public TerminalNode COMMA(int i) { @@ -3111,63 +3252,63 @@ public T accept(ParseTreeVisitor visitor) { public final KeepCommandContext keepCommand() throws RecognitionException { KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_keepCommand); + enterRule(_localctx, 58, RULE_keepCommand); try { int _alt; - setState(398); + setState(412); _errHandler.sync(this); switch (_input.LA(1)) { case KEEP: enterOuterAlt(_localctx, 1); { - setState(380); + setState(394); match(KEEP); - setState(381); - sourceIdentifier(); - setState(386); + setState(395); + qualifiedNamePattern(); + setState(400); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(382); + setState(396); match(COMMA); - setState(383); - sourceIdentifier(); + setState(397); + qualifiedNamePattern(); } } } - setState(388); + setState(402); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,37,_ctx); } } break; case PROJECT: enterOuterAlt(_localctx, 2); { - setState(389); + setState(403); match(PROJECT); - setState(390); - sourceIdentifier(); - setState(395); + setState(404); + qualifiedNamePattern(); + setState(409); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,38,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(391); + setState(405); match(COMMA); - setState(392); - sourceIdentifier(); + setState(406); + qualifiedNamePattern(); } } } - setState(397); + setState(411); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,38,_ctx); } } break; @@ -3189,11 +3330,11 @@ public final KeepCommandContext keepCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class DropCommandContext extends ParserRuleContext { public TerminalNode DROP() { return getToken(EsqlBaseParser.DROP, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List qualifiedNamePattern() { + return getRuleContexts(QualifiedNamePatternContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public QualifiedNamePatternContext qualifiedNamePattern(int i) { + return getRuleContext(QualifiedNamePatternContext.class,i); } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } public TerminalNode COMMA(int i) { @@ -3221,32 +3362,32 @@ public T accept(ParseTreeVisitor visitor) { public final DropCommandContext dropCommand() throws RecognitionException { DropCommandContext _localctx = new DropCommandContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_dropCommand); + enterRule(_localctx, 60, RULE_dropCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(400); + setState(414); match(DROP); - setState(401); - sourceIdentifier(); - setState(406); + setState(415); + qualifiedNamePattern(); + setState(420); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,39,_ctx); + _alt = getInterpreter().adaptivePredict(_input,40,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(402); + setState(416); match(COMMA); - setState(403); - sourceIdentifier(); + setState(417); + qualifiedNamePattern(); } } } - setState(408); + setState(422); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,39,_ctx); + _alt = getInterpreter().adaptivePredict(_input,40,_ctx); } } } @@ -3296,32 +3437,32 @@ public T accept(ParseTreeVisitor visitor) { public final RenameCommandContext renameCommand() throws RecognitionException { RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_renameCommand); + enterRule(_localctx, 62, RULE_renameCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(409); + setState(423); match(RENAME); - setState(410); + setState(424); renameClause(); - setState(415); + setState(429); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,40,_ctx); + _alt = getInterpreter().adaptivePredict(_input,41,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(411); + setState(425); match(COMMA); - setState(412); + setState(426); renameClause(); } } } - setState(417); + setState(431); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,40,_ctx); + _alt = getInterpreter().adaptivePredict(_input,41,_ctx); } } } @@ -3338,14 +3479,14 @@ public final RenameCommandContext renameCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class RenameClauseContext extends ParserRuleContext { - public SourceIdentifierContext oldName; - public SourceIdentifierContext newName; + public QualifiedNamePatternContext oldName; + public QualifiedNamePatternContext newName; public TerminalNode AS() { return getToken(EsqlBaseParser.AS, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public List qualifiedNamePattern() { + return getRuleContexts(QualifiedNamePatternContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public QualifiedNamePatternContext qualifiedNamePattern(int i) { + return getRuleContext(QualifiedNamePatternContext.class,i); } @SuppressWarnings("this-escape") public RenameClauseContext(ParserRuleContext parent, int invokingState) { @@ -3369,16 +3510,16 @@ public T accept(ParseTreeVisitor visitor) { public final RenameClauseContext renameClause() throws RecognitionException { RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_renameClause); + enterRule(_localctx, 64, RULE_renameClause); try { enterOuterAlt(_localctx, 1); { - setState(418); - ((RenameClauseContext)_localctx).oldName = sourceIdentifier(); - setState(419); + setState(432); + ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); + setState(433); match(AS); - setState(420); - ((RenameClauseContext)_localctx).newName = sourceIdentifier(); + setState(434); + ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } catch (RecognitionException re) { @@ -3426,22 +3567,22 @@ public T accept(ParseTreeVisitor visitor) { public final DissectCommandContext dissectCommand() throws RecognitionException { DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_dissectCommand); + enterRule(_localctx, 66, RULE_dissectCommand); try { enterOuterAlt(_localctx, 1); { - setState(422); + setState(436); match(DISSECT); - setState(423); + setState(437); primaryExpression(); - setState(424); + setState(438); string(); - setState(426); + setState(440); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { case 1: { - setState(425); + setState(439); commandOptions(); } break; @@ -3490,15 +3631,15 @@ public T accept(ParseTreeVisitor visitor) { public final GrokCommandContext grokCommand() throws RecognitionException { GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_grokCommand); + enterRule(_localctx, 68, RULE_grokCommand); try { enterOuterAlt(_localctx, 1); { - setState(428); + setState(442); match(GROK); - setState(429); + setState(443); primaryExpression(); - setState(430); + setState(444); string(); } } @@ -3516,8 +3657,8 @@ public final GrokCommandContext grokCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class MvExpandCommandContext extends ParserRuleContext { public TerminalNode MV_EXPAND() { return getToken(EsqlBaseParser.MV_EXPAND, 0); } - public SourceIdentifierContext sourceIdentifier() { - return getRuleContext(SourceIdentifierContext.class,0); + public QualifiedNameContext qualifiedName() { + return getRuleContext(QualifiedNameContext.class,0); } @SuppressWarnings("this-escape") public MvExpandCommandContext(ParserRuleContext parent, int invokingState) { @@ -3541,14 +3682,14 @@ public T accept(ParseTreeVisitor visitor) { public final MvExpandCommandContext mvExpandCommand() throws RecognitionException { MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_mvExpandCommand); + enterRule(_localctx, 70, RULE_mvExpandCommand); try { enterOuterAlt(_localctx, 1); { - setState(432); + setState(446); match(MV_EXPAND); - setState(433); - sourceIdentifier(); + setState(447); + qualifiedName(); } } catch (RecognitionException re) { @@ -3596,30 +3737,30 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionsContext commandOptions() throws RecognitionException { CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_commandOptions); + enterRule(_localctx, 72, RULE_commandOptions); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(435); + setState(449); commandOption(); - setState(440); + setState(454); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,43,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(436); + setState(450); match(COMMA); - setState(437); + setState(451); commandOption(); } } } - setState(442); + setState(456); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,43,_ctx); } } } @@ -3665,15 +3806,15 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionContext commandOption() throws RecognitionException { CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_commandOption); + enterRule(_localctx, 74, RULE_commandOption); try { enterOuterAlt(_localctx, 1); { - setState(443); + setState(457); identifier(); - setState(444); + setState(458); match(ASSIGN); - setState(445); + setState(459); constant(); } } @@ -3714,12 +3855,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_booleanValue); + enterRule(_localctx, 76, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(447); + setState(461); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -3772,22 +3913,22 @@ public T accept(ParseTreeVisitor visitor) { public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_numericValue); + enterRule(_localctx, 78, RULE_numericValue); try { - setState(451); + setState(465); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(449); + setState(463); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(450); + setState(464); integerValue(); } break; @@ -3831,17 +3972,17 @@ public T accept(ParseTreeVisitor visitor) { public final DecimalValueContext decimalValue() throws RecognitionException { DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_decimalValue); + enterRule(_localctx, 80, RULE_decimalValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(454); + setState(468); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(453); + setState(467); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -3854,7 +3995,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(456); + setState(470); match(DECIMAL_LITERAL); } } @@ -3896,17 +4037,17 @@ public T accept(ParseTreeVisitor visitor) { public final IntegerValueContext integerValue() throws RecognitionException { IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState()); - enterRule(_localctx, 78, RULE_integerValue); + enterRule(_localctx, 82, RULE_integerValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(459); + setState(473); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(458); + setState(472); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -3919,7 +4060,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(461); + setState(475); match(INTEGER_LITERAL); } } @@ -3959,11 +4100,11 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 80, RULE_string); + enterRule(_localctx, 84, RULE_string); try { enterOuterAlt(_localctx, 1); { - setState(463); + setState(477); match(STRING); } } @@ -4008,14 +4149,14 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 82, RULE_comparisonOperator); + enterRule(_localctx, 86, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(465); + setState(479); _la = _input.LA(1); - if ( !(((_la) & ~0x3f) == 0 && ((1L << _la) & 1134907106097364992L) != 0) ) { + if ( !(((_la) & ~0x3f) == 0 && ((1L << _la) & 283726776524341248L) != 0) ) { _errHandler.recoverInline(this); } else { @@ -4064,13 +4205,13 @@ public T accept(ParseTreeVisitor visitor) { public final ExplainCommandContext explainCommand() throws RecognitionException { ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState()); - enterRule(_localctx, 84, RULE_explainCommand); + enterRule(_localctx, 88, RULE_explainCommand); try { enterOuterAlt(_localctx, 1); { - setState(467); + setState(481); match(EXPLAIN); - setState(468); + setState(482); subqueryExpression(); } } @@ -4114,15 +4255,15 @@ public T accept(ParseTreeVisitor visitor) { public final SubqueryExpressionContext subqueryExpression() throws RecognitionException { SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState()); - enterRule(_localctx, 86, RULE_subqueryExpression); + enterRule(_localctx, 90, RULE_subqueryExpression); try { enterOuterAlt(_localctx, 1); { - setState(470); + setState(484); match(OPENING_BRACKET); - setState(471); + setState(485); query(0); - setState(472); + setState(486); match(CLOSING_BRACKET); } } @@ -4194,18 +4335,18 @@ public T accept(ParseTreeVisitor visitor) { public final ShowCommandContext showCommand() throws RecognitionException { ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState()); - enterRule(_localctx, 88, RULE_showCommand); + enterRule(_localctx, 92, RULE_showCommand); try { - setState(478); + setState(492); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { case 1: _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(474); + setState(488); match(SHOW); - setState(475); + setState(489); match(INFO); } break; @@ -4213,9 +4354,9 @@ public final ShowCommandContext showCommand() throws RecognitionException { _localctx = new ShowFunctionsContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(476); + setState(490); match(SHOW); - setState(477); + setState(491); match(FUNCTIONS); } break; @@ -4234,14 +4375,11 @@ public final ShowCommandContext showCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class EnrichCommandContext extends ParserRuleContext { - public SourceIdentifierContext policyName; - public SourceIdentifierContext matchField; + public FromIdentifierContext policyName; + public QualifiedNamePatternContext matchField; public TerminalNode ENRICH() { return getToken(EsqlBaseParser.ENRICH, 0); } - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); - } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public FromIdentifierContext fromIdentifier() { + return getRuleContext(FromIdentifierContext.class,0); } public TerminalNode ON() { return getToken(EsqlBaseParser.ON, 0); } public TerminalNode WITH() { return getToken(EsqlBaseParser.WITH, 0); } @@ -4251,6 +4389,9 @@ public List enrichWithClause() { public EnrichWithClauseContext enrichWithClause(int i) { return getRuleContext(EnrichWithClauseContext.class,i); } + public QualifiedNamePatternContext qualifiedNamePattern() { + return getRuleContext(QualifiedNamePatternContext.class,0); + } public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } public TerminalNode COMMA(int i) { return getToken(EsqlBaseParser.COMMA, i); @@ -4277,53 +4418,53 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 90, RULE_enrichCommand); + enterRule(_localctx, 94, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(480); + setState(494); match(ENRICH); - setState(481); - ((EnrichCommandContext)_localctx).policyName = sourceIdentifier(); - setState(484); + setState(495); + ((EnrichCommandContext)_localctx).policyName = fromIdentifier(); + setState(498); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { case 1: { - setState(482); + setState(496); match(ON); - setState(483); - ((EnrichCommandContext)_localctx).matchField = sourceIdentifier(); + setState(497); + ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(495); + setState(509); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { case 1: { - setState(486); + setState(500); match(WITH); - setState(487); + setState(501); enrichWithClause(); - setState(492); + setState(506); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(488); + setState(502); match(COMMA); - setState(489); + setState(503); enrichWithClause(); } } } - setState(494); + setState(508); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); } } break; @@ -4343,13 +4484,13 @@ public final EnrichCommandContext enrichCommand() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class EnrichWithClauseContext extends ParserRuleContext { - public SourceIdentifierContext newName; - public SourceIdentifierContext enrichField; - public List sourceIdentifier() { - return getRuleContexts(SourceIdentifierContext.class); + public QualifiedNamePatternContext newName; + public QualifiedNamePatternContext enrichField; + public List qualifiedNamePattern() { + return getRuleContexts(QualifiedNamePatternContext.class); } - public SourceIdentifierContext sourceIdentifier(int i) { - return getRuleContext(SourceIdentifierContext.class,i); + public QualifiedNamePatternContext qualifiedNamePattern(int i) { + return getRuleContext(QualifiedNamePatternContext.class,i); } public TerminalNode ASSIGN() { return getToken(EsqlBaseParser.ASSIGN, 0); } @SuppressWarnings("this-escape") @@ -4374,24 +4515,24 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 92, RULE_enrichWithClause); + enterRule(_localctx, 96, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(500); + setState(514); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { case 1: { - setState(497); - ((EnrichWithClauseContext)_localctx).newName = sourceIdentifier(); - setState(498); + setState(511); + ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); + setState(512); match(ASSIGN); } break; } - setState(502); - ((EnrichWithClauseContext)_localctx).enrichField = sourceIdentifier(); + setState(516); + ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } catch (RecognitionException re) { @@ -4443,7 +4584,7 @@ private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, } public static final String _serializedATN = - "\u0004\u0001Q\u01f9\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ + "\u0004\u0001b\u0207\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ @@ -4456,319 +4597,327 @@ private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, "\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002"+ "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002"+ "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ - "-\u0007-\u0002.\u0007.\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001"+ - "h\b\u0001\n\u0001\f\u0001k\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0003\u0002q\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003"+ + "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u0001\u0000\u0001\u0000"+ + "\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0005\u0001l\b\u0001\n\u0001\f\u0001o\t\u0001\u0001\u0002"+ + "\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002u\b\u0002\u0001\u0003"+ + "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u0080\b\u0003"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u008c\b\u0005"+ + "\u0003\u0003\u0084\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0003\u0005\u0090\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0005\u0005\u0097\b\u0005\n\u0005\f\u0005\u009a\t\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00a1"+ + "\b\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00a5\b\u0005\u0001\u0005"+ "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005"+ - "\u0093\b\u0005\n\u0005\f\u0005\u0096\t\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u009d\b\u0005\u0001\u0005\u0001"+ - "\u0005\u0003\u0005\u00a1\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00a9\b\u0005\n\u0005\f\u0005"+ - "\u00ac\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00b0\b\u0006\u0001"+ - "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00b7"+ - "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00bc\b\u0006"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007"+ - "\u00c3\b\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00c9\b\b\u0001"+ - "\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0005\b\u00d1\b\b\n\b\f\b\u00d4"+ - "\t\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00dd"+ - "\b\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0005\n\u00e5\b\n"+ - "\n\n\f\n\u00e8\t\n\u0003\n\u00ea\b\n\u0001\n\u0001\n\u0001\u000b\u0001"+ - "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0005\f\u00f4\b\f\n\f\f\f\u00f7"+ - "\t\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0003\r\u00fe\b\r\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0001\u000e\u0005\u000e\u0104\b\u000e\n\u000e"+ - "\f\u000e\u0107\t\u000e\u0001\u000e\u0003\u000e\u010a\b\u000e\u0001\u000f"+ - "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0005\u000f\u0111\b\u000f"+ - "\n\u000f\f\u000f\u0114\t\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001"+ - "\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0003\u0011\u011d\b\u0011\u0001"+ - "\u0011\u0001\u0011\u0003\u0011\u0121\b\u0011\u0001\u0012\u0001\u0012\u0001"+ - "\u0012\u0001\u0012\u0003\u0012\u0127\b\u0012\u0001\u0013\u0001\u0013\u0001"+ - "\u0013\u0005\u0013\u012c\b\u0013\n\u0013\f\u0013\u012f\t\u0013\u0001\u0014"+ - "\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u0136\b\u0015"+ - "\n\u0015\f\u0015\u0139\t\u0015\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0005"+ - "\u0017\u014a\b\u0017\n\u0017\f\u0017\u014d\t\u0017\u0001\u0017\u0001\u0017"+ - "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u0155\b\u0017"+ - "\n\u0017\f\u0017\u0158\t\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u0160\b\u0017\n\u0017\f\u0017"+ - "\u0163\t\u0017\u0001\u0017\u0001\u0017\u0003\u0017\u0167\b\u0017\u0001"+ - "\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001"+ - "\u0019\u0005\u0019\u0170\b\u0019\n\u0019\f\u0019\u0173\t\u0019\u0001\u001a"+ - "\u0001\u001a\u0003\u001a\u0177\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a"+ - "\u017b\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b"+ - "\u0181\b\u001b\n\u001b\f\u001b\u0184\t\u001b\u0001\u001b\u0001\u001b\u0001"+ - "\u001b\u0001\u001b\u0005\u001b\u018a\b\u001b\n\u001b\f\u001b\u018d\t\u001b"+ - "\u0003\u001b\u018f\b\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c"+ - "\u0005\u001c\u0195\b\u001c\n\u001c\f\u001c\u0198\t\u001c\u0001\u001d\u0001"+ - "\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u019e\b\u001d\n\u001d\f\u001d"+ - "\u01a1\t\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u01ab\b\u001f\u0001 "+ - "\u0001 \u0001 \u0001 \u0001!\u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0005"+ - "\"\u01b7\b\"\n\"\f\"\u01ba\t\"\u0001#\u0001#\u0001#\u0001#\u0001$\u0001"+ - "$\u0001%\u0001%\u0003%\u01c4\b%\u0001&\u0003&\u01c7\b&\u0001&\u0001&\u0001"+ - "\'\u0003\'\u01cc\b\'\u0001\'\u0001\'\u0001(\u0001(\u0001)\u0001)\u0001"+ - "*\u0001*\u0001*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001"+ - ",\u0003,\u01df\b,\u0001-\u0001-\u0001-\u0001-\u0003-\u01e5\b-\u0001-\u0001"+ - "-\u0001-\u0001-\u0005-\u01eb\b-\n-\f-\u01ee\t-\u0003-\u01f0\b-\u0001."+ - "\u0001.\u0001.\u0003.\u01f5\b.\u0001.\u0001.\u0001.\u0000\u0003\u0002"+ - "\n\u0010/\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016"+ - "\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\\u0000\b\u0001"+ - "\u0000<=\u0001\u0000>@\u0001\u0000LM\u0001\u0000CD\u0002\u0000 ##\u0001"+ - "\u0000&\'\u0002\u0000%%33\u0001\u00006;\u0217\u0000^\u0001\u0000\u0000"+ - "\u0000\u0002a\u0001\u0000\u0000\u0000\u0004p\u0001\u0000\u0000\u0000\u0006"+ - "\u007f\u0001\u0000\u0000\u0000\b\u0081\u0001\u0000\u0000\u0000\n\u00a0"+ - "\u0001\u0000\u0000\u0000\f\u00bb\u0001\u0000\u0000\u0000\u000e\u00c2\u0001"+ - "\u0000\u0000\u0000\u0010\u00c8\u0001\u0000\u0000\u0000\u0012\u00dc\u0001"+ - "\u0000\u0000\u0000\u0014\u00de\u0001\u0000\u0000\u0000\u0016\u00ed\u0001"+ - "\u0000\u0000\u0000\u0018\u00f0\u0001\u0000\u0000\u0000\u001a\u00fd\u0001"+ - "\u0000\u0000\u0000\u001c\u00ff\u0001\u0000\u0000\u0000\u001e\u010b\u0001"+ - "\u0000\u0000\u0000 \u0117\u0001\u0000\u0000\u0000\"\u011a\u0001\u0000"+ - "\u0000\u0000$\u0122\u0001\u0000\u0000\u0000&\u0128\u0001\u0000\u0000\u0000"+ - "(\u0130\u0001\u0000\u0000\u0000*\u0132\u0001\u0000\u0000\u0000,\u013a"+ - "\u0001\u0000\u0000\u0000.\u0166\u0001\u0000\u0000\u00000\u0168\u0001\u0000"+ - "\u0000\u00002\u016b\u0001\u0000\u0000\u00004\u0174\u0001\u0000\u0000\u0000"+ - "6\u018e\u0001\u0000\u0000\u00008\u0190\u0001\u0000\u0000\u0000:\u0199"+ - "\u0001\u0000\u0000\u0000<\u01a2\u0001\u0000\u0000\u0000>\u01a6\u0001\u0000"+ - "\u0000\u0000@\u01ac\u0001\u0000\u0000\u0000B\u01b0\u0001\u0000\u0000\u0000"+ - "D\u01b3\u0001\u0000\u0000\u0000F\u01bb\u0001\u0000\u0000\u0000H\u01bf"+ - "\u0001\u0000\u0000\u0000J\u01c3\u0001\u0000\u0000\u0000L\u01c6\u0001\u0000"+ - "\u0000\u0000N\u01cb\u0001\u0000\u0000\u0000P\u01cf\u0001\u0000\u0000\u0000"+ - "R\u01d1\u0001\u0000\u0000\u0000T\u01d3\u0001\u0000\u0000\u0000V\u01d6"+ - "\u0001\u0000\u0000\u0000X\u01de\u0001\u0000\u0000\u0000Z\u01e0\u0001\u0000"+ - "\u0000\u0000\\\u01f4\u0001\u0000\u0000\u0000^_\u0003\u0002\u0001\u0000"+ - "_`\u0005\u0000\u0000\u0001`\u0001\u0001\u0000\u0000\u0000ab\u0006\u0001"+ - "\uffff\uffff\u0000bc\u0003\u0004\u0002\u0000ci\u0001\u0000\u0000\u0000"+ - "de\n\u0001\u0000\u0000ef\u0005\u001a\u0000\u0000fh\u0003\u0006\u0003\u0000"+ - "gd\u0001\u0000\u0000\u0000hk\u0001\u0000\u0000\u0000ig\u0001\u0000\u0000"+ - "\u0000ij\u0001\u0000\u0000\u0000j\u0003\u0001\u0000\u0000\u0000ki\u0001"+ - "\u0000\u0000\u0000lq\u0003T*\u0000mq\u0003\u001c\u000e\u0000nq\u0003\u0016"+ - "\u000b\u0000oq\u0003X,\u0000pl\u0001\u0000\u0000\u0000pm\u0001\u0000\u0000"+ - "\u0000pn\u0001\u0000\u0000\u0000po\u0001\u0000\u0000\u0000q\u0005\u0001"+ - "\u0000\u0000\u0000r\u0080\u0003 \u0010\u0000s\u0080\u0003$\u0012\u0000"+ - "t\u0080\u00030\u0018\u0000u\u0080\u00036\u001b\u0000v\u0080\u00032\u0019"+ - "\u0000w\u0080\u0003\"\u0011\u0000x\u0080\u0003\b\u0004\u0000y\u0080\u0003"+ - "8\u001c\u0000z\u0080\u0003:\u001d\u0000{\u0080\u0003>\u001f\u0000|\u0080"+ - "\u0003@ \u0000}\u0080\u0003Z-\u0000~\u0080\u0003B!\u0000\u007fr\u0001"+ - "\u0000\u0000\u0000\u007fs\u0001\u0000\u0000\u0000\u007ft\u0001\u0000\u0000"+ - "\u0000\u007fu\u0001\u0000\u0000\u0000\u007fv\u0001\u0000\u0000\u0000\u007f"+ - "w\u0001\u0000\u0000\u0000\u007fx\u0001\u0000\u0000\u0000\u007fy\u0001"+ - "\u0000\u0000\u0000\u007fz\u0001\u0000\u0000\u0000\u007f{\u0001\u0000\u0000"+ - "\u0000\u007f|\u0001\u0000\u0000\u0000\u007f}\u0001\u0000\u0000\u0000\u007f"+ - "~\u0001\u0000\u0000\u0000\u0080\u0007\u0001\u0000\u0000\u0000\u0081\u0082"+ - "\u0005\u0012\u0000\u0000\u0082\u0083\u0003\n\u0005\u0000\u0083\t\u0001"+ - "\u0000\u0000\u0000\u0084\u0085\u0006\u0005\uffff\uffff\u0000\u0085\u0086"+ - "\u0005,\u0000\u0000\u0086\u00a1\u0003\n\u0005\u0007\u0087\u00a1\u0003"+ - "\u000e\u0007\u0000\u0088\u00a1\u0003\f\u0006\u0000\u0089\u008b\u0003\u000e"+ - "\u0007\u0000\u008a\u008c\u0005,\u0000\u0000\u008b\u008a\u0001\u0000\u0000"+ - "\u0000\u008b\u008c\u0001\u0000\u0000\u0000\u008c\u008d\u0001\u0000\u0000"+ - "\u0000\u008d\u008e\u0005)\u0000\u0000\u008e\u008f\u0005(\u0000\u0000\u008f"+ - "\u0094\u0003\u000e\u0007\u0000\u0090\u0091\u0005\"\u0000\u0000\u0091\u0093"+ - "\u0003\u000e\u0007\u0000\u0092\u0090\u0001\u0000\u0000\u0000\u0093\u0096"+ - "\u0001\u0000\u0000\u0000\u0094\u0092\u0001\u0000\u0000\u0000\u0094\u0095"+ - "\u0001\u0000\u0000\u0000\u0095\u0097\u0001\u0000\u0000\u0000\u0096\u0094"+ - "\u0001\u0000\u0000\u0000\u0097\u0098\u00052\u0000\u0000\u0098\u00a1\u0001"+ - "\u0000\u0000\u0000\u0099\u009a\u0003\u000e\u0007\u0000\u009a\u009c\u0005"+ - "*\u0000\u0000\u009b\u009d\u0005,\u0000\u0000\u009c\u009b\u0001\u0000\u0000"+ - "\u0000\u009c\u009d\u0001\u0000\u0000\u0000\u009d\u009e\u0001\u0000\u0000"+ - "\u0000\u009e\u009f\u0005-\u0000\u0000\u009f\u00a1\u0001\u0000\u0000\u0000"+ - "\u00a0\u0084\u0001\u0000\u0000\u0000\u00a0\u0087\u0001\u0000\u0000\u0000"+ - "\u00a0\u0088\u0001\u0000\u0000\u0000\u00a0\u0089\u0001\u0000\u0000\u0000"+ - "\u00a0\u0099\u0001\u0000\u0000\u0000\u00a1\u00aa\u0001\u0000\u0000\u0000"+ - "\u00a2\u00a3\n\u0004\u0000\u0000\u00a3\u00a4\u0005\u001f\u0000\u0000\u00a4"+ - "\u00a9\u0003\n\u0005\u0005\u00a5\u00a6\n\u0003\u0000\u0000\u00a6\u00a7"+ - "\u0005/\u0000\u0000\u00a7\u00a9\u0003\n\u0005\u0004\u00a8\u00a2\u0001"+ - "\u0000\u0000\u0000\u00a8\u00a5\u0001\u0000\u0000\u0000\u00a9\u00ac\u0001"+ - "\u0000\u0000\u0000\u00aa\u00a8\u0001\u0000\u0000\u0000\u00aa\u00ab\u0001"+ - "\u0000\u0000\u0000\u00ab\u000b\u0001\u0000\u0000\u0000\u00ac\u00aa\u0001"+ - "\u0000\u0000\u0000\u00ad\u00af\u0003\u000e\u0007\u0000\u00ae\u00b0\u0005"+ - ",\u0000\u0000\u00af\u00ae\u0001\u0000\u0000\u0000\u00af\u00b0\u0001\u0000"+ - "\u0000\u0000\u00b0\u00b1\u0001\u0000\u0000\u0000\u00b1\u00b2\u0005+\u0000"+ - "\u0000\u00b2\u00b3\u0003P(\u0000\u00b3\u00bc\u0001\u0000\u0000\u0000\u00b4"+ - "\u00b6\u0003\u000e\u0007\u0000\u00b5\u00b7\u0005,\u0000\u0000\u00b6\u00b5"+ - "\u0001\u0000\u0000\u0000\u00b6\u00b7\u0001\u0000\u0000\u0000\u00b7\u00b8"+ - "\u0001\u0000\u0000\u0000\u00b8\u00b9\u00051\u0000\u0000\u00b9\u00ba\u0003"+ - "P(\u0000\u00ba\u00bc\u0001\u0000\u0000\u0000\u00bb\u00ad\u0001\u0000\u0000"+ - "\u0000\u00bb\u00b4\u0001\u0000\u0000\u0000\u00bc\r\u0001\u0000\u0000\u0000"+ - "\u00bd\u00c3\u0003\u0010\b\u0000\u00be\u00bf\u0003\u0010\b\u0000\u00bf"+ - "\u00c0\u0003R)\u0000\u00c0\u00c1\u0003\u0010\b\u0000\u00c1\u00c3\u0001"+ - "\u0000\u0000\u0000\u00c2\u00bd\u0001\u0000\u0000\u0000\u00c2\u00be\u0001"+ - "\u0000\u0000\u0000\u00c3\u000f\u0001\u0000\u0000\u0000\u00c4\u00c5\u0006"+ - "\b\uffff\uffff\u0000\u00c5\u00c9\u0003\u0012\t\u0000\u00c6\u00c7\u0007"+ - "\u0000\u0000\u0000\u00c7\u00c9\u0003\u0010\b\u0003\u00c8\u00c4\u0001\u0000"+ - "\u0000\u0000\u00c8\u00c6\u0001\u0000\u0000\u0000\u00c9\u00d2\u0001\u0000"+ - "\u0000\u0000\u00ca\u00cb\n\u0002\u0000\u0000\u00cb\u00cc\u0007\u0001\u0000"+ - "\u0000\u00cc\u00d1\u0003\u0010\b\u0003\u00cd\u00ce\n\u0001\u0000\u0000"+ - "\u00ce\u00cf\u0007\u0000\u0000\u0000\u00cf\u00d1\u0003\u0010\b\u0002\u00d0"+ - "\u00ca\u0001\u0000\u0000\u0000\u00d0\u00cd\u0001\u0000\u0000\u0000\u00d1"+ - "\u00d4\u0001\u0000\u0000\u0000\u00d2\u00d0\u0001\u0000\u0000\u0000\u00d2"+ - "\u00d3\u0001\u0000\u0000\u0000\u00d3\u0011\u0001\u0000\u0000\u0000\u00d4"+ - "\u00d2\u0001\u0000\u0000\u0000\u00d5\u00dd\u0003.\u0017\u0000\u00d6\u00dd"+ - "\u0003*\u0015\u0000\u00d7\u00dd\u0003\u0014\n\u0000\u00d8\u00d9\u0005"+ - "(\u0000\u0000\u00d9\u00da\u0003\n\u0005\u0000\u00da\u00db\u00052\u0000"+ - "\u0000\u00db\u00dd\u0001\u0000\u0000\u0000\u00dc\u00d5\u0001\u0000\u0000"+ - "\u0000\u00dc\u00d6\u0001\u0000\u0000\u0000\u00dc\u00d7\u0001\u0000\u0000"+ - "\u0000\u00dc\u00d8\u0001\u0000\u0000\u0000\u00dd\u0013\u0001\u0000\u0000"+ - "\u0000\u00de\u00df\u0003,\u0016\u0000\u00df\u00e9\u0005(\u0000\u0000\u00e0"+ - "\u00ea\u0005>\u0000\u0000\u00e1\u00e6\u0003\n\u0005\u0000\u00e2\u00e3"+ - "\u0005\"\u0000\u0000\u00e3\u00e5\u0003\n\u0005\u0000\u00e4\u00e2\u0001"+ - "\u0000\u0000\u0000\u00e5\u00e8\u0001\u0000\u0000\u0000\u00e6\u00e4\u0001"+ - "\u0000\u0000\u0000\u00e6\u00e7\u0001\u0000\u0000\u0000\u00e7\u00ea\u0001"+ - "\u0000\u0000\u0000\u00e8\u00e6\u0001\u0000\u0000\u0000\u00e9\u00e0\u0001"+ - "\u0000\u0000\u0000\u00e9\u00e1\u0001\u0000\u0000\u0000\u00e9\u00ea\u0001"+ - "\u0000\u0000\u0000\u00ea\u00eb\u0001\u0000\u0000\u0000\u00eb\u00ec\u0005"+ - "2\u0000\u0000\u00ec\u0015\u0001\u0000\u0000\u0000\u00ed\u00ee\u0005\u000e"+ - "\u0000\u0000\u00ee\u00ef\u0003\u0018\f\u0000\u00ef\u0017\u0001\u0000\u0000"+ - "\u0000\u00f0\u00f5\u0003\u001a\r\u0000\u00f1\u00f2\u0005\"\u0000\u0000"+ - "\u00f2\u00f4\u0003\u001a\r\u0000\u00f3\u00f1\u0001\u0000\u0000\u0000\u00f4"+ - "\u00f7\u0001\u0000\u0000\u0000\u00f5\u00f3\u0001\u0000\u0000\u0000\u00f5"+ - "\u00f6\u0001\u0000\u0000\u0000\u00f6\u0019\u0001\u0000\u0000\u0000\u00f7"+ - "\u00f5\u0001\u0000\u0000\u0000\u00f8\u00fe\u0003\n\u0005\u0000\u00f9\u00fa"+ - "\u0003*\u0015\u0000\u00fa\u00fb\u0005!\u0000\u0000\u00fb\u00fc\u0003\n"+ - "\u0005\u0000\u00fc\u00fe\u0001\u0000\u0000\u0000\u00fd\u00f8\u0001\u0000"+ - "\u0000\u0000\u00fd\u00f9\u0001\u0000\u0000\u0000\u00fe\u001b\u0001\u0000"+ - "\u0000\u0000\u00ff\u0100\u0005\u0006\u0000\u0000\u0100\u0105\u0003(\u0014"+ - "\u0000\u0101\u0102\u0005\"\u0000\u0000\u0102\u0104\u0003(\u0014\u0000"+ - "\u0103\u0101\u0001\u0000\u0000\u0000\u0104\u0107\u0001\u0000\u0000\u0000"+ - "\u0105\u0103\u0001\u0000\u0000\u0000\u0105\u0106\u0001\u0000\u0000\u0000"+ - "\u0106\u0109\u0001\u0000\u0000\u0000\u0107\u0105\u0001\u0000\u0000\u0000"+ - "\u0108\u010a\u0003\u001e\u000f\u0000\u0109\u0108\u0001\u0000\u0000\u0000"+ - "\u0109\u010a\u0001\u0000\u0000\u0000\u010a\u001d\u0001\u0000\u0000\u0000"+ - "\u010b\u010c\u0005A\u0000\u0000\u010c\u010d\u0005I\u0000\u0000\u010d\u0112"+ - "\u0003(\u0014\u0000\u010e\u010f\u0005\"\u0000\u0000\u010f\u0111\u0003"+ - "(\u0014\u0000\u0110\u010e\u0001\u0000\u0000\u0000\u0111\u0114\u0001\u0000"+ - "\u0000\u0000\u0112\u0110\u0001\u0000\u0000\u0000\u0112\u0113\u0001\u0000"+ - "\u0000\u0000\u0113\u0115\u0001\u0000\u0000\u0000\u0114\u0112\u0001\u0000"+ - "\u0000\u0000\u0115\u0116\u0005B\u0000\u0000\u0116\u001f\u0001\u0000\u0000"+ - "\u0000\u0117\u0118\u0005\u0004\u0000\u0000\u0118\u0119\u0003\u0018\f\u0000"+ - "\u0119!\u0001\u0000\u0000\u0000\u011a\u011c\u0005\u0011\u0000\u0000\u011b"+ - "\u011d\u0003\u0018\f\u0000\u011c\u011b\u0001\u0000\u0000\u0000\u011c\u011d"+ - "\u0001\u0000\u0000\u0000\u011d\u0120\u0001\u0000\u0000\u0000\u011e\u011f"+ - "\u0005\u001e\u0000\u0000\u011f\u0121\u0003&\u0013\u0000\u0120\u011e\u0001"+ - "\u0000\u0000\u0000\u0120\u0121\u0001\u0000\u0000\u0000\u0121#\u0001\u0000"+ - "\u0000\u0000\u0122\u0123\u0005\b\u0000\u0000\u0123\u0126\u0003\u0018\f"+ - "\u0000\u0124\u0125\u0005\u001e\u0000\u0000\u0125\u0127\u0003&\u0013\u0000"+ - "\u0126\u0124\u0001\u0000\u0000\u0000\u0126\u0127\u0001\u0000\u0000\u0000"+ - "\u0127%\u0001\u0000\u0000\u0000\u0128\u012d\u0003*\u0015\u0000\u0129\u012a"+ - "\u0005\"\u0000\u0000\u012a\u012c\u0003*\u0015\u0000\u012b\u0129\u0001"+ - "\u0000\u0000\u0000\u012c\u012f\u0001\u0000\u0000\u0000\u012d\u012b\u0001"+ - "\u0000\u0000\u0000\u012d\u012e\u0001\u0000\u0000\u0000\u012e\'\u0001\u0000"+ - "\u0000\u0000\u012f\u012d\u0001\u0000\u0000\u0000\u0130\u0131\u0007\u0002"+ - "\u0000\u0000\u0131)\u0001\u0000\u0000\u0000\u0132\u0137\u0003,\u0016\u0000"+ - "\u0133\u0134\u0005$\u0000\u0000\u0134\u0136\u0003,\u0016\u0000\u0135\u0133"+ - "\u0001\u0000\u0000\u0000\u0136\u0139\u0001\u0000\u0000\u0000\u0137\u0135"+ - "\u0001\u0000\u0000\u0000\u0137\u0138\u0001\u0000\u0000\u0000\u0138+\u0001"+ - "\u0000\u0000\u0000\u0139\u0137\u0001\u0000\u0000\u0000\u013a\u013b\u0007"+ - "\u0003\u0000\u0000\u013b-\u0001\u0000\u0000\u0000\u013c\u0167\u0005-\u0000"+ - "\u0000\u013d\u013e\u0003N\'\u0000\u013e\u013f\u0005C\u0000\u0000\u013f"+ - "\u0167\u0001\u0000\u0000\u0000\u0140\u0167\u0003L&\u0000\u0141\u0167\u0003"+ - "N\'\u0000\u0142\u0167\u0003H$\u0000\u0143\u0167\u00050\u0000\u0000\u0144"+ - "\u0167\u0003P(\u0000\u0145\u0146\u0005A\u0000\u0000\u0146\u014b\u0003"+ - "J%\u0000\u0147\u0148\u0005\"\u0000\u0000\u0148\u014a\u0003J%\u0000\u0149"+ - "\u0147\u0001\u0000\u0000\u0000\u014a\u014d\u0001\u0000\u0000\u0000\u014b"+ - "\u0149\u0001\u0000\u0000\u0000\u014b\u014c\u0001\u0000\u0000\u0000\u014c"+ - "\u014e\u0001\u0000\u0000\u0000\u014d\u014b\u0001\u0000\u0000\u0000\u014e"+ - "\u014f\u0005B\u0000\u0000\u014f\u0167\u0001\u0000\u0000\u0000\u0150\u0151"+ - "\u0005A\u0000\u0000\u0151\u0156\u0003H$\u0000\u0152\u0153\u0005\"\u0000"+ - "\u0000\u0153\u0155\u0003H$\u0000\u0154\u0152\u0001\u0000\u0000\u0000\u0155"+ - "\u0158\u0001\u0000\u0000\u0000\u0156\u0154\u0001\u0000\u0000\u0000\u0156"+ - "\u0157\u0001\u0000\u0000\u0000\u0157\u0159\u0001\u0000\u0000\u0000\u0158"+ - "\u0156\u0001\u0000\u0000\u0000\u0159\u015a\u0005B\u0000\u0000\u015a\u0167"+ - "\u0001\u0000\u0000\u0000\u015b\u015c\u0005A\u0000\u0000\u015c\u0161\u0003"+ - "P(\u0000\u015d\u015e\u0005\"\u0000\u0000\u015e\u0160\u0003P(\u0000\u015f"+ - "\u015d\u0001\u0000\u0000\u0000\u0160\u0163\u0001\u0000\u0000\u0000\u0161"+ - "\u015f\u0001\u0000\u0000\u0000\u0161\u0162\u0001\u0000\u0000\u0000\u0162"+ - "\u0164\u0001\u0000\u0000\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0164"+ - "\u0165\u0005B\u0000\u0000\u0165\u0167\u0001\u0000\u0000\u0000\u0166\u013c"+ - "\u0001\u0000\u0000\u0000\u0166\u013d\u0001\u0000\u0000\u0000\u0166\u0140"+ - "\u0001\u0000\u0000\u0000\u0166\u0141\u0001\u0000\u0000\u0000\u0166\u0142"+ - "\u0001\u0000\u0000\u0000\u0166\u0143\u0001\u0000\u0000\u0000\u0166\u0144"+ - "\u0001\u0000\u0000\u0000\u0166\u0145\u0001\u0000\u0000\u0000\u0166\u0150"+ - "\u0001\u0000\u0000\u0000\u0166\u015b\u0001\u0000\u0000\u0000\u0167/\u0001"+ - "\u0000\u0000\u0000\u0168\u0169\u0005\n\u0000\u0000\u0169\u016a\u0005\u001c"+ - "\u0000\u0000\u016a1\u0001\u0000\u0000\u0000\u016b\u016c\u0005\u0010\u0000"+ - "\u0000\u016c\u0171\u00034\u001a\u0000\u016d\u016e\u0005\"\u0000\u0000"+ - "\u016e\u0170\u00034\u001a\u0000\u016f\u016d\u0001\u0000\u0000\u0000\u0170"+ - "\u0173\u0001\u0000\u0000\u0000\u0171\u016f\u0001\u0000\u0000\u0000\u0171"+ - "\u0172\u0001\u0000\u0000\u0000\u01723\u0001\u0000\u0000\u0000\u0173\u0171"+ - "\u0001\u0000\u0000\u0000\u0174\u0176\u0003\n\u0005\u0000\u0175\u0177\u0007"+ - "\u0004\u0000\u0000\u0176\u0175\u0001\u0000\u0000\u0000\u0176\u0177\u0001"+ - "\u0000\u0000\u0000\u0177\u017a\u0001\u0000\u0000\u0000\u0178\u0179\u0005"+ - ".\u0000\u0000\u0179\u017b\u0007\u0005\u0000\u0000\u017a\u0178\u0001\u0000"+ - "\u0000\u0000\u017a\u017b\u0001\u0000\u0000\u0000\u017b5\u0001\u0000\u0000"+ - "\u0000\u017c\u017d\u0005\t\u0000\u0000\u017d\u0182\u0003(\u0014\u0000"+ - "\u017e\u017f\u0005\"\u0000\u0000\u017f\u0181\u0003(\u0014\u0000\u0180"+ - "\u017e\u0001\u0000\u0000\u0000\u0181\u0184\u0001\u0000\u0000\u0000\u0182"+ - "\u0180\u0001\u0000\u0000\u0000\u0182\u0183\u0001\u0000\u0000\u0000\u0183"+ - "\u018f\u0001\u0000\u0000\u0000\u0184\u0182\u0001\u0000\u0000\u0000\u0185"+ - "\u0186\u0005\f\u0000\u0000\u0186\u018b\u0003(\u0014\u0000\u0187\u0188"+ - "\u0005\"\u0000\u0000\u0188\u018a\u0003(\u0014\u0000\u0189\u0187\u0001"+ - "\u0000\u0000\u0000\u018a\u018d\u0001\u0000\u0000\u0000\u018b\u0189\u0001"+ - "\u0000\u0000\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c\u018f\u0001"+ - "\u0000\u0000\u0000\u018d\u018b\u0001\u0000\u0000\u0000\u018e\u017c\u0001"+ - "\u0000\u0000\u0000\u018e\u0185\u0001\u0000\u0000\u0000\u018f7\u0001\u0000"+ - "\u0000\u0000\u0190\u0191\u0005\u0002\u0000\u0000\u0191\u0196\u0003(\u0014"+ - "\u0000\u0192\u0193\u0005\"\u0000\u0000\u0193\u0195\u0003(\u0014\u0000"+ - "\u0194\u0192\u0001\u0000\u0000\u0000\u0195\u0198\u0001\u0000\u0000\u0000"+ - "\u0196\u0194\u0001\u0000\u0000\u0000\u0196\u0197\u0001\u0000\u0000\u0000"+ - "\u01979\u0001\u0000\u0000\u0000\u0198\u0196\u0001\u0000\u0000\u0000\u0199"+ - "\u019a\u0005\r\u0000\u0000\u019a\u019f\u0003<\u001e\u0000\u019b\u019c"+ - "\u0005\"\u0000\u0000\u019c\u019e\u0003<\u001e\u0000\u019d\u019b\u0001"+ - "\u0000\u0000\u0000\u019e\u01a1\u0001\u0000\u0000\u0000\u019f\u019d\u0001"+ - "\u0000\u0000\u0000\u019f\u01a0\u0001\u0000\u0000\u0000\u01a0;\u0001\u0000"+ - "\u0000\u0000\u01a1\u019f\u0001\u0000\u0000\u0000\u01a2\u01a3\u0003(\u0014"+ - "\u0000\u01a3\u01a4\u0005H\u0000\u0000\u01a4\u01a5\u0003(\u0014\u0000\u01a5"+ - "=\u0001\u0000\u0000\u0000\u01a6\u01a7\u0005\u0001\u0000\u0000\u01a7\u01a8"+ - "\u0003\u0012\t\u0000\u01a8\u01aa\u0003P(\u0000\u01a9\u01ab\u0003D\"\u0000"+ - "\u01aa\u01a9\u0001\u0000\u0000\u0000\u01aa\u01ab\u0001\u0000\u0000\u0000"+ - "\u01ab?\u0001\u0000\u0000\u0000\u01ac\u01ad\u0005\u0007\u0000\u0000\u01ad"+ - "\u01ae\u0003\u0012\t\u0000\u01ae\u01af\u0003P(\u0000\u01afA\u0001\u0000"+ - "\u0000\u0000\u01b0\u01b1\u0005\u000b\u0000\u0000\u01b1\u01b2\u0003(\u0014"+ - "\u0000\u01b2C\u0001\u0000\u0000\u0000\u01b3\u01b8\u0003F#\u0000\u01b4"+ - "\u01b5\u0005\"\u0000\u0000\u01b5\u01b7\u0003F#\u0000\u01b6\u01b4\u0001"+ - "\u0000\u0000\u0000\u01b7\u01ba\u0001\u0000\u0000\u0000\u01b8\u01b6\u0001"+ - "\u0000\u0000\u0000\u01b8\u01b9\u0001\u0000\u0000\u0000\u01b9E\u0001\u0000"+ - "\u0000\u0000\u01ba\u01b8\u0001\u0000\u0000\u0000\u01bb\u01bc\u0003,\u0016"+ - "\u0000\u01bc\u01bd\u0005!\u0000\u0000\u01bd\u01be\u0003.\u0017\u0000\u01be"+ - "G\u0001\u0000\u0000\u0000\u01bf\u01c0\u0007\u0006\u0000\u0000\u01c0I\u0001"+ - "\u0000\u0000\u0000\u01c1\u01c4\u0003L&\u0000\u01c2\u01c4\u0003N\'\u0000"+ - "\u01c3\u01c1\u0001\u0000\u0000\u0000\u01c3\u01c2\u0001\u0000\u0000\u0000"+ - "\u01c4K\u0001\u0000\u0000\u0000\u01c5\u01c7\u0007\u0000\u0000\u0000\u01c6"+ - "\u01c5\u0001\u0000\u0000\u0000\u01c6\u01c7\u0001\u0000\u0000\u0000\u01c7"+ - "\u01c8\u0001\u0000\u0000\u0000\u01c8\u01c9\u0005\u001d\u0000\u0000\u01c9"+ - "M\u0001\u0000\u0000\u0000\u01ca\u01cc\u0007\u0000\u0000\u0000\u01cb\u01ca"+ - "\u0001\u0000\u0000\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc\u01cd"+ - "\u0001\u0000\u0000\u0000\u01cd\u01ce\u0005\u001c\u0000\u0000\u01ceO\u0001"+ - "\u0000\u0000\u0000\u01cf\u01d0\u0005\u001b\u0000\u0000\u01d0Q\u0001\u0000"+ - "\u0000\u0000\u01d1\u01d2\u0007\u0007\u0000\u0000\u01d2S\u0001\u0000\u0000"+ - "\u0000\u01d3\u01d4\u0005\u0005\u0000\u0000\u01d4\u01d5\u0003V+\u0000\u01d5"+ - "U\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005A\u0000\u0000\u01d7\u01d8\u0003"+ - "\u0002\u0001\u0000\u01d8\u01d9\u0005B\u0000\u0000\u01d9W\u0001\u0000\u0000"+ - "\u0000\u01da\u01db\u0005\u000f\u0000\u0000\u01db\u01df\u00054\u0000\u0000"+ - "\u01dc\u01dd\u0005\u000f\u0000\u0000\u01dd\u01df\u00055\u0000\u0000\u01de"+ - "\u01da\u0001\u0000\u0000\u0000\u01de\u01dc\u0001\u0000\u0000\u0000\u01df"+ - "Y\u0001\u0000\u0000\u0000\u01e0\u01e1\u0005\u0003\u0000\u0000\u01e1\u01e4"+ - "\u0003(\u0014\u0000\u01e2\u01e3\u0005J\u0000\u0000\u01e3\u01e5\u0003("+ - "\u0014\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e4\u01e5\u0001\u0000"+ - "\u0000\u0000\u01e5\u01ef\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005K\u0000"+ - "\u0000\u01e7\u01ec\u0003\\.\u0000\u01e8\u01e9\u0005\"\u0000\u0000\u01e9"+ - "\u01eb\u0003\\.\u0000\u01ea\u01e8\u0001\u0000\u0000\u0000\u01eb\u01ee"+ - "\u0001\u0000\u0000\u0000\u01ec\u01ea\u0001\u0000\u0000\u0000\u01ec\u01ed"+ - "\u0001\u0000\u0000\u0000\u01ed\u01f0\u0001\u0000\u0000\u0000\u01ee\u01ec"+ - "\u0001\u0000\u0000\u0000\u01ef\u01e6\u0001\u0000\u0000\u0000\u01ef\u01f0"+ - "\u0001\u0000\u0000\u0000\u01f0[\u0001\u0000\u0000\u0000\u01f1\u01f2\u0003"+ - "(\u0014\u0000\u01f2\u01f3\u0005!\u0000\u0000\u01f3\u01f5\u0001\u0000\u0000"+ - "\u0000\u01f4\u01f1\u0001\u0000\u0000\u0000\u01f4\u01f5\u0001\u0000\u0000"+ - "\u0000\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0003(\u0014\u0000"+ - "\u01f7]\u0001\u0000\u0000\u00003ip\u007f\u008b\u0094\u009c\u00a0\u00a8"+ - "\u00aa\u00af\u00b6\u00bb\u00c2\u00c8\u00d0\u00d2\u00dc\u00e6\u00e9\u00f5"+ - "\u00fd\u0105\u0109\u0112\u011c\u0120\u0126\u012d\u0137\u014b\u0156\u0161"+ - "\u0166\u0171\u0176\u017a\u0182\u018b\u018e\u0196\u019f\u01aa\u01b8\u01c3"+ - "\u01c6\u01cb\u01de\u01e4\u01ec\u01ef\u01f4"; + "\u00ad\b\u0005\n\u0005\f\u0005\u00b0\t\u0005\u0001\u0006\u0001\u0006\u0003"+ + "\u0006\u00b4\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0003\u0006\u00bb\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003"+ + "\u0006\u00c0\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0003\u0007\u00c7\b\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0003"+ + "\b\u00cd\b\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0005\b\u00d5"+ + "\b\b\n\b\f\b\u00d8\t\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t"+ + "\u0001\t\u0003\t\u00e1\b\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0005\n\u00e9\b\n\n\n\f\n\u00ec\t\n\u0003\n\u00ee\b\n\u0001\n\u0001"+ + "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0005\f"+ + "\u00f8\b\f\n\f\f\f\u00fb\t\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0003"+ + "\r\u0102\b\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0005\u000e"+ + "\u0108\b\u000e\n\u000e\f\u000e\u010b\t\u000e\u0001\u000e\u0003\u000e\u010e"+ + "\b\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0005"+ + "\u000f\u0115\b\u000f\n\u000f\f\u000f\u0118\t\u000f\u0001\u000f\u0001\u000f"+ + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0003\u0011"+ + "\u0121\b\u0011\u0001\u0011\u0001\u0011\u0003\u0011\u0125\b\u0011\u0001"+ + "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u012b\b\u0012\u0001"+ + "\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u0130\b\u0013\n\u0013\f\u0013"+ + "\u0133\t\u0013\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015"+ + "\u0005\u0015\u013a\b\u0015\n\u0015\f\u0015\u013d\t\u0015\u0001\u0016\u0001"+ + "\u0016\u0001\u0016\u0005\u0016\u0142\b\u0016\n\u0016\f\u0016\u0145\t\u0016"+ + "\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019"+ + "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019"+ + "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0005\u0019"+ + "\u0158\b\u0019\n\u0019\f\u0019\u015b\t\u0019\u0001\u0019\u0001\u0019\u0001"+ + "\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0005\u0019\u0163\b\u0019\n"+ + "\u0019\f\u0019\u0166\t\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001"+ + "\u0019\u0001\u0019\u0001\u0019\u0005\u0019\u016e\b\u0019\n\u0019\f\u0019"+ + "\u0171\t\u0019\u0001\u0019\u0001\u0019\u0003\u0019\u0175\b\u0019\u0001"+ + "\u001a\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001"+ + "\u001b\u0005\u001b\u017e\b\u001b\n\u001b\f\u001b\u0181\t\u001b\u0001\u001c"+ + "\u0001\u001c\u0003\u001c\u0185\b\u001c\u0001\u001c\u0001\u001c\u0003\u001c"+ + "\u0189\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d"+ + "\u018f\b\u001d\n\u001d\f\u001d\u0192\t\u001d\u0001\u001d\u0001\u001d\u0001"+ + "\u001d\u0001\u001d\u0005\u001d\u0198\b\u001d\n\u001d\f\u001d\u019b\t\u001d"+ + "\u0003\u001d\u019d\b\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e"+ + "\u0005\u001e\u01a3\b\u001e\n\u001e\f\u001e\u01a6\t\u001e\u0001\u001f\u0001"+ + "\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01ac\b\u001f\n\u001f\f\u001f"+ + "\u01af\t\u001f\u0001 \u0001 \u0001 \u0001 \u0001!\u0001!\u0001!\u0001"+ + "!\u0003!\u01b9\b!\u0001\"\u0001\"\u0001\"\u0001\"\u0001#\u0001#\u0001"+ + "#\u0001$\u0001$\u0001$\u0005$\u01c5\b$\n$\f$\u01c8\t$\u0001%\u0001%\u0001"+ + "%\u0001%\u0001&\u0001&\u0001\'\u0001\'\u0003\'\u01d2\b\'\u0001(\u0003"+ + "(\u01d5\b(\u0001(\u0001(\u0001)\u0003)\u01da\b)\u0001)\u0001)\u0001*\u0001"+ + "*\u0001+\u0001+\u0001,\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001"+ + ".\u0001.\u0001.\u0001.\u0003.\u01ed\b.\u0001/\u0001/\u0001/\u0001/\u0003"+ + "/\u01f3\b/\u0001/\u0001/\u0001/\u0001/\u0005/\u01f9\b/\n/\f/\u01fc\t/"+ + "\u0003/\u01fe\b/\u00010\u00010\u00010\u00030\u0203\b0\u00010\u00010\u0001"+ + "0\u0000\u0003\u0002\n\u00101\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ + "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ + "TVXZ\\^`\u0000\t\u0001\u0000:;\u0001\u0000<>\u0002\u0000BBGG\u0001\u0000"+ + "AB\u0002\u0000BBKK\u0002\u0000 ##\u0001\u0000&\'\u0002\u0000%%33\u0001"+ + "\u000049\u0224\u0000b\u0001\u0000\u0000\u0000\u0002e\u0001\u0000\u0000"+ + "\u0000\u0004t\u0001\u0000\u0000\u0000\u0006\u0083\u0001\u0000\u0000\u0000"+ + "\b\u0085\u0001\u0000\u0000\u0000\n\u00a4\u0001\u0000\u0000\u0000\f\u00bf"+ + "\u0001\u0000\u0000\u0000\u000e\u00c6\u0001\u0000\u0000\u0000\u0010\u00cc"+ + "\u0001\u0000\u0000\u0000\u0012\u00e0\u0001\u0000\u0000\u0000\u0014\u00e2"+ + "\u0001\u0000\u0000\u0000\u0016\u00f1\u0001\u0000\u0000\u0000\u0018\u00f4"+ + "\u0001\u0000\u0000\u0000\u001a\u0101\u0001\u0000\u0000\u0000\u001c\u0103"+ + "\u0001\u0000\u0000\u0000\u001e\u010f\u0001\u0000\u0000\u0000 \u011b\u0001"+ + "\u0000\u0000\u0000\"\u011e\u0001\u0000\u0000\u0000$\u0126\u0001\u0000"+ + "\u0000\u0000&\u012c\u0001\u0000\u0000\u0000(\u0134\u0001\u0000\u0000\u0000"+ + "*\u0136\u0001\u0000\u0000\u0000,\u013e\u0001\u0000\u0000\u0000.\u0146"+ + "\u0001\u0000\u0000\u00000\u0148\u0001\u0000\u0000\u00002\u0174\u0001\u0000"+ + "\u0000\u00004\u0176\u0001\u0000\u0000\u00006\u0179\u0001\u0000\u0000\u0000"+ + "8\u0182\u0001\u0000\u0000\u0000:\u019c\u0001\u0000\u0000\u0000<\u019e"+ + "\u0001\u0000\u0000\u0000>\u01a7\u0001\u0000\u0000\u0000@\u01b0\u0001\u0000"+ + "\u0000\u0000B\u01b4\u0001\u0000\u0000\u0000D\u01ba\u0001\u0000\u0000\u0000"+ + "F\u01be\u0001\u0000\u0000\u0000H\u01c1\u0001\u0000\u0000\u0000J\u01c9"+ + "\u0001\u0000\u0000\u0000L\u01cd\u0001\u0000\u0000\u0000N\u01d1\u0001\u0000"+ + "\u0000\u0000P\u01d4\u0001\u0000\u0000\u0000R\u01d9\u0001\u0000\u0000\u0000"+ + "T\u01dd\u0001\u0000\u0000\u0000V\u01df\u0001\u0000\u0000\u0000X\u01e1"+ + "\u0001\u0000\u0000\u0000Z\u01e4\u0001\u0000\u0000\u0000\\\u01ec\u0001"+ + "\u0000\u0000\u0000^\u01ee\u0001\u0000\u0000\u0000`\u0202\u0001\u0000\u0000"+ + "\u0000bc\u0003\u0002\u0001\u0000cd\u0005\u0000\u0000\u0001d\u0001\u0001"+ + "\u0000\u0000\u0000ef\u0006\u0001\uffff\uffff\u0000fg\u0003\u0004\u0002"+ + "\u0000gm\u0001\u0000\u0000\u0000hi\n\u0001\u0000\u0000ij\u0005\u001a\u0000"+ + "\u0000jl\u0003\u0006\u0003\u0000kh\u0001\u0000\u0000\u0000lo\u0001\u0000"+ + "\u0000\u0000mk\u0001\u0000\u0000\u0000mn\u0001\u0000\u0000\u0000n\u0003"+ + "\u0001\u0000\u0000\u0000om\u0001\u0000\u0000\u0000pu\u0003X,\u0000qu\u0003"+ + "\u001c\u000e\u0000ru\u0003\u0016\u000b\u0000su\u0003\\.\u0000tp\u0001"+ + "\u0000\u0000\u0000tq\u0001\u0000\u0000\u0000tr\u0001\u0000\u0000\u0000"+ + "ts\u0001\u0000\u0000\u0000u\u0005\u0001\u0000\u0000\u0000v\u0084\u0003"+ + " \u0010\u0000w\u0084\u0003$\u0012\u0000x\u0084\u00034\u001a\u0000y\u0084"+ + "\u0003:\u001d\u0000z\u0084\u00036\u001b\u0000{\u0084\u0003\"\u0011\u0000"+ + "|\u0084\u0003\b\u0004\u0000}\u0084\u0003<\u001e\u0000~\u0084\u0003>\u001f"+ + "\u0000\u007f\u0084\u0003B!\u0000\u0080\u0084\u0003D\"\u0000\u0081\u0084"+ + "\u0003^/\u0000\u0082\u0084\u0003F#\u0000\u0083v\u0001\u0000\u0000\u0000"+ + "\u0083w\u0001\u0000\u0000\u0000\u0083x\u0001\u0000\u0000\u0000\u0083y"+ + "\u0001\u0000\u0000\u0000\u0083z\u0001\u0000\u0000\u0000\u0083{\u0001\u0000"+ + "\u0000\u0000\u0083|\u0001\u0000\u0000\u0000\u0083}\u0001\u0000\u0000\u0000"+ + "\u0083~\u0001\u0000\u0000\u0000\u0083\u007f\u0001\u0000\u0000\u0000\u0083"+ + "\u0080\u0001\u0000\u0000\u0000\u0083\u0081\u0001\u0000\u0000\u0000\u0083"+ + "\u0082\u0001\u0000\u0000\u0000\u0084\u0007\u0001\u0000\u0000\u0000\u0085"+ + "\u0086\u0005\u0012\u0000\u0000\u0086\u0087\u0003\n\u0005\u0000\u0087\t"+ + "\u0001\u0000\u0000\u0000\u0088\u0089\u0006\u0005\uffff\uffff\u0000\u0089"+ + "\u008a\u0005,\u0000\u0000\u008a\u00a5\u0003\n\u0005\u0007\u008b\u00a5"+ + "\u0003\u000e\u0007\u0000\u008c\u00a5\u0003\f\u0006\u0000\u008d\u008f\u0003"+ + "\u000e\u0007\u0000\u008e\u0090\u0005,\u0000\u0000\u008f\u008e\u0001\u0000"+ + "\u0000\u0000\u008f\u0090\u0001\u0000\u0000\u0000\u0090\u0091\u0001\u0000"+ + "\u0000\u0000\u0091\u0092\u0005)\u0000\u0000\u0092\u0093\u0005(\u0000\u0000"+ + "\u0093\u0098\u0003\u000e\u0007\u0000\u0094\u0095\u0005\"\u0000\u0000\u0095"+ + "\u0097\u0003\u000e\u0007\u0000\u0096\u0094\u0001\u0000\u0000\u0000\u0097"+ + "\u009a\u0001\u0000\u0000\u0000\u0098\u0096\u0001\u0000\u0000\u0000\u0098"+ + "\u0099\u0001\u0000\u0000\u0000\u0099\u009b\u0001\u0000\u0000\u0000\u009a"+ + "\u0098\u0001\u0000\u0000\u0000\u009b\u009c\u00052\u0000\u0000\u009c\u00a5"+ + "\u0001\u0000\u0000\u0000\u009d\u009e\u0003\u000e\u0007\u0000\u009e\u00a0"+ + "\u0005*\u0000\u0000\u009f\u00a1\u0005,\u0000\u0000\u00a0\u009f\u0001\u0000"+ + "\u0000\u0000\u00a0\u00a1\u0001\u0000\u0000\u0000\u00a1\u00a2\u0001\u0000"+ + "\u0000\u0000\u00a2\u00a3\u0005-\u0000\u0000\u00a3\u00a5\u0001\u0000\u0000"+ + "\u0000\u00a4\u0088\u0001\u0000\u0000\u0000\u00a4\u008b\u0001\u0000\u0000"+ + "\u0000\u00a4\u008c\u0001\u0000\u0000\u0000\u00a4\u008d\u0001\u0000\u0000"+ + "\u0000\u00a4\u009d\u0001\u0000\u0000\u0000\u00a5\u00ae\u0001\u0000\u0000"+ + "\u0000\u00a6\u00a7\n\u0004\u0000\u0000\u00a7\u00a8\u0005\u001f\u0000\u0000"+ + "\u00a8\u00ad\u0003\n\u0005\u0005\u00a9\u00aa\n\u0003\u0000\u0000\u00aa"+ + "\u00ab\u0005/\u0000\u0000\u00ab\u00ad\u0003\n\u0005\u0004\u00ac\u00a6"+ + "\u0001\u0000\u0000\u0000\u00ac\u00a9\u0001\u0000\u0000\u0000\u00ad\u00b0"+ + "\u0001\u0000\u0000\u0000\u00ae\u00ac\u0001\u0000\u0000\u0000\u00ae\u00af"+ + "\u0001\u0000\u0000\u0000\u00af\u000b\u0001\u0000\u0000\u0000\u00b0\u00ae"+ + "\u0001\u0000\u0000\u0000\u00b1\u00b3\u0003\u000e\u0007\u0000\u00b2\u00b4"+ + "\u0005,\u0000\u0000\u00b3\u00b2\u0001\u0000\u0000\u0000\u00b3\u00b4\u0001"+ + "\u0000\u0000\u0000\u00b4\u00b5\u0001\u0000\u0000\u0000\u00b5\u00b6\u0005"+ + "+\u0000\u0000\u00b6\u00b7\u0003T*\u0000\u00b7\u00c0\u0001\u0000\u0000"+ + "\u0000\u00b8\u00ba\u0003\u000e\u0007\u0000\u00b9\u00bb\u0005,\u0000\u0000"+ + "\u00ba\u00b9\u0001\u0000\u0000\u0000\u00ba\u00bb\u0001\u0000\u0000\u0000"+ + "\u00bb\u00bc\u0001\u0000\u0000\u0000\u00bc\u00bd\u00051\u0000\u0000\u00bd"+ + "\u00be\u0003T*\u0000\u00be\u00c0\u0001\u0000\u0000\u0000\u00bf\u00b1\u0001"+ + "\u0000\u0000\u0000\u00bf\u00b8\u0001\u0000\u0000\u0000\u00c0\r\u0001\u0000"+ + "\u0000\u0000\u00c1\u00c7\u0003\u0010\b\u0000\u00c2\u00c3\u0003\u0010\b"+ + "\u0000\u00c3\u00c4\u0003V+\u0000\u00c4\u00c5\u0003\u0010\b\u0000\u00c5"+ + "\u00c7\u0001\u0000\u0000\u0000\u00c6\u00c1\u0001\u0000\u0000\u0000\u00c6"+ + "\u00c2\u0001\u0000\u0000\u0000\u00c7\u000f\u0001\u0000\u0000\u0000\u00c8"+ + "\u00c9\u0006\b\uffff\uffff\u0000\u00c9\u00cd\u0003\u0012\t\u0000\u00ca"+ + "\u00cb\u0007\u0000\u0000\u0000\u00cb\u00cd\u0003\u0010\b\u0003\u00cc\u00c8"+ + "\u0001\u0000\u0000\u0000\u00cc\u00ca\u0001\u0000\u0000\u0000\u00cd\u00d6"+ + "\u0001\u0000\u0000\u0000\u00ce\u00cf\n\u0002\u0000\u0000\u00cf\u00d0\u0007"+ + "\u0001\u0000\u0000\u00d0\u00d5\u0003\u0010\b\u0003\u00d1\u00d2\n\u0001"+ + "\u0000\u0000\u00d2\u00d3\u0007\u0000\u0000\u0000\u00d3\u00d5\u0003\u0010"+ + "\b\u0002\u00d4\u00ce\u0001\u0000\u0000\u0000\u00d4\u00d1\u0001\u0000\u0000"+ + "\u0000\u00d5\u00d8\u0001\u0000\u0000\u0000\u00d6\u00d4\u0001\u0000\u0000"+ + "\u0000\u00d6\u00d7\u0001\u0000\u0000\u0000\u00d7\u0011\u0001\u0000\u0000"+ + "\u0000\u00d8\u00d6\u0001\u0000\u0000\u0000\u00d9\u00e1\u00032\u0019\u0000"+ + "\u00da\u00e1\u0003*\u0015\u0000\u00db\u00e1\u0003\u0014\n\u0000\u00dc"+ + "\u00dd\u0005(\u0000\u0000\u00dd\u00de\u0003\n\u0005\u0000\u00de\u00df"+ + "\u00052\u0000\u0000\u00df\u00e1\u0001\u0000\u0000\u0000\u00e0\u00d9\u0001"+ + "\u0000\u0000\u0000\u00e0\u00da\u0001\u0000\u0000\u0000\u00e0\u00db\u0001"+ + "\u0000\u0000\u0000\u00e0\u00dc\u0001\u0000\u0000\u0000\u00e1\u0013\u0001"+ + "\u0000\u0000\u0000\u00e2\u00e3\u0003.\u0017\u0000\u00e3\u00ed\u0005(\u0000"+ + "\u0000\u00e4\u00ee\u0005<\u0000\u0000\u00e5\u00ea\u0003\n\u0005\u0000"+ + "\u00e6\u00e7\u0005\"\u0000\u0000\u00e7\u00e9\u0003\n\u0005\u0000\u00e8"+ + "\u00e6\u0001\u0000\u0000\u0000\u00e9\u00ec\u0001\u0000\u0000\u0000\u00ea"+ + "\u00e8\u0001\u0000\u0000\u0000\u00ea\u00eb\u0001\u0000\u0000\u0000\u00eb"+ + "\u00ee\u0001\u0000\u0000\u0000\u00ec\u00ea\u0001\u0000\u0000\u0000\u00ed"+ + "\u00e4\u0001\u0000\u0000\u0000\u00ed\u00e5\u0001\u0000\u0000\u0000\u00ed"+ + "\u00ee\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001\u0000\u0000\u0000\u00ef"+ + "\u00f0\u00052\u0000\u0000\u00f0\u0015\u0001\u0000\u0000\u0000\u00f1\u00f2"+ + "\u0005\u000e\u0000\u0000\u00f2\u00f3\u0003\u0018\f\u0000\u00f3\u0017\u0001"+ + "\u0000\u0000\u0000\u00f4\u00f9\u0003\u001a\r\u0000\u00f5\u00f6\u0005\""+ + "\u0000\u0000\u00f6\u00f8\u0003\u001a\r\u0000\u00f7\u00f5\u0001\u0000\u0000"+ + "\u0000\u00f8\u00fb\u0001\u0000\u0000\u0000\u00f9\u00f7\u0001\u0000\u0000"+ + "\u0000\u00f9\u00fa\u0001\u0000\u0000\u0000\u00fa\u0019\u0001\u0000\u0000"+ + "\u0000\u00fb\u00f9\u0001\u0000\u0000\u0000\u00fc\u0102\u0003\n\u0005\u0000"+ + "\u00fd\u00fe\u0003*\u0015\u0000\u00fe\u00ff\u0005!\u0000\u0000\u00ff\u0100"+ + "\u0003\n\u0005\u0000\u0100\u0102\u0001\u0000\u0000\u0000\u0101\u00fc\u0001"+ + "\u0000\u0000\u0000\u0101\u00fd\u0001\u0000\u0000\u0000\u0102\u001b\u0001"+ + "\u0000\u0000\u0000\u0103\u0104\u0005\u0006\u0000\u0000\u0104\u0109\u0003"+ + "(\u0014\u0000\u0105\u0106\u0005\"\u0000\u0000\u0106\u0108\u0003(\u0014"+ + "\u0000\u0107\u0105\u0001\u0000\u0000\u0000\u0108\u010b\u0001\u0000\u0000"+ + "\u0000\u0109\u0107\u0001\u0000\u0000\u0000\u0109\u010a\u0001\u0000\u0000"+ + "\u0000\u010a\u010d\u0001\u0000\u0000\u0000\u010b\u0109\u0001\u0000\u0000"+ + "\u0000\u010c\u010e\u0003\u001e\u000f\u0000\u010d\u010c\u0001\u0000\u0000"+ + "\u0000\u010d\u010e\u0001\u0000\u0000\u0000\u010e\u001d\u0001\u0000\u0000"+ + "\u0000\u010f\u0110\u0005?\u0000\u0000\u0110\u0111\u0005F\u0000\u0000\u0111"+ + "\u0116\u0003(\u0014\u0000\u0112\u0113\u0005\"\u0000\u0000\u0113\u0115"+ + "\u0003(\u0014\u0000\u0114\u0112\u0001\u0000\u0000\u0000\u0115\u0118\u0001"+ + "\u0000\u0000\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0116\u0117\u0001"+ + "\u0000\u0000\u0000\u0117\u0119\u0001\u0000\u0000\u0000\u0118\u0116\u0001"+ + "\u0000\u0000\u0000\u0119\u011a\u0005@\u0000\u0000\u011a\u001f\u0001\u0000"+ + "\u0000\u0000\u011b\u011c\u0005\u0004\u0000\u0000\u011c\u011d\u0003\u0018"+ + "\f\u0000\u011d!\u0001\u0000\u0000\u0000\u011e\u0120\u0005\u0011\u0000"+ + "\u0000\u011f\u0121\u0003\u0018\f\u0000\u0120\u011f\u0001\u0000\u0000\u0000"+ + "\u0120\u0121\u0001\u0000\u0000\u0000\u0121\u0124\u0001\u0000\u0000\u0000"+ + "\u0122\u0123\u0005\u001e\u0000\u0000\u0123\u0125\u0003&\u0013\u0000\u0124"+ + "\u0122\u0001\u0000\u0000\u0000\u0124\u0125\u0001\u0000\u0000\u0000\u0125"+ + "#\u0001\u0000\u0000\u0000\u0126\u0127\u0005\b\u0000\u0000\u0127\u012a"+ + "\u0003\u0018\f\u0000\u0128\u0129\u0005\u001e\u0000\u0000\u0129\u012b\u0003"+ + "&\u0013\u0000\u012a\u0128\u0001\u0000\u0000\u0000\u012a\u012b\u0001\u0000"+ + "\u0000\u0000\u012b%\u0001\u0000\u0000\u0000\u012c\u0131\u0003*\u0015\u0000"+ + "\u012d\u012e\u0005\"\u0000\u0000\u012e\u0130\u0003*\u0015\u0000\u012f"+ + "\u012d\u0001\u0000\u0000\u0000\u0130\u0133\u0001\u0000\u0000\u0000\u0131"+ + "\u012f\u0001\u0000\u0000\u0000\u0131\u0132\u0001\u0000\u0000\u0000\u0132"+ + "\'\u0001\u0000\u0000\u0000\u0133\u0131\u0001\u0000\u0000\u0000\u0134\u0135"+ + "\u0007\u0002\u0000\u0000\u0135)\u0001\u0000\u0000\u0000\u0136\u013b\u0003"+ + ".\u0017\u0000\u0137\u0138\u0005$\u0000\u0000\u0138\u013a\u0003.\u0017"+ + "\u0000\u0139\u0137\u0001\u0000\u0000\u0000\u013a\u013d\u0001\u0000\u0000"+ + "\u0000\u013b\u0139\u0001\u0000\u0000\u0000\u013b\u013c\u0001\u0000\u0000"+ + "\u0000\u013c+\u0001\u0000\u0000\u0000\u013d\u013b\u0001\u0000\u0000\u0000"+ + "\u013e\u0143\u00030\u0018\u0000\u013f\u0140\u0005$\u0000\u0000\u0140\u0142"+ + "\u00030\u0018\u0000\u0141\u013f\u0001\u0000\u0000\u0000\u0142\u0145\u0001"+ + "\u0000\u0000\u0000\u0143\u0141\u0001\u0000\u0000\u0000\u0143\u0144\u0001"+ + "\u0000\u0000\u0000\u0144-\u0001\u0000\u0000\u0000\u0145\u0143\u0001\u0000"+ + "\u0000\u0000\u0146\u0147\u0007\u0003\u0000\u0000\u0147/\u0001\u0000\u0000"+ + "\u0000\u0148\u0149\u0007\u0004\u0000\u0000\u01491\u0001\u0000\u0000\u0000"+ + "\u014a\u0175\u0005-\u0000\u0000\u014b\u014c\u0003R)\u0000\u014c\u014d"+ + "\u0005A\u0000\u0000\u014d\u0175\u0001\u0000\u0000\u0000\u014e\u0175\u0003"+ + "P(\u0000\u014f\u0175\u0003R)\u0000\u0150\u0175\u0003L&\u0000\u0151\u0175"+ + "\u00050\u0000\u0000\u0152\u0175\u0003T*\u0000\u0153\u0154\u0005?\u0000"+ + "\u0000\u0154\u0159\u0003N\'\u0000\u0155\u0156\u0005\"\u0000\u0000\u0156"+ + "\u0158\u0003N\'\u0000\u0157\u0155\u0001\u0000\u0000\u0000\u0158\u015b"+ + "\u0001\u0000\u0000\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u0159\u015a"+ + "\u0001\u0000\u0000\u0000\u015a\u015c\u0001\u0000\u0000\u0000\u015b\u0159"+ + "\u0001\u0000\u0000\u0000\u015c\u015d\u0005@\u0000\u0000\u015d\u0175\u0001"+ + "\u0000\u0000\u0000\u015e\u015f\u0005?\u0000\u0000\u015f\u0164\u0003L&"+ + "\u0000\u0160\u0161\u0005\"\u0000\u0000\u0161\u0163\u0003L&\u0000\u0162"+ + "\u0160\u0001\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164"+ + "\u0162\u0001\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000\u0000\u0165"+ + "\u0167\u0001\u0000\u0000\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167"+ + "\u0168\u0005@\u0000\u0000\u0168\u0175\u0001\u0000\u0000\u0000\u0169\u016a"+ + "\u0005?\u0000\u0000\u016a\u016f\u0003T*\u0000\u016b\u016c\u0005\"\u0000"+ + "\u0000\u016c\u016e\u0003T*\u0000\u016d\u016b\u0001\u0000\u0000\u0000\u016e"+ + "\u0171\u0001\u0000\u0000\u0000\u016f\u016d\u0001\u0000\u0000\u0000\u016f"+ + "\u0170\u0001\u0000\u0000\u0000\u0170\u0172\u0001\u0000\u0000\u0000\u0171"+ + "\u016f\u0001\u0000\u0000\u0000\u0172\u0173\u0005@\u0000\u0000\u0173\u0175"+ + "\u0001\u0000\u0000\u0000\u0174\u014a\u0001\u0000\u0000\u0000\u0174\u014b"+ + "\u0001\u0000\u0000\u0000\u0174\u014e\u0001\u0000\u0000\u0000\u0174\u014f"+ + "\u0001\u0000\u0000\u0000\u0174\u0150\u0001\u0000\u0000\u0000\u0174\u0151"+ + "\u0001\u0000\u0000\u0000\u0174\u0152\u0001\u0000\u0000\u0000\u0174\u0153"+ + "\u0001\u0000\u0000\u0000\u0174\u015e\u0001\u0000\u0000\u0000\u0174\u0169"+ + "\u0001\u0000\u0000\u0000\u01753\u0001\u0000\u0000\u0000\u0176\u0177\u0005"+ + "\n\u0000\u0000\u0177\u0178\u0005\u001c\u0000\u0000\u01785\u0001\u0000"+ + "\u0000\u0000\u0179\u017a\u0005\u0010\u0000\u0000\u017a\u017f\u00038\u001c"+ + "\u0000\u017b\u017c\u0005\"\u0000\u0000\u017c\u017e\u00038\u001c\u0000"+ + "\u017d\u017b\u0001\u0000\u0000\u0000\u017e\u0181\u0001\u0000\u0000\u0000"+ + "\u017f\u017d\u0001\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000\u0000"+ + "\u01807\u0001\u0000\u0000\u0000\u0181\u017f\u0001\u0000\u0000\u0000\u0182"+ + "\u0184\u0003\n\u0005\u0000\u0183\u0185\u0007\u0005\u0000\u0000\u0184\u0183"+ + "\u0001\u0000\u0000\u0000\u0184\u0185\u0001\u0000\u0000\u0000\u0185\u0188"+ + "\u0001\u0000\u0000\u0000\u0186\u0187\u0005.\u0000\u0000\u0187\u0189\u0007"+ + "\u0006\u0000\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0188\u0189\u0001"+ + "\u0000\u0000\u0000\u01899\u0001\u0000\u0000\u0000\u018a\u018b\u0005\t"+ + "\u0000\u0000\u018b\u0190\u0003,\u0016\u0000\u018c\u018d\u0005\"\u0000"+ + "\u0000\u018d\u018f\u0003,\u0016\u0000\u018e\u018c\u0001\u0000\u0000\u0000"+ + "\u018f\u0192\u0001\u0000\u0000\u0000\u0190\u018e\u0001\u0000\u0000\u0000"+ + "\u0190\u0191\u0001\u0000\u0000\u0000\u0191\u019d\u0001\u0000\u0000\u0000"+ + "\u0192\u0190\u0001\u0000\u0000\u0000\u0193\u0194\u0005\f\u0000\u0000\u0194"+ + "\u0199\u0003,\u0016\u0000\u0195\u0196\u0005\"\u0000\u0000\u0196\u0198"+ + "\u0003,\u0016\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0198\u019b\u0001"+ + "\u0000\u0000\u0000\u0199\u0197\u0001\u0000\u0000\u0000\u0199\u019a\u0001"+ + "\u0000\u0000\u0000\u019a\u019d\u0001\u0000\u0000\u0000\u019b\u0199\u0001"+ + "\u0000\u0000\u0000\u019c\u018a\u0001\u0000\u0000\u0000\u019c\u0193\u0001"+ + "\u0000\u0000\u0000\u019d;\u0001\u0000\u0000\u0000\u019e\u019f\u0005\u0002"+ + "\u0000\u0000\u019f\u01a4\u0003,\u0016\u0000\u01a0\u01a1\u0005\"\u0000"+ + "\u0000\u01a1\u01a3\u0003,\u0016\u0000\u01a2\u01a0\u0001\u0000\u0000\u0000"+ + "\u01a3\u01a6\u0001\u0000\u0000\u0000\u01a4\u01a2\u0001\u0000\u0000\u0000"+ + "\u01a4\u01a5\u0001\u0000\u0000\u0000\u01a5=\u0001\u0000\u0000\u0000\u01a6"+ + "\u01a4\u0001\u0000\u0000\u0000\u01a7\u01a8\u0005\r\u0000\u0000\u01a8\u01ad"+ + "\u0003@ \u0000\u01a9\u01aa\u0005\"\u0000\u0000\u01aa\u01ac\u0003@ \u0000"+ + "\u01ab\u01a9\u0001\u0000\u0000\u0000\u01ac\u01af\u0001\u0000\u0000\u0000"+ + "\u01ad\u01ab\u0001\u0000\u0000\u0000\u01ad\u01ae\u0001\u0000\u0000\u0000"+ + "\u01ae?\u0001\u0000\u0000\u0000\u01af\u01ad\u0001\u0000\u0000\u0000\u01b0"+ + "\u01b1\u0003,\u0016\u0000\u01b1\u01b2\u0005O\u0000\u0000\u01b2\u01b3\u0003"+ + ",\u0016\u0000\u01b3A\u0001\u0000\u0000\u0000\u01b4\u01b5\u0005\u0001\u0000"+ + "\u0000\u01b5\u01b6\u0003\u0012\t\u0000\u01b6\u01b8\u0003T*\u0000\u01b7"+ + "\u01b9\u0003H$\u0000\u01b8\u01b7\u0001\u0000\u0000\u0000\u01b8\u01b9\u0001"+ + "\u0000\u0000\u0000\u01b9C\u0001\u0000\u0000\u0000\u01ba\u01bb\u0005\u0007"+ + "\u0000\u0000\u01bb\u01bc\u0003\u0012\t\u0000\u01bc\u01bd\u0003T*\u0000"+ + "\u01bdE\u0001\u0000\u0000\u0000\u01be\u01bf\u0005\u000b\u0000\u0000\u01bf"+ + "\u01c0\u0003*\u0015\u0000\u01c0G\u0001\u0000\u0000\u0000\u01c1\u01c6\u0003"+ + "J%\u0000\u01c2\u01c3\u0005\"\u0000\u0000\u01c3\u01c5\u0003J%\u0000\u01c4"+ + "\u01c2\u0001\u0000\u0000\u0000\u01c5\u01c8\u0001\u0000\u0000\u0000\u01c6"+ + "\u01c4\u0001\u0000\u0000\u0000\u01c6\u01c7\u0001\u0000\u0000\u0000\u01c7"+ + "I\u0001\u0000\u0000\u0000\u01c8\u01c6\u0001\u0000\u0000\u0000\u01c9\u01ca"+ + "\u0003.\u0017\u0000\u01ca\u01cb\u0005!\u0000\u0000\u01cb\u01cc\u00032"+ + "\u0019\u0000\u01ccK\u0001\u0000\u0000\u0000\u01cd\u01ce\u0007\u0007\u0000"+ + "\u0000\u01ceM\u0001\u0000\u0000\u0000\u01cf\u01d2\u0003P(\u0000\u01d0"+ + "\u01d2\u0003R)\u0000\u01d1\u01cf\u0001\u0000\u0000\u0000\u01d1\u01d0\u0001"+ + "\u0000\u0000\u0000\u01d2O\u0001\u0000\u0000\u0000\u01d3\u01d5\u0007\u0000"+ + "\u0000\u0000\u01d4\u01d3\u0001\u0000\u0000\u0000\u01d4\u01d5\u0001\u0000"+ + "\u0000\u0000\u01d5\u01d6\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005\u001d"+ + "\u0000\u0000\u01d7Q\u0001\u0000\u0000\u0000\u01d8\u01da\u0007\u0000\u0000"+ + "\u0000\u01d9\u01d8\u0001\u0000\u0000\u0000\u01d9\u01da\u0001\u0000\u0000"+ + "\u0000\u01da\u01db\u0001\u0000\u0000\u0000\u01db\u01dc\u0005\u001c\u0000"+ + "\u0000\u01dcS\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\u001b\u0000\u0000"+ + "\u01deU\u0001\u0000\u0000\u0000\u01df\u01e0\u0007\b\u0000\u0000\u01e0"+ + "W\u0001\u0000\u0000\u0000\u01e1\u01e2\u0005\u0005\u0000\u0000\u01e2\u01e3"+ + "\u0003Z-\u0000\u01e3Y\u0001\u0000\u0000\u0000\u01e4\u01e5\u0005?\u0000"+ + "\u0000\u01e5\u01e6\u0003\u0002\u0001\u0000\u01e6\u01e7\u0005@\u0000\u0000"+ + "\u01e7[\u0001\u0000\u0000\u0000\u01e8\u01e9\u0005\u000f\u0000\u0000\u01e9"+ + "\u01ed\u0005^\u0000\u0000\u01ea\u01eb\u0005\u000f\u0000\u0000\u01eb\u01ed"+ + "\u0005_\u0000\u0000\u01ec\u01e8\u0001\u0000\u0000\u0000\u01ec\u01ea\u0001"+ + "\u0000\u0000\u0000\u01ed]\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005\u0003"+ + "\u0000\u0000\u01ef\u01f2\u0003(\u0014\u0000\u01f0\u01f1\u0005S\u0000\u0000"+ + "\u01f1\u01f3\u0003,\u0016\u0000\u01f2\u01f0\u0001\u0000\u0000\u0000\u01f2"+ + "\u01f3\u0001\u0000\u0000\u0000\u01f3\u01fd\u0001\u0000\u0000\u0000\u01f4"+ + "\u01f5\u0005T\u0000\u0000\u01f5\u01fa\u0003`0\u0000\u01f6\u01f7\u0005"+ + "\"\u0000\u0000\u01f7\u01f9\u0003`0\u0000\u01f8\u01f6\u0001\u0000\u0000"+ + "\u0000\u01f9\u01fc\u0001\u0000\u0000\u0000\u01fa\u01f8\u0001\u0000\u0000"+ + "\u0000\u01fa\u01fb\u0001\u0000\u0000\u0000\u01fb\u01fe\u0001\u0000\u0000"+ + "\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000\u01fd\u01f4\u0001\u0000\u0000"+ + "\u0000\u01fd\u01fe\u0001\u0000\u0000\u0000\u01fe_\u0001\u0000\u0000\u0000"+ + "\u01ff\u0200\u0003,\u0016\u0000\u0200\u0201\u0005!\u0000\u0000\u0201\u0203"+ + "\u0001\u0000\u0000\u0000\u0202\u01ff\u0001\u0000\u0000\u0000\u0202\u0203"+ + "\u0001\u0000\u0000\u0000\u0203\u0204\u0001\u0000\u0000\u0000\u0204\u0205"+ + "\u0003,\u0016\u0000\u0205a\u0001\u0000\u0000\u00004mt\u0083\u008f\u0098"+ + "\u00a0\u00a4\u00ac\u00ae\u00b3\u00ba\u00bf\u00c6\u00cc\u00d4\u00d6\u00e0"+ + "\u00ea\u00ed\u00f9\u0101\u0109\u010d\u0116\u0120\u0124\u012a\u0131\u013b"+ + "\u0143\u0159\u0164\u016f\u0174\u017f\u0184\u0188\u0190\u0199\u019c\u01a4"+ + "\u01ad\u01b8\u01c6\u01d1\u01d4\u01d9\u01ec\u01f2\u01fa\u01fd\u0202"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 3137eff0b6550..73b529cd2be92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -401,13 +401,13 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { * *

The default implementation does nothing.

*/ - @Override public void enterSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx) { } + @Override public void enterFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx) { } /** * {@inheritDoc} * *

The default implementation does nothing.

*/ - @Override public void exitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx) { } + @Override public void exitFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx) { } /** * {@inheritDoc} * @@ -420,6 +420,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitQualifiedName(EsqlBaseParser.QualifiedNameContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { } /** * {@inheritDoc} * @@ -432,6 +444,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitIdentifier(EsqlBaseParser.IdentifierContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void enterIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { } + /** + * {@inheritDoc} + * + *

The default implementation does nothing.

+ */ + @Override public void exitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index d7b2f359e3c83..d35481745cecc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -242,7 +242,7 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im *

The default implementation returns the result of calling * {@link #visitChildren} on {@code ctx}.

*/ - @Override public T visitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx) { return visitChildren(ctx); } + @Override public T visitFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -250,6 +250,13 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitQualifiedName(EsqlBaseParser.QualifiedNameContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -257,6 +264,13 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitIdentifier(EsqlBaseParser.IdentifierContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index dd6cdaacddbef..6c8cd7272d8dc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -362,15 +362,15 @@ public interface EsqlBaseParserListener extends ParseTreeListener { */ void exitGrouping(EsqlBaseParser.GroupingContext ctx); /** - * Enter a parse tree produced by {@link EsqlBaseParser#sourceIdentifier}. + * Enter a parse tree produced by {@link EsqlBaseParser#fromIdentifier}. * @param ctx the parse tree */ - void enterSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx); + void enterFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx); /** - * Exit a parse tree produced by {@link EsqlBaseParser#sourceIdentifier}. + * Exit a parse tree produced by {@link EsqlBaseParser#fromIdentifier}. * @param ctx the parse tree */ - void exitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx); + void exitFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#qualifiedName}. * @param ctx the parse tree @@ -381,6 +381,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitQualifiedName(EsqlBaseParser.QualifiedNameContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#qualifiedNamePattern}. + * @param ctx the parse tree + */ + void enterQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#qualifiedNamePattern}. + * @param ctx the parse tree + */ + void exitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#identifier}. * @param ctx the parse tree @@ -391,6 +401,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitIdentifier(EsqlBaseParser.IdentifierContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#identifierPattern}. + * @param ctx the parse tree + */ + void enterIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#identifierPattern}. + * @param ctx the parse tree + */ + void exitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx); /** * Enter a parse tree produced by the {@code nullLiteral} * labeled alternative in {@link EsqlBaseParser#constant}. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index 35297f3d4f336..2fe5de566dbaf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -220,23 +220,35 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { */ T visitGrouping(EsqlBaseParser.GroupingContext ctx); /** - * Visit a parse tree produced by {@link EsqlBaseParser#sourceIdentifier}. + * Visit a parse tree produced by {@link EsqlBaseParser#fromIdentifier}. * @param ctx the parse tree * @return the visitor result */ - T visitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx); + T visitFromIdentifier(EsqlBaseParser.FromIdentifierContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#qualifiedName}. * @param ctx the parse tree * @return the visitor result */ T visitQualifiedName(EsqlBaseParser.QualifiedNameContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#qualifiedNamePattern}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#identifier}. * @param ctx the parse tree * @return the visitor result */ T visitIdentifier(EsqlBaseParser.IdentifierContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#identifierPattern}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx); /** * Visit a parse tree produced by the {@code nullLiteral} * labeled alternative in {@link EsqlBaseParser#constant}. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index f24324fac2fbd..3b1ef475350b1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -207,10 +207,21 @@ public UnresolvedAttribute visitQualifiedName(EsqlBaseParser.QualifiedNameContex return null; } - return new UnresolvedAttribute( - source(ctx), - Strings.collectionToDelimitedString(visitList(this, ctx.identifier(), String.class), ".") - ); + List strings = visitList(this, ctx.identifier(), String.class); + return new UnresolvedAttribute(source(ctx), Strings.collectionToDelimitedString(strings, ".")); + } + + @Override + public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { + if (ctx == null) { + return null; + } + + List strings = visitList(this, ctx.identifierPattern(), String.class); + var src = source(ctx); + return strings.size() == 1 && strings.get(0).equals(WILDCARD) + ? new UnresolvedStar(src, null) + : new UnresolvedAttribute(src, Strings.collectionToDelimitedString(strings, ".")); } @Override @@ -366,22 +377,32 @@ public Order visitOrderExpression(EsqlBaseParser.OrderExpressionContext ctx) { ); } - public NamedExpression visitProjectExpression(EsqlBaseParser.SourceIdentifierContext ctx) { - Source src = source(ctx); - String identifier = visitSourceIdentifier(ctx); - return identifier.equals(WILDCARD) ? new UnresolvedStar(src, null) : new UnresolvedAttribute(src, identifier); - } - @Override public Alias visitRenameClause(EsqlBaseParser.RenameClauseContext ctx) { Source src = source(ctx); - String newName = visitSourceIdentifier(ctx.newName); - String oldName = visitSourceIdentifier(ctx.oldName); - if (newName.contains(WILDCARD) || oldName.contains(WILDCARD)) { + NamedExpression newName = visitQualifiedNamePattern(ctx.newName); + NamedExpression oldName = visitQualifiedNamePattern(ctx.oldName); + if (newName.name().contains(WILDCARD) || oldName.name().contains(WILDCARD)) { throw new ParsingException(src, "Using wildcards (*) in renaming projections is not allowed [{}]", src.text()); } - return new Alias(src, newName, new UnresolvedAttribute(source(ctx.oldName), oldName)); + return new Alias(src, newName.name(), oldName); + } + + @Override + public NamedExpression visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseContext ctx) { + Source src = source(ctx); + NamedExpression enrichField = enrichFieldName(ctx.enrichField); + NamedExpression newName = enrichFieldName(ctx.newName); + return newName == null ? enrichField : new Alias(src, newName.name(), enrichField); + } + + private NamedExpression enrichFieldName(EsqlBaseParser.QualifiedNamePatternContext ctx) { + var name = visitQualifiedNamePattern(ctx); + if (name != null && name.name().contains(WILDCARD)) { + throw new ParsingException(source(ctx), "Using wildcards (*) in ENRICH WITH projections is not allowed [{}]", name.name()); + } + return name; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java index 296206b1079b2..2039dc633f6cf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java @@ -9,6 +9,8 @@ import org.antlr.v4.runtime.tree.TerminalNode; import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.FromIdentifierContext; +import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.IdentifierContext; import java.util.List; @@ -17,27 +19,32 @@ abstract class IdentifierBuilder extends AbstractBuilder { @Override - public String visitIdentifier(EsqlBaseParser.IdentifierContext ctx) { - return unquoteIdentifier(ctx.QUOTED_IDENTIFIER(), ctx.UNQUOTED_IDENTIFIER()); + public String visitIdentifier(IdentifierContext ctx) { + return ctx == null ? null : unquoteIdentifier(ctx.QUOTED_IDENTIFIER(), ctx.UNQUOTED_IDENTIFIER()); } @Override - public String visitSourceIdentifier(EsqlBaseParser.SourceIdentifierContext ctx) { - return unquoteIdentifier(ctx.SRC_QUOTED_IDENTIFIER(), ctx.SRC_UNQUOTED_IDENTIFIER()); + public String visitIdentifierPattern(EsqlBaseParser.IdentifierPatternContext ctx) { + return unquoteIdentifier(ctx.QUOTED_IDENTIFIER(), ctx.PROJECT_UNQUOTED_IDENTIFIER()); } - private static String unquoteIdentifier(TerminalNode quotedNode, TerminalNode unquotedNode) { + @Override + public String visitFromIdentifier(FromIdentifierContext ctx) { + return ctx == null ? null : unquoteIdentifier(ctx.QUOTED_IDENTIFIER(), ctx.FROM_UNQUOTED_IDENTIFIER()); + } + + static String unquoteIdentifier(TerminalNode quotedNode, TerminalNode unquotedNode) { String result; if (quotedNode != null) { String identifier = quotedNode.getText(); - result = identifier.substring(1, identifier.length() - 1); + result = identifier.substring(1, identifier.length() - 1).replace("``", "`"); } else { result = unquotedNode.getText(); } return result; } - public String visitSourceIdentifiers(List ctx) { + public String visitFromIdentifiers(List ctx) { return Strings.collectionToDelimitedString(visitList(this, ctx, String.class), ","); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index d5763f28f6394..f9d1a252afe42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -12,6 +12,7 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.elasticsearch.dissect.DissectException; import org.elasticsearch.dissect.DissectParser; +import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.QualifiedNamePatternContext; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -61,7 +62,6 @@ import static org.elasticsearch.xpack.ql.parser.ParserUtils.source; import static org.elasticsearch.xpack.ql.parser.ParserUtils.typedParsing; import static org.elasticsearch.xpack.ql.parser.ParserUtils.visitList; -import static org.elasticsearch.xpack.ql.util.StringUtils.WILDCARD; public class LogicalPlanBuilder extends ExpressionBuilder { @@ -149,9 +149,9 @@ public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx) @Override public PlanFactory visitMvExpandCommand(EsqlBaseParser.MvExpandCommandContext ctx) { - String identifier = visitSourceIdentifier(ctx.sourceIdentifier()); + UnresolvedAttribute field = visitQualifiedName(ctx.qualifiedName()); Source src = source(ctx); - return child -> new MvExpand(src, child, new UnresolvedAttribute(src, identifier), new UnresolvedAttribute(src, identifier)); + return child -> new MvExpand(src, child, field, new UnresolvedAttribute(src, field.name())); } @@ -175,11 +175,11 @@ public LogicalPlan visitRowCommand(EsqlBaseParser.RowCommandContext ctx) { @Override public LogicalPlan visitFromCommand(EsqlBaseParser.FromCommandContext ctx) { Source source = source(ctx); - TableIdentifier table = new TableIdentifier(source, null, visitSourceIdentifiers(ctx.sourceIdentifier())); + TableIdentifier table = new TableIdentifier(source, null, visitFromIdentifiers(ctx.fromIdentifier())); Map metadataMap = new LinkedHashMap<>(); if (ctx.metadata() != null) { - for (var c : ctx.metadata().sourceIdentifier()) { - String id = visitSourceIdentifier(c); + for (var c : ctx.metadata().fromIdentifier()) { + String id = visitFromIdentifier(c); Source src = source(c); if (MetadataAttribute.isSupported(id) == false) { throw new ParsingException(src, "unsupported metadata field [" + id + "]"); @@ -254,16 +254,16 @@ public Explain visitExplainCommand(EsqlBaseParser.ExplainCommandContext ctx) { @Override public PlanFactory visitDropCommand(EsqlBaseParser.DropCommandContext ctx) { - var identifiers = ctx.sourceIdentifier(); + var identifiers = ctx.qualifiedNamePattern(); List removals = new ArrayList<>(identifiers.size()); - for (EsqlBaseParser.SourceIdentifierContext idCtx : identifiers) { - Source src = source(idCtx); - String identifier = visitSourceIdentifier(idCtx); - if (identifier.equals(WILDCARD)) { + for (QualifiedNamePatternContext patternContext : identifiers) { + NamedExpression ne = visitQualifiedNamePattern(patternContext); + if (ne instanceof UnresolvedStar) { + var src = ne.source(); throw new ParsingException(src, "Removing all fields is not allowed [{}]", src.text()); } - removals.add(new UnresolvedAttribute(src, identifier)); + removals.add(ne); } return child -> new Drop(source(ctx), child, removals); @@ -280,13 +280,15 @@ public PlanFactory visitKeepCommand(EsqlBaseParser.KeepCommandContext ctx) { if (ctx.PROJECT() != null) { addWarning("PROJECT command is no longer supported, please use KEEP instead"); } - List projections = new ArrayList<>(ctx.sourceIdentifier().size()); + var identifiers = ctx.qualifiedNamePattern(); + List projections = new ArrayList<>(identifiers.size()); boolean hasSeenStar = false; - for (var srcIdCtx : ctx.sourceIdentifier()) { - NamedExpression ne = visitProjectExpression(srcIdCtx); + for (QualifiedNamePatternContext patternContext : identifiers) { + NamedExpression ne = visitQualifiedNamePattern(patternContext); if (ne instanceof UnresolvedStar) { if (hasSeenStar) { - throw new ParsingException(ne.source(), "Cannot specify [*] more than once", ne.source().text()); + var src = ne.source(); + throw new ParsingException(src, "Cannot specify [*] more than once", src.text()); } else { hasSeenStar = true; } @@ -309,11 +311,9 @@ public LogicalPlan visitShowFunctions(EsqlBaseParser.ShowFunctionsContext ctx) { @Override public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) { return p -> { - final String policyName = visitSourceIdentifier(ctx.policyName); + String policyName = visitFromIdentifier(ctx.policyName); var source = source(ctx); - NamedExpression matchField = ctx.ON() != null - ? new UnresolvedAttribute(source(ctx.matchField), visitSourceIdentifier(ctx.matchField)) - : new EmptyAttribute(source); + NamedExpression matchField = ctx.ON() != null ? visitQualifiedNamePattern(ctx.matchField) : new EmptyAttribute(source); if (matchField.name().contains("*")) { throw new ParsingException( source(ctx), @@ -321,6 +321,7 @@ public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) { matchField.name() ); } + List keepClauses = visitList(this, ctx.enrichWithClause(), NamedExpression.class); return new Enrich( source, @@ -333,22 +334,5 @@ public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) { }; } - @Override - public NamedExpression visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseContext ctx) { - Source src = source(ctx); - String enrichField = enrichFieldName(ctx.enrichField); - String newName = enrichFieldName(ctx.newName); - UnresolvedAttribute enrichAttr = new UnresolvedAttribute(src, enrichField); - return newName == null ? enrichAttr : new Alias(src, newName, enrichAttr); - } - - private String enrichFieldName(EsqlBaseParser.SourceIdentifierContext ctx) { - String name = ctx == null ? null : visitSourceIdentifier(ctx); - if (name != null && name.contains(WILDCARD)) { - throw new ParsingException(source(ctx), "Using wildcards (*) in ENRICH WITH projections is not allowed [{}]", name); - } - return name; - } - interface PlanFactory extends Function {} } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 17ed0c1223636..a5e3033b6e1e3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -229,7 +229,7 @@ protected final boolean enableWarningsCheck() { } public boolean logResults() { - return true; + return false; } private void doTest() throws Exception { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 53e2a2e412fcd..4b4cbdf50e702 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.EmptyAttribute; +import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.Order; @@ -41,6 +42,7 @@ import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.OrderBy; +import org.elasticsearch.xpack.ql.plan.logical.Project; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.versionfield.Version; @@ -53,6 +55,7 @@ import java.util.List; import java.util.function.Function; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.ql.expression.Literal.FALSE; import static org.elasticsearch.xpack.ql.expression.Literal.TRUE; import static org.elasticsearch.xpack.ql.expression.function.FunctionResolutionStrategy.DEFAULT; @@ -60,6 +63,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.util.NumericUtils.asLongUnsigned; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -576,13 +580,10 @@ public void testDeprecatedIsNullFunction() { public void testMetadataFieldOnOtherSources() { expectError( "row a = 1 [metadata _index]", - "1:11: mismatched input '[' expecting {, PIPE, 'and', COMMA, 'or', '+', '-', '*', '/', '%'}" - ); - expectError("show functions [metadata _index]", "line 1:16: mismatched input '[' expecting {, PIPE}"); - expectError( - "explain [from foo] [metadata _index]", - "line 1:20: mismatched input '[' expecting {PIPE, COMMA, OPENING_BRACKET, ']'}" + "1:11: mismatched input '[' expecting {, '|', 'and', ',', 'or', '+', '-', '*', '/', '%'}" ); + expectError("show functions [metadata _index]", "line 1:16: token recognition error at: '['"); + expectError("explain [from foo] [metadata _index]", "line 1:20: mismatched input '[' expecting {'|', ',', OPENING_BRACKET, ']'}"); } public void testMetadataFieldMultipleDeclarations() { @@ -799,6 +800,29 @@ public void testMissingInputParams() { expectError("row x = ?, y = ?", List.of(new TypedParamValue("integer", 1)), "Not enough actual parameters 1"); } + public void testFieldContainingDotsAndNumbers() { + LogicalPlan where = processingCommand("where `a.b.1m.4321`"); + assertThat(where, instanceOf(Filter.class)); + Filter w = (Filter) where; + assertThat(w.child(), equalTo(PROCESSING_CMD_INPUT)); + assertThat(Expressions.name(w.condition()), equalTo("a.b.1m.4321")); + } + + public void testFieldQualifiedName() { + LogicalPlan where = processingCommand("where a.b.`1m`.`4321`"); + assertThat(where, instanceOf(Filter.class)); + Filter w = (Filter) where; + assertThat(w.child(), equalTo(PROCESSING_CMD_INPUT)); + assertThat(Expressions.name(w.condition()), equalTo("a.b.1m.4321")); + } + + public void testQuotedName() { + // row `my-field`=123 | stats count(`my-field`) | eval x = `count(`my-field`)` + LogicalPlan plan = processingCommand("stats count(`my-field`) | keep `count(``my-field``)`"); + var project = as(plan, Project.class); + assertThat(Expressions.names(project.projections()), contains("count(`my-field`)")); + } + private void assertIdentifierAsIndexPattern(String identifier, String statement) { LogicalPlan from = statement(statement); assertThat(from, instanceOf(EsqlUnresolvedRelation.class)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 0aaf4a1a18e32..fb5135d1de54c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -198,7 +198,7 @@ public void testEvalDateTruncGrouping() { | eval y = date_trunc(hire_date, 1 year) | stats count(emp_no) by y | sort y - | keep y, count(emp_no) + | keep y, `count(emp_no)` | limit 5""", Set.of("hire_date", "hire_date.*", "emp_no", "emp_no.*")); } From 2c8f6eba8c0d0f41cde23a029134834b8505bf5a Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 12 Dec 2023 23:34:17 -0800 Subject: [PATCH 04/54] ESQL: Simpify IS NULL/IS NOT NULL evaluation (#103099) The IS NULL, IS NOT NULL predicates only care about the nullability not the value of an expression. Given the expression x + 1 / 2, the actual result does not matter only if it's null or not - that is, it only matters if x is null or not. So x + 1 / 2 IS NULL becomes x IS NULL - which can be opportunistically pushed down or evaluated. Preserve the original expression to cope with under/overflow or mv fields Fix #103097 --- docs/changelog/103099.yaml | 6 ++ .../optimizer/LocalLogicalPlanOptimizer.java | 18 +++- .../LocalLogicalPlanOptimizerTests.java | 40 +++++++++ .../xpack/ql/optimizer/OptimizerRules.java | 90 +++++++++++++++++++ .../ql/optimizer/OptimizerRulesTests.java | 84 +++++++++++++++++ 5 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/103099.yaml diff --git a/docs/changelog/103099.yaml b/docs/changelog/103099.yaml new file mode 100644 index 0000000000000..c3fd3f9d7b8e4 --- /dev/null +++ b/docs/changelog/103099.yaml @@ -0,0 +1,6 @@ +pr: 103099 +summary: "ESQL: Simpify IS NULL/IS NOT NULL evaluation" +area: ES|QL +type: enhancement +issues: + - 103097 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 3451a3981d3e3..e05dd9a00c567 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -7,10 +7,12 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.stats.SearchStats; import org.elasticsearch.xpack.ql.expression.Alias; +import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.NamedExpression; @@ -37,7 +39,13 @@ public LocalLogicalPlanOptimizer(LocalLogicalOptimizerContext localLogicalOptimi @Override protected List> batches() { - var local = new Batch<>("Local rewrite", new ReplaceTopNWithLimitAndSort(), new ReplaceMissingFieldWithNull()); + var local = new Batch<>( + "Local rewrite", + Limiter.ONCE, + new ReplaceTopNWithLimitAndSort(), + new ReplaceMissingFieldWithNull(), + new InferIsNotNull() + ); var rules = new ArrayList>(); rules.add(local); @@ -116,6 +124,14 @@ else if (plan instanceof Project project) { } } + static class InferIsNotNull extends OptimizerRules.InferIsNotNull { + + @Override + protected boolean skipExpression(Expression e) { + return e instanceof Coalesce; + } + } + abstract static class ParameterizedOptimizerRule extends ParameterizedRule { public final LogicalPlan apply(LogicalPlan plan, P context) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index bc46189e13827..ac2426f485fcc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; @@ -21,9 +22,11 @@ import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.ReferenceAttribute; +import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.plan.logical.EsRelation; +import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.Project; @@ -35,6 +38,7 @@ import java.util.Map; import static org.elasticsearch.xpack.esql.EsqlTestUtils.L; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_SEARCH_STATS; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; @@ -263,6 +267,38 @@ public void testMissingFieldInFilterNoProjection() { ); } + public void testIsNotNullOnCoalesce() { + var plan = localPlan(""" + from test + | where coalesce(emp_no, salary) is not null + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var inn = as(filter.condition(), IsNotNull.class); + var coalesce = as(inn.children().get(0), Coalesce.class); + assertThat(Expressions.names(coalesce.children()), contains("emp_no", "salary")); + var source = as(filter.child(), EsRelation.class); + } + + public void testIsNotNullOnExpression() { + var plan = localPlan(""" + from test + | eval x = emp_no + 1 + | where x is not null + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var inn = as(filter.condition(), IsNotNull.class); + assertThat(Expressions.names(inn.children()), contains("x")); + var eval = as(filter.child(), Eval.class); + filter = as(eval.child(), Filter.class); + inn = as(filter.condition(), IsNotNull.class); + assertThat(Expressions.names(inn.children()), contains("emp_no")); + var source = as(filter.child(), EsRelation.class); + } + private LocalRelation asEmptyRelation(Object o) { var empty = as(o, LocalRelation.class); assertThat(empty.supplier(), is(LocalSupplier.EMPTY)); @@ -285,6 +321,10 @@ private LogicalPlan localPlan(LogicalPlan plan, SearchStats searchStats) { return localPlan; } + private LogicalPlan localPlan(String query) { + return localPlan(plan(query), TEST_SEARCH_STATS); + } + @Override protected List filteredWarnings() { return withDefaultLimitWarning(super.filteredWarnings()); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java index 5d9736726b46f..f084b5cda4abe 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java @@ -8,6 +8,8 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.xpack.ql.expression.Alias; +import org.elasticsearch.xpack.ql.expression.Attribute; +import org.elasticsearch.xpack.ql.expression.AttributeMap; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; @@ -69,6 +71,7 @@ import static java.lang.Math.signum; import static java.util.Arrays.asList; +import static java.util.Collections.emptySet; import static org.elasticsearch.xpack.ql.expression.Literal.FALSE; import static org.elasticsearch.xpack.ql.expression.Literal.TRUE; import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.combineAnd; @@ -1785,6 +1788,93 @@ protected Expression nonNullify(Expression exp, Expression nonNullExp) { } } + /** + * Simplify IsNotNull targets by resolving the underlying expression to its root fields with unknown + * nullability. + * e.g. + * (x + 1) / 2 IS NOT NULL --> x IS NOT NULL AND (x+1) / 2 IS NOT NULL + * SUBSTRING(x, 3) > 4 IS NOT NULL --> x IS NOT NULL AND SUBSTRING(x, 3) > 4 IS NOT NULL + * When dealing with multiple fields, a conjunction/disjunction based on the predicate: + * (x + y) / 4 IS NOT NULL --> x IS NOT NULL AND y IS NOT NULL AND (x + y) / 4 IS NOT NULL + * This handles the case of fields nested inside functions or expressions in order to avoid: + * - having to evaluate the whole expression + * - not pushing down the filter due to expression evaluation + * IS NULL cannot be simplified since it leads to a disjunction which prevents the filter to be + * pushed down: + * (x + 1) IS NULL --> x IS NULL OR x + 1 IS NULL + * and x IS NULL cannot be pushed down + *
+ * Implementation-wise this rule goes bottom-up, keeping an alias up to date to the current plan + * and then looks for replacing the target. + */ + public static class InferIsNotNull extends Rule { + + @Override + public LogicalPlan apply(LogicalPlan plan) { + // the alias map is shared across the whole plan + AttributeMap aliases = new AttributeMap<>(); + // traverse bottom-up to pick up the aliases as we go + plan = plan.transformUp(p -> inspectPlan(p, aliases)); + return plan; + } + + private LogicalPlan inspectPlan(LogicalPlan plan, AttributeMap aliases) { + // inspect just this plan properties + plan.forEachExpression(Alias.class, a -> aliases.put(a.toAttribute(), a.child())); + // now go about finding isNull/isNotNull + LogicalPlan newPlan = plan.transformExpressionsOnlyUp(IsNotNull.class, inn -> inferNotNullable(inn, aliases)); + return newPlan; + } + + private Expression inferNotNullable(IsNotNull inn, AttributeMap aliases) { + Expression result = inn; + Set refs = resolveExpressionAsRootAttributes(inn.field(), aliases); + // no refs found or could not detect - return the original function + if (refs.size() > 0) { + // add IsNull for the filters along with the initial inn + var innList = CollectionUtils.combine(refs.stream().map(r -> (Expression) new IsNotNull(inn.source(), r)).toList(), inn); + result = Predicates.combineAnd(innList); + } + return result; + } + + /** + * Unroll the expression to its references to get to the root fields + * that really matter for filtering. + */ + protected Set resolveExpressionAsRootAttributes(Expression exp, AttributeMap aliases) { + Set resolvedExpressions = new LinkedHashSet<>(); + boolean changed = doResolve(exp, aliases, resolvedExpressions); + return changed ? resolvedExpressions : emptySet(); + } + + private boolean doResolve(Expression exp, AttributeMap aliases, Set resolvedExpressions) { + boolean changed = false; + // check if the expression can be skipped or is not nullabe + if (skipExpression(exp) || exp.nullable() == Nullability.FALSE) { + resolvedExpressions.add(exp); + } else { + for (Expression e : exp.references()) { + Expression resolved = aliases.resolve(e, e); + // found a root attribute, bail out + if (resolved instanceof Attribute a && resolved == e) { + resolvedExpressions.add(a); + // don't mark things as change if the original expression hasn't been broken down + changed |= resolved != exp; + } else { + // go further + changed |= doResolve(resolved, aliases, resolvedExpressions); + } + } + } + return changed; + } + + protected boolean skipExpression(Expression e) { + return false; + } + } + public static final class SetAsOptimized extends Rule { @Override diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java index b14e46a96a9e6..1cab7dd87195b 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.Nullability; import org.elasticsearch.xpack.ql.expression.function.aggregate.Count; +import org.elasticsearch.xpack.ql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.ql.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.ql.expression.predicate.Predicates; import org.elasticsearch.xpack.ql.expression.predicate.Range; @@ -1768,7 +1769,90 @@ public void testPushDownFilterThroughAgg() throws Exception { // expected Filter expected = new Filter(EMPTY, new Aggregate(EMPTY, combinedFilter, emptyList(), emptyList()), aggregateCondition); assertEquals(expected, new PushDownAndCombineFilters().apply(fb)); + } + + public void testIsNotNullOnIsNullField() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + Expression inn = isNotNull(fieldA); + Filter f = new Filter(EMPTY, relation, inn); + + assertEquals(f, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public void testIsNotNullOnOperatorWithOneField() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + Expression inn = isNotNull(new Add(EMPTY, fieldA, ONE)); + Filter f = new Filter(EMPTY, relation, inn); + Filter expected = new Filter(EMPTY, relation, new And(EMPTY, isNotNull(fieldA), inn)); + + assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public void testIsNotNullOnOperatorWithTwoFields() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + var fieldB = getFieldAttribute("b"); + Expression inn = isNotNull(new Add(EMPTY, fieldA, fieldB)); + Filter f = new Filter(EMPTY, relation, inn); + Filter expected = new Filter(EMPTY, relation, new And(EMPTY, new And(EMPTY, isNotNull(fieldA), isNotNull(fieldB)), inn)); + + assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public void testIsNotNullOnFunctionWithOneField() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + var pattern = L("abc"); + Expression inn = isNotNull( + new And(EMPTY, new TestStartsWith(EMPTY, fieldA, pattern, false), greaterThanOf(new Add(EMPTY, ONE, TWO), THREE)) + ); + + Filter f = new Filter(EMPTY, relation, inn); + Filter expected = new Filter(EMPTY, relation, new And(EMPTY, isNotNull(fieldA), inn)); + + assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public void testIsNotNullOnFunctionWithTwoFields() { + EsRelation relation = relation(); + var fieldA = getFieldAttribute("a"); + var fieldB = getFieldAttribute("b"); + var pattern = L("abc"); + Expression inn = isNotNull(new TestStartsWith(EMPTY, fieldA, fieldB, false)); + + Filter f = new Filter(EMPTY, relation, inn); + Filter expected = new Filter(EMPTY, relation, new And(EMPTY, new And(EMPTY, isNotNull(fieldA), isNotNull(fieldB)), inn)); + + assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f)); + } + + public static class TestStartsWith extends StartsWith { + + public TestStartsWith(Source source, Expression input, Expression pattern, boolean caseInsensitive) { + super(source, input, pattern, caseInsensitive); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new TestStartsWith(source(), newChildren.get(0), newChildren.get(1), isCaseInsensitive()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, TestStartsWith::new, input(), pattern(), isCaseInsensitive()); + } + } + + public void testIsNotNullOnFunctionWithTwoField() {} + + private IsNotNull isNotNull(Expression field) { + return new IsNotNull(EMPTY, field); + } + private IsNull isNull(Expression field) { + return new IsNull(EMPTY, field); } private Literal nullOf(DataType dataType) { From 9b674777f137d94a6c6317b1ed105f2df7f549f7 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 13 Dec 2023 09:13:07 +0100 Subject: [PATCH 05/54] ESQL: Fix resolution of MV_EXPAND after KEEP * (#103339) --- docs/changelog/103339.yaml | 6 ++++++ .../src/main/resources/mv_expand.csv-spec | 8 ++++++++ .../xpack/esql/plan/logical/MvExpand.java | 6 ++++-- .../xpack/esql/parser/StatementParserTests.java | 12 ++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/103339.yaml diff --git a/docs/changelog/103339.yaml b/docs/changelog/103339.yaml new file mode 100644 index 0000000000000..6ea1ab0cf799a --- /dev/null +++ b/docs/changelog/103339.yaml @@ -0,0 +1,6 @@ +pr: 103339 +summary: "ESQL: Fix resolution of MV_EXPAND after KEEP *" +area: ES|QL +type: bug +issues: + - 103331 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index a3bc9c6c6dcf6..3a1ae3985e129 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -316,3 +316,11 @@ a:keyword | e:keyword a | a ; + +//see https://github.com/elastic/elasticsearch/issues/103331 +keepStarMvExpand#[skip:-8.12.99] +from employees | where emp_no == 10001 | keep * | mv_expand first_name; + +avg_worked_seconds:long | birth_date:date | emp_no:integer | first_name:keyword | gender:keyword | height:double | height.float:double | height.half_float:double | height.scaled_float:double | hire_date:date | is_rehired:boolean | job_positions:keyword | languages:integer | languages.byte:integer | languages.long:long | languages.short:integer | last_name:keyword | salary:integer | salary_change:double | salary_change.int:integer | salary_change.keyword:keyword | salary_change.long:long | still_hired:boolean +268728049 | 1953-09-02T00:00:00.000Z | 10001 | Georgi | M | 2.03 | 2.0299999713897705 | 2.029296875 | 2.0300000000000002 | 1986-06-26T00:00:00.000Z | [false, true] | [Accountant, Senior Python Developer] | 2 | 2 | 2 | 2 | Facello | 57305 | 1.19 | 1 | 1.19 | 1 | true +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java index 17f669b5d30b3..0cdcd4af00026 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java @@ -22,13 +22,12 @@ public class MvExpand extends UnaryPlan { private final NamedExpression target; private final Attribute expanded; - private final List output; + private List output; public MvExpand(Source source, LogicalPlan child, NamedExpression target, Attribute expanded) { super(source, child); this.target = target; this.expanded = expanded; - this.output = calculateOutput(child.output(), target, expanded); } public static List calculateOutput(List input, NamedExpression target, Attribute expanded) { @@ -63,6 +62,9 @@ public UnaryPlan replaceChild(LogicalPlan newChild) { @Override public List output() { + if (output == null) { + output = calculateOutput(child().output(), target, expanded); + } return output; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 4b4cbdf50e702..b20d166beb22e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.MvExpand; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.ql.capabilities.UnresolvedException; import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.EmptyAttribute; import org.elasticsearch.xpack.ql.expression.Expressions; @@ -710,6 +711,17 @@ public void testMvExpand() { assertThat(expand.target(), equalTo(attribute("a"))); } + // see https://github.com/elastic/elasticsearch/issues/103331 + public void testKeepStarMvExpand() { + try { + String query = "from test | keep * | mv_expand a"; + var plan = statement(query); + } catch (UnresolvedException e) { + fail(e, "Regression: https://github.com/elastic/elasticsearch/issues/103331"); + } + + } + public void testUsageOfProject() { processingCommand("project a"); assertWarnings("PROJECT command is no longer supported, please use KEEP instead"); From 36eb1b7c810f02dcd4db9dcc3699f4334bec6f6c Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 13 Dec 2023 09:27:32 +0100 Subject: [PATCH 06/54] [Connector API] Document get, list and delete endpoints (#103306) --- .../connector/apis/connector-apis.asciidoc | 6 ++ .../apis/delete-connector-api.asciidoc | 66 ++++++++++++++++ .../connector/apis/get-connector-api.asciidoc | 63 +++++++++++++++ .../apis/list-connectors-api.asciidoc | 77 +++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 docs/reference/connector/apis/delete-connector-api.asciidoc create mode 100644 docs/reference/connector/apis/get-connector-api.asciidoc create mode 100644 docs/reference/connector/apis/list-connectors-api.asciidoc diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index a777d5919f71a..f3a5148f8497c 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -23,6 +23,9 @@ You can use these APIs to create, get, delete and update connectors. Use the following APIs to manage connectors: * <> +* <> +* <> +* <> [discrete] @@ -35,3 +38,6 @@ Use the following APIs to manage sync jobs: include::create-connector-api.asciidoc[] +include::delete-connector-api.asciidoc[] +include::get-connector-api.asciidoc[] +include::list-connectors-api.asciidoc[] diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc new file mode 100644 index 0000000000000..2bda7da72cb72 --- /dev/null +++ b/docs/reference/connector/apis/delete-connector-api.asciidoc @@ -0,0 +1,66 @@ +[[delete-connector-api]] +=== Delete connector API + +preview::[] + +++++ +Delete connector +++++ + +Removes a connector and its associated data. +This is a destructive action that is not recoverable. + +[[delete-connector-api-request]] +==== {api-request-title} + +`DELETE _connector/` + +[[delete-connector-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[delete-connector-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[delete-connector-api-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `connector_id` was not provided. + +`404` (Missing resources):: +No connector matching `connector_id` could be found. + +[[delete-connector-api-example]] +==== {api-examples-title} + +The following example deletes the connector with ID `my-connector`: + +//// +[source, console] +-------------------------------------------------- +PUT _connector/my-connector +{ + "index_name": "search-google-drive", + "name": "My Connector", + "service_type": "google_drive" +} +-------------------------------------------------- +// TESTSETUP +//// + +[source,console] +---- +DELETE _connector/my-connector +---- + +[source,console-result] +---- +{ + "acknowledged": true +} +---- diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc new file mode 100644 index 0000000000000..ab4a2758ce4f1 --- /dev/null +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -0,0 +1,63 @@ +[[get-connector-api]] +=== Get connector API +preview::[] +++++ +Get connector +++++ + +Retrieves the details about a connector. + +[[get-connector-api-request]] +==== {api-request-title} + +`GET _connector/` + +[[get-connector-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[get-connector-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[get-connector-api-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `connector_id` was not provided. + +`404` (Missing resources):: +No connector matching `connector_id` could be found. + +[[get-connector-api-example]] +==== {api-examples-title} + +The following example gets the connector `my-connector`: + +//// +[source,console] +-------------------------------------------------- +PUT _connector/my-connector +{ + "index_name": "search-google-drive", + "name": "Google Drive Connector", + "service_type": "google_drive" +} + +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _connector/my-connector +-------------------------------------------------- +// TEARDOWN +//// + +[source,console] +---- +GET _connector/my-connector +---- diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc new file mode 100644 index 0000000000000..57d3cc47aeb7a --- /dev/null +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -0,0 +1,77 @@ +[role="xpack"] +[[list-connector-api]] +=== List connectors API + +preview::[] + +++++ +List connectors +++++ + +Returns information about all stored connectors. + + +[[list-connector-api-request]] +==== {api-request-title} + +`GET _connector` + +[[list-connector-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[list-connector-api-path-params]] +==== {api-path-parms-title} + +`size`:: +(Optional, integer) Maximum number of results to retrieve. + +`from`:: +(Optional, integer) The offset from the first result to fetch. + +[[list-connector-api-example]] +==== {api-examples-title} + +The following example lists all connectors: + +//// +[source,console] +-------------------------------------------------- +PUT _connector/connector-1 +{ + "index_name": "search-google-drive", + "name": "Google Drive Connector", + "service_type": "google_drive" +} + +PUT _connector/connector-2 +{ + "index_name": "search-sharepoint-online", + "name": "Sharepoint Online Connector", + "service_type": "sharepoint_online" +} + +-------------------------------------------------- +// TESTSETUP + +[source,console] +-------------------------------------------------- +DELETE _connector/connector-1 + +DELETE _connector/connector-2 +-------------------------------------------------- +// TEARDOWN +//// + +[source,console] +---- +GET _connector +---- + +The following example lists the first two connectors: + +[source,console] +---- +GET _connector/?from=0&size=2 +---- From be9882f7305ac7e807d6c10afdf3a2bc905e80c3 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 13 Dec 2023 09:44:20 +0100 Subject: [PATCH 07/54] Review KEEP logic to prevent duplicate column names (#103316) --- docs/changelog/103316.yaml | 5 ++ .../xpack/esql/analysis/Analyzer.java | 76 ++++++++++++++----- .../xpack/esql/analysis/AnalyzerTests.java | 63 ++++++++++++++- 3 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 docs/changelog/103316.yaml diff --git a/docs/changelog/103316.yaml b/docs/changelog/103316.yaml new file mode 100644 index 0000000000000..47eddcc34d924 --- /dev/null +++ b/docs/changelog/103316.yaml @@ -0,0 +1,5 @@ +pr: 103316 +summary: Review KEEP logic to prevent duplicate column names +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 6959c04345d31..674a32db1f0fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -68,6 +68,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -416,6 +417,40 @@ private LogicalPlan resolveEval(Eval eval, List childOutput) { return changed ? new Eval(eval.source(), eval.child(), newFields) : eval; } + /** + * resolve each item manually. + * + * Fields are added in the order they appear. + * + * If one field matches multiple expressions, the following precedence rules apply (higher to lower): + * 1. complete field name (ie. no wildcards) + * 2. partial wildcard expressions (eg. fieldNam*) + * 3. wildcard only (ie. *) + * + * If a field name matches multiple expressions with the same precedence, last one is used. + * + * A few examples below: + * + * // full name + * row foo = 1, bar = 2 | keep foo, bar, foo -> bar, foo + * + * // the full name has precedence on wildcard expression + * row foo = 1, bar = 2 | keep foo, bar, foo* -> foo, bar + * + * // the two wildcard expressions have the same priority, even though the first one is more specific + * // so last one wins + * row foo = 1, bar = 2 | keep foo*, bar, fo* -> bar, foo + * + * // * has the lowest priority + * row foo = 1, bar = 2 | keep *, foo -> bar, foo + * row foo = 1, bar = 2 | keep foo, * -> foo, bar + * row foo = 1, bar = 2 | keep bar*, foo, * -> bar, foo + * + * + * @param p + * @param childOutput + * @return + */ private LogicalPlan resolveKeep(Project p, List childOutput) { List resolvedProjections = new ArrayList<>(); var projections = p.projections(); @@ -427,26 +462,31 @@ private LogicalPlan resolveKeep(Project p, List childOutput) { } // otherwise resolve them else { - var starPosition = -1; // no star - // resolve each item manually while paying attention to: - // 1. name patterns a*, *b, a*b - // 2. star * - which can only appear once and signifies "everything else" - this will be added at the end - for (var ne : projections) { - if (ne instanceof UnresolvedStar) { - starPosition = resolvedProjections.size(); - } else if (ne instanceof UnresolvedAttribute ua) { - resolvedProjections.addAll(resolveAgainstList(ua, childOutput)); - } else { - // if this gets here it means it was already resolved - resolvedProjections.add(ne); + Map priorities = new LinkedHashMap<>(); + for (Attribute attribute : childOutput) { + for (var proj : projections) { + List resolved; + int priority; + if (proj instanceof UnresolvedStar) { + resolved = childOutput; + priority = 2; + } else if (proj instanceof UnresolvedAttribute ua) { + resolved = resolveAgainstList(ua, childOutput); + priority = Regex.isSimpleMatchPattern(ua.name()) ? 1 : 0; + } else { + resolved = List.of(attribute); + priority = 0; + } + for (Attribute attr : resolved) { + Integer previousPrio = priorities.get(attr); + if (previousPrio == null || previousPrio >= priority) { + priorities.remove(attr); + priorities.put(attr, priority); + } + } } } - // compute star if specified and add it to the list - if (starPosition >= 0) { - var remainingProjections = new ArrayList<>(childOutput); - remainingProjections.removeAll(resolvedProjections); - resolvedProjections.addAll(starPosition, remainingProjections); - } + resolvedProjections = new ArrayList<>(priorities.keySet()); } return new EsqlProject(p.source(), p.child(), resolvedProjections); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 93456ff30c4cd..90e45a0a8b5a7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolution; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.aggregate.Max; +import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Row; @@ -266,11 +267,68 @@ public void testNoProjection() { ); } - public void testProjectOrder() { + public void testDuplicateProjections() { + assertProjection(""" + from test + | keep first_name, first_name + """, "first_name"); + assertProjection(""" + from test + | keep first_name, first_name, last_name, first_name + """, "last_name", "first_name"); + } + + public void testProjectWildcard() { assertProjection(""" from test | keep first_name, *, last_name """, "first_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name"); + assertProjection(""" + from test + | keep first_name, last_name, * + """, "first_name", "last_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary"); + assertProjection(""" + from test + | keep *, first_name, last_name + """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "first_name", "last_name"); + + var e = expectThrows(ParsingException.class, () -> analyze(""" + from test + | keep *, first_name, last_name, * + """)); + assertThat(e.getMessage(), containsString("Cannot specify [*] more than once")); + + } + + public void testProjectMixedWildcard() { + assertProjection(""" + from test + | keep *name, first* + """, "last_name", "first_name"); + assertProjection(""" + from test + | keep first_name, *name, first* + """, "first_name", "last_name"); + assertProjection(""" + from test + | keep *ob*, first_name, *name, first* + """, "job", "job.raw", "first_name", "last_name"); + assertProjection(""" + from test + | keep first_name, *, *name + """, "first_name", "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name"); + assertProjection(""" + from test + | keep first*, *, last_name, first_name + """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name", "first_name"); + assertProjection(""" + from test + | keep first*, *, last_name, fir* + """, "_meta_field", "emp_no", "gender", "job", "job.raw", "languages", "long_noidx", "salary", "last_name", "first_name"); + assertProjection(""" + from test + | keep *, job* + """, "_meta_field", "emp_no", "first_name", "gender", "languages", "last_name", "long_noidx", "salary", "job", "job.raw"); } public void testProjectThenDropName() { @@ -1429,9 +1487,6 @@ public void testMissingAttributeException_InChainedEval() { public void testUnresolvedMvExpand() { var e = expectThrows(VerificationException.class, () -> analyze("row foo = 1 | mv_expand bar")); assertThat(e.getMessage(), containsString("Unknown column [bar]")); - - e = expectThrows(VerificationException.class, () -> analyze("row foo = 1 | keep foo, foo | mv_expand foo")); - assertThat(e.getMessage(), containsString("Reference [foo] is ambiguous (to disambiguate use quotes or qualifiers)")); } private void verifyUnsupported(String query, String errorMessage) { From 40b817e63314e511b6dc63b08246c3db7f2c0c17 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 13 Dec 2023 10:54:04 +0100 Subject: [PATCH 08/54] [Connectors API] Unify Response object for all _update endpoints (#103267) --- .../rest-api-spec/api/connector.post.json | 2 +- .../action/ConnectorUpdateActionResponse.java | 69 +++++++++++++++++++ ...estUpdateConnectorConfigurationAction.java | 2 +- .../RestUpdateConnectorErrorAction.java | 2 +- .../RestUpdateConnectorFilteringAction.java | 2 +- .../RestUpdateConnectorLastSeenAction.java | 2 +- ...estUpdateConnectorLastSyncStatsAction.java | 2 +- .../action/RestUpdateConnectorNameAction.java | 2 +- .../RestUpdateConnectorPipelineAction.java | 2 +- .../RestUpdateConnectorSchedulingAction.java | 2 +- ...ortUpdateConnectorConfigurationAction.java | 9 +-- .../TransportUpdateConnectorErrorAction.java | 6 +- ...ansportUpdateConnectorFilteringAction.java | 9 +-- ...ransportUpdateConnectorLastSeenAction.java | 9 +-- ...ortUpdateConnectorLastSyncStatsAction.java | 9 +-- .../TransportUpdateConnectorNameAction.java | 10 +-- ...ransportUpdateConnectorPipelineAction.java | 9 +-- ...nsportUpdateConnectorSchedulingAction.java | 9 +-- .../UpdateConnectorConfigurationAction.java | 54 +-------------- .../action/UpdateConnectorErrorAction.java | 54 +-------------- .../UpdateConnectorFilteringAction.java | 54 +-------------- .../action/UpdateConnectorLastSeenAction.java | 54 +-------------- .../UpdateConnectorLastSyncStatsAction.java | 54 +-------------- .../action/UpdateConnectorNameAction.java | 54 +-------------- .../action/UpdateConnectorPipelineAction.java | 55 +-------------- .../UpdateConnectorSchedulingAction.java | 54 +-------------- ...dateActionResponseBWCSerializingTests.java | 38 ++++++++++ ...tionActionResponseBWCSerializingTests.java | 43 ------------ ...rrorActionResponseBWCSerializingTests.java | 42 ----------- ...ringActionResponseBWCSerializingTests.java | 42 ----------- ...SeenActionResponseBWCSerializingTests.java | 42 ----------- ...tatsActionResponseBWCSerializingTests.java | 43 ------------ ...NameActionResponseBWCSerializingTests.java | 42 ----------- ...lineActionResponseBWCSerializingTests.java | 41 ----------- ...lingActionResponseBWCSerializingTests.java | 43 ------------ 35 files changed, 156 insertions(+), 810 deletions(-) create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java create mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponseBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionResponseBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionResponseBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenActionResponseBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionResponseBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionResponseBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionResponseBWCSerializingTests.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.post.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.post.json index e76124bbecf7d..aadb59e99af7a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.post.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.post.json @@ -26,7 +26,7 @@ }, "body": { "description": "The connector configuration.", - "required": false + "required": true } } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java new file mode 100644 index 0000000000000..3d8367dfdeee0 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Represents a response for update actions related to connectors and sync jobs. + * The response encapsulates the result of the update action, represented by a {@link DocWriteResponse.Result}. + */ +public class ConnectorUpdateActionResponse extends ActionResponse implements ToXContentObject { + final DocWriteResponse.Result result; + + public ConnectorUpdateActionResponse(StreamInput in) throws IOException { + super(in); + result = DocWriteResponse.Result.readFrom(in); + } + + public ConnectorUpdateActionResponse(DocWriteResponse.Result result) { + this.result = result; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + this.result.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("result", this.result.getLowercase()); + builder.endObject(); + return builder; + } + + public RestStatus status() { + return switch (result) { + case NOT_FOUND -> RestStatus.NOT_FOUND; + default -> RestStatus.OK; + }; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectorUpdateActionResponse that = (ConnectorUpdateActionResponse) o; + return Objects.equals(result, that.result); + } + + @Override + public int hashCode() { + return Objects.hash(result); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorConfigurationAction.java index aa46353d47999..12c96d212f77a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorConfigurationAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorConfigurationAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorConfigurationAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java index ea8bd1b4ee50f..8b4b70b994ec1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorErrorAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorErrorAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorErrorAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFilteringAction.java index 63ae3e81fe563..4908e9e09d73f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFilteringAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorFilteringAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorFilteringAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSeenAction.java index b2ebaa74984b1..c2c6ee12a7767 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSeenAction.java @@ -35,7 +35,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorLastSeenAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorLastSeenAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSyncStatsAction.java index 8e373ce48caf3..ff3ba53e34a9d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorLastSyncStatsAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorLastSyncStatsAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorLastSyncStatsAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorNameAction.java index 54ce2c9af79e8..c51744e57b1df 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorNameAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorNameAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorNameAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorPipelineAction.java index ba83bd42dac11..8192099b832dd 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorPipelineAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorPipelineAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorPipelineAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorSchedulingAction.java index 06a6cb527544e..fda9fa03af913 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorSchedulingAction.java @@ -39,7 +39,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorSchedulingAction.INSTANCE, request, - new RestToXContentListener<>(channel, UpdateConnectorSchedulingAction.Response::status, r -> null) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java index 211c3b5a3a670..d4a7e0cf58df2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorConfigurationAction extends HandledTransportAction< UpdateConnectorConfigurationAction.Request, - UpdateConnectorConfigurationAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorConfigurationAction( protected void doExecute( Task task, UpdateConnectorConfigurationAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorConfiguration( - request, - listener.map(r -> new UpdateConnectorConfigurationAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorConfiguration(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java index 629fd14861cf6..5d9be2ec93f45 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorErrorAction extends HandledTransportAction< UpdateConnectorErrorAction.Request, - UpdateConnectorErrorAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,8 +45,8 @@ public TransportUpdateConnectorErrorAction( protected void doExecute( Task task, UpdateConnectorErrorAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorError(request, listener.map(r -> new UpdateConnectorErrorAction.Response(r.getResult()))); + connectorIndexService.updateConnectorError(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java index e871eb4bb79e5..658a8075121af 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorFilteringAction extends HandledTransportAction< UpdateConnectorFilteringAction.Request, - UpdateConnectorFilteringAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorFilteringAction( protected void doExecute( Task task, UpdateConnectorFilteringAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorFiltering( - request, - listener.map(r -> new UpdateConnectorFilteringAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorFiltering(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java index 3d3d2c9ee04b7..60c75bce8314a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorLastSeenAction extends HandledTransportAction< UpdateConnectorLastSeenAction.Request, - UpdateConnectorLastSeenAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorLastSeenAction( protected void doExecute( Task task, UpdateConnectorLastSeenAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorLastSeen( - request, - listener.map(r -> new UpdateConnectorLastSeenAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorLastSeen(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java index 9ec0105668fbc..ad934b04c772e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorLastSyncStatsAction extends HandledTransportAction< UpdateConnectorLastSyncStatsAction.Request, - UpdateConnectorLastSyncStatsAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorLastSyncStatsAction( protected void doExecute( Task task, UpdateConnectorLastSyncStatsAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorLastSyncStats( - request, - listener.map(r -> new UpdateConnectorLastSyncStatsAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorLastSyncStats(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java index 252734aab1c51..db79ed7be2836 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorNameAction extends HandledTransportAction< UpdateConnectorNameAction.Request, - UpdateConnectorNameAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -42,14 +42,10 @@ public TransportUpdateConnectorNameAction( } @Override - protected void doExecute( - Task task, - UpdateConnectorNameAction.Request request, - ActionListener listener - ) { + protected void doExecute(Task task, UpdateConnectorNameAction.Request request, ActionListener listener) { connectorIndexService.updateConnectorNameOrDescription( request, - listener.map(r -> new UpdateConnectorNameAction.Response(r.getResult())) + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java index c54d3db1215bc..11862d568c4ff 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorPipelineAction extends HandledTransportAction< UpdateConnectorPipelineAction.Request, - UpdateConnectorPipelineAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorPipelineAction( protected void doExecute( Task task, UpdateConnectorPipelineAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorPipeline( - request, - listener.map(r -> new UpdateConnectorPipelineAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorPipeline(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java index 186edb2328f38..bb5e1f858bd94 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java @@ -20,7 +20,7 @@ public class TransportUpdateConnectorSchedulingAction extends HandledTransportAction< UpdateConnectorSchedulingAction.Request, - UpdateConnectorSchedulingAction.Response> { + ConnectorUpdateActionResponse> { protected final ConnectorIndexService connectorIndexService; @@ -45,11 +45,8 @@ public TransportUpdateConnectorSchedulingAction( protected void doExecute( Task task, UpdateConnectorSchedulingAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorIndexService.updateConnectorScheduling( - request, - listener.map(r -> new UpdateConnectorSchedulingAction.Response(r.getResult())) - ); + connectorIndexService.updateConnectorScheduling(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java index 6b5f52f3afda7..19e7628746485 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ToXContentObject; @@ -37,13 +34,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class UpdateConnectorConfigurationAction extends ActionType { +public class UpdateConnectorConfigurationAction extends ActionType { public static final UpdateConnectorConfigurationAction INSTANCE = new UpdateConnectorConfigurationAction(); public static final String NAME = "cluster:admin/xpack/connector/update_configuration"; public UpdateConnectorConfigurationAction() { - super(NAME, UpdateConnectorConfigurationAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -152,51 +149,4 @@ public int hashCode() { return Objects.hash(connectorId, configuration); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java index c9e48dac08cd5..ad2036ecbaf81 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java @@ -10,16 +10,13 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -34,13 +31,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class UpdateConnectorErrorAction extends ActionType { +public class UpdateConnectorErrorAction extends ActionType { public static final UpdateConnectorErrorAction INSTANCE = new UpdateConnectorErrorAction(); public static final String NAME = "cluster:admin/xpack/connector/update_error"; public UpdateConnectorErrorAction() { - super(NAME, UpdateConnectorErrorAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -136,51 +133,4 @@ public int hashCode() { return Objects.hash(connectorId, error); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java index 68c644cb9d9db..dabb87f2afc22 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,13 +32,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class UpdateConnectorFilteringAction extends ActionType { +public class UpdateConnectorFilteringAction extends ActionType { public static final UpdateConnectorFilteringAction INSTANCE = new UpdateConnectorFilteringAction(); public static final String NAME = "cluster:admin/xpack/connector/update_filtering"; public UpdateConnectorFilteringAction() { - super(NAME, UpdateConnectorFilteringAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -141,51 +138,4 @@ public int hashCode() { return Objects.hash(connectorId, filtering); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java index 976be76ba84af..bd20513e47033 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java @@ -9,13 +9,10 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.application.connector.Connector; @@ -26,13 +23,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class UpdateConnectorLastSeenAction extends ActionType { +public class UpdateConnectorLastSeenAction extends ActionType { public static final UpdateConnectorLastSeenAction INSTANCE = new UpdateConnectorLastSeenAction(); public static final String NAME = "cluster:admin/xpack/connector/update_last_seen"; public UpdateConnectorLastSeenAction() { - super(NAME, UpdateConnectorLastSeenAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -97,51 +94,4 @@ public int hashCode() { return Objects.hash(connectorId, lastSeen); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java index 328831cf0b840..7d82c28ca4af1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ToXContentObject; @@ -36,13 +33,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class UpdateConnectorLastSyncStatsAction extends ActionType { +public class UpdateConnectorLastSyncStatsAction extends ActionType { public static final UpdateConnectorLastSyncStatsAction INSTANCE = new UpdateConnectorLastSyncStatsAction(); public static final String NAME = "cluster:admin/xpack/connector/update_last_sync_stats"; public UpdateConnectorLastSyncStatsAction() { - super(NAME, UpdateConnectorLastSyncStatsAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -190,51 +187,4 @@ public int hashCode() { return Objects.hash(connectorId, syncInfo); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java index 1db9bbe3aad9d..6b5c580e396ad 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java @@ -10,16 +10,13 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,13 +32,13 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class UpdateConnectorNameAction extends ActionType { +public class UpdateConnectorNameAction extends ActionType { public static final UpdateConnectorNameAction INSTANCE = new UpdateConnectorNameAction(); public static final String NAME = "cluster:admin/xpack/connector/update_name"; public UpdateConnectorNameAction() { - super(NAME, UpdateConnectorNameAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -159,51 +156,4 @@ public int hashCode() { return Objects.hash(connectorId, name, description); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java index 68babb2d4b517..ba5b0e702bf0e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -34,13 +31,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class UpdateConnectorPipelineAction extends ActionType { +public class UpdateConnectorPipelineAction extends ActionType { public static final UpdateConnectorPipelineAction INSTANCE = new UpdateConnectorPipelineAction(); public static final String NAME = "cluster:admin/xpack/connector/update_pipeline"; public UpdateConnectorPipelineAction() { - super(NAME, UpdateConnectorPipelineAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -139,52 +136,4 @@ public int hashCode() { return Objects.hash(connectorId, pipeline); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java index 9867830c5d211..df76e9a09547a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java @@ -10,15 +10,12 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -34,13 +31,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class UpdateConnectorSchedulingAction extends ActionType { +public class UpdateConnectorSchedulingAction extends ActionType { public static final UpdateConnectorSchedulingAction INSTANCE = new UpdateConnectorSchedulingAction(); public static final String NAME = "cluster:admin/xpack/connector/update_scheduling"; public UpdateConnectorSchedulingAction() { - super(NAME, UpdateConnectorSchedulingAction.Response::new); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { @@ -139,51 +136,4 @@ public int hashCode() { return Objects.hash(connectorId, scheduling); } } - - public static class Response extends ActionResponse implements ToXContentObject { - - final DocWriteResponse.Result result; - - public Response(StreamInput in) throws IOException { - super(in); - result = DocWriteResponse.Result.readFrom(in); - } - - public Response(DocWriteResponse.Result result) { - this.result = result; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - this.result.writeTo(out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("result", this.result.getLowercase()); - builder.endObject(); - return builder; - } - - public RestStatus status() { - return switch (result) { - case NOT_FOUND -> RestStatus.NOT_FOUND; - default -> RestStatus.OK; - }; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Response that = (Response) o; - return Objects.equals(result, that.result); - } - - @Override - public int hashCode() { - return Objects.hash(result); - } - } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponseBWCSerializingTests.java new file mode 100644 index 0000000000000..d70aa892788ce --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponseBWCSerializingTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; + +public class ConnectorUpdateActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return ConnectorUpdateActionResponse::new; + } + + @Override + protected ConnectorUpdateActionResponse createTestInstance() { + return new ConnectorUpdateActionResponse(randomFrom(DocWriteResponse.Result.values())); + } + + @Override + protected ConnectorUpdateActionResponse mutateInstance(ConnectorUpdateActionResponse instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected ConnectorUpdateActionResponse mutateInstanceForVersion(ConnectorUpdateActionResponse instance, TransportVersion version) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionResponseBWCSerializingTests.java deleted file mode 100644 index d4aa4f12b36d3..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorConfigurationActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorConfigurationAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorConfigurationAction.Response::new; - } - - @Override - protected UpdateConnectorConfigurationAction.Response createTestInstance() { - return new UpdateConnectorConfigurationAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorConfigurationAction.Response mutateInstance(UpdateConnectorConfigurationAction.Response instance) - throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorConfigurationAction.Response mutateInstanceForVersion( - UpdateConnectorConfigurationAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java deleted file mode 100644 index a39fcac3d2f04..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorErrorActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorErrorAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorErrorAction.Response::new; - } - - @Override - protected UpdateConnectorErrorAction.Response createTestInstance() { - return new UpdateConnectorErrorAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorErrorAction.Response mutateInstance(UpdateConnectorErrorAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorErrorAction.Response mutateInstanceForVersion( - UpdateConnectorErrorAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionResponseBWCSerializingTests.java deleted file mode 100644 index 0f33eeac8dfb5..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorFilteringActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorFilteringAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorFilteringAction.Response::new; - } - - @Override - protected UpdateConnectorFilteringAction.Response createTestInstance() { - return new UpdateConnectorFilteringAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorFilteringAction.Response mutateInstance(UpdateConnectorFilteringAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorFilteringAction.Response mutateInstanceForVersion( - UpdateConnectorFilteringAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenActionResponseBWCSerializingTests.java deleted file mode 100644 index d992f1b5f188e..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorLastSeenActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorLastSeenAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorLastSeenAction.Response::new; - } - - @Override - protected UpdateConnectorLastSeenAction.Response createTestInstance() { - return new UpdateConnectorLastSeenAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorLastSeenAction.Response mutateInstance(UpdateConnectorLastSeenAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorLastSeenAction.Response mutateInstanceForVersion( - UpdateConnectorLastSeenAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests.java deleted file mode 100644 index dd214e10699ef..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorLastSyncStatsActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorLastSyncStatsAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorLastSyncStatsAction.Response::new; - } - - @Override - protected UpdateConnectorLastSyncStatsAction.Response createTestInstance() { - return new UpdateConnectorLastSyncStatsAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorLastSyncStatsAction.Response mutateInstance(UpdateConnectorLastSyncStatsAction.Response instance) - throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorLastSyncStatsAction.Response mutateInstanceForVersion( - UpdateConnectorLastSyncStatsAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionResponseBWCSerializingTests.java deleted file mode 100644 index 2297ccb565b5e..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorNameActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorNameAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorNameAction.Response::new; - } - - @Override - protected UpdateConnectorNameAction.Response createTestInstance() { - return new UpdateConnectorNameAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorNameAction.Response mutateInstance(UpdateConnectorNameAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorNameAction.Response mutateInstanceForVersion( - UpdateConnectorNameAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionResponseBWCSerializingTests.java deleted file mode 100644 index 065dafcaf00a4..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorPipelineActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorPipelineAction.Response> { - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorPipelineAction.Response::new; - } - - @Override - protected UpdateConnectorPipelineAction.Response createTestInstance() { - return new UpdateConnectorPipelineAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorPipelineAction.Response mutateInstance(UpdateConnectorPipelineAction.Response instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorPipelineAction.Response mutateInstanceForVersion( - UpdateConnectorPipelineAction.Response instance, - TransportVersion version - ) { - return instance; - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionResponseBWCSerializingTests.java deleted file mode 100644 index a03713fa61a36..0000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionResponseBWCSerializingTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; - -import java.io.IOException; - -public class UpdateConnectorSchedulingActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< - UpdateConnectorSchedulingAction.Response> { - - @Override - protected Writeable.Reader instanceReader() { - return UpdateConnectorSchedulingAction.Response::new; - } - - @Override - protected UpdateConnectorSchedulingAction.Response createTestInstance() { - return new UpdateConnectorSchedulingAction.Response(randomFrom(DocWriteResponse.Result.values())); - } - - @Override - protected UpdateConnectorSchedulingAction.Response mutateInstance(UpdateConnectorSchedulingAction.Response instance) - throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected UpdateConnectorSchedulingAction.Response mutateInstanceForVersion( - UpdateConnectorSchedulingAction.Response instance, - TransportVersion version - ) { - return instance; - } -} From 6e36ea841d059aea2d63e47f8bf572c4c3ec3d03 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 13 Dec 2023 11:38:24 +0100 Subject: [PATCH 09/54] Port idp-fixture to testcontainers (#103320) This ports idp-fixture to test container and updates downstream tests accordingly. --- .../packer/CacheCacheableTestFixtures.java | 13 +- docs/build.gradle | 5 +- settings.gradle | 1 + test/fixtures/minio-fixture/build.gradle | 2 +- .../fixtures/minio/MinioTestContainer.java | 15 +- .../fixtures/testcontainer-utils/build.gradle | 15 ++ .../test/fixtures/CacheableTestFixture.java | 0 .../test/fixtures/ResourceUtils.java | 36 ++++ .../DockerEnvironmentAwareTestContainer.java | 20 ++- .../TestContainersThreadFilter.java | 4 +- .../authc/ldap/GroupsResolverTestCase.java | 4 + x-pack/qa/oidc-op-tests/build.gradle | 5 +- .../security/authc/jwt/JwtWithOidcAuthIT.java | 2 +- .../security/authc/oidc/C2IdOpTestCase.java | 91 +++++----- .../authc/oidc/OpenIdConnectAuthIT.java | 4 +- x-pack/qa/openldap-tests/build.gradle | 16 +- .../org/elasticsearch/test/OpenLdapTests.java | 36 ++-- ...OpenLdapUserSearchSessionFactoryTests.java | 14 +- .../authc/ldap/SearchGroupsResolverTests.java | 20 ++- x-pack/qa/saml-idp-tests/build.gradle | 35 +--- .../authc/saml/SamlAuthenticationIT.java | 38 +++- x-pack/test/idp-fixture/README.txt | 2 +- x-pack/test/idp-fixture/build.gradle | 53 +++--- x-pack/test/idp-fixture/docker-compose.yml | 67 ------- .../fixtures/idp/HttpProxyTestContainer.java | 40 +++++ .../test/fixtures/idp/IdpTestContainer.java | 163 ++++++++++++++++++ .../idp/OidcProviderTestContainer.java | 66 +++++++ .../fixtures/idp/OpenLdapTestContainer.java | 109 ++++++++++++ .../{ => src/main/resources}/idp/Dockerfile | 0 .../main/resources}/idp/bin/init-idp.sh | 0 .../main/resources}/idp/bin/run-jetty.sh | 0 .../main/resources}/idp/jetty-custom/keystore | Bin .../main/resources}/idp/jetty-custom/ssl.mod | 0 .../shib-jetty-base/etc/jetty-backchannel.xml | 0 .../idp/shib-jetty-base/etc/jetty-logging.xml | 0 .../shib-jetty-base/etc/jetty-requestlog.xml | 0 .../idp/shib-jetty-base/etc/jetty-rewrite.xml | 0 .../shib-jetty-base/etc/jetty-ssl-context.xml | 0 .../shib-jetty-base/modules/backchannel.mod | 0 .../resources/logback-access.xml | 0 .../idp/shib-jetty-base/resources/logback.xml | 0 .../shib-jetty-base/start.d/backchannel.ini | 0 .../idp/shib-jetty-base/start.d/ssl.ini | 0 .../resources}/idp/shib-jetty-base/start.ini | 0 .../idp/shib-jetty-base/webapps/idp.xml | 0 .../shibboleth-idp/conf/access-control.xml | 0 .../conf/admin/general-admin.xml | 0 .../idp/shibboleth-idp/conf/admin/metrics.xml | 0 .../shibboleth-idp/conf/attribute-filter.xml | 0 .../conf/attribute-resolver.xml | 0 .../idp/shibboleth-idp/conf/audit.xml | 0 .../conf/authn/authn-comparison.xml | 0 .../conf/authn/authn-events-flow.xml | 0 .../conf/authn/duo-authn-config.xml | 0 .../shibboleth-idp/conf/authn/duo.properties | 0 .../conf/authn/external-authn-config.xml | 0 .../conf/authn/function-authn-config.xml | 0 .../conf/authn/general-authn.xml | 0 .../conf/authn/ipaddress-authn-config.xml | 0 .../conf/authn/jaas-authn-config.xml | 0 .../idp/shibboleth-idp/conf/authn/jaas.config | 0 .../conf/authn/krb5-authn-config.xml | 0 .../conf/authn/ldap-authn-config.xml | 0 .../conf/authn/mfa-authn-config.xml | 0 .../conf/authn/password-authn-config.xml | 0 .../conf/authn/remoteuser-authn-config.xml | 0 .../remoteuser-internal-authn-config.xml | 0 .../conf/authn/spnego-authn-config.xml | 0 .../conf/authn/x509-authn-config.xml | 0 .../conf/authn/x509-internal-authn-config.xml | 0 .../attribute-sourced-subject-c14n-config.xml | 0 .../conf/c14n/simple-subject-c14n-config.xml | 0 .../conf/c14n/subject-c14n-events-flow.xml | 0 .../shibboleth-idp/conf/c14n/subject-c14n.xml | 0 .../conf/c14n/x500-subject-c14n-config.xml | 0 .../idp/shibboleth-idp/conf/cas-protocol.xml | 0 .../idp/shibboleth-idp/conf/credentials.xml | 0 .../idp/shibboleth-idp/conf/errors.xml | 0 .../idp/shibboleth-idp/conf/global.xml | 0 .../idp/shibboleth-idp/conf/idp.properties | 0 .../intercept/consent-intercept-config.xml | 0 .../context-check-intercept-config.xml | 0 .../expiring-password-intercept-config.xml | 0 .../impersonate-intercept-config.xml | 0 .../conf/intercept/intercept-events-flow.xml | 0 .../conf/intercept/profile-intercept.xml | 0 .../idp/shibboleth-idp/conf/ldap.properties | 0 .../idp/shibboleth-idp/conf/logback.xml | 0 .../conf/metadata-providers.xml | 0 .../idp/shibboleth-idp/conf/relying-party.xml | 0 .../conf/saml-nameid.properties | 0 .../idp/shibboleth-idp/conf/saml-nameid.xml | 0 .../shibboleth-idp/conf/services.properties | 0 .../idp/shibboleth-idp/conf/services.xml | 0 .../shibboleth-idp/conf/session-manager.xml | 0 .../idp/shibboleth-idp/credentials/README | 0 .../shibboleth-idp/credentials/ca_server.pem | 0 .../credentials/idp-backchannel.crt | 0 .../credentials/idp-backchannel.p12 | Bin .../credentials/idp-browser.key | 0 .../credentials/idp-browser.p12 | Bin .../credentials/idp-browser.pem | 0 .../credentials/idp-encryption.crt | 0 .../credentials/idp-encryption.key | 0 .../credentials/idp-signing.crt | 0 .../credentials/idp-signing.key | 0 .../idp/shibboleth-idp/credentials/sealer.jks | Bin .../shibboleth-idp/credentials/sealer.kver | 0 .../shibboleth-idp/credentials/sp-signing.crt | 0 .../shibboleth-idp/credentials/sp-signing.key | 0 .../shibboleth-idp/metadata/README.asciidoc | 0 .../metadata/idp-docs-metadata.xml | 0 .../shibboleth-idp/metadata/idp-metadata.xml | 0 .../shibboleth-idp/metadata/sp-metadata.xml | 0 .../shibboleth-idp/metadata/sp-metadata2.xml | 0 .../shibboleth-idp/metadata/sp-metadata3.xml | 0 .../shibboleth-idp/views/admin/unlock-keys.vm | 0 .../client-storage/client-storage-read.vm | 0 .../client-storage/client-storage-write.vm | 0 .../idp/shibboleth-idp/views/duo.vm | 0 .../idp/shibboleth-idp/views/error.vm | 0 .../views/intercept/attribute-release.vm | 0 .../views/intercept/expiring-password.vm | 0 .../views/intercept/impersonate.vm | 0 .../views/intercept/terms-of-use.vm | 0 .../idp/shibboleth-idp/views/login-error.vm | 0 .../idp/shibboleth-idp/views/login.vm | 0 .../shibboleth-idp/views/logout-complete.vm | 0 .../shibboleth-idp/views/logout-propagate.vm | 0 .../idp/shibboleth-idp/views/logout.vm | 0 .../views/spnego-unavailable.vm | 0 .../idp/shibboleth-idp/views/user-prefs.vm | 0 .../idp/shibboleth-idp/webapp/css/consent.css | 0 .../idp/shibboleth-idp/webapp/css/logout.css | 0 .../idp/shibboleth-idp/webapp/css/main.css | 0 .../webapp/images/dummylogo-mobile.png | Bin .../webapp/images/dummylogo.png | Bin .../webapp/images/failure-32x32.png | Bin .../webapp/images/success-32x32.png | Bin .../{ => src/main/resources}/oidc/Dockerfile | 0 .../{ => src/main/resources}/oidc/nginx.conf | 0 .../main/resources}/oidc/op-jwks.json | 0 .../oidc/override.properties.template | 0 .../{ => src/main/resources}/oidc/setup.sh | 0 .../main/resources}/openldap/certs/README | 0 .../main/resources}/openldap/certs/ca.jks | Bin .../resources}/openldap/certs/ca_server.key | 0 .../resources}/openldap/certs/ca_server.pem | 0 .../resources}/openldap/certs/dhparam.pem | 0 .../resources}/openldap/certs/ldap_server.csr | 0 .../resources}/openldap/certs/ldap_server.key | 0 .../resources}/openldap/certs/ldap_server.pem | 0 .../openldap/certs/templates/ca_server.conf | 0 .../openldap/certs/templates/ldap_server.conf | 0 .../main/resources}/openldap/ldif/config.ldif | 0 .../main/resources}/openldap/ldif/users.ldif | 0 156 files changed, 629 insertions(+), 247 deletions(-) create mode 100644 test/fixtures/testcontainer-utils/build.gradle rename test/fixtures/{minio-fixture => testcontainer-utils}/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java (100%) create mode 100644 test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java rename test/fixtures/{minio-fixture => testcontainer-utils}/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java (88%) rename test/fixtures/{minio-fixture => testcontainer-utils}/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java (80%) delete mode 100644 x-pack/test/idp-fixture/docker-compose.yml create mode 100644 x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/HttpProxyTestContainer.java create mode 100644 x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/IdpTestContainer.java create mode 100644 x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OidcProviderTestContainer.java create mode 100644 x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OpenLdapTestContainer.java rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/Dockerfile (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/bin/init-idp.sh (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/bin/run-jetty.sh (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/jetty-custom/keystore (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/jetty-custom/ssl.mod (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/etc/jetty-backchannel.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/etc/jetty-logging.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/etc/jetty-requestlog.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/etc/jetty-rewrite.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/etc/jetty-ssl-context.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/modules/backchannel.mod (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/resources/logback-access.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/resources/logback.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/start.d/backchannel.ini (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/start.d/ssl.ini (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/start.ini (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shib-jetty-base/webapps/idp.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/access-control.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/admin/general-admin.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/admin/metrics.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/attribute-filter.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/attribute-resolver.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/audit.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/authn-comparison.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/authn-events-flow.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/duo-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/duo.properties (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/external-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/function-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/general-authn.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/jaas.config (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/password-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/x509-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/c14n/subject-c14n.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/cas-protocol.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/credentials.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/errors.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/global.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/idp.properties (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/intercept/profile-intercept.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/ldap.properties (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/logback.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/metadata-providers.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/relying-party.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/saml-nameid.properties (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/saml-nameid.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/services.properties (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/services.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/conf/session-manager.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/README (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/ca_server.pem (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-backchannel.crt (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-backchannel.p12 (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-browser.key (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-browser.p12 (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-browser.pem (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-encryption.crt (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-encryption.key (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-signing.crt (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/idp-signing.key (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/sealer.jks (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/sealer.kver (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/sp-signing.crt (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/credentials/sp-signing.key (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/metadata/README.asciidoc (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/metadata/idp-docs-metadata.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/metadata/idp-metadata.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/metadata/sp-metadata.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/metadata/sp-metadata2.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/metadata/sp-metadata3.xml (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/admin/unlock-keys.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/client-storage/client-storage-read.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/client-storage/client-storage-write.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/duo.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/error.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/intercept/attribute-release.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/intercept/expiring-password.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/intercept/impersonate.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/intercept/terms-of-use.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/login-error.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/login.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/logout-complete.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/logout-propagate.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/logout.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/spnego-unavailable.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/views/user-prefs.vm (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/webapp/css/consent.css (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/webapp/css/logout.css (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/webapp/css/main.css (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/webapp/images/dummylogo.png (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/webapp/images/failure-32x32.png (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/idp/shibboleth-idp/webapp/images/success-32x32.png (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/oidc/Dockerfile (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/oidc/nginx.conf (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/oidc/op-jwks.json (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/oidc/override.properties.template (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/oidc/setup.sh (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/README (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/ca.jks (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/ca_server.key (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/ca_server.pem (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/dhparam.pem (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/ldap_server.csr (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/ldap_server.key (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/ldap_server.pem (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/templates/ca_server.conf (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/certs/templates/ldap_server.conf (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/ldif/config.ldif (100%) rename x-pack/test/idp-fixture/{ => src/main/resources}/openldap/ldif/users.ldif (100%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java index a01b1c28a851f..bfc52adcdecfd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java @@ -25,7 +25,9 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Constructor; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; @@ -74,9 +76,14 @@ public void execute() { Set> classes = (Set>) reflections.getSubTypesOf(ifClass); for (Class cacheableTestFixtureClazz : classes) { - Object o = cacheableTestFixtureClazz.getDeclaredConstructor().newInstance(); - Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache"); - cacheMethod.invoke(o); + if (Modifier.isAbstract(cacheableTestFixtureClazz.getModifiers()) == false) { + Constructor declaredConstructor = cacheableTestFixtureClazz.getDeclaredConstructor(); + declaredConstructor.setAccessible(true); + Object o = declaredConstructor.newInstance(); + Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache"); + System.out.println("Caching resources from " + cacheableTestFixtureClazz.getName()); + cacheMethod.invoke(o); + } } } catch (Exception e) { throw new RuntimeException(e); diff --git a/docs/build.gradle b/docs/build.gradle index ddd2a38b5160b..b6f696f0aae6a 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -113,8 +113,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach { requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0") - extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("oidc/op-jwks.json") - extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml") + // TODO Rene: clean up this kind of cross project file references + extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("src/main/resources/oidc/op-jwks.json") + extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("src/main/resources/idp/shibboleth-idp/metadata/idp-docs-metadata.xml") extraConfigFile 'testClient.crt', project(':x-pack:plugin:security').file("src/test/resources/org/elasticsearch/xpack/security/action/pki_delegation/testClient.crt") setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' diff --git a/settings.gradle b/settings.gradle index b3a33e11c4ec4..90422913ef441 100644 --- a/settings.gradle +++ b/settings.gradle @@ -97,6 +97,7 @@ List projects = [ 'test:fixtures:minio-fixture', 'test:fixtures:old-elasticsearch', 'test:fixtures:s3-fixture', + 'test:fixtures:testcontainer-utils', 'test:fixtures:geoip-fixture', 'test:fixtures:url-fixture', 'test:fixtures:nginx-fixture', diff --git a/test/fixtures/minio-fixture/build.gradle b/test/fixtures/minio-fixture/build.gradle index 66613809068f7..9a71387d7c6b7 100644 --- a/test/fixtures/minio-fixture/build.gradle +++ b/test/fixtures/minio-fixture/build.gradle @@ -5,7 +5,6 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'java' apply plugin: 'elasticsearch.java' apply plugin: 'elasticsearch.cache-test-fixtures' @@ -19,6 +18,7 @@ dependencies { testImplementation project(':test:framework') api "junit:junit:${versions.junit}" + api project(':test:fixtures:testcontainer-utils') api "org.testcontainers:testcontainers:${versions.testcontainer}" implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" implementation "org.slf4j:slf4j-api:${versions.slf4j}" diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java index a7e6ba8d785a1..671632f2e0125 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java @@ -8,12 +8,10 @@ package org.elasticsearch.test.fixtures.minio; -import org.elasticsearch.test.fixtures.CacheableTestFixture; import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; -import org.junit.rules.TestRule; import org.testcontainers.images.builder.ImageFromDockerfile; -public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer implements TestRule, CacheableTestFixture { +public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer { private static final int servicePort = 9000; public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2021-03-01T04-20-55Z"; @@ -25,7 +23,7 @@ public MinioTestContainer() { public MinioTestContainer(boolean enabled) { super( - new ImageFromDockerfile().withDockerfileFromBuilder( + new ImageFromDockerfile("es-minio-testfixture").withDockerfileFromBuilder( builder -> builder.from(DOCKER_BASE_IMAGE) .env("MINIO_ACCESS_KEY", "s3_test_access_key") .env("MINIO_SECRET_KEY", "s3_test_secret_key") @@ -50,13 +48,4 @@ public void start() { public String getAddress() { return "http://127.0.0.1:" + getMappedPort(servicePort); } - - public void cache() { - try { - start(); - stop(); - } catch (RuntimeException e) { - logger().warn("Error while caching container images.", e); - } - } } diff --git a/test/fixtures/testcontainer-utils/build.gradle b/test/fixtures/testcontainer-utils/build.gradle new file mode 100644 index 0000000000000..80886d99087c9 --- /dev/null +++ b/test/fixtures/testcontainer-utils/build.gradle @@ -0,0 +1,15 @@ +apply plugin: 'elasticsearch.java' + + +configurations.all { + transitive = false +} + +dependencies { + testImplementation project(':test:framework') + api "junit:junit:${versions.junit}" + api "org.testcontainers:testcontainers:${versions.testcontainer}" + implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" +} diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java similarity index 100% rename from test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java rename to test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/CacheableTestFixture.java diff --git a/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java new file mode 100644 index 0000000000000..8fe64ea34d8d4 --- /dev/null +++ b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test.fixtures; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +public final class ResourceUtils { + + public static Path copyResourceToFile(Class clazz, Path targetFolder, String resourcePath) { + try { + ClassLoader classLoader = clazz.getClassLoader(); + URL resourceUrl = classLoader.getResource(resourcePath); + if (resourceUrl == null) { + throw new RuntimeException("Failed to load " + resourcePath + " from classpath"); + } + InputStream inputStream = resourceUrl.openStream(); + File outputFile = new File(targetFolder.toFile(), resourcePath.substring(resourcePath.lastIndexOf("/"))); + Files.copy(inputStream, outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + return outputFile.toPath(); + } catch (IOException e) { + throw new RuntimeException("Failed to load ca.jks from classpath", e); + } + } +} diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java similarity index 88% rename from test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java rename to test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java index c0fb83e5206f4..e894c767d6217 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java +++ b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java @@ -8,12 +8,14 @@ package org.elasticsearch.test.fixtures.testcontainers; -import org.elasticsearch.test.fixtures.minio.MinioTestContainer; +import org.elasticsearch.test.fixtures.CacheableTestFixture; import org.junit.Assume; +import org.junit.rules.TestRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.images.builder.ImageFromDockerfile; import java.io.File; @@ -27,7 +29,10 @@ import java.util.Map; import java.util.stream.Collectors; -public class DockerEnvironmentAwareTestContainer extends GenericContainer { +public abstract class DockerEnvironmentAwareTestContainer extends GenericContainer + implements + TestRule, + CacheableTestFixture { protected static final Logger LOGGER = LoggerFactory.getLogger(DockerEnvironmentAwareTestContainer.class); private static final String DOCKER_ON_LINUX_EXCLUSIONS_FILE = ".ci/dockerOnLinuxExclusions"; @@ -53,6 +58,7 @@ private static boolean isDockerAvailable() { public DockerEnvironmentAwareTestContainer(ImageFromDockerfile imageFromDockerfile) { super(imageFromDockerfile); + withLogConsumer(new Slf4jLogConsumer(logger())); } @Override @@ -62,6 +68,16 @@ public void start() { super.start(); } + @Override + public void cache() { + try { + start(); + stop(); + } catch (RuntimeException e) { + logger().warn("Error while caching container images.", e); + } + } + static String deriveId(Map values) { return values.get("ID") + "-" + values.get("VERSION_ID"); } diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java similarity index 80% rename from test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java rename to test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java index 1b0dacbacfd1a..f36a3264bffbb 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java +++ b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java @@ -17,6 +17,8 @@ public class TestContainersThreadFilter implements ThreadFilter { @Override public boolean reject(Thread t) { - return t.getName().startsWith("testcontainers-") || t.getName().startsWith("ducttape"); + return t.getName().startsWith("testcontainers-") + || t.getName().startsWith("ducttape") + || t.getName().startsWith("ForkJoinPool.commonPool-worker-1"); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java index 57bda2ad9cc1d..09a7e33e51901 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java @@ -54,6 +54,10 @@ protected static RealmConfig config(RealmConfig.RealmIdentifier realmId, Setting @Before public void setUpLdapConnection() throws Exception { + doSetupLdapConnection(); + } + + protected void doSetupLdapConnection() throws Exception { Path trustPath = getDataPath(trustPath()); this.ldapConnection = LdapTestUtils.openConnection(ldapUrl(), bindDN(), bindPassword(), trustPath); } diff --git a/x-pack/qa/oidc-op-tests/build.gradle b/x-pack/qa/oidc-op-tests/build.gradle index 3987073b1e6ad..8f46613d5d9f0 100644 --- a/x-pack/qa/oidc-op-tests/build.gradle +++ b/x-pack/qa/oidc-op-tests/build.gradle @@ -1,16 +1,13 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-java-rest-test' -apply plugin: 'elasticsearch.test.fixtures' dependencies { javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation(testArtifact(project(xpackModule('security')))) + javaRestTestImplementation project(":x-pack:test:idp-fixture") } -testFixtures.useFixture ":x-pack:test:idp-fixture", "http-proxy" -testFixtures.useFixture ":x-pack:test:idp-fixture", "oidc-provider" - tasks.named('javaRestTest') { usesDefaultDistribution() // test suite uses jks which is not supported in fips mode diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java index 2d3fc611758b0..18224c887c663 100644 --- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java +++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java @@ -106,7 +106,7 @@ public void testAuthenticateWithOidcIssuedJwt() throws Exception { new Scope(OIDCScopeValue.OPENID), new ClientID(clientId), new URI(redirectUri) - ).endpointURI(new URI(C2ID_AUTH_ENDPOINT)).state(new State(state)).nonce(new Nonce(nonce)).build(); + ).endpointURI(new URI(c2id.getC2OPUrl() + "/c2id-login")).state(new State(state)).nonce(new Nonce(nonce)).build(); final String implicitFlowURI = authenticateAtOP(oidcAuthRequest.toURI()); diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java index 106e1d27910f2..56d3bbe77c78a 100644 --- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java +++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java @@ -10,6 +10,8 @@ import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; @@ -38,11 +40,17 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.fixtures.idp.HttpProxyTestContainer; +import org.elasticsearch.test.fixtures.idp.OidcProviderTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.common.socket.SocketAccess; import org.hamcrest.Matchers; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; +import org.testcontainers.containers.Network; import java.io.FileNotFoundException; import java.io.IOException; @@ -56,25 +64,21 @@ import static org.hamcrest.Matchers.equalTo; +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public abstract class C2IdOpTestCase extends ESRestTestCase { protected static final String TEST_SUBJECT_ID = "alice"; - // URLs for accessing the C2id OP - private static final String C2OP_PORT = getEphemeralTcpPortFromProperty("oidc-provider", "8080"); - private static final String C2ID_HOST = "http://127.0.0.1:" + C2OP_PORT; - protected static final String C2ID_ISSUER = C2ID_HOST + "/c2id"; - private static final String PROXY_PORT = getEphemeralTcpPortFromProperty("http-proxy", "8888"); - private static final String C2ID_LOGIN_API = C2ID_HOST + "/c2id-login/api/"; - private static final String C2ID_REGISTRATION_URL = C2ID_HOST + "/c2id/clients"; - protected static final String C2ID_AUTH_ENDPOINT = C2ID_HOST + "/c2id-login"; - // SHA256 of this is defined in x-pack/test/idp-fixture/oidc/override.properties private static final String OP_API_BEARER_TOKEN = "811fa888f3e0fdc9e01d4201bfeee46a"; private static Path HTTP_TRUSTED_CERT; private static final String CLIENT_SECRET = "b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2"; - @ClassRule + + private static Network network = Network.newNetwork(); + protected static OidcProviderTestContainer c2id = new OidcProviderTestContainer(network); + protected static HttpProxyTestContainer proxy = new HttpProxyTestContainer(network); + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .nodes(1) @@ -86,10 +90,10 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.file.file.order", "0") .setting("xpack.security.authc.realms.native.native.order", "1") .setting("xpack.security.authc.realms.oidc.c2id.order", "2") - .setting("xpack.security.authc.realms.oidc.c2id.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id.rp.client_id", "https://my.elasticsearch.org/rp") @@ -99,10 +103,10 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id.claims.groups", "groups") .setting("xpack.security.authc.realms.oidc.c2id-implicit.order", "3") - .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id-implicit.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id-implicit.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id-implicit.rp.client_id", "elasticsearch-rp") @@ -112,10 +116,10 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id-implicit.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id-implicit.claims.groups", "groups") .setting("xpack.security.authc.realms.oidc.c2id-proxy.order", "4") - .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id-proxy.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id-proxy.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id-proxy.rp.client_id", "https://my.elasticsearch.org/rp") @@ -125,12 +129,12 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id-proxy.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id-proxy.claims.groups", "groups") .setting("xpack.security.authc.realms.oidc.c2id-proxy.http.proxy.host", "127.0.0.1") - .setting("xpack.security.authc.realms.oidc.c2id-proxy.http.proxy.port", PROXY_PORT) + .setting("xpack.security.authc.realms.oidc.c2id-proxy.http.proxy.port", () -> proxy.getProxyPort().toString()) .setting("xpack.security.authc.realms.oidc.c2id-post.order", "5") - .setting("xpack.security.authc.realms.oidc.c2id-post.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id-post.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id-post.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id-post.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id-post.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id-post.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id-post.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id-post.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id-post.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id-post.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id-post.rp.client_id", "elasticsearch-post") @@ -141,10 +145,10 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id-post.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id-post.claims.groups", "groups") .setting("xpack.security.authc.realms.oidc.c2id-jwt.order", "6") - .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.issuer", C2ID_ISSUER) - .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.authorization_endpoint", C2ID_HOST + "/c2id-login") - .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.token_endpoint", C2ID_HOST + "/c2id/token") - .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.userinfo_endpoint", C2ID_HOST + "/c2id/userinfo") + .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.issuer", () -> c2id.getC2IssuerUrl()) + .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.authorization_endpoint", () -> c2id.getC2OPUrl() + "/c2id-login") + .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.token_endpoint", () -> c2id.getC2OPUrl() + "/c2id/token") + .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.userinfo_endpoint", () -> c2id.getC2OPUrl() + "/c2id/userinfo") .setting("xpack.security.authc.realms.oidc.c2id-jwt.op.jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.oidc.c2id-jwt.rp.redirect_uri", "https://my.fantastic.rp/cb") .setting("xpack.security.authc.realms.oidc.c2id-jwt.rp.client_id", "elasticsearch-post-jwt") @@ -155,7 +159,7 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .setting("xpack.security.authc.realms.oidc.c2id-jwt.claims.mail", "email") .setting("xpack.security.authc.realms.oidc.c2id-jwt.claims.groups", "groups") .setting("xpack.security.authc.realms.jwt.op-jwt.order", "7") - .setting("xpack.security.authc.realms.jwt.op-jwt.allowed_issuer", C2ID_ISSUER) + .setting("xpack.security.authc.realms.jwt.op-jwt.allowed_issuer", () -> c2id.getC2IssuerUrl()) .setting("xpack.security.authc.realms.jwt.op-jwt.allowed_audiences", "elasticsearch-jwt1,elasticsearch-jwt2") .setting("xpack.security.authc.realms.jwt.op-jwt.pkc_jwkset_path", "op-jwks.json") .setting("xpack.security.authc.realms.jwt.op-jwt.claims.principal", "sub") @@ -174,6 +178,9 @@ public abstract class C2IdOpTestCase extends ESRestTestCase { .user("x_pack_rest_user", "x-pack-test-password", "superuser", false) .build(); + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(network).around(c2id).around(proxy).around(cluster); + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); @@ -193,24 +200,17 @@ public static void readTrustedCert() throws Exception { HTTP_TRUSTED_CERT = PathUtils.get(resource.toURI()); } - protected static String getEphemeralTcpPortFromProperty(String service, String port) { - String key = "test.fixtures." + service + ".tcp." + port; - final String value = System.getProperty(key); - assertNotNull("Expected the actual value for port " + port + " to be in system property " + key, value); - return value; - } - /** * Register one or more OIDC clients on the C2id server. This should be done once (per client) only. * C2id server only supports dynamic registration, so we can't pre-seed its config with our client data. */ protected static void registerClients(String... jsonBody) throws IOException { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + String ci2dRegistrationUrl = c2id.getC2OPUrl() + "/c2id/clients"; final BasicHttpContext context = new BasicHttpContext(); - final List requests = new ArrayList<>(jsonBody.length); for (String body : jsonBody) { - HttpPost httpPost = new HttpPost(C2ID_REGISTRATION_URL); + HttpPost httpPost = new HttpPost(ci2dRegistrationUrl); httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); httpPost.setHeader("Accept", "application/json"); httpPost.setHeader("Content-type", "application/json"); @@ -240,12 +240,13 @@ protected Settings restAdminSettings() { } protected String authenticateAtOP(URI opAuthUri) throws Exception { + String c2LoginApi = c2id.getC2OPUrl() + "/c2id-login/api/"; // C2ID doesn't have a non JS login page :/, so use their API directly // see https://connect2id.com/products/server/docs/guides/login-page try (CloseableHttpClient httpClient = HttpClients.createDefault()) { final BasicHttpContext context = new BasicHttpContext(); // Initiate the authentication process - HttpPost httpPost = new HttpPost(C2ID_LOGIN_API + "initAuthRequest"); + HttpPost httpPost = new HttpPost(c2LoginApi + "initAuthRequest"); String initJson = Strings.format(""" {"qs":"%s"} """, opAuthUri.getRawQuery()); @@ -258,7 +259,7 @@ protected String authenticateAtOP(URI opAuthUri) throws Exception { final String sid = initResponse.getAsString("sid"); // Actually authenticate the user with ldapAuth HttpPost loginHttpPost = new HttpPost( - C2ID_LOGIN_API + "authenticateSubject?cacheBuster=" + randomAlphaOfLength(8) + "&authSessionId=" + sid + c2LoginApi + "authenticateSubject?cacheBuster=" + randomAlphaOfLength(8) + "&authSessionId=" + sid ); String loginJson = """ {"username":"alice","password":"secret"}"""; @@ -268,9 +269,7 @@ protected String authenticateAtOP(URI opAuthUri) throws Exception { return parseJsonResponse(response); }); - HttpPut consentHttpPut = new HttpPut( - C2ID_LOGIN_API + "updateAuthRequest" + "/" + sid + "?cacheBuster=" + randomAlphaOfLength(8) - ); + HttpPut consentHttpPut = new HttpPut(c2LoginApi + "updateAuthRequest" + "/" + sid + "?cacheBuster=" + randomAlphaOfLength(8)); String consentJson = """ {"claims":["name", "email"],"scope":["openid"]}"""; configureJsonRequest(consentHttpPut, consentJson); diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java index 9302f731ff285..cd37d86626333 100644 --- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java +++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java @@ -182,7 +182,7 @@ private void verifyElasticsearchAccessTokenForCodeFlow(String accessToken) throw assertThat(map.get("metadata"), instanceOf(Map.class)); final Map metadata = (Map) map.get("metadata"); assertThat(metadata.get("oidc(sub)"), equalTo("alice")); - assertThat(metadata.get("oidc(iss)"), equalTo(C2ID_ISSUER)); + assertThat(metadata.get("oidc(iss)"), equalTo(c2id.getC2IssuerUrl())); } private void verifyElasticsearchAccessTokenForImplicitFlow(String accessToken) throws Exception { @@ -194,7 +194,7 @@ private void verifyElasticsearchAccessTokenForImplicitFlow(String accessToken) t assertThat(map.get("metadata"), instanceOf(Map.class)); final Map metadata = (Map) map.get("metadata"); assertThat(metadata.get("oidc(sub)"), equalTo("alice")); - assertThat(metadata.get("oidc(iss)"), equalTo(C2ID_ISSUER)); + assertThat(metadata.get("oidc(iss)"), equalTo(c2id.getC2IssuerUrl())); } private PrepareAuthResponse getRedirectedFromFacilitator(String realmName) throws Exception { diff --git a/x-pack/qa/openldap-tests/build.gradle b/x-pack/qa/openldap-tests/build.gradle index 844f47c9a53f5..78a03c556bc11 100644 --- a/x-pack/qa/openldap-tests/build.gradle +++ b/x-pack/qa/openldap-tests/build.gradle @@ -1,18 +1,16 @@ apply plugin: 'elasticsearch.standalone-test' -apply plugin: 'elasticsearch.test.fixtures' dependencies { testImplementation(testArtifact(project(xpackModule('security')))) testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(":x-pack:test:idp-fixture") + testImplementation "junit:junit:${versions.junit}" + testImplementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" } -testFixtures.useFixture ":x-pack:test:idp-fixture", "openldap" -Project idpFixtureProject = project(":x-pack:test:idp-fixture") -String outputDir = "${project.buildDir}/generated-resources/${project.name}" -def copyIdpTrust = tasks.register("copyIdpTrust", Copy) { - from idpFixtureProject.file('openldap/certs/ca.jks'); - from idpFixtureProject.file('openldap/certs/ca_server.pem'); - into outputDir +tasks.named('test') { + // test suite uses jks which is not supported in fips mode + systemProperty 'tests.security.manager', 'false' } -project.sourceSets.test.output.dir(outputDir, builtBy: copyIdpTrust) diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java index c1375823548df..7d1610d2ccc0f 100644 --- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java +++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.test; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.unboundid.ldap.sdk.LDAPConnection; import com.unboundid.ldap.sdk.LDAPException; @@ -19,6 +20,8 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.test.fixtures.idp.OpenLdapTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.test.junit.annotations.Network; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -37,8 +40,8 @@ import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory; import org.junit.After; import org.junit.Before; +import org.junit.ClassRule; -import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -53,10 +56,9 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.startsWith; +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public class OpenLdapTests extends ESTestCase { - public static final String OPEN_LDAP_DNS_URL = "ldaps://localhost:" + getFromProperty("636"); - /** * * ip.es.io is magic that will resolve any IP-like DNS name into the embedded IP @@ -67,11 +69,12 @@ public class OpenLdapTests extends ESTestCase { * so in order to have a "not-valid-hostname" failure, we need a second * hostname that isn't in the certificate's Subj Alt Name list */ - private static final String OPEN_LDAP_ES_IO_URL = "ldaps://127.0.0.1.ip.es.io:" + getFromProperty("636"); + + @ClassRule + public static final OpenLdapTestContainer openLdap = new OpenLdapTestContainer(); public static final String PASSWORD = "NickFuryHeartsES"; private static final String HAWKEYE_DN = "uid=hawkeye,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; - public static final String LDAPTRUST_PATH = "/ca.jks"; private static final SecureString PASSWORD_SECURE_STRING = new SecureString(PASSWORD.toCharArray()); public static final String REALM_NAME = "oldap-test"; @@ -96,7 +99,6 @@ public boolean enableWarningsCheck() { @Before public void initializeSslSocketFactory() throws Exception { - Path truststore = getDataPath(LDAPTRUST_PATH); /* * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext. * If we re-use an SSLContext, previously connected sessions can get re-established which breaks hostname @@ -105,14 +107,14 @@ public void initializeSslSocketFactory() throws Exception { MockSecureSettings mockSecureSettings = new MockSecureSettings(); Settings.Builder builder = Settings.builder().put("path.home", createTempDir()); // fake realms so ssl will get loaded - builder.put("xpack.security.authc.realms.ldap.foo.ssl.truststore.path", truststore); + builder.put("xpack.security.authc.realms.ldap.foo.ssl.truststore.path", openLdap.getJavaKeyStorePath()); mockSecureSettings.setString("xpack.security.authc.realms.ldap.foo.ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.ldap.foo.ssl.verification_mode", SslVerificationMode.FULL); - builder.put("xpack.security.authc.realms.ldap." + REALM_NAME + ".ssl.truststore.path", truststore); + builder.put("xpack.security.authc.realms.ldap." + REALM_NAME + ".ssl.truststore.path", openLdap.getJavaKeyStorePath()); mockSecureSettings.setString("xpack.security.authc.realms.ldap." + REALM_NAME + ".ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.ldap." + REALM_NAME + ".ssl.verification_mode", SslVerificationMode.CERTIFICATE); - builder.put("xpack.security.authc.realms.ldap.vmode_full.ssl.truststore.path", truststore); + builder.put("xpack.security.authc.realms.ldap.vmode_full.ssl.truststore.path", openLdap.getJavaKeyStorePath()); mockSecureSettings.setString("xpack.security.authc.realms.ldap.vmode_full.ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.ldap.vmode_full.ssl.verification_mode", SslVerificationMode.FULL); globalSettings = builder.setSecureSettings(mockSecureSettings).build(); @@ -127,7 +129,7 @@ public void testConnect() throws Exception { final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", "oldap-test"); RealmConfig config = new RealmConfig( realmId, - buildLdapSettings(realmId, OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL), + buildLdapSettings(realmId, openLdap.getLdapUrl(), userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL), TestEnvironment.newEnvironment(globalSettings), new ThreadContext(Settings.EMPTY) ); @@ -150,7 +152,7 @@ public void testGroupSearchScopeBase() throws Exception { final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", REALM_NAME); RealmConfig config = new RealmConfig( realmId, - buildLdapSettings(realmId, OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.BASE), + buildLdapSettings(realmId, openLdap.getLdapUrl(), userTemplate, groupSearchBase, LdapSearchScope.BASE), TestEnvironment.newEnvironment(globalSettings), new ThreadContext(Settings.EMPTY) ); @@ -169,7 +171,7 @@ public void testCustomFilter() throws Exception { String userTemplate = "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", "oldap-test"); Settings settings = Settings.builder() - .put(buildLdapSettings(realmId, OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) + .put(buildLdapSettings(realmId, openLdap.getLdapUrl(), userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) .put(getFullSettingKey(realmId.getName(), SearchGroupsResolverSettings.FILTER), "(&(objectclass=posixGroup)(memberUid={0}))") .put(getFullSettingKey(realmId.getName(), SearchGroupsResolverSettings.USER_ATTRIBUTE), "uid") .build(); @@ -192,9 +194,10 @@ public void testStandardLdapConnectionHostnameVerificationFailure() throws Excep String groupSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; String userTemplate = "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", "vmode_full"); + String openLdapEsIoURL = "ldaps://127.0.0.1.ip.es.io:" + openLdap.getDefaultPort(); Settings settings = Settings.builder() // The certificate used in the vagrant box is valid for "localhost", but not for "*.ip.es.io" - .put(buildLdapSettings(realmId, OPEN_LDAP_ES_IO_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) + .put(buildLdapSettings(realmId, openLdapEsIoURL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) .build(); final Environment env = TestEnvironment.newEnvironment(globalSettings); RealmConfig config = new RealmConfig(realmId, settings, env, new ThreadContext(Settings.EMPTY)); @@ -220,7 +223,7 @@ public void testStandardLdapConnectionHostnameVerificationSuccess() throws Excep final RealmConfig.RealmIdentifier realmId = new RealmConfig.RealmIdentifier("ldap", "vmode_full"); Settings settings = Settings.builder() // The certificate used in the vagrant box is valid for "localhost" (but not for "*.ip.es.io") - .put(buildLdapSettings(realmId, OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) + .put(buildLdapSettings(realmId, openLdap.getLdapUrl(), userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) .build(); RealmConfig config = new RealmConfig( @@ -320,7 +323,7 @@ private Settings buildLdapSettings( Settings.Builder builder = Settings.builder() .put(LdapTestCase.buildLdapSettings(realmId, urls, templates, groupSearchBase, scope, null, false)); builder.put(getFullSettingKey(realmId.getName(), SearchGroupsResolverSettings.USER_ATTRIBUTE), "uid"); - return builder.put(SSLConfigurationSettings.TRUSTSTORE_PATH.realm(realmId).getKey(), getDataPath(LDAPTRUST_PATH)) + return builder.put(SSLConfigurationSettings.TRUSTSTORE_PATH.realm(realmId).getKey(), openLdap.getJavaKeyStorePath()) .put(SSLConfigurationSettings.LEGACY_TRUSTSTORE_PASSWORD.realm(realmId).getKey(), "changeit") .put(globalSettings) .put(getFullSettingKey(realmId, RealmSettings.ORDER_SETTING), 0) @@ -340,8 +343,7 @@ private List groups(LdapSession ldapSession) { } private LDAPConnection setupOpenLdapConnection() throws Exception { - Path truststore = getDataPath(LDAPTRUST_PATH); - return LdapTestUtils.openConnection(OpenLdapTests.OPEN_LDAP_DNS_URL, HAWKEYE_DN, OpenLdapTests.PASSWORD, truststore); + return LdapTestUtils.openConnection(openLdap.getLdapUrl(), HAWKEYE_DN, OpenLdapTests.PASSWORD, openLdap.getJavaKeyStorePath()); } private Map resolve(LDAPConnection connection, LdapMetadataResolver resolver) throws Exception { diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java index eb3365010b550..60bff29bfb58d 100644 --- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java +++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.security.authc.ldap; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; @@ -16,6 +18,8 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.OpenLdapTests; +import org.elasticsearch.test.fixtures.idp.OpenLdapTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.security.authc.RealmConfig; @@ -31,6 +35,7 @@ import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory; import org.junit.After; import org.junit.Before; +import org.junit.ClassRule; import java.nio.file.Path; import java.text.MessageFormat; @@ -44,15 +49,18 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public class OpenLdapUserSearchSessionFactoryTests extends ESTestCase { private Settings globalSettings; private ThreadPool threadPool; - private static final String LDAPCACERT_PATH = "/ca_server.pem"; + + @ClassRule + public static final OpenLdapTestContainer openLdapContainer = new OpenLdapTestContainer(); @Before public void init() { - Path caPath = getDataPath(LDAPCACERT_PATH); + Path caPath = openLdapContainer.getCaCertPath(); /* * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext. * If we re-use an SSLContext, previously connected sessions can get re-established which breaks hostname @@ -79,7 +87,7 @@ public void testUserSearchWithBindUserOpenLDAP() throws Exception { .put( LdapTestCase.buildLdapSettings( realmId, - new String[] { OpenLdapTests.OPEN_LDAP_DNS_URL }, + new String[] { openLdapContainer.getLdapUrl() }, Strings.EMPTY_ARRAY, groupSearchBase, LdapSearchScope.ONE_LEVEL, diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java index feec06f4b3b6d..4cc12e5af448e 100644 --- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java +++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java @@ -6,15 +6,21 @@ */ package org.elasticsearch.xpack.security.authc.ldap; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.OpenLdapTests; +import org.elasticsearch.test.fixtures.idp.OpenLdapTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings; import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope; import org.elasticsearch.xpack.core.security.support.NoOpLogger; +import org.junit.ClassRule; +import java.nio.file.Path; import java.util.List; import static org.elasticsearch.xpack.core.security.authc.RealmSettings.getFullSettingKey; @@ -24,11 +30,15 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public class SearchGroupsResolverTests extends GroupsResolverTestCase { private static final String BRUCE_BANNER_DN = "uid=hulk,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; private static final RealmConfig.RealmIdentifier REALM_ID = new RealmConfig.RealmIdentifier("ldap", "my-ldap-realm"); + @ClassRule + public static final OpenLdapTestContainer openLdapContainer = new OpenLdapTestContainer(); + public void testResolveSubTree() throws Exception { Settings settings = Settings.builder() .put(getFullSettingKey(REALM_ID, SearchGroupsResolverSettings.BASE_DN), "dc=oldap,dc=test,dc=elasticsearch,dc=com") @@ -202,7 +212,7 @@ public void testReadBinaryUserAttribute() throws Exception { @Override protected String ldapUrl() { - return OpenLdapTests.OPEN_LDAP_DNS_URL; + return openLdapContainer.getLdapUrl(); } @Override @@ -217,6 +227,12 @@ protected String bindPassword() { @Override protected String trustPath() { - return "/ca.jks"; + return "/openldap/certs/ca.jks"; + } + + @Override + protected void doSetupLdapConnection() throws Exception { + Path trustPath = openLdapContainer.getJavaKeyStorePath(); + this.ldapConnection = LdapTestUtils.openConnection(ldapUrl(), bindDN(), bindPassword(), trustPath); } } diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 6027a421b62f2..6a7d60f88a1d7 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -1,46 +1,13 @@ -import org.elasticsearch.gradle.LazyPropertyMap - -Project idpFixtureProject = project(':x-pack:test:idp-fixture') - apply plugin: 'elasticsearch.internal-java-rest-test' -apply plugin: 'elasticsearch.test.fixtures' dependencies { javaRestTestImplementation testArtifact(project(xpackModule('core'))) javaRestTestImplementation "com.google.jimfs:jimfs:${versions.jimfs}" javaRestTestImplementation "com.google.guava:guava:${versions.jimfs_guava}" + javaRestTestImplementation project(":x-pack:test:idp-fixture") } -testFixtures.useFixture ":x-pack:test:idp-fixture" - -String outputDir = "${project.buildDir}/generated-resources/${project.name}" -tasks.register("copyIdpFiles", Sync) { - dependsOn idpFixtureProject.postProcessFixture - // Don't attempt to get ephemeral ports when Docker is not available - onlyIf(idpFixtureProject.postProcessFixture.path + " not skipped") { - idpFixtureProject.postProcessFixture.state.skipped == false - } - from idpFixtureProject.files('idp/shibboleth-idp/credentials/idp-browser.pem', 'idp/shibboleth-idp/metadata/idp-metadata.xml', - 'idp/shibboleth-idp/credentials/sp-signing.key', 'idp/shibboleth-idp/credentials/sp-signing.crt'); - into outputDir - def expandProps = new LazyPropertyMap<>("lazy port config") - expandProps.put("port", () -> idpFixtureProject.postProcessFixture.ext."test.fixtures.shibboleth-idp.tcp.4443") - inputs.properties(expandProps) - filesMatching("idp-metadata.xml") { - expand(expandProps) - } -} - -normalization { - runtimeClasspath { - ignore 'idp-metadata.xml' - } -} tasks.named("javaRestTest").configure { usesDefaultDistribution() - classpath += files(tasks.named("copyIdpFiles")) - onlyIf(idpFixtureProject.postProcessFixture.path + " not skipped") { - idpFixtureProject.postProcessFixture.state.skipped == false - } } diff --git a/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java b/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java index 625c0ffae167e..c8b3b3fc3aed2 100644 --- a/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java +++ b/x-pack/qa/saml-idp-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.security.authc.saml; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -42,6 +44,9 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.fixtures.idp.IdpTestContainer; +import org.elasticsearch.test.fixtures.idp.OpenLdapTestContainer; +import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -52,6 +57,10 @@ import org.hamcrest.Matchers; import org.junit.Before; import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; +import org.testcontainers.containers.Network; +import org.testcontainers.shaded.org.apache.commons.io.IOUtils; import java.io.IOException; import java.io.InputStream; @@ -82,13 +91,17 @@ /** * An integration test for validating SAML authentication against a real Identity Provider (Shibboleth) */ +@ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) public class SamlAuthenticationIT extends ESRestTestCase { private static final String SAML_RESPONSE_FIELD = "SAMLResponse"; private static final String KIBANA_PASSWORD = "K1b@na K1b@na K1b@na"; - @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + private static Network network = Network.newNetwork(); + private static OpenLdapTestContainer openLdapTestContainer = new OpenLdapTestContainer(network); + private static IdpTestContainer idpFixture = new IdpTestContainer(network); + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") @@ -131,12 +144,25 @@ public class SamlAuthenticationIT extends ESRestTestCase { .setting("xpack.security.authc.realms.native.native.order", "4") .setting("xpack.ml.enabled", "false") .setting("logger.org.elasticsearch.xpack.security", "TRACE") - .configFile("sp-signing.key", Resource.fromClasspath("sp-signing.key")) - .configFile("idp-metadata.xml", Resource.fromClasspath("idp-metadata.xml")) - .configFile("sp-signing.crt", Resource.fromClasspath("sp-signing.crt")) + .configFile("sp-signing.key", Resource.fromClasspath("/idp/shibboleth-idp/credentials/sp-signing.key")) + .configFile("idp-metadata.xml", Resource.fromString(SamlAuthenticationIT::calculateIdpMetaData)) + .configFile("sp-signing.crt", Resource.fromClasspath("/idp/shibboleth-idp/credentials/sp-signing.crt")) .user("test_admin", "x-pack-test-password") .build(); + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(network).around(openLdapTestContainer).around(idpFixture).around(cluster); + + private static String calculateIdpMetaData() { + Resource resource = Resource.fromClasspath("/idp/shibboleth-idp/metadata/idp-metadata.xml"); + try (InputStream stream = resource.asStream()) { + String metadata = IOUtils.toString(stream, "UTF-8"); + return metadata.replace("${port}", String.valueOf(idpFixture.getDefaultPort())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); @@ -526,7 +552,7 @@ private CloseableHttpClient getHttpClient() throws Exception { } private SSLContext getClientSslContext() throws Exception { - final Path pem = getDataPath("/idp-browser.pem"); + final Path pem = idpFixture.getBrowserPem(); final X509ExtendedTrustManager trustManager = CertParsingUtils.getTrustManagerFromPEM(List.of(pem)); SSLContext context = SSLContext.getInstance("TLS"); context.init(new KeyManager[0], new TrustManager[] { trustManager }, new SecureRandom()); diff --git a/x-pack/test/idp-fixture/README.txt b/x-pack/test/idp-fixture/README.txt index 8e42bb142e4ee..c05f53772ed65 100644 --- a/x-pack/test/idp-fixture/README.txt +++ b/x-pack/test/idp-fixture/README.txt @@ -1 +1 @@ -Provisions OpenLDAP + shibboleth IDP 3.4.2 using docker compose +Provisions OpenLDAP + shibboleth IDP 3.4.2 testcontainer fixtures diff --git a/x-pack/test/idp-fixture/build.gradle b/x-pack/test/idp-fixture/build.gradle index 0f5363a278f60..407fb520fcae1 100644 --- a/x-pack/test/idp-fixture/build.gradle +++ b/x-pack/test/idp-fixture/build.gradle @@ -1,40 +1,27 @@ -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.Architecture -import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER; +apply plugin: 'elasticsearch.java' +apply plugin: 'elasticsearch.cache-test-fixtures' -apply plugin: 'elasticsearch.test.fixtures' - -dockerCompose { - composeAdditionalArgs = ['--compatibility'] +configurations.all { + transitive = false } -tasks.named("preProcessFixture").configure { - file("${testFixturesDir}/shared/oidc").mkdirs() -} +dependencies { + testImplementation project(':test:framework') -tasks.register("copyFiles", Sync) { - from file("oidc/override.properties.template") - into "${buildDir}/config" - doLast { - file("${buildDir}/config").setReadable(true, false) - file("${buildDir}/config/override.properties.template").setReadable(true, false) - } -} + api project(':test:fixtures:testcontainer-utils') + api "junit:junit:${versions.junit}" + api "org.testcontainers:testcontainers:${versions.testcontainer}" + implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" -tasks.named("postProcessFixture").configure { - dependsOn "copyFiles" - inputs.dir("${testFixturesDir}/shared/oidc") - File confTemplate = file("${buildDir}/config/override.properties.template") - File confFile = file("${testFixturesDir}/shared/oidc/override.properties") - outputs.file(confFile) - doLast { - assert confTemplate.exists() - String confContents = confTemplate.text - .replace("\${MAPPED_PORT}", "${ext."test.fixtures.oidc-provider.tcp.8080"}") - confFile.text = confContents - } -} + runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}" + runtimeOnly "com.github.docker-java:docker-java-transport:${versions.dockerJava}" + runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}" + runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}" + runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}" -tasks.named('composePull').configure { - enabled = false // this task fails due to docker-compose oddities + // ensure we have proper logging during when used in tests + runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" + runtimeOnly "org.hamcrest:hamcrest:${versions.hamcrest}" } diff --git a/x-pack/test/idp-fixture/docker-compose.yml b/x-pack/test/idp-fixture/docker-compose.yml deleted file mode 100644 index e431fa4ede611..0000000000000 --- a/x-pack/test/idp-fixture/docker-compose.yml +++ /dev/null @@ -1,67 +0,0 @@ -version: "3.7" -services: - openldap: - command: --copy-service --loglevel debug - image: "osixia/openldap:1.4.0" - ports: - - "389" - - "636" - environment: - LDAP_ADMIN_PASSWORD: "NickFuryHeartsES" - LDAP_DOMAIN: "oldap.test.elasticsearch.com" - LDAP_BASE_DN: "DC=oldap,DC=test,DC=elasticsearch,DC=com" - LDAP_TLS: "true" - LDAP_TLS_CRT_FILENAME: "ldap_server.pem" - LDAP_TLS_CA_CRT_FILENAME: "ca_server.pem" - LDAP_TLS_KEY_FILENAME: "ldap_server.key" - LDAP_TLS_VERIFY_CLIENT: "never" - LDAP_TLS_CIPHER_SUITE: "NORMAL" - LDAP_LOG_LEVEL: 256 - volumes: - - ./openldap/ldif/users.ldif:/container/service/slapd/assets/config/bootstrap/ldif/custom/20-bootstrap-users.ldif - - ./openldap/ldif/config.ldif:/container/service/slapd/assets/config/bootstrap/ldif/custom/10-bootstrap-config.ldif - - ./openldap/certs:/container/service/slapd/assets/certs - - shibboleth-idp: - build: - context: . - dockerfile: ./idp/Dockerfile - depends_on: - - openldap - environment: - - JETTY_MAX_HEAP=64m - ports: - - 4443 - expose: - - 4443 - links: - - openldap:openldap - restart: always #ensure ephemeral port mappings are properly updated - healthcheck: - test: curl -f -s --http0.9 http://localhost:4443 --connect-timeout 10 --max-time 10 --output - > /dev/null - interval: 5s - timeout: 20s - retries: 60 - start_period: 10s - - oidc-provider: - build: - context: . - dockerfile: ./oidc/Dockerfile - depends_on: - - http-proxy - ports: - - "8080" - expose: - - "8080" - volumes: - - ./testfixtures_shared/shared/oidc/:/config/c2id/ - - http-proxy: - image: "nginx:latest" - volumes: - - ./oidc/nginx.conf:/etc/nginx/nginx.conf - ports: - - "8888" - expose: - - "8888" diff --git a/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/HttpProxyTestContainer.java b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/HttpProxyTestContainer.java new file mode 100644 index 0000000000000..e517c2a9fe2c3 --- /dev/null +++ b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/HttpProxyTestContainer.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.fixtures.idp; + +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; + +public final class HttpProxyTestContainer extends DockerEnvironmentAwareTestContainer { + + public static final String DOCKER_BASE_IMAGE = "nginx:latest"; + private static final Integer PORT = 8888; + + /** + * for packer caching only + * */ + public HttpProxyTestContainer() { + this(Network.newNetwork()); + } + + public HttpProxyTestContainer(Network network) { + super( + new ImageFromDockerfile("es-http-proxy-fixture").withDockerfileFromBuilder( + builder -> builder.from(DOCKER_BASE_IMAGE).copy("oidc/nginx.conf", "/etc/nginx/nginx.conf").build() + ).withFileFromClasspath("oidc/nginx.conf", "/oidc/nginx.conf") + ); + addExposedPort(PORT); + withNetwork(network); + + } + + public Integer getProxyPort() { + return getMappedPort(PORT); + } +} diff --git a/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/IdpTestContainer.java b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/IdpTestContainer.java new file mode 100644 index 0000000000000..ed19dc997fd8e --- /dev/null +++ b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/IdpTestContainer.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.fixtures.idp; + +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.junit.rules.TemporaryFolder; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.elasticsearch.test.fixtures.ResourceUtils.copyResourceToFile; + +public final class IdpTestContainer extends DockerEnvironmentAwareTestContainer { + + public static final String DOCKER_BASE_IMAGE = "openjdk:11.0.16-jre"; + + private final TemporaryFolder temporaryFolder = new TemporaryFolder(); + private Path certsPath; + + /** + * for packer caching only + * */ + protected IdpTestContainer() { + this(Network.newNetwork()); + } + + public IdpTestContainer(Network network) { + super( + new ImageFromDockerfile("es-idp-testfixture").withDockerfileFromBuilder( + builder -> builder.from(DOCKER_BASE_IMAGE) + .env("jetty_version", "9.3.27.v20190418") + .env("jetty_hash", "7c7c80dd1c9f921771e2b1a05deeeec652d5fcaa") + .env("idp_version", "3.4.3") + .env("idp_hash", "eb86bc7b6366ce2a44f97cae1b014d307b84257e3149469b22b2d091007309db") + .env("dta_hash", "2f547074b06952b94c35631398f36746820a7697") + .env("slf4j_version", "1.7.25") + .env("slf4j_hash", "da76ca59f6a57ee3102f8f9bd9cee742973efa8a") + .env("logback_version", "1.2.3") + .env("logback_classic_hash", "7c4f3c474fb2c041d8028740440937705ebb473a") + .env("logback_core_hash", "864344400c3d4d92dfeb0a305dc87d953677c03c") + .env("logback_access_hash", "e8a841cb796f6423c7afd8738df6e0e4052bf24a") + + .env("JETTY_HOME", "/opt/jetty-home") + .env("JETTY_BASE", "/opt/shib-jetty-base") + .env("PATH", "$PATH:$JAVA_HOME/bin") + .env("JETTY_BROWSER_SSL_KEYSTORE_PASSWORD", "secret") + .env("JETTY_BACKCHANNEL_SSL_KEYSTORE_PASSWORD", "secret") + .env("JETTY_MAX_HEAP", "64m") + // Manually override the jetty keystore otherwise it will attempt to download and fail + .run("mkdir -p /opt/shib-jetty-base/modules") + .copy("idp/jetty-custom/ssl.mod", "/opt/shib-jetty-base/modules/ssl.mod") + .copy("idp/jetty-custom/keystore", "/opt/shib-jetty-base/etc/keystore") + // Download Jetty, verify the hash, and install, initialize a new base + .run( + "wget -q https://repo.maven.apache.org/maven2/org/eclipse/jetty/jetty-distribution/$jetty_version/jetty-distribution-$jetty_version.tar.gz" + + " && echo \"$jetty_hash jetty-distribution-$jetty_version.tar.gz\" | sha1sum -c -" + + " && tar -zxvf jetty-distribution-$jetty_version.tar.gz -C /opt" + + " && ln -s /opt/jetty-distribution-$jetty_version/ /opt/jetty-home" + ) + // Config Jetty + .run( + "mkdir -p /opt/shib-jetty-base/modules /opt/shib-jetty-base/lib/ext /opt/shib-jetty-base/lib/logging /opt/shib-jetty-base/resources" + + " && cd /opt/shib-jetty-base" + + " && touch start.ini" + + " && java -jar ../jetty-home/start.jar --add-to-startd=http,https,deploy,ext,annotations,jstl,rewrite" + ) + // Download Shibboleth IdP, verify the hash, and install + .run( + "wget -q https://shibboleth.net/downloads/identity-provider/archive/$idp_version/shibboleth-identity-provider-$idp_version.tar.gz" + + " && echo \"$idp_hash shibboleth-identity-provider-$idp_version.tar.gz\" | sha256sum -c -" + + " && tar -zxvf shibboleth-identity-provider-$idp_version.tar.gz -C /opt" + + " && ln -s /opt/shibboleth-identity-provider-$idp_version/ /opt/shibboleth-idp" + ) + // Download the library to allow SOAP Endpoints, verify the hash, and place + .run( + "wget -q https://build.shibboleth.net/nexus/content/repositories/releases/net/shibboleth/utilities/jetty9/jetty9-dta-ssl/1.0.0/jetty9-dta-ssl-1.0.0.jar" + + " && echo \"$dta_hash jetty9-dta-ssl-1.0.0.jar\" | sha1sum -c -" + + " && mv jetty9-dta-ssl-1.0.0.jar /opt/shib-jetty-base/lib/ext/" + ) + // Download the slf4j library for Jetty logging, verify the hash, and place + .run( + "wget -q https://repo.maven.apache.org/maven2/org/slf4j/slf4j-api/$slf4j_version/slf4j-api-$slf4j_version.jar" + + " && echo \"$slf4j_hash slf4j-api-$slf4j_version.jar\" | sha1sum -c -" + + " && mv slf4j-api-$slf4j_version.jar /opt/shib-jetty-base/lib/logging/" + ) + // Download the logback_classic library for Jetty logging, verify the hash, and place + .run( + "wget -q https://repo.maven.apache.org/maven2/ch/qos/logback/logback-classic/$logback_version/logback-classic-$logback_version.jar" + + " && echo \"$logback_classic_hash logback-classic-$logback_version.jar\" | sha1sum -c -" + + " && mv logback-classic-$logback_version.jar /opt/shib-jetty-base/lib/logging/" + ) + // Download the logback-core library for Jetty logging, verify the hash, and place + .run( + "wget -q https://repo.maven.apache.org/maven2/ch/qos/logback/logback-core/$logback_version/logback-core-$logback_version.jar" + + " && echo \"$logback_core_hash logback-core-$logback_version.jar\" | sha1sum -c -" + + " && mv logback-core-$logback_version.jar /opt/shib-jetty-base/lib/logging/" + ) + // Download the logback-access library for Jetty logging, verify the hash, and place + .run( + "wget -q https://repo.maven.apache.org/maven2/ch/qos/logback/logback-access/$logback_version/logback-access-$logback_version.jar" + + " && echo \"$logback_access_hash logback-access-$logback_version.jar\" | sha1sum -c -" + + " && mv logback-access-$logback_version.jar /opt/shib-jetty-base/lib/logging/" + ) + // ## Copy local files + .copy("idp/shib-jetty-base/", "/opt/shib-jetty-base/") + .copy("idp/shibboleth-idp/", "/opt/shibboleth-idp/") + .copy("idp/bin/", "/usr/local/bin/") + // Setting owner ownership and permissions + .run( + "useradd jetty -U -s /bin/false" + + " && chown -R root:jetty /opt" + + " && chmod -R 640 /opt" + + " && chown -R root:jetty /opt/shib-jetty-base" + + " && chmod -R 640 /opt/shib-jetty-base" + + " && chmod -R 750 /opt/shibboleth-idp/bin" + ) + .run("chmod 750 /usr/local/bin/run-jetty.sh /usr/local/bin/init-idp.sh") + .run("chmod +x /opt/jetty-home/bin/jetty.sh") + // Opening 4443 (browser TLS), 8443 (mutual auth TLS) + .cmd("run-jetty.sh") + // .expose(4443) + .build() + + ) + .withFileFromClasspath("idp/jetty-custom/ssl.mod", "/idp/jetty-custom/ssl.mod") + .withFileFromClasspath("idp/jetty-custom/keystore", "/idp/jetty-custom/keystore") + .withFileFromClasspath("idp/shib-jetty-base/", "/idp/shib-jetty-base/") + .withFileFromClasspath("idp/shibboleth-idp/", "/idp/shibboleth-idp/") + .withFileFromClasspath("idp/bin/", "/idp/bin/") + + ); + withNetworkAliases("idp"); + withNetwork(network); + addExposedPorts(4443, 8443); + } + + @Override + public void stop() { + super.stop(); + temporaryFolder.delete(); + } + + public Path getBrowserPem() { + try { + temporaryFolder.create(); + certsPath = temporaryFolder.newFolder("certs").toPath(); + return copyResourceToFile(getClass(), certsPath, "idp/shibboleth-idp/credentials/idp-browser.pem"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Integer getDefaultPort() { + return getMappedPort(4443); + } +} diff --git a/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OidcProviderTestContainer.java b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OidcProviderTestContainer.java new file mode 100644 index 0000000000000..89090fa6e11bc --- /dev/null +++ b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OidcProviderTestContainer.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.fixtures.idp; + +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.images.builder.Transferable; + +public final class OidcProviderTestContainer extends DockerEnvironmentAwareTestContainer { + + private static final int PORT = 8080; + + /** + * for packer caching only + * */ + protected OidcProviderTestContainer() { + this(Network.newNetwork()); + } + + public OidcProviderTestContainer(Network network) { + super( + new ImageFromDockerfile("es-oidc-provider-fixture").withFileFromClasspath("oidc/setup.sh", "/oidc/setup.sh") + // we cannot make use of docker file builder + // as it does not support multi-stage builds + .withFileFromClasspath("Dockerfile", "oidc/Dockerfile") + ); + withNetworkAliases("oidc-provider"); + withNetwork(network); + addExposedPort(PORT); + } + + @Override + public void start() { + super.start(); + copyFileToContainer( + Transferable.of( + "op.issuer=http://127.0.0.1:" + + getMappedPort(PORT) + + "/c2id\n" + + "op.authz.endpoint=http://127.0.0.1:" + + getMappedPort(PORT) + + "/c2id-login/\n" + + "op.reg.apiAccessTokenSHA256=d1c4fa70d9ee708d13cfa01daa0e060a05a2075a53c5cc1ad79e460e96ab5363\n" + + "jose.jwkSer=RnVsbCBrZXk6CnsKICAia2V5cyI6IFsKICAgIHsKICAgICAgInAiOiAiLXhhN2d2aW5tY3N3QXU3Vm1mV2loZ2o3U3gzUzhmd2dFSTdMZEVveW5FU1RzcElaeUY5aHc0NVhQZmI5VHlpbzZsOHZTS0F5RmU4T2lOalpkNE1Ra0ttYlJzTmxxR1Y5VlBoWF84UG1JSm5mcGVhb3E5YnZfU0k1blZHUl9zYUUzZE9sTEE2VWpaS0lsRVBNb0ZuRlZCMUFaUU9qQlhRRzZPTDg2eDZ2NHMwIiwKICAgICAgImt0eSI6ICJSU0EiLAogICAgICAicSI6ICJ2Q3pDQUlpdHV0MGx1V0djQloyLUFabURLc1RxNkkxcUp0RmlEYkIyZFBNQVlBNldOWTdaWEZoVWxsSjJrT2ZELWdlYjlkYkN2ODBxNEwyajVZSjZoOTBUc1NRWWVHRlljN1lZMGdCMU5VR3l5cXctb29QN0EtYlJmMGI3b3I4ajZJb0hzQTZKa2JranN6c3otbkJ2U2RmUURlZkRNSVc3Ni1ZWjN0c2hsY2MiLAogICAgICAiZCI6ICJtbFBOcm1zVVM5UmJtX1I5SElyeHdmeFYzZnJ2QzlaQktFZzRzc1ZZaThfY09lSjV2U1hyQV9laEtwa2g4QVhYaUdWUGpQbVlyd29xQzFVUksxUkZmLVg0dG10emV2OUVHaU12Z0JCaEF5RkdTSUd0VUNla2x4Q2dhb3BpMXdZSU1Bd0M0STZwMUtaZURxTVNCWVZGeHA5ZWlJZ2pwb05JbV9lR3hXUUs5VHNnYmk5T3lyc1VqaE9KLVczN2JVMEJWUU56UXpxODhCcGxmNzM3VmV1dy1FeDZaMk1iWXR3SWdfZ0JVb0JEZ0NrZkhoOVE4MElYcEZRV0x1RzgwenFrdkVwTHZ0RWxLbDRvQ3BHVnBjcmFUOFNsOGpYc3FDT1k0dnVRT19LRVUzS2VPNUNJbHd4eEhJYXZjQTE5cHFpSWJ5cm1LbThxS0ZEWHluUFJMSGFNZ1EiLAogICAgICAiZSI6ICJBUUFCIiwKICAgICAgImtpZCI6ICJyc2EzODRfMjA0OCIsCiAgICAgICJxaSI6ICJzMldTamVrVDl3S2JPbk9neGNoaDJPY3VubzE2Y20wS281Z3hoUWJTdVMyMldfUjJBR2ZVdkRieGF0cTRLakQ3THo3X1k2TjdTUkwzUVpudVhoZ1djeXgyNGhrUGppQUZLNmlkYVZKQzJqQmgycEZTUDVTNXZxZ0lsME12eWY4NjlwdkN4S0NzaGRKMGdlRWhveE93VkRPYXJqdTl2Zm9IQV90LWJoRlZrUnciLAogICAgICAiZHAiOiAiQlJhQTFqYVRydG9mTHZBSUJBYW1OSEVhSm51RU9zTVJJMFRCZXFuR1BNUm0tY2RjSG1OUVo5WUtqb2JpdXlmbnhGZ0piVDlSeElBRG0ySkpoZEp5RTN4Y1dTSzhmSjBSM1Jick1aT1dwako0QmJTVzFtU1VtRnlKTGxib3puRFhZR2RaZ1hzS0o1UkFrRUNQZFBCY3YwZVlkbk9NYWhfZndfaFZoNjRuZ2tFIiwKICAgICAgImFsZyI6ICJSU0EzODQiLAogICAgICAiZHEiOiAiUFJoVERKVlR3cDNXaDZfWFZrTjIwMUlpTWhxcElrUDN1UTYyUlRlTDNrQ2ZXSkNqMkZPLTRxcVRIQk0tQjZJWUVPLXpoVWZyQnhiMzJ1djNjS2JDWGFZN3BJSFJxQlFEQWQ2WGhHYzlwc0xqNThXd3VGY2RncERJYUFpRjNyc3NUMjJ4UFVvYkJFTVdBalV3bFJrNEtNTjItMnpLQk5FR3lIcDIzOUpKdnpVIiwKICAgICAgIm4iOiAidUpDWDVDbEZpM0JnTXBvOWhRSVZ2SDh0Vi1jLTVFdG5OeUZxVm91R3NlNWwyUG92MWJGb0tsRllsU25YTzNWUE9KRWR3azNDdl9VT0UtQzlqZERYRHpvS3Z4RURaTVM1TDZWMFpIVEJoNndIOV9iN3JHSlBxLV9RdlNkejczSzZxbHpGaUtQamRvdTF6VlFYTmZfblBZbnRnQkdNRUtBc1pRNGp0cWJCdE5lV0h0MF9UM001cEktTV9KNGVlRWpCTW95TkZuU2ExTEZDVmZRNl9YVnpjelp1TlRGMlh6UmdRWkFmcmJGRXZ6eXR1TzVMZTNTTXFrUUFJeDhFQmkwYXVlRUNqNEQ4cDNVNXFVRG92NEF2VnRJbUZlbFJvb1pBMHJtVW1KRHJ4WExrVkhuVUpzaUF6ZW9TLTNBSnV1bHJkMGpuNjJ5VjZHV2dFWklZMVNlZVd3IgogICAgfQogIF0KfQo\n" + + "op.authz.alwaysPromptForConsent=true\n" + + "op.authz.alwaysPromptForAuth=true" + ), + "config/c2id/override.properties" + ); + } + + public String getC2OPUrl() { + return "http://127.0.0.1:" + getMappedPort(PORT); + } + + public String getC2IssuerUrl() { + return getC2OPUrl() + "/c2id"; + } + +} diff --git a/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OpenLdapTestContainer.java b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OpenLdapTestContainer.java new file mode 100644 index 0000000000000..2f65134f2ec72 --- /dev/null +++ b/x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OpenLdapTestContainer.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.fixtures.idp; + +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; +import org.junit.rules.TemporaryFolder; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.elasticsearch.test.fixtures.ResourceUtils.copyResourceToFile; + +public final class OpenLdapTestContainer extends DockerEnvironmentAwareTestContainer { + + public static final String DOCKER_BASE_IMAGE = "osixia/openldap:1.4.0"; + + private final TemporaryFolder temporaryFolder = new TemporaryFolder(); + private Path certsPath; + + public OpenLdapTestContainer() { + this(Network.newNetwork()); + } + + public OpenLdapTestContainer(Network network) { + super( + new ImageFromDockerfile("es-openldap-testfixture").withDockerfileFromBuilder( + builder -> builder.from(DOCKER_BASE_IMAGE) + .env("LDAP_ADMIN_PASSWORD", "NickFuryHeartsES") + .env("LDAP_DOMAIN", "oldap.test.elasticsearch.com") + .env("LDAP_BASE_DN", "DC=oldap,DC=test,DC=elasticsearch,DC=com") + .env("LDAP_TLS", "true") + .env("LDAP_TLS_CRT_FILENAME", "ldap_server.pem") + .env("LDAP_TLS_CA_CRT_FILENAME", "ca_server.pem") + .env("LDAP_TLS_KEY_FILENAME", "ldap_server.key") + .env("LDAP_TLS_VERIFY_CLIENT", "never") + .env("LDAP_TLS_CIPHER_SUITE", "NORMAL") + .env("LDAP_LOG_LEVEL", "256") + .copy( + "openldap/ldif/users.ldif", + "/container/service/slapd/assets/config/bootstrap/ldif/custom/20-bootstrap-users.ldif" + ) + .copy( + "openldap/ldif/config.ldif", + "/container/service/slapd/assets/config/bootstrap/ldif/custom/10-bootstrap-config.ldif" + ) + .copy("openldap/certs", "/container/service/slapd/assets/certs") + + .build() + ) + .withFileFromClasspath("openldap/certs", "/openldap/certs/") + .withFileFromClasspath("openldap/ldif/users.ldif", "/openldap/ldif/users.ldif") + .withFileFromClasspath("openldap/ldif/config.ldif", "/openldap/ldif/config.ldif") + ); + // withLogConsumer(new Slf4jLogConsumer(logger())); + withNetworkAliases("openldap"); + withNetwork(network); + withExposedPorts(389, 636); + } + + public String getLdapUrl() { + return "ldaps://localhost:" + getMappedPort(636); + } + + @Override + public void start() { + super.start(); + setupCerts(); + } + + @Override + public void stop() { + super.stop(); + temporaryFolder.delete(); + } + + private void setupCerts() { + try { + temporaryFolder.create(); + certsPath = temporaryFolder.newFolder("certs").toPath(); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ca.jks"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ca_server.key"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ca_server.pem"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/dhparam.pem"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ldap_server.key"); + copyResourceToFile(getClass(), certsPath, "openldap/certs/ldap_server.pem"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Path getJavaKeyStorePath() { + return certsPath.resolve("ca.jks"); + } + + public Path getCaCertPath() { + return certsPath.resolve("ca_server.pem"); + } + + public Integer getDefaultPort() { + return getMappedPort(636); + } +} diff --git a/x-pack/test/idp-fixture/idp/Dockerfile b/x-pack/test/idp-fixture/src/main/resources/idp/Dockerfile similarity index 100% rename from x-pack/test/idp-fixture/idp/Dockerfile rename to x-pack/test/idp-fixture/src/main/resources/idp/Dockerfile diff --git a/x-pack/test/idp-fixture/idp/bin/init-idp.sh b/x-pack/test/idp-fixture/src/main/resources/idp/bin/init-idp.sh similarity index 100% rename from x-pack/test/idp-fixture/idp/bin/init-idp.sh rename to x-pack/test/idp-fixture/src/main/resources/idp/bin/init-idp.sh diff --git a/x-pack/test/idp-fixture/idp/bin/run-jetty.sh b/x-pack/test/idp-fixture/src/main/resources/idp/bin/run-jetty.sh similarity index 100% rename from x-pack/test/idp-fixture/idp/bin/run-jetty.sh rename to x-pack/test/idp-fixture/src/main/resources/idp/bin/run-jetty.sh diff --git a/x-pack/test/idp-fixture/idp/jetty-custom/keystore b/x-pack/test/idp-fixture/src/main/resources/idp/jetty-custom/keystore similarity index 100% rename from x-pack/test/idp-fixture/idp/jetty-custom/keystore rename to x-pack/test/idp-fixture/src/main/resources/idp/jetty-custom/keystore diff --git a/x-pack/test/idp-fixture/idp/jetty-custom/ssl.mod b/x-pack/test/idp-fixture/src/main/resources/idp/jetty-custom/ssl.mod similarity index 100% rename from x-pack/test/idp-fixture/idp/jetty-custom/ssl.mod rename to x-pack/test/idp-fixture/src/main/resources/idp/jetty-custom/ssl.mod diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-backchannel.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-backchannel.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-backchannel.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-backchannel.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-logging.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-logging.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-logging.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-logging.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-requestlog.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-requestlog.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-requestlog.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-requestlog.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-rewrite.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-rewrite.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-rewrite.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-rewrite.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-ssl-context.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-ssl-context.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/etc/jetty-ssl-context.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/etc/jetty-ssl-context.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/modules/backchannel.mod b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/modules/backchannel.mod similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/modules/backchannel.mod rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/modules/backchannel.mod diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/resources/logback-access.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/resources/logback-access.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/resources/logback-access.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/resources/logback-access.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/resources/logback.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/resources/logback.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/resources/logback.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/resources/logback.xml diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/start.d/backchannel.ini b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.d/backchannel.ini similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/start.d/backchannel.ini rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.d/backchannel.ini diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/start.d/ssl.ini b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.d/ssl.ini similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/start.d/ssl.ini rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.d/ssl.ini diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/start.ini b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.ini similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/start.ini rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/start.ini diff --git a/x-pack/test/idp-fixture/idp/shib-jetty-base/webapps/idp.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/webapps/idp.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shib-jetty-base/webapps/idp.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shib-jetty-base/webapps/idp.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/access-control.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/access-control.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/access-control.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/access-control.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/admin/general-admin.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/admin/general-admin.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/admin/general-admin.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/admin/general-admin.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/admin/metrics.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/admin/metrics.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/admin/metrics.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/admin/metrics.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/attribute-filter.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/attribute-filter.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/attribute-filter.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/attribute-filter.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/attribute-resolver.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/attribute-resolver.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/attribute-resolver.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/attribute-resolver.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/audit.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/audit.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/audit.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/audit.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/authn-comparison.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/authn-comparison.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/authn-comparison.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/authn-comparison.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/authn-events-flow.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/authn-events-flow.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/authn-events-flow.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/authn-events-flow.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/duo-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/duo-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/duo-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/duo-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/duo.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/duo.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/duo.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/duo.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/external-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/external-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/external-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/external-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/function-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/function-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/function-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/function-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/general-authn.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/general-authn.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/general-authn.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/general-authn.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/ipaddress-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/jaas-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/jaas.config b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/jaas.config similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/jaas.config rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/jaas.config diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/krb5-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/ldap-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/mfa-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/password-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/password-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/password-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/password-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/remoteuser-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/remoteuser-internal-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/spnego-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/x509-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/x509-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/x509-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/x509-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/authn/x509-internal-authn-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/attribute-sourced-subject-c14n-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/simple-subject-c14n-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/subject-c14n-events-flow.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/subject-c14n.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/subject-c14n.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/subject-c14n.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/subject-c14n.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/c14n/x500-subject-c14n-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/cas-protocol.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/cas-protocol.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/cas-protocol.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/cas-protocol.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/credentials.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/credentials.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/credentials.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/credentials.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/errors.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/errors.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/errors.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/errors.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/global.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/global.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/global.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/global.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/idp.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/idp.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/idp.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/idp.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/consent-intercept-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/context-check-intercept-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/expiring-password-intercept-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/impersonate-intercept-config.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/intercept-events-flow.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/profile-intercept.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/profile-intercept.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/intercept/profile-intercept.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/intercept/profile-intercept.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/ldap.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/ldap.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/ldap.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/ldap.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/logback.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/logback.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/logback.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/logback.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/metadata-providers.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/metadata-providers.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/metadata-providers.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/metadata-providers.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/relying-party.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/relying-party.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/relying-party.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/relying-party.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/saml-nameid.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/saml-nameid.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/saml-nameid.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/saml-nameid.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/saml-nameid.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/saml-nameid.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/saml-nameid.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/saml-nameid.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/services.properties b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/services.properties similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/services.properties rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/services.properties diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/services.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/services.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/services.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/services.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/conf/session-manager.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/session-manager.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/conf/session-manager.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/conf/session-manager.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/README b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/README similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/README rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/README diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/ca_server.pem b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/ca_server.pem similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/ca_server.pem rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/ca_server.pem diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-backchannel.crt b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-backchannel.crt similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-backchannel.crt rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-backchannel.crt diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-backchannel.p12 b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-backchannel.p12 similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-backchannel.p12 rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-backchannel.p12 diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.key b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.key similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.key rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.key diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.p12 b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.p12 similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.p12 rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.p12 diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.pem b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.pem similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-browser.pem rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-browser.pem diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-encryption.crt b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-encryption.crt similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-encryption.crt rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-encryption.crt diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-encryption.key b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-encryption.key similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-encryption.key rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-encryption.key diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-signing.crt b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-signing.crt similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-signing.crt rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-signing.crt diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-signing.key b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-signing.key similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/idp-signing.key rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/idp-signing.key diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sealer.jks b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sealer.jks similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sealer.jks rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sealer.jks diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sealer.kver b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sealer.kver similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sealer.kver rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sealer.kver diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sp-signing.crt b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sp-signing.crt similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sp-signing.crt rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sp-signing.crt diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sp-signing.key b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sp-signing.key similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/credentials/sp-signing.key rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/credentials/sp-signing.key diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/README.asciidoc b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/README.asciidoc similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/README.asciidoc rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/README.asciidoc diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/idp-docs-metadata.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/idp-docs-metadata.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/idp-docs-metadata.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/idp-docs-metadata.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/idp-metadata.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/idp-metadata.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/idp-metadata.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/idp-metadata.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata2.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata2.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata2.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata2.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata3.xml b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata3.xml similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/metadata/sp-metadata3.xml rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/metadata/sp-metadata3.xml diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/admin/unlock-keys.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/admin/unlock-keys.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/admin/unlock-keys.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/admin/unlock-keys.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/client-storage/client-storage-read.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/client-storage/client-storage-read.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/client-storage/client-storage-read.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/client-storage/client-storage-read.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/client-storage/client-storage-write.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/client-storage/client-storage-write.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/client-storage/client-storage-write.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/client-storage/client-storage-write.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/duo.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/duo.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/duo.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/duo.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/error.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/error.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/error.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/error.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/attribute-release.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/attribute-release.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/attribute-release.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/attribute-release.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/expiring-password.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/expiring-password.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/expiring-password.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/expiring-password.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/impersonate.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/impersonate.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/impersonate.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/impersonate.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/terms-of-use.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/terms-of-use.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/intercept/terms-of-use.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/intercept/terms-of-use.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/login-error.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/login-error.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/login-error.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/login-error.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/login.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/login.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/login.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/login.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout-complete.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout-complete.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout-complete.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout-complete.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout-propagate.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout-propagate.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout-propagate.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout-propagate.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/logout.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/logout.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/spnego-unavailable.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/spnego-unavailable.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/spnego-unavailable.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/spnego-unavailable.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/views/user-prefs.vm b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/user-prefs.vm similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/views/user-prefs.vm rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/views/user-prefs.vm diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/consent.css b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/consent.css similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/consent.css rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/consent.css diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/logout.css b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/logout.css similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/logout.css rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/logout.css diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/main.css b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/main.css similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/css/main.css rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/css/main.css diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/dummylogo-mobile.png diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/dummylogo.png b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/dummylogo.png similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/dummylogo.png rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/dummylogo.png diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/failure-32x32.png b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/failure-32x32.png similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/failure-32x32.png rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/failure-32x32.png diff --git a/x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/success-32x32.png b/x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/success-32x32.png similarity index 100% rename from x-pack/test/idp-fixture/idp/shibboleth-idp/webapp/images/success-32x32.png rename to x-pack/test/idp-fixture/src/main/resources/idp/shibboleth-idp/webapp/images/success-32x32.png diff --git a/x-pack/test/idp-fixture/oidc/Dockerfile b/x-pack/test/idp-fixture/src/main/resources/oidc/Dockerfile similarity index 100% rename from x-pack/test/idp-fixture/oidc/Dockerfile rename to x-pack/test/idp-fixture/src/main/resources/oidc/Dockerfile diff --git a/x-pack/test/idp-fixture/oidc/nginx.conf b/x-pack/test/idp-fixture/src/main/resources/oidc/nginx.conf similarity index 100% rename from x-pack/test/idp-fixture/oidc/nginx.conf rename to x-pack/test/idp-fixture/src/main/resources/oidc/nginx.conf diff --git a/x-pack/test/idp-fixture/oidc/op-jwks.json b/x-pack/test/idp-fixture/src/main/resources/oidc/op-jwks.json similarity index 100% rename from x-pack/test/idp-fixture/oidc/op-jwks.json rename to x-pack/test/idp-fixture/src/main/resources/oidc/op-jwks.json diff --git a/x-pack/test/idp-fixture/oidc/override.properties.template b/x-pack/test/idp-fixture/src/main/resources/oidc/override.properties.template similarity index 100% rename from x-pack/test/idp-fixture/oidc/override.properties.template rename to x-pack/test/idp-fixture/src/main/resources/oidc/override.properties.template diff --git a/x-pack/test/idp-fixture/oidc/setup.sh b/x-pack/test/idp-fixture/src/main/resources/oidc/setup.sh similarity index 100% rename from x-pack/test/idp-fixture/oidc/setup.sh rename to x-pack/test/idp-fixture/src/main/resources/oidc/setup.sh diff --git a/x-pack/test/idp-fixture/openldap/certs/README b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/README similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/README rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/README diff --git a/x-pack/test/idp-fixture/openldap/certs/ca.jks b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca.jks similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ca.jks rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca.jks diff --git a/x-pack/test/idp-fixture/openldap/certs/ca_server.key b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca_server.key similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ca_server.key rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca_server.key diff --git a/x-pack/test/idp-fixture/openldap/certs/ca_server.pem b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca_server.pem similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ca_server.pem rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ca_server.pem diff --git a/x-pack/test/idp-fixture/openldap/certs/dhparam.pem b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/dhparam.pem similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/dhparam.pem rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/dhparam.pem diff --git a/x-pack/test/idp-fixture/openldap/certs/ldap_server.csr b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.csr similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ldap_server.csr rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.csr diff --git a/x-pack/test/idp-fixture/openldap/certs/ldap_server.key b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.key similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ldap_server.key rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.key diff --git a/x-pack/test/idp-fixture/openldap/certs/ldap_server.pem b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.pem similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/ldap_server.pem rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/ldap_server.pem diff --git a/x-pack/test/idp-fixture/openldap/certs/templates/ca_server.conf b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/templates/ca_server.conf similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/templates/ca_server.conf rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/templates/ca_server.conf diff --git a/x-pack/test/idp-fixture/openldap/certs/templates/ldap_server.conf b/x-pack/test/idp-fixture/src/main/resources/openldap/certs/templates/ldap_server.conf similarity index 100% rename from x-pack/test/idp-fixture/openldap/certs/templates/ldap_server.conf rename to x-pack/test/idp-fixture/src/main/resources/openldap/certs/templates/ldap_server.conf diff --git a/x-pack/test/idp-fixture/openldap/ldif/config.ldif b/x-pack/test/idp-fixture/src/main/resources/openldap/ldif/config.ldif similarity index 100% rename from x-pack/test/idp-fixture/openldap/ldif/config.ldif rename to x-pack/test/idp-fixture/src/main/resources/openldap/ldif/config.ldif diff --git a/x-pack/test/idp-fixture/openldap/ldif/users.ldif b/x-pack/test/idp-fixture/src/main/resources/openldap/ldif/users.ldif similarity index 100% rename from x-pack/test/idp-fixture/openldap/ldif/users.ldif rename to x-pack/test/idp-fixture/src/main/resources/openldap/ldif/users.ldif From 684645372c0b741538ce3908126cf53e3f20817a Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 13 Dec 2023 13:50:34 +0200 Subject: [PATCH 10/54] EQL: Use the eql query filter for the open-pit request (#103212) --- docs/changelog/103212.yaml | 5 + .../execution/search/PITAwareQueryClient.java | 3 + .../search/PITAwareQueryClientTests.java | 285 ++++++++++++++++++ 3 files changed, 293 insertions(+) create mode 100644 docs/changelog/103212.yaml create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java diff --git a/docs/changelog/103212.yaml b/docs/changelog/103212.yaml new file mode 100644 index 0000000000000..3cbbddc8f2229 --- /dev/null +++ b/docs/changelog/103212.yaml @@ -0,0 +1,5 @@ +pr: 103212 +summary: Use the eql query filter for the open-pit request +area: EQL +type: enhancement +issues: [] diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java index 707964a93ab9e..4e4817d4c041d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClient.java @@ -46,10 +46,12 @@ public class PITAwareQueryClient extends BasicQueryClient { private String pitId; private final TimeValue keepAlive; + private final QueryBuilder filter; public PITAwareQueryClient(EqlSession eqlSession) { super(eqlSession); this.keepAlive = eqlSession.configuration().requestTimeout(); + this.filter = eqlSession.configuration().filter(); } @Override @@ -131,6 +133,7 @@ private ActionListener pitListener(Function void openPIT(ActionListener listener, Runnable runnable) { OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).indicesOptions(IndexResolver.FIELD_CAPS_INDICES_OPTIONS) .keepAlive(keepAlive); + request.indexFilter(filter); client.execute(TransportOpenPointInTimeAction.TYPE, request, listener.delegateFailureAndWrap((l, r) -> { pitId = r.getPointInTimeId(); runnable.run(); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java new file mode 100644 index 0000000000000..e12eec4833199 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java @@ -0,0 +1,285 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.eql.execution.search; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.ClosePointInTimeResponse; +import org.elasticsearch.action.search.OpenPointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchSortValues; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.async.AsyncExecutionId; +import org.elasticsearch.xpack.eql.action.EqlSearchAction; +import org.elasticsearch.xpack.eql.action.EqlSearchTask; +import org.elasticsearch.xpack.eql.analysis.PostAnalyzer; +import org.elasticsearch.xpack.eql.analysis.PreAnalyzer; +import org.elasticsearch.xpack.eql.analysis.Verifier; +import org.elasticsearch.xpack.eql.execution.assembler.BoxedQueryRequest; +import org.elasticsearch.xpack.eql.execution.assembler.SequenceCriterion; +import org.elasticsearch.xpack.eql.execution.search.extractor.ImplicitTiebreakerHitExtractor; +import org.elasticsearch.xpack.eql.execution.sequence.SequenceMatcher; +import org.elasticsearch.xpack.eql.execution.sequence.TumblingWindow; +import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry; +import org.elasticsearch.xpack.eql.optimizer.Optimizer; +import org.elasticsearch.xpack.eql.planner.Planner; +import org.elasticsearch.xpack.eql.session.EqlConfiguration; +import org.elasticsearch.xpack.eql.session.EqlSession; +import org.elasticsearch.xpack.eql.stats.Metrics; +import org.elasticsearch.xpack.ql.execution.search.extractor.HitExtractor; +import org.elasticsearch.xpack.ql.index.IndexResolver; +import org.elasticsearch.xpack.ql.type.DefaultDataTypeRegistry; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.index.query.QueryBuilders.idsQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; +import static org.elasticsearch.xpack.eql.EqlTestUtils.booleanArrayOf; + +public class PITAwareQueryClientTests extends ESTestCase { + + private final List keyExtractors = emptyList(); + private static final QueryBuilder[] FILTERS = new QueryBuilder[] { + rangeQuery("some_timestamp_field").gte("2023-12-07"), + termQuery("tier", "hot"), + idsQuery().addIds("1", "2", "3") }; + private static final String[] INDICES = new String[] { "test1", "test2", "test3" }; + + public void testQueryFilterUsedInPitAndSearches() { + try (var threadPool = createThreadPool()) { + final var filter = frequently() ? randomFrom(FILTERS) : null; + int stages = randomIntBetween(2, 5); + final var esClient = new ESMockClient(threadPool, filter, stages); + + EqlConfiguration eqlConfiguration = new EqlConfiguration( + INDICES, + org.elasticsearch.xpack.ql.util.DateUtils.UTC, + "nobody", + "cluster", + filter, + emptyMap(), + null, + TimeValue.timeValueSeconds(30), + null, + 123, + 1, + "", + new TaskId("test", 123), + new EqlSearchTask( + randomLong(), + "transport", + EqlSearchAction.NAME, + "", + null, + emptyMap(), + emptyMap(), + new AsyncExecutionId("", new TaskId(randomAlphaOfLength(10), 1)), + TimeValue.timeValueDays(5) + ) + ); + IndexResolver indexResolver = new IndexResolver(esClient, "cluster", DefaultDataTypeRegistry.INSTANCE, () -> emptySet()); + CircuitBreaker cb = new NoopCircuitBreaker("testcb"); + EqlSession eqlSession = new EqlSession( + esClient, + eqlConfiguration, + indexResolver, + new PreAnalyzer(), + new PostAnalyzer(), + new EqlFunctionRegistry(), + new Verifier(new Metrics()), + new Optimizer(), + new Planner(), + cb + ); + QueryClient eqlClient = new PITAwareQueryClient(eqlSession) { + @Override + public void fetchHits(Iterable> refs, ActionListener>> listener) { + List> searchHits = new ArrayList<>(); + for (List ref : refs) { + List hits = new ArrayList<>(ref.size()); + for (HitReference hitRef : ref) { + hits.add(new SearchHit(-1, hitRef.id())); + } + searchHits.add(hits); + } + listener.onResponse(searchHits); + } + }; + + List criteria = new ArrayList<>(stages); + for (int i = 0; i < stages; i++) { + final int j = i; + criteria.add( + new SequenceCriterion( + i, + new BoxedQueryRequest( + () -> SearchSourceBuilder.searchSource().size(10).query(matchAllQuery()).terminateAfter(j), + "@timestamp", + emptyList(), + emptySet() + ), + keyExtractors, + TimestampExtractor.INSTANCE, + null, + ImplicitTiebreakerHitExtractor.INSTANCE, + false, + false + ) + ); + } + + SequenceMatcher matcher = new SequenceMatcher(stages, false, TimeValue.MINUS_ONE, null, booleanArrayOf(stages, false), cb); + TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList()); + window.execute(wrap(response -> { + // do nothing, we don't care about the query results + }, ex -> { fail("Shouldn't have failed"); })); + } + } + + /** + * This class is used by {@code PITFailureTests.testPitCloseOnFailure} method + * to test that PIT close is never (wrongly) invoked if PIT open failed. + */ + private class ESMockClient extends NoOpClient { + private final QueryBuilder filter; + private final String pitId = "test_pit_id"; + private boolean openedPIT = false; + private int searchRequestsRemainingCount; + + ESMockClient(ThreadPool threadPool, QueryBuilder filter, int stages) { + super(threadPool); + this.filter = filter; + this.searchRequestsRemainingCount = stages; + } + + @SuppressWarnings("unchecked") + @Override + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + if (request instanceof OpenPointInTimeRequest openPIT) { + assertFalse(openedPIT); + assertEquals(filter, openPIT.indexFilter()); // check that the filter passed on to the eql query is used in opening the pit + assertArrayEquals(INDICES, openPIT.indices()); // indices for opening pit should be the same as for the eql query itself + + openedPIT = true; + OpenPointInTimeResponse response = new OpenPointInTimeResponse(pitId); + listener.onResponse((Response) response); + } else if (request instanceof ClosePointInTimeRequest closePIT) { + assertTrue(openedPIT); + assertEquals(pitId, closePIT.getId()); + + openedPIT = false; + ClosePointInTimeResponse response = new ClosePointInTimeResponse(true, 1); + listener.onResponse((Response) response); + } else if (request instanceof SearchRequest searchRequest) { + assertTrue(openedPIT); + searchRequestsRemainingCount--; + assertTrue(searchRequestsRemainingCount >= 0); + + assertEquals(pitId, searchRequest.source().pointInTimeBuilder().getEncodedId()); + assertEquals(0, searchRequest.indices().length); // no indices set in the search request + assertEquals(1, searchRequest.source().subSearches().size()); + + BoolQueryBuilder actualQuery = (BoolQueryBuilder) searchRequest.source().subSearches().get(0).getQueryBuilder(); + assertEquals(3, actualQuery.filter().size()); + assertTrue(actualQuery.filter().get(0) instanceof MatchAllQueryBuilder); // the match_all we used when building the criteria + assertTrue(actualQuery.filter().get(1) instanceof RangeQueryBuilder); + QueryBuilder expectedQuery = termsQuery("_index", INDICES); // indices should be used as a filter further on + assertEquals(expectedQuery, actualQuery.filter().get(2)); + + handleSearchRequest(listener, searchRequest); + } else { + super.doExecute(action, request, listener); + } + } + + @SuppressWarnings("unchecked") + void handleSearchRequest(ActionListener listener, SearchRequest searchRequest) { + int ordinal = searchRequest.source().terminateAfter(); + SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal)); + searchHit.sortValues( + new SearchSortValues(new Long[] { (long) ordinal, 1L }, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }) + ); + + SearchHits searchHits = new SearchHits(new SearchHit[] { searchHit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 0.0f); + SearchResponseSections internal = new SearchResponseSections(searchHits, null, null, false, false, null, 0); + SearchResponse response = new SearchResponse( + internal, + null, + 2, + 0, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY, + searchRequest.pointInTimeBuilder().getEncodedId() + ); + + ActionListener.respondAndRelease(listener, (Response) response); + } + } + + private static class TimestampExtractor implements HitExtractor { + + static final TimestampExtractor INSTANCE = new TimestampExtractor(); + + @Override + public String getWriteableName() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public String hitName() { + return null; + } + + @Override + public Timestamp extract(SearchHit hit) { + return Timestamp.of(String.valueOf(hit.docId())); + } + } +} From 1612ad1d659696f2e6a8f7a99016456aec9d9aff Mon Sep 17 00:00:00 2001 From: Abdon Pijpelink Date: Wed, 13 Dec 2023 13:17:00 +0100 Subject: [PATCH 11/54] fix typo (#103149) (#103381) Fixed a typo and a small grammatical error in the explanation of the `null_value` option (cherry picked from commit fa52f8283893fc40f7f03ce93fa9325026b78683) Co-authored-by: Nimrod Dolev --- docs/reference/mapping/types/flattened.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc index 87f5cebe21993..0a72ebc98ecef 100644 --- a/docs/reference/mapping/types/flattened.asciidoc +++ b/docs/reference/mapping/types/flattened.asciidoc @@ -294,8 +294,8 @@ The following mapping parameters are accepted: <>:: A string value which is substituted for any explicit `null` values within - the flattened object field. Defaults to `null`, which means null sields are - treated as if it were missing. + the flattened object field. Defaults to `null`, which means null fields are + treated as if they were missing. <>:: From 16a4d00999916e1c3c0d1b9c58c0506f182fbd3b Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 13 Dec 2023 14:32:23 +0100 Subject: [PATCH 12/54] [Connectors API] Add create connector sync job docs (#103275) Adds documentation for the create connector sync job API endpoint. --- .../connector/apis/connector-apis.asciidoc | 3 + .../create-connector-sync-job-api.asciidoc | 69 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 docs/reference/connector/apis/create-connector-sync-job-api.asciidoc diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index f3a5148f8497c..0018f0e0aa158 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -36,8 +36,11 @@ You can use these APIs to create, cancel, delete and update sync jobs. Use the following APIs to manage sync jobs: +* <> + include::create-connector-api.asciidoc[] +include::create-connector-sync-job-api.asciidoc[] include::delete-connector-api.asciidoc[] include::get-connector-api.asciidoc[] include::list-connectors-api.asciidoc[] diff --git a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..e8c2c364797c4 --- /dev/null +++ b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc @@ -0,0 +1,69 @@ +[[create-connector-sync-job-api]] +=== Create connector sync job API +++++ +Create connector sync job +++++ + +Creates a connector sync job. + +[source, console] +-------------------------------------------------- +POST _connector/_sync_job +{ + "id": "connector-id", + "job_type": "full", + "trigger_method": "on_demand" +} +-------------------------------------------------- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the id ahead of time] + + +[[create-connector-sync-job-api-request]] +==== {api-request-title} +`POST _connector/_sync_job` + + +[[create-connector-sync-job-api-prereqs]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `id` parameter should reference an existing connector. + +[[create-connector-sync-job-api-desc]] +==== {api-description-title} + +Creates a connector sync job document in the internal index and initializes its counters and timestamps with default values. +Certain values can be updated via the API. + +[role="child_attributes"] +[[create-connector-sync-job-api-request-body]] +==== {api-request-body-title} + +`id`:: +(Required, string) The id of the connector to create the sync job for. + +`job_type`:: +(Optional, string) The job type of the created sync job. Defaults to `full`. + +`trigger_method`:: +(Optional, string) The trigger method of the created sync job. Defaults to `on_demand`. + + +[role="child_attributes"] +[[create-connector-sync-job-api-response-body]] +==== {api-response-body-title} + +`id`:: +(string) The ID associated with the connector sync job document. + +[[create-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`201`:: +Indicates that the connector sync job was created successfully. + +`400`:: +Indicates that the request was malformed. + +`404`:: +Indicates that either the index or the referenced connector is missing. From 92eae448e9921f7ed55858af8fea1b2de557ff6d Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 13 Dec 2023 13:41:30 +0000 Subject: [PATCH 13/54] Clarify that we need stack dumps of the main process (#103391) ES comprises more than one Java process, but it's the main one which matters when looking at stack dumps. --- docs/reference/troubleshooting/network-timeouts.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/troubleshooting/network-timeouts.asciidoc b/docs/reference/troubleshooting/network-timeouts.asciidoc index ab60eeff1b1a9..1920dafe62210 100644 --- a/docs/reference/troubleshooting/network-timeouts.asciidoc +++ b/docs/reference/troubleshooting/network-timeouts.asciidoc @@ -34,9 +34,9 @@ end::troubleshooting-network-timeouts-packet-capture-fault-detection[] tag::troubleshooting-network-timeouts-threads[] * Long waits for particular threads to be available can be identified by taking -stack dumps (for example, using `jstack`) or a profiling trace (for example, -using Java Flight Recorder) in the few seconds leading up to the relevant log -message. +stack dumps of the main {es} process (for example, using `jstack`) or a +profiling trace (for example, using Java Flight Recorder) in the few seconds +leading up to the relevant log message. + The <> API sometimes yields useful information, but bear in mind that this API also requires a number of `transport_worker` and From 21f059166458978a5f3289686f1d218f0814c4ff Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 13 Dec 2023 10:46:00 -0500 Subject: [PATCH 14/54] Bump versions after 7.17.16 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- .../src/main/java/org/elasticsearch/Version.java | 1 + 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 762790e8816c0..3271007a00077 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -48,7 +48,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.16", "8.11.4", "8.12.0", "8.13.0"] + BWC_VERSION: ["7.17.17", "8.11.4", "8.12.0", "8.13.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 414f06dda49e6..66eb1fc79e3ca 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1089,6 +1089,22 @@ steps: env: BWC_VERSION: 7.17.16 + - label: "{{matrix.image}} / 7.17.17 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.17 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.17 + - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 5d5fd6be4a6a5..3ce048533d131 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -662,6 +662,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 7.17.16 + - label: 7.17.17 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.17#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.17 - label: 8.0.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest timeout_in_minutes: 300 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 8fd326a0b1bd7..569caf22ae830 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -65,6 +65,7 @@ BWC_VERSION: - "7.17.14" - "7.17.15" - "7.17.16" + - "7.17.17" - "8.0.0" - "8.0.1" - "8.1.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index f3c524d844892..98bfd6b50d24b 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - - "7.17.16" + - "7.17.17" - "8.11.4" - "8.12.0" - "8.13.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 303f81ab43063..4181b077cb185 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -116,6 +116,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_7_17_14 = new Version(7_17_14_99); public static final Version V_7_17_15 = new Version(7_17_15_99); public static final Version V_7_17_16 = new Version(7_17_16_99); + public static final Version V_7_17_17 = new Version(7_17_17_99); public static final Version V_8_0_0 = new Version(8_00_00_99); public static final Version V_8_0_1 = new Version(8_00_01_99); public static final Version V_8_1_0 = new Version(8_01_00_99); From 3520584aace932d61419ca42d37af532f07a5015 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Wed, 13 Dec 2023 14:53:13 -0500 Subject: [PATCH 15/54] Add optional pruning config (weighted terms scoring) to text expansion query (#102862) Co-authored-by: Jim Ferenczi Co-authored-by: Elastic Machine --- docs/changelog/102862.yaml | 5 + .../query-dsl/text-expansion-query.asciidoc | 137 +++++- .../org/elasticsearch/TransportVersions.java | 1 + .../test/AbstractQueryTestCase.java | 18 +- .../xpack/ml/MachineLearning.java | 6 + .../ml/queries/TextExpansionQueryBuilder.java | 59 ++- .../xpack/ml/queries/TokenPruningConfig.java | 177 +++++++ .../queries/WeightedTokensQueryBuilder.java | 264 +++++++++++ .../TextExpansionQueryBuilderTests.java | 108 ++++- .../ml/queries/TokenPruningConfigTests.java | 41 ++ .../WeightedTokensQueryBuilderTests.java | 439 ++++++++++++++++++ .../test/ml/text_expansion_search.yml | 180 +++++++ 12 files changed, 1416 insertions(+), 19 deletions(-) create mode 100644 docs/changelog/102862.yaml create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfig.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilder.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfigTests.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilderTests.java diff --git a/docs/changelog/102862.yaml b/docs/changelog/102862.yaml new file mode 100644 index 0000000000000..bb453163009d5 --- /dev/null +++ b/docs/changelog/102862.yaml @@ -0,0 +1,5 @@ +pr: 102862 +summary: Add optional pruning configuration (weighted terms scoring) to text expansion query +area: "Machine Learning" +type: enhancement +issues: [] diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index e924cc05376d9..46a9aafdd1af8 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -52,10 +52,42 @@ text. (Required, string) The query text you want to use for search. +`pruning_config` :::: +(Optional, object) +preview:[] +Optional pruning configuration. If enabled, this will omit non-significant tokens from the query in order to improve query performance. +Default: Disabled. ++ +-- +Parameters for `` are: + +`tokens_freq_ratio_threshold`:: +(Optional, float) +preview:[] +Tokens whose frequency is more than `tokens_freq_ratio_threshold` times the average frequency of all tokens in the specified field are considered outliers and pruned. +This value must between 1 and 100. +Default: `5`. + +`tokens_weight_threshold`:: +(Optional, float) +preview:[] +Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +This value must be between 0 and 1. +Default: `0.4`. + +`only_score_pruned_tokens`:: +(Optional, boolean) +preview:[] +If `true` we only input pruned tokens into scoring, and discard non-pruned tokens. +It is strongly recommended to set this to `false` for the main query, but this can be set to `true` for a rescore query to get more relevant results. +Default: `false`. + +NOTE: The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSER that provided the most optimal results. +-- [discrete] [[text-expansion-query-example]] -=== Example +=== Example ELSER query The following is an example of the `text_expansion` query that references the ELSER model to perform semantic search. For a more detailed description of how @@ -69,7 +101,7 @@ GET my-index/_search "query":{ "text_expansion":{ "ml.tokens":{ - "model_id":".elser_model_1", + "model_id":".elser_model_2", "model_text":"How is the weather in Jamaica?" } } @@ -78,7 +110,108 @@ GET my-index/_search ---- // TEST[skip: TBD] +[discrete] +[[text-expansion-query-with-pruning-config-example]] +=== Example ELSER query with pruning configuration + +The following is an extension to the above example that adds a preview:[] pruning configuration to the `text_expansion` query. +The pruning configuration identifies non-significant tokens to prune from the query in order to improve query performance. +[source,console] +---- +GET my-index/_search +{ + "query":{ + "text_expansion":{ + "ml.tokens":{ + "model_id":".elser_model_2", + "model_text":"How is the weather in Jamaica?" + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } + } + } +} +---- +// TEST[skip: TBD] + +[discrete] +[[text-expansion-query-with-pruning-config-and-rescore-example]] +=== Example ELSER query with pruning configuration and rescore + +The following is an extension to the above example that adds a <> function on top of the preview:[] pruning configuration to the `text_expansion` query. +The pruning configuration identifies non-significant tokens to prune from the query in order to improve query performance. +Rescoring the query with the tokens that were originally pruned from the query may improve overall search relevance when using this pruning strategy. + +[source,console] +---- +GET my-index/_search +{ + "query":{ + "text_expansion":{ + "ml.tokens":{ + "model_id":".elser_model_2", + "model_text":"How is the weather in Jamaica?" + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } + } + }, + "rescore": { + "window_size": 100, + "query": { + "rescore_query": { + "text_expansion": { + "ml.tokens": { + "model_id": ".elser_model_2", + "model_text": "How is the weather in Jamaica?" + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } + } + } + } + } +} +---- +//TEST[skip: TBD] + [NOTE] ==== Depending on your data, the text expansion query may be faster with `track_total_hits: false`. ==== + +[discrete] +[[weighted-tokens-query-example]] +=== Example Weighted token query + +In order to quickly iterate during tests, we exposed a new preview:[] `weighted_tokens` query for evaluation of tokenized datasets. +While this is not a query that is intended for production use, it can be used to quickly evaluate relevance using various pruning configurations. + +[source,console] +---- +POST /docs/_search +{ + "query": { + "weighted_tokens": { + "query_expansion": { + "tokens": {"2161": 0.4679, "2621": 0.307, "2782": 0.1299, "2851": 0.1056, "3088": 0.3041, "3376": 0.1038, "3467": 0.4873, "3684": 0.8958, "4380": 0.334, "4542": 0.4636, "4633": 2.2805, "4785": 1.2628, "4860": 1.0655, "5133": 1.0709, "7139": 1.0016, "7224": 0.2486, "7387": 0.0985, "7394": 0.0542, "8915": 0.369, "9156": 2.8947, "10505": 0.2771, "11464": 0.3996, "13525": 0.0088, "14178": 0.8161, "16893": 0.1376, "17851": 1.5348, "19939": 0.6012}, + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } + } + } + } +} +---- +//TEST[skip: TBD] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index ad29384b16f45..625871d25734b 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -192,6 +192,7 @@ static TransportVersion def(int id) { public static final TransportVersion INFERENCE_SERVICE_EMBEDDING_SIZE_ADDED = def(8_559_00_0); public static final TransportVersion ENRICH_ELASTICSEARCH_VERSION_REMOVED = def(8_560_00_0); public static final TransportVersion NODE_STATS_REQUEST_SIMPLIFIED = def(8_561_00_0); + public static final TransportVersion TEXT_EXPANSION_TOKEN_PRUNING_CONFIG_ADDED = def(8_562_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 6137b2b0aad18..0a3316b87bd04 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -281,7 +281,7 @@ static List> alterateQueries(Set queries, Map levels = new LinkedList<>(); @@ -388,7 +388,7 @@ private void queryWrappedInArrayTest(String queryName, String validQuery) { + "[" + validQuery.substring(insertionPosition, endArrayPosition) + "]" - + validQuery.substring(endArrayPosition, validQuery.length()); + + validQuery.substring(endArrayPosition); ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(testQuery)); assertEquals("[" + queryName + "] query malformed, no start_object after query name", e.getMessage()); @@ -464,13 +464,15 @@ public void testToQuery() throws IOException { assertNotNull("toQuery should not return null", firstLuceneQuery); assertLuceneQuery(firstQuery, firstLuceneQuery, context); // remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well - assertTrue( + assertEquals( "query is not equal to its copy after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, - firstQuery.equals(controlQuery) + firstQuery, + controlQuery ); - assertTrue( + assertEquals( "equals is not symmetric after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, - controlQuery.equals(firstQuery) + controlQuery, + firstQuery ); assertThat( "query copy's hashcode is different from original hashcode after calling toQuery, firstQuery: " @@ -552,7 +554,7 @@ protected boolean supportsQueryName() { * and {@link SearchExecutionContext}. Verifies that named queries and boost are properly handled and delegates to * {@link #doAssertLuceneQuery(AbstractQueryBuilder, Query, SearchExecutionContext)} for query specific checks. */ - private void assertLuceneQuery(QB queryBuilder, Query query, SearchExecutionContext context) throws IOException { + protected void assertLuceneQuery(QB queryBuilder, Query query, SearchExecutionContext context) throws IOException { if (queryBuilder.queryName() != null && query instanceof MatchNoDocsQuery == false) { Query namedQuery = context.copyNamedQueries().get(queryBuilder.queryName()); assertThat(namedQuery, equalTo(query)); @@ -671,7 +673,7 @@ protected QB changeNameOrBoost(QB original) throws IOException { // we use the streaming infra to create a copy of the query provided as argument @SuppressWarnings("unchecked") - private QB copyQuery(QB query) throws IOException { + protected QB copyQuery(QB query) throws IOException { Reader reader = (Reader) namedWriteableRegistry().getReader(QueryBuilder.class, query.getWriteableName()); return copyWriteable(query, namedWriteableRegistry(), reader); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 1031d45facf85..20efc5d681cc3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -373,6 +373,7 @@ import org.elasticsearch.xpack.ml.process.NativeController; import org.elasticsearch.xpack.ml.process.NativeStorageProvider; import org.elasticsearch.xpack.ml.queries.TextExpansionQueryBuilder; +import org.elasticsearch.xpack.ml.queries.WeightedTokensQueryBuilder; import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction; import org.elasticsearch.xpack.ml.rest.RestMlInfoAction; import org.elasticsearch.xpack.ml.rest.RestMlMemoryAction; @@ -1720,6 +1721,11 @@ public List> getQueries() { TextExpansionQueryBuilder.NAME, TextExpansionQueryBuilder::new, TextExpansionQueryBuilder::fromXContent + ), + new QuerySpec( + WeightedTokensQueryBuilder.NAME, + WeightedTokensQueryBuilder::new, + WeightedTokensQueryBuilder::fromXContent ) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java index 12019e93ba713..24383e51b0ed2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilder.java @@ -15,8 +15,8 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryRewriteContext; @@ -41,6 +41,7 @@ public class TextExpansionQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "text_expansion"; + public static final ParseField PRUNING_CONFIG = new ParseField("pruning_config"); public static final ParseField MODEL_TEXT = new ParseField("model_text"); public static final ParseField MODEL_ID = new ParseField("model_id"); @@ -48,8 +49,13 @@ public class TextExpansionQueryBuilder extends AbstractQueryBuilder weightedTokensSupplier; + private final TokenPruningConfig tokenPruningConfig; public TextExpansionQueryBuilder(String fieldName, String modelText, String modelId) { + this(fieldName, modelText, modelId, null); + } + + public TextExpansionQueryBuilder(String fieldName, String modelText, String modelId, @Nullable TokenPruningConfig tokenPruningConfig) { if (fieldName == null) { throw new IllegalArgumentException("[" + NAME + "] requires a fieldName"); } @@ -59,10 +65,10 @@ public TextExpansionQueryBuilder(String fieldName, String modelText, String mode if (modelId == null) { throw new IllegalArgumentException("[" + NAME + "] requires a " + MODEL_ID.getPreferredName() + " value"); } - this.fieldName = fieldName; this.modelText = modelText; this.modelId = modelId; + this.tokenPruningConfig = tokenPruningConfig; } public TextExpansionQueryBuilder(StreamInput in) throws IOException { @@ -70,12 +76,18 @@ public TextExpansionQueryBuilder(StreamInput in) throws IOException { this.fieldName = in.readString(); this.modelText = in.readString(); this.modelId = in.readString(); + if (in.getTransportVersion().onOrAfter(TransportVersions.TEXT_EXPANSION_TOKEN_PRUNING_CONFIG_ADDED)) { + this.tokenPruningConfig = in.readOptionalWriteable(TokenPruningConfig::new); + } else { + this.tokenPruningConfig = null; + } } private TextExpansionQueryBuilder(TextExpansionQueryBuilder other, SetOnce weightedTokensSupplier) { this.fieldName = other.fieldName; this.modelText = other.modelText; this.modelId = other.modelId; + this.tokenPruningConfig = other.tokenPruningConfig; this.boost = other.boost; this.queryName = other.queryName; this.weightedTokensSupplier = weightedTokensSupplier; @@ -85,6 +97,10 @@ String getFieldName() { return fieldName; } + public TokenPruningConfig getTokenPruningConfig() { + return tokenPruningConfig; + } + @Override public String getWriteableName() { return NAME; @@ -103,6 +119,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); out.writeString(modelText); out.writeString(modelId); + if (out.getTransportVersion().onOrAfter(TransportVersions.TEXT_EXPANSION_TOKEN_PRUNING_CONFIG_ADDED)) { + out.writeOptionalWriteable(tokenPruningConfig); + } } @Override @@ -111,6 +130,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.startObject(fieldName); builder.field(MODEL_TEXT.getPreferredName(), modelText); builder.field(MODEL_ID.getPreferredName(), modelId); + if (tokenPruningConfig != null) { + builder.field(PRUNING_CONFIG.getPreferredName(), tokenPruningConfig); + } boostAndQueryNameToXContent(builder); builder.endObject(); builder.endObject(); @@ -174,21 +196,33 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws return new TextExpansionQueryBuilder(this, textExpansionResultsSupplier); } - static BoolQueryBuilder weightedTokensToQuery( + private QueryBuilder weightedTokensToQuery( String fieldName, TextExpansionResults textExpansionResults, QueryRewriteContext queryRewriteContext - ) throws IOException { + ) { + if (tokenPruningConfig != null) { + WeightedTokensQueryBuilder weightedTokensQueryBuilder = new WeightedTokensQueryBuilder( + fieldName, + textExpansionResults.getWeightedTokens(), + tokenPruningConfig + ); + weightedTokensQueryBuilder.queryName(queryName); + weightedTokensQueryBuilder.boost(boost); + return weightedTokensQueryBuilder; + } var boolQuery = QueryBuilders.boolQuery(); for (var weightedToken : textExpansionResults.getWeightedTokens()) { boolQuery.should(QueryBuilders.termQuery(fieldName, weightedToken.token()).boost(weightedToken.weight())); } boolQuery.minimumShouldMatch(1); + boolQuery.boost(this.boost); + boolQuery.queryName(this.queryName); return boolQuery; } @Override - protected Query doToQuery(SearchExecutionContext context) throws IOException { + protected Query doToQuery(SearchExecutionContext context) { throw new IllegalStateException("text_expansion should have been rewritten to another query type"); } @@ -197,18 +231,20 @@ protected boolean doEquals(TextExpansionQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(modelText, other.modelText) && Objects.equals(modelId, other.modelId) + && Objects.equals(tokenPruningConfig, other.tokenPruningConfig) && Objects.equals(weightedTokensSupplier, other.weightedTokensSupplier); } @Override protected int doHashCode() { - return Objects.hash(fieldName, modelText, modelId, weightedTokensSupplier); + return Objects.hash(fieldName, modelText, modelId, tokenPruningConfig, weightedTokensSupplier); } public static TextExpansionQueryBuilder fromXContent(XContentParser parser) throws IOException { String fieldName = null; String modelText = null; String modelId = null; + TokenPruningConfig tokenPruningConfig = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; String queryName = null; String currentFieldName = null; @@ -222,6 +258,15 @@ public static TextExpansionQueryBuilder fromXContent(XContentParser parser) thro while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (PRUNING_CONFIG.match(currentFieldName, parser.getDeprecationHandler())) { + tokenPruningConfig = TokenPruningConfig.fromXContent(parser); + } else { + throw new ParsingException( + parser.getTokenLocation(), + "[" + NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]" + ); + } } else if (token.isValue()) { if (MODEL_TEXT.match(currentFieldName, parser.getDeprecationHandler())) { modelText = parser.text(); @@ -259,7 +304,7 @@ public static TextExpansionQueryBuilder fromXContent(XContentParser parser) thro throw new ParsingException(parser.getTokenLocation(), "No fieldname specified for query"); } - TextExpansionQueryBuilder queryBuilder = new TextExpansionQueryBuilder(fieldName, modelText, modelId); + TextExpansionQueryBuilder queryBuilder = new TextExpansionQueryBuilder(fieldName, modelText, modelId, tokenPruningConfig); queryBuilder.queryName(queryName); queryBuilder.boost(boost); return queryBuilder; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfig.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfig.java new file mode 100644 index 0000000000000..d789a645fd9c4 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfig.java @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.queries; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; + +import static org.elasticsearch.xpack.ml.queries.TextExpansionQueryBuilder.PRUNING_CONFIG; + +public class TokenPruningConfig implements Writeable, ToXContentObject { + public static final ParseField TOKENS_FREQ_RATIO_THRESHOLD = new ParseField("tokens_freq_ratio_threshold"); + public static final ParseField TOKENS_WEIGHT_THRESHOLD = new ParseField("tokens_weight_threshold"); + public static final ParseField ONLY_SCORE_PRUNED_TOKENS_FIELD = new ParseField("only_score_pruned_tokens"); + + // Tokens whose frequency is more than 5 times the average frequency of all tokens in the specified field are considered outliers. + public static final float DEFAULT_TOKENS_FREQ_RATIO_THRESHOLD = 5; + public static final float MAX_TOKENS_FREQ_RATIO_THRESHOLD = 100; + // A token's weight should be > 40% of the best weight in the query to be considered significant. + public static final float DEFAULT_TOKENS_WEIGHT_THRESHOLD = 0.4f; + + private final float tokensFreqRatioThreshold; + private final float tokensWeightThreshold; + private final boolean onlyScorePrunedTokens; + + public TokenPruningConfig() { + this(DEFAULT_TOKENS_FREQ_RATIO_THRESHOLD, DEFAULT_TOKENS_WEIGHT_THRESHOLD, false); + } + + public TokenPruningConfig(float tokensFreqRatioThreshold, float tokensWeightThreshold, boolean onlyScorePrunedTokens) { + if (tokensFreqRatioThreshold < 1 || tokensFreqRatioThreshold > MAX_TOKENS_FREQ_RATIO_THRESHOLD) { + throw new IllegalArgumentException( + "[" + + TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName() + + "] must be between [1.0] and [" + + String.format(Locale.ROOT, "%.1f", MAX_TOKENS_FREQ_RATIO_THRESHOLD) + + "], got " + + tokensFreqRatioThreshold + ); + } + if (tokensWeightThreshold < 0 || tokensWeightThreshold > 1) { + throw new IllegalArgumentException("[" + TOKENS_WEIGHT_THRESHOLD.getPreferredName() + "] must be between 0 and 1"); + } + this.tokensFreqRatioThreshold = tokensFreqRatioThreshold; + this.tokensWeightThreshold = tokensWeightThreshold; + this.onlyScorePrunedTokens = onlyScorePrunedTokens; + } + + public TokenPruningConfig(StreamInput in) throws IOException { + this.tokensFreqRatioThreshold = in.readFloat(); + this.tokensWeightThreshold = in.readFloat(); + this.onlyScorePrunedTokens = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeFloat(tokensFreqRatioThreshold); + out.writeFloat(tokensWeightThreshold); + out.writeBoolean(onlyScorePrunedTokens); + } + + /** + * Returns the frequency ratio threshold to apply on the query. + * Tokens whose frequency is more than ratio_threshold times the average frequency of all tokens in the specified + * field are considered outliers and may be subject to removal from the query. + */ + public float getTokensFreqRatioThreshold() { + return tokensFreqRatioThreshold; + } + + /** + * Returns the weight threshold to apply on the query. + * Tokens whose weight is more than (weightThreshold * best_weight) of the highest weight in the query are not + * considered outliers, even if their frequency exceeds the specified ratio_threshold. + * This threshold ensures that important tokens, as indicated by their weight, are retained in the query. + */ + public float getTokensWeightThreshold() { + return tokensWeightThreshold; + } + + /** + * Returns whether the filtering process retains tokens identified as non-relevant based on the specified thresholds + * (ratio and weight). When {@code true}, only non-relevant tokens are considered for matching and scoring documents. + * Enabling this option is valuable for re-scoring top hits retrieved from a {@link WeightedTokensQueryBuilder} with + * active thresholds. + */ + public boolean isOnlyScorePrunedTokens() { + return onlyScorePrunedTokens; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TokenPruningConfig that = (TokenPruningConfig) o; + return Float.compare(that.tokensFreqRatioThreshold, tokensFreqRatioThreshold) == 0 + && Float.compare(that.tokensWeightThreshold, tokensWeightThreshold) == 0 + && onlyScorePrunedTokens == that.onlyScorePrunedTokens; + } + + @Override + public int hashCode() { + return Objects.hash(tokensFreqRatioThreshold, tokensWeightThreshold, onlyScorePrunedTokens); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), tokensFreqRatioThreshold); + builder.field(TOKENS_WEIGHT_THRESHOLD.getPreferredName(), tokensWeightThreshold); + if (onlyScorePrunedTokens) { + builder.field(ONLY_SCORE_PRUNED_TOKENS_FIELD.getPreferredName(), onlyScorePrunedTokens); + } + builder.endObject(); + return builder; + } + + public static TokenPruningConfig fromXContent(XContentParser parser) throws IOException { + String currentFieldName = null; + XContentParser.Token token; + float ratioThreshold = DEFAULT_TOKENS_FREQ_RATIO_THRESHOLD; + float weightThreshold = DEFAULT_TOKENS_WEIGHT_THRESHOLD; + boolean onlyScorePrunedTokens = false; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.START_OBJECT) { + continue; + } + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if (Set.of( + TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), + TOKENS_WEIGHT_THRESHOLD.getPreferredName(), + ONLY_SCORE_PRUNED_TOKENS_FIELD.getPreferredName() + ).contains(currentFieldName) == false) { + throw new ParsingException( + parser.getTokenLocation(), + "[" + PRUNING_CONFIG.getPreferredName() + "] unknown token [" + currentFieldName + "]" + ); + } + } else if (token.isValue()) { + if (TOKENS_FREQ_RATIO_THRESHOLD.match(currentFieldName, parser.getDeprecationHandler())) { + ratioThreshold = parser.intValue(); + } else if (TOKENS_WEIGHT_THRESHOLD.match(currentFieldName, parser.getDeprecationHandler())) { + weightThreshold = parser.floatValue(); + } else if (ONLY_SCORE_PRUNED_TOKENS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + onlyScorePrunedTokens = parser.booleanValue(); + } else { + throw new ParsingException( + parser.getTokenLocation(), + "[" + PRUNING_CONFIG.getPreferredName() + "] does not support [" + currentFieldName + "]" + ); + } + } else { + throw new ParsingException( + parser.getTokenLocation(), + "[" + PRUNING_CONFIG.getPreferredName() + "] unknown token [" + token + "] after [" + currentFieldName + "]" + ); + } + } + return new TokenPruningConfig(ratioThreshold, weightThreshold, onlyScorePrunedTokens); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilder.java new file mode 100644 index 0000000000000..a09bcadaacfc0 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilder.java @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.queries; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults.WeightedToken; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.xpack.ml.queries.TextExpansionQueryBuilder.PRUNING_CONFIG; + +public class WeightedTokensQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "weighted_tokens"; + + public static final ParseField TOKENS_FIELD = new ParseField("tokens"); + private final String fieldName; + private final List tokens; + @Nullable + private final TokenPruningConfig tokenPruningConfig; + + public WeightedTokensQueryBuilder(String fieldName, List tokens) { + this(fieldName, tokens, null); + } + + public WeightedTokensQueryBuilder(String fieldName, List tokens, @Nullable TokenPruningConfig tokenPruningConfig) { + this.fieldName = Objects.requireNonNull(fieldName, "[" + NAME + "] requires a fieldName"); + this.tokens = Objects.requireNonNull(tokens, "[" + NAME + "] requires tokens"); + if (tokens.isEmpty()) { + throw new IllegalArgumentException("[" + NAME + "] requires at least one token"); + } + this.tokenPruningConfig = tokenPruningConfig; + } + + public WeightedTokensQueryBuilder(StreamInput in) throws IOException { + super(in); + this.fieldName = in.readString(); + this.tokens = in.readCollectionAsList(WeightedToken::new); + this.tokenPruningConfig = in.readOptionalWriteable(TokenPruningConfig::new); + } + + public String getFieldName() { + return fieldName; + } + + @Nullable + public TokenPruningConfig getTokenPruningConfig() { + return tokenPruningConfig; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(fieldName); + out.writeCollection(tokens); + out.writeOptionalWriteable(tokenPruningConfig); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.startObject(fieldName); + builder.startObject(TOKENS_FIELD.getPreferredName()); + for (var token : tokens) { + token.toXContent(builder, params); + } + builder.endObject(); + if (tokenPruningConfig != null) { + builder.field(PRUNING_CONFIG.getPreferredName(), tokenPruningConfig); + } + boostAndQueryNameToXContent(builder); + builder.endObject(); + builder.endObject(); + } + + /** + * We calculate the maximum number of unique tokens for any shard of data. The maximum is used to compute + * average token frequency since we don't have a unique inter-segment token count. + * Once we have the maximum number of unique tokens, we use the total count of tokens in the index to calculate + * the average frequency ratio. + * + * @param reader + * @param fieldDocCount + * @return float + * @throws IOException + */ + private float getAverageTokenFreqRatio(IndexReader reader, int fieldDocCount) throws IOException { + int numUniqueTokens = 0; + for (var leaf : reader.getContext().leaves()) { + var terms = leaf.reader().terms(fieldName); + if (terms != null) { + numUniqueTokens = (int) Math.max(terms.size(), numUniqueTokens); + } + } + if (numUniqueTokens == 0) { + return 0; + } + return (float) reader.getSumDocFreq(fieldName) / fieldDocCount / numUniqueTokens; + } + + /** + * Returns true if the token should be queried based on the {@code tokensFreqRatioThreshold} and {@code tokensWeightThreshold} + * set on the query. + */ + private boolean shouldKeepToken( + IndexReader reader, + WeightedToken token, + int fieldDocCount, + float averageTokenFreqRatio, + float bestWeight + ) throws IOException { + if (this.tokenPruningConfig == null) { + return true; + } + int docFreq = reader.docFreq(new Term(fieldName, token.token())); + if (docFreq == 0) { + return false; + } + float tokenFreqRatio = (float) docFreq / fieldDocCount; + return tokenFreqRatio < this.tokenPruningConfig.getTokensFreqRatioThreshold() * averageTokenFreqRatio + || token.weight() > this.tokenPruningConfig.getTokensWeightThreshold() * bestWeight; + } + + @Override + protected Query doToQuery(SearchExecutionContext context) throws IOException { + final MappedFieldType ft = context.getFieldType(fieldName); + if (ft == null) { + return new MatchNoDocsQuery("The \"" + getName() + "\" query is against a field that does not exist"); + } + var qb = new BooleanQuery.Builder(); + int fieldDocCount = context.getIndexReader().getDocCount(fieldName); + float bestWeight = 0f; + for (var t : tokens) { + bestWeight = Math.max(t.weight(), bestWeight); + } + float averageTokenFreqRatio = getAverageTokenFreqRatio(context.getIndexReader(), fieldDocCount); + if (averageTokenFreqRatio == 0) { + return new MatchNoDocsQuery("The \"" + getName() + "\" query is against an empty field"); + } + for (var token : tokens) { + boolean keep = shouldKeepToken(context.getIndexReader(), token, fieldDocCount, averageTokenFreqRatio, bestWeight); + if (this.tokenPruningConfig != null) { + keep ^= this.tokenPruningConfig.isOnlyScorePrunedTokens(); + } + if (keep) { + qb.add(new BoostQuery(ft.termQuery(token.token(), context), token.weight()), BooleanClause.Occur.SHOULD); + } + } + qb.setMinimumNumberShouldMatch(1); + return qb.build(); + } + + @Override + protected boolean doEquals(WeightedTokensQueryBuilder other) { + return Objects.equals(fieldName, other.fieldName) + && Objects.equals(tokenPruningConfig, other.tokenPruningConfig) + && tokens.equals(other.tokens); + } + + @Override + protected int doHashCode() { + return Objects.hash(fieldName, tokens, tokenPruningConfig); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.TEXT_EXPANSION_TOKEN_PRUNING_CONFIG_ADDED; + } + + private static float parseWeight(String token, Object weight) throws IOException { + if (weight instanceof Number asNumber) { + return asNumber.floatValue(); + } + if (weight instanceof String asString) { + return Float.parseFloat(asString); + } + throw new ElasticsearchParseException( + "Illegal weight for token: [" + token + "], expected floating point got " + weight.getClass().getSimpleName() + ); + } + + public static WeightedTokensQueryBuilder fromXContent(XContentParser parser) throws IOException { + String currentFieldName = null; + String fieldName = null; + List tokens = new ArrayList<>(); + TokenPruningConfig tokenPruningConfig = null; + float boost = AbstractQueryBuilder.DEFAULT_BOOST; + String queryName = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, currentFieldName); + fieldName = currentFieldName; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (PRUNING_CONFIG.match(currentFieldName, parser.getDeprecationHandler())) { + if (token != XContentParser.Token.START_OBJECT) { + throw new ParsingException( + parser.getTokenLocation(), + "[" + PRUNING_CONFIG.getPreferredName() + "] should be an object" + ); + } + tokenPruningConfig = TokenPruningConfig.fromXContent(parser); + } else if (TOKENS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + var tokensMap = parser.map(); + for (var e : tokensMap.entrySet()) { + tokens.add(new WeightedToken(e.getKey(), parseWeight(e.getKey(), e.getValue()))); + } + } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + boost = parser.floatValue(); + } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + queryName = parser.text(); + } else { + throw new ParsingException(parser.getTokenLocation(), "unknown field [" + currentFieldName + "]"); + } + } + } else { + throw new IllegalArgumentException("invalid query"); + } + } + + if (fieldName == null) { + throw new ParsingException(parser.getTokenLocation(), "No fieldname specified for query"); + } + + var qb = new WeightedTokensQueryBuilder(fieldName, tokens, tokenPruningConfig); + qb.queryName(queryName); + qb.boost(boost); + return qb; + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java index 5e414a7f997d5..13f12f3cdc1e1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TextExpansionQueryBuilderTests.java @@ -7,10 +7,15 @@ package org.elasticsearch.xpack.ml.queries; +import org.apache.lucene.document.Document; import org.apache.lucene.document.FeatureField; +import org.apache.lucene.document.FloatDocValuesField; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionType; @@ -20,6 +25,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.Plugin; @@ -36,6 +42,7 @@ import java.util.Collection; import java.util.List; +import static org.elasticsearch.xpack.ml.queries.WeightedTokensQueryBuilder.TOKENS_FIELD; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.hasSize; @@ -43,11 +50,19 @@ public class TextExpansionQueryBuilderTests extends AbstractQueryTestCase { private static final String RANK_FEATURES_FIELD = "rank"; - private static int NUM_TOKENS = 10; + private static final int NUM_TOKENS = 10; @Override protected TextExpansionQueryBuilder doCreateTestQueryBuilder() { - var builder = new TextExpansionQueryBuilder(RANK_FEATURES_FIELD, randomAlphaOfLength(4), randomAlphaOfLength(4)); + TokenPruningConfig tokenPruningConfig = randomBoolean() + ? new TokenPruningConfig(randomIntBetween(1, 100), randomFloat(), randomBoolean()) + : null; + var builder = new TextExpansionQueryBuilder( + RANK_FEATURES_FIELD, + randomAlphaOfLength(4), + randomAlphaOfLength(4), + tokenPruningConfig + ); if (randomBoolean()) { builder.boost((float) randomDoubleBetween(0.1, 10.0, true)); } @@ -126,6 +141,44 @@ protected void doAssertLuceneQuery(TextExpansionQueryBuilder queryBuilder, Query } } + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testCacheability() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new FloatDocValuesField(RANK_FEATURES_FIELD, 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + TextExpansionQueryBuilder queryBuilder = createTestQueryBuilder(); + QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new SearchExecutionContext(context)); + + assertNotNull(rewriteQuery.toQuery(context)); + assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable()); + } + } + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader}; this query should always be rewritten + */ + @Override + public void testToQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new FloatDocValuesField(RANK_FEATURES_FIELD, 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + TextExpansionQueryBuilder queryBuilder = createTestQueryBuilder(); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> queryBuilder.toQuery(context)); + assertEquals("text_expansion should have been rewritten to another query type", e.getMessage()); + } + } + } + public void testIllegalValues() { { IllegalArgumentException e = expectThrows( @@ -162,4 +215,55 @@ public void testToXContent() throws IOException { } }""", query); } + + public void testToXContentWithThresholds() throws IOException { + QueryBuilder query = new TextExpansionQueryBuilder("foo", "bar", "baz", new TokenPruningConfig(4, 0.3f, false)); + checkGeneratedJson(""" + { + "text_expansion": { + "foo": { + "model_text": "bar", + "model_id": "baz", + "pruning_config": { + "tokens_freq_ratio_threshold": 4.0, + "tokens_weight_threshold": 0.3 + } + } + } + }""", query); + } + + public void testToXContentWithThresholdsAndOnlyScorePrunedTokens() throws IOException { + QueryBuilder query = new TextExpansionQueryBuilder("foo", "bar", "baz", new TokenPruningConfig(4, 0.3f, true)); + checkGeneratedJson(""" + { + "text_expansion": { + "foo": { + "model_text": "bar", + "model_id": "baz", + "pruning_config": { + "tokens_freq_ratio_threshold": 4.0, + "tokens_weight_threshold": 0.3, + "only_score_pruned_tokens": true + } + } + } + }""", query); + } + + @Override + protected String[] shuffleProtectedFields() { + return new String[] { TOKENS_FIELD.getPreferredName() }; + } + + public void testThatTokensAreCorrectlyPruned() { + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + TextExpansionQueryBuilder queryBuilder = createTestQueryBuilder(); + QueryBuilder rewrittenQueryBuilder = rewriteAndFetch(queryBuilder, searchExecutionContext); + if (queryBuilder.getTokenPruningConfig() == null) { + assertTrue(rewrittenQueryBuilder instanceof BoolQueryBuilder); + } else { + assertTrue(rewrittenQueryBuilder instanceof WeightedTokensQueryBuilder); + } + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfigTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfigTests.java new file mode 100644 index 0000000000000..3f38a2ee891d5 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/TokenPruningConfigTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.queries; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +public class TokenPruningConfigTests extends AbstractXContentSerializingTestCase { + + public static TokenPruningConfig testInstance() { + return new TokenPruningConfig(randomIntBetween(1, 100), randomFloat(), randomBoolean()); + } + + @Override + protected Writeable.Reader instanceReader() { + return TokenPruningConfig::new; + } + + @Override + protected TokenPruningConfig createTestInstance() { + return testInstance(); + } + + @Override + protected TokenPruningConfig mutateInstance(TokenPruningConfig instance) throws IOException { + return null; + } + + @Override + protected TokenPruningConfig doParseInstance(XContentParser parser) throws IOException { + return TokenPruningConfig.fromXContent(parser); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilderTests.java new file mode 100644 index 0000000000000..4d91c66de4b9e --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/queries/WeightedTokensQueryBuilderTests.java @@ -0,0 +1,439 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.queries; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FeatureField; +import org.apache.lucene.document.FloatDocValuesField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.xpack.core.ml.action.InferModelAction; +import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.ml.MachineLearning; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults.WeightedToken; +import static org.elasticsearch.xpack.ml.queries.WeightedTokensQueryBuilder.TOKENS_FIELD; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.hasSize; + +public class WeightedTokensQueryBuilderTests extends AbstractQueryTestCase { + + private static final String RANK_FEATURES_FIELD = "rank"; + private static final List WEIGHTED_TOKENS = List.of(new TextExpansionResults.WeightedToken("foo", .42f)); + private static final int NUM_TOKENS = WEIGHTED_TOKENS.size(); + + @Override + protected WeightedTokensQueryBuilder doCreateTestQueryBuilder() { + return createTestQueryBuilder(randomBoolean()); + } + + private WeightedTokensQueryBuilder createTestQueryBuilder(boolean onlyScorePrunedTokens) { + TokenPruningConfig tokenPruningConfig = randomBoolean() + ? new TokenPruningConfig(randomIntBetween(1, 100), randomFloat(), onlyScorePrunedTokens) + : null; + + var builder = new WeightedTokensQueryBuilder(RANK_FEATURES_FIELD, WEIGHTED_TOKENS, tokenPruningConfig); + if (randomBoolean()) { + builder.boost((float) randomDoubleBetween(0.1, 10.0, true)); + } + if (randomBoolean()) { + builder.queryName(randomAlphaOfLength(4)); + } + return builder; + } + + @Override + protected Collection> getPlugins() { + return List.of(MachineLearning.class, MapperExtrasPlugin.class); + } + + @Override + protected boolean canSimulateMethod(Method method, Object[] args) throws NoSuchMethodException { + return method.equals(Client.class.getMethod("execute", ActionType.class, ActionRequest.class, ActionListener.class)) + && (args[0] instanceof InferModelAction); + } + + @Override + protected Object simulateMethod(Method method, Object[] args) { + InferModelAction.Request request = (InferModelAction.Request) args[1]; + assertEquals(InferModelAction.Request.DEFAULT_TIMEOUT_FOR_API, request.getInferenceTimeout()); + assertEquals(TrainedModelPrefixStrings.PrefixType.SEARCH, request.getPrefixType()); + + // Randomisation of tokens cannot be used here as {@code #doAssertLuceneQuery} + // asserts that 2 rewritten queries are the same + var response = InferModelAction.Response.builder() + .setId(request.getId()) + .addInferenceResults(List.of(new TextExpansionResults("foo", WEIGHTED_TOKENS.stream().toList(), randomBoolean()))) + .build(); + @SuppressWarnings("unchecked") // We matched the method above. + ActionListener listener = (ActionListener) args[2]; + listener.onResponse(response); + return null; + } + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + mapperService.merge( + "_doc", + new CompressedXContent(Strings.toString(PutMappingRequest.simpleMapping(RANK_FEATURES_FIELD, "type=rank_features"))), + MapperService.MergeReason.MAPPING_UPDATE + ); + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testToQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + // Index at least one document so we have a freq > 0 + Document document = new Document(); + document.add(new FeatureField(RANK_FEATURES_FIELD, "foo", 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + // We need to force token pruning config here, to get repeatable lucene queries for comparison + WeightedTokensQueryBuilder firstQuery = createTestQueryBuilder(false); + WeightedTokensQueryBuilder controlQuery = copyQuery(firstQuery); + QueryBuilder rewritten = rewriteQuery(firstQuery, context); + Query firstLuceneQuery = rewritten.toQuery(context); + assertNotNull("toQuery should not return null", firstLuceneQuery); + assertLuceneQuery(firstQuery, firstLuceneQuery, context); + assertEquals( + "query is not equal to its copy after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, + firstQuery, + controlQuery + ); + assertEquals( + "equals is not symmetric after calling toQuery, firstQuery: " + firstQuery + ", secondQuery: " + controlQuery, + controlQuery, + firstQuery + ); + assertThat( + "query copy's hashcode is different from original hashcode after calling toQuery, firstQuery: " + + firstQuery + + ", secondQuery: " + + controlQuery, + controlQuery.hashCode(), + equalTo(firstQuery.hashCode()) + ); + WeightedTokensQueryBuilder secondQuery = copyQuery(firstQuery); + + // query _name never should affect the result of toQuery, we randomly set it to make sure + if (randomBoolean()) { + secondQuery.queryName( + secondQuery.queryName() == null + ? randomAlphaOfLengthBetween(1, 30) + : secondQuery.queryName() + randomAlphaOfLengthBetween(1, 10) + ); + } + context = new SearchExecutionContext(context); + Query secondLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context); + assertNotNull("toQuery should not return null", secondLuceneQuery); + assertLuceneQuery(secondQuery, secondLuceneQuery, context); + + if (builderGeneratesCacheableQueries()) { + assertEquals( + "two equivalent query builders lead to different lucene queries hashcode", + secondLuceneQuery.hashCode(), + firstLuceneQuery.hashCode() + ); + assertEquals( + "two equivalent query builders lead to different lucene queries", + rewrite(secondLuceneQuery), + rewrite(firstLuceneQuery) + ); + } + + if (supportsBoost() && firstLuceneQuery instanceof MatchNoDocsQuery == false) { + secondQuery.boost(firstQuery.boost() + 1f + randomFloat()); + Query thirdLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context); + assertNotEquals( + "modifying the boost doesn't affect the corresponding lucene query", + rewrite(firstLuceneQuery), + rewrite(thirdLuceneQuery) + ); + } + + } + } + } + + public void testPruningIsAppliedCorrectly() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + List documents = List.of( + createDocument( + List.of("the", "quick", "brown", "fox", "jumped", "over", "lazy", "dog", "me"), + List.of(.2f, 1.8f, 1.75f, 5.9f, 1.6f, 1.4f, .4f, 4.8f, 2.1f) + ), + createDocument( + List.of("the", "rains", "in", "spain", "fall", "mainly", "on", "plain", "me"), + List.of(.1f, 3.6f, .1f, 4.8f, .6f, .3f, .1f, 2.6f, 2.1f) + ), + createDocument( + List.of("betty", "bought", "butter", "but", "the", "was", "bitter", "me"), + List.of(6.8f, 1.4f, .5f, 3.2f, .1f, 3.2f, .6f, 2.1f) + ), + createDocument( + List.of("she", "sells", "seashells", "by", "the", "seashore", "me"), + List.of(.2f, 1.4f, 5.9f, .1f, .1f, 3.6f, 2.1f) + ) + ); + iw.addDocuments(documents); + + List inputTokens = List.of( + new WeightedToken("the", .1f), // Will be pruned - score too low, freq too high + new WeightedToken("black", 5.3f), // Will be pruned - does not exist in index + new WeightedToken("dog", 7.5f), // Will be kept - high score and low freq + new WeightedToken("jumped", 4.5f), // Will be kept - high score and low freq + new WeightedToken("on", .1f), // Will be kept - low score but also low freq + new WeightedToken("me", 3.8f) // Will be kept - high freq but also high score + ); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + + WeightedTokensQueryBuilder noPruningQuery = new WeightedTokensQueryBuilder(RANK_FEATURES_FIELD, inputTokens, null); + Query query = noPruningQuery.doToQuery(context); + assertCorrectLuceneQuery("noPruningQuery", query, List.of("the", "black", "dog", "jumped", "on", "me")); + + WeightedTokensQueryBuilder queryThatShouldBePruned = new WeightedTokensQueryBuilder( + RANK_FEATURES_FIELD, + inputTokens, + new TokenPruningConfig(1.5f, 0.5f, false) + ); + query = queryThatShouldBePruned.doToQuery(context); + assertCorrectLuceneQuery("queryThatShouldBePruned", query, List.of("dog", "jumped", "on", "me")); + + WeightedTokensQueryBuilder onlyScorePrunedTokensQuery = new WeightedTokensQueryBuilder( + RANK_FEATURES_FIELD, + inputTokens, + new TokenPruningConfig(1.5f, 0.5f, true) + ); + query = onlyScorePrunedTokensQuery.doToQuery(context); + assertCorrectLuceneQuery("onlyScorePrunedTokensQuery", query, List.of("the", "black")); + } + } + } + + private void assertCorrectLuceneQuery(String name, Query query, List expectedFeatureFields) { + assertTrue(query instanceof BooleanQuery); + List booleanClauses = ((BooleanQuery) query).clauses(); + assertEquals( + name + " had " + booleanClauses.size() + " clauses, expected " + expectedFeatureFields.size(), + expectedFeatureFields.size(), + booleanClauses.size() + ); + for (int i = 0; i < booleanClauses.size(); i++) { + Query clauseQuery = booleanClauses.get(i).getQuery(); + assertTrue(name + " query " + query + " expected to be a BoostQuery", clauseQuery instanceof BoostQuery); + // FeatureQuery is not visible so we check the String representation + assertTrue(name + " query " + query + " expected to be a FeatureQuery", clauseQuery.toString().contains("FeatureQuery")); + assertTrue( + name + " query " + query + " expected to have field " + expectedFeatureFields.get(i), + clauseQuery.toString().contains("feature=" + expectedFeatureFields.get(i)) + ); + } + } + + private Document createDocument(List tokens, List weights) { + if (tokens.size() != weights.size()) { + throw new IllegalArgumentException( + "tokens and weights must have the same size. Got " + tokens.size() + " and " + weights.size() + "." + ); + } + Document document = new Document(); + for (int i = 0; i < tokens.size(); i++) { + document.add(new FeatureField(RANK_FEATURES_FIELD, tokens.get(i), weights.get(i))); + } + return document; + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testCacheability() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new FloatDocValuesField(RANK_FEATURES_FIELD, 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + WeightedTokensQueryBuilder queryBuilder = createTestQueryBuilder(); + QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new SearchExecutionContext(context)); + + assertNotNull(rewriteQuery.toQuery(context)); + assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable()); + } + } + } + + /** + * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader} + */ + @Override + public void testMustRewrite() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + Document document = new Document(); + document.add(new FloatDocValuesField(RANK_FEATURES_FIELD, 1.0f)); + iw.addDocument(document); + try (IndexReader reader = iw.getReader()) { + SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader)); + context.setAllowUnmappedFields(true); + WeightedTokensQueryBuilder queryBuilder = createTestQueryBuilder(); + queryBuilder.toQuery(context); + } + } + } + + @Override + protected void doAssertLuceneQuery(WeightedTokensQueryBuilder queryBuilder, Query query, SearchExecutionContext context) { + assertThat(query, instanceOf(BooleanQuery.class)); + BooleanQuery booleanQuery = (BooleanQuery) query; + assertEquals(booleanQuery.getMinimumNumberShouldMatch(), 1); + assertThat(booleanQuery.clauses(), hasSize(NUM_TOKENS)); + + Class featureQueryClass = FeatureField.newLinearQuery("", "", 0.5f).getClass(); + // if the weight is 1.0f a BoostQuery is returned + Class boostQueryClass = FeatureField.newLinearQuery("", "", 1.0f).getClass(); + + for (var clause : booleanQuery.clauses()) { + assertEquals(BooleanClause.Occur.SHOULD, clause.getOccur()); + assertThat(clause.getQuery(), either(instanceOf(featureQueryClass)).or(instanceOf(boostQueryClass))); + } + } + + public void testIllegalValues() { + List weightedTokens = List.of(new WeightedToken("foo", 1.0f)); + { + NullPointerException e = expectThrows( + NullPointerException.class, + () -> new WeightedTokensQueryBuilder(null, weightedTokens, null) + ); + assertEquals("[weighted_tokens] requires a fieldName", e.getMessage()); + } + { + NullPointerException e = expectThrows( + NullPointerException.class, + () -> new WeightedTokensQueryBuilder("field name", null, null) + ); + assertEquals("[weighted_tokens] requires tokens", e.getMessage()); + } + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new WeightedTokensQueryBuilder("field name", List.of(), null) + ); + assertEquals("[weighted_tokens] requires at least one token", e.getMessage()); + } + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new WeightedTokensQueryBuilder("field name", weightedTokens, new TokenPruningConfig(-1f, 0.0f, false)) + ); + assertEquals("[tokens_freq_ratio_threshold] must be between [1.0] and [100.0], got -1.0", e.getMessage()); + } + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new WeightedTokensQueryBuilder("field name", weightedTokens, new TokenPruningConfig(101f, 0.0f, false)) + ); + assertEquals("[tokens_freq_ratio_threshold] must be between [1.0] and [100.0], got 101.0", e.getMessage()); + } + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new WeightedTokensQueryBuilder("field name", weightedTokens, new TokenPruningConfig(5f, 5f, false)) + ); + assertEquals("[tokens_weight_threshold] must be between 0 and 1", e.getMessage()); + } + } + + public void testToXContent() throws Exception { + QueryBuilder query = new WeightedTokensQueryBuilder("foo", WEIGHTED_TOKENS, null); + checkGeneratedJson(""" + { + "weighted_tokens": { + "foo": { + "tokens": { + "foo": 0.42 + } + } + } + }""", query); + } + + public void testToXContentWithThresholds() throws Exception { + QueryBuilder query = new WeightedTokensQueryBuilder("foo", WEIGHTED_TOKENS, new TokenPruningConfig(4, 0.4f, false)); + checkGeneratedJson(""" + { + "weighted_tokens": { + "foo": { + "tokens": { + "foo": 0.42 + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 4.0, + "tokens_weight_threshold": 0.4 + } + } + } + }""", query); + } + + public void testToXContentWithThresholdsAndOnlyScorePrunedTokens() throws Exception { + QueryBuilder query = new WeightedTokensQueryBuilder("foo", WEIGHTED_TOKENS, new TokenPruningConfig(4, 0.4f, true)); + checkGeneratedJson(""" + { + "weighted_tokens": { + "foo": { + "tokens": { + "foo": 0.42 + }, + "pruning_config": { + "tokens_freq_ratio_threshold": 4.0, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": true + } + } + } + }""", query); + } + + @Override + protected String[] shuffleProtectedFields() { + return new String[] { TOKENS_FIELD.getPreferredName() }; + } +} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml index a099e327c32f0..fe25d8957216c 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml @@ -107,3 +107,183 @@ setup: model_text: "octopus comforter smells" - match: { hits.total.value: 4 } - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test text expansion search with pruning config": + - skip: + version: " - 8.12.99" + reason: "pruning introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test named, boosted text expansion search with pruning config": + - skip: + version: " - 8.12.99" + reason: "pruning introduced in 8.13.0" + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + - match: { hits.hits.0._score: 3.0 } + + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + _name: i-like-naming-my-queries + boost: 100.0 + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + - match: { hits.hits.0.matched_queries: ["i-like-naming-my-queries"] } + - match: { hits.hits.0._score: 300.0 } + +--- +"Test text expansion search with default pruning config": + - skip: + version: " - 8.12.99" + reason: "pruning introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: {} + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test text expansion search with weighted tokens rescoring only pruned tokens": + - skip: + version: " - 8.12.99" + reason: "pruning introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + text_expansion: + ml.tokens: + model_id: text_expansion_model + model_text: "octopus comforter smells" + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + only_score_pruned_tokens: true + - match: { hits.total.value: 0 } + +--- +"Test weighted tokens search": + - skip: + version: " - 8.12.99" + reason: "weighted token search introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + weighted_tokens: + ml.tokens: + tokens: [{"the": 1.0}, {"octopus":1.0}, {"comforter":1.0}] + pruning_config: + tokens_freq_ratio_threshold: 1 + tokens_weight_threshold: 0.4 + only_score_pruned_tokens: false + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test weighted tokens search with default pruning config": + - skip: + version: " - 8.12.99" + reason: "weighted token search introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + weighted_tokens: + ml.tokens: + tokens: [{"the": 1.0}, {"octopus":1.0}, {"comforter":1.0}] + pruning_config: {} + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } + +--- +"Test weighted tokens search only scoring pruned tokens": + - skip: + version: " - 8.12.99" + reason: "weighted token search introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + weighted_tokens: + ml.tokens: + tokens: [{"the": 1.0}, {"octopus":1.0}, {"comforter":1.0}] + pruning_config: + tokens_freq_ratio_threshold: 4 + tokens_weight_threshold: 0.4 + only_score_pruned_tokens: true + - match: { hits.total.value: 0 } + +--- +"Test weighted tokens search that prunes tokens based on frequency": + - skip: + version: " - 8.12.99" + reason: "weighted token search introduced in 8.13.0" + + - do: + search: + index: index-with-rank-features + body: + query: + weighted_tokens: + ml.tokens: + tokens: [{"the": 1.0}, {"octopus":1.0}, {"comforter":1.0}, {"is": 1.0}, {"the": 1.0}, {"best": 1.0}, {"of": 1.0}, {"the": 1.0}, {"bunch": 1.0}] + pruning_config: + tokens_freq_ratio_threshold: 3 + tokens_weight_threshold: 0.4 + only_score_pruned_tokens: true + - match: { hits.total.value: 0 } From a1a6ddb8d9df64f7890389b0fac460013851e356 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Wed, 13 Dec 2023 12:52:42 -0800 Subject: [PATCH 16/54] Propagate 'isCI' system property to BWC build tasks (#103409) --- .../org/elasticsearch/gradle/internal/BwcSetupExtension.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index d71c893cdd20f..3d6d37575eca9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -109,6 +109,10 @@ private TaskProvider createRunBwcGradleTask( loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl); } + if (System.getProperty("isCI") != null) { + loggedExec.args("-DisCI"); + } + loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED"); final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel(); List nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG); From b7ce82df399bc9dbc738264b19f0ef69775538ff Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 13 Dec 2023 17:40:18 -0500 Subject: [PATCH 17/54] [ci] Add Buildkite packer cache script (#103410) --- .buildkite/packer_cache.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 .buildkite/packer_cache.sh diff --git a/.buildkite/packer_cache.sh b/.buildkite/packer_cache.sh new file mode 100755 index 0000000000000..752914ba55c23 --- /dev/null +++ b/.buildkite/packer_cache.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd) + +branches=($(cat "$ROOT_DIR/branches.json" | jq -r '.branches[].branch')) +for branch in "${branches[@]}"; do + echo "Resolving dependencies for ${branch} branch" + rm -rf "checkout/$branch" + git clone /opt/git-mirrors/elastic-elasticsearch --branch "$branch" --single-branch "checkout/$branch" + + CHECKOUT_DIR=$(cd "./checkout/${branch}" && pwd) + CI_DIR="$CHECKOUT_DIR/.ci" + + if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then + ## On ARM we use a different properties file for setting java home + ## Also, we don't bother attempting to resolve dependencies for the 6.8 branch + source "$CI_DIR/java-versions-aarch64.properties" + export JAVA16_HOME="$HOME/.java/jdk16" + else + source "$CI_DIR/java-versions.properties" + ## We are caching BWC versions too, need these so we can build those + export JAVA8_HOME="$HOME/.java/java8" + export JAVA11_HOME="$HOME/.java/java11" + export JAVA12_HOME="$HOME/.java/openjdk12" + export JAVA13_HOME="$HOME/.java/openjdk13" + export JAVA14_HOME="$HOME/.java/openjdk14" + export JAVA15_HOME="$HOME/.java/openjdk15" + export JAVA16_HOME="$HOME/.java/openjdk16" + fi + + export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA" + "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI + rm -rf "checkout/${branch}" +done From 3c28dc0d69bf73ac0724c7d5924b2a6ae89e286f Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Thu, 14 Dec 2023 07:28:47 +0800 Subject: [PATCH 18/54] CODEOWNERS: reduce scope of elastic/apm-server (#103368) Cross-cutting refactoring often pings the apm-server team and we're not really the best placed for reviewing these changes. The apm-data plugin is structured so the apm-server team can work on the templates and ingest pipelines without touching any Java code, so reduce the scope to just the assets and YAML REST tests. --- .github/CODEOWNERS | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 10ca81c9ada3c..a5e959e795c07 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -23,5 +23,6 @@ x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet # Kibana Security x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security -# APM -x-pack/plugin/apm-data @elastic/apm-server +# APM Data index templates, etc. +x-pack/plugin/apm-data/src/main/resources @elastic/apm-server +x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/apm-server From 0e5f485058e727007451ad3f9e96893d82229600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 14 Dec 2023 07:37:50 +0100 Subject: [PATCH 19/54] ServerProcess refactoring (separate options construction from ServerProcess start) (#102973) * Refactoring: move command line and environment build to a separate class * Refactoring: adding a builder * Moving tmp dir setup and JVM option parsing outside of builder --- .../server/cli/JvmOptionsParser.java | 23 +- .../elasticsearch/server/cli/ServerCli.java | 11 +- .../server/cli/ServerProcess.java | 170 +------------- .../server/cli/ServerProcessBuilder.java | 208 ++++++++++++++++++ .../server/cli/ServerProcessUtils.java | 66 ++++++ .../server/cli/SystemJvmOptions.java | 7 +- .../server/cli/JvmOptionsParserTests.java | 14 +- .../server/cli/ServerCliTests.java | 109 +++++---- .../server/cli/ServerProcessTests.java | 137 +++++------- .../server/cli/ServerProcessUtilsTests.java | 82 +++++++ .../windows/service/WindowsServiceDaemon.java | 12 +- 11 files changed, 518 insertions(+), 321 deletions(-) create mode 100644 distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java create mode 100644 distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java create mode 100644 distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java index 29650e4b74114..d312fae4456f1 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java @@ -10,6 +10,7 @@ import org.elasticsearch.bootstrap.ServerArgs; import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.ProcessInfo; import org.elasticsearch.cli.UserException; import java.io.BufferedReader; @@ -39,7 +40,7 @@ /** * Parses JVM options from a file and prints a single line with all JVM options to standard output. */ -final class JvmOptionsParser { +public final class JvmOptionsParser { static class JvmOptionsFileParserException extends Exception { @@ -59,7 +60,6 @@ SortedMap invalidLines() { this.jvmOptionsFile = jvmOptionsFile; this.invalidLines = invalidLines; } - } /** @@ -70,25 +70,27 @@ SortedMap invalidLines() { * variable. * * @param args the start-up arguments - * @param configDir the ES config dir + * @param processInfo information about the CLI process. * @param tmpDir the directory that should be passed to {@code -Djava.io.tmpdir} - * @param envOptions the options passed through the ES_JAVA_OPTS env var * @return the list of options to put on the Java command line * @throws InterruptedException if the java subprocess is interrupted * @throws IOException if there is a problem reading any of the files * @throws UserException if there is a problem parsing the `jvm.options` file or `jvm.options.d` files */ - static List determineJvmOptions(ServerArgs args, Path configDir, Path tmpDir, String envOptions) throws InterruptedException, + public static List determineJvmOptions(ServerArgs args, ProcessInfo processInfo, Path tmpDir) throws InterruptedException, IOException, UserException { - final JvmOptionsParser parser = new JvmOptionsParser(); final Map substitutions = new HashMap<>(); substitutions.put("ES_TMPDIR", tmpDir.toString()); - substitutions.put("ES_PATH_CONF", configDir.toString()); + substitutions.put("ES_PATH_CONF", args.configDir().toString()); + + final String envOptions = processInfo.envVars().get("ES_JAVA_OPTS"); try { - return parser.jvmOptions(args, configDir, tmpDir, envOptions, substitutions); + return Collections.unmodifiableList( + parser.jvmOptions(args, args.configDir(), tmpDir, envOptions, substitutions, processInfo.sysprops()) + ); } catch (final JvmOptionsFileParserException e) { final String errorMessage = String.format( Locale.ROOT, @@ -122,7 +124,8 @@ private List jvmOptions( final Path config, Path tmpDir, final String esJavaOpts, - final Map substitutions + final Map substitutions, + final Map cliSysprops ) throws InterruptedException, IOException, JvmOptionsFileParserException, UserException { final List jvmOptions = readJvmOptionsFiles(config); @@ -137,7 +140,7 @@ private List jvmOptions( ); substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions)); final List ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions); - final List systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings()); + final List systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings(), cliSysprops); final List apmOptions = APMJvmOptions.apmJvmOptions(args.nodeSettings(), args.secrets(), args.logsDir(), tmpDir); diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java index ea2df72fb2c0b..aac5f718081b4 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java @@ -243,8 +243,15 @@ protected Command loadTool(String toolname, String libs) { } // protected to allow tests to override - protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws UserException { - return ServerProcess.start(terminal, processInfo, args); + protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception { + var tempDir = ServerProcessUtils.setupTempDir(processInfo); + var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir); + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(processInfo) + .withServerArgs(args) + .withTempDir(tempDir) + .withJvmOptions(jvmOptions); + return serverProcessBuilder.start(); } // protected to allow tests to override diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java index d4b4d57977f5d..3972095a3a5c0 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java @@ -9,34 +9,17 @@ package org.elasticsearch.server.cli; import org.elasticsearch.bootstrap.BootstrapInfo; -import org.elasticsearch.bootstrap.ServerArgs; -import org.elasticsearch.cli.ExitCodes; -import org.elasticsearch.cli.ProcessInfo; -import org.elasticsearch.cli.Terminal; -import org.elasticsearch.cli.UserException; -import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.core.PathUtils; -import org.elasticsearch.core.SuppressForbidden; import java.io.IOException; import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import static org.elasticsearch.server.cli.ProcessUtil.nonInterruptible; /** * A helper to control a {@link Process} running the main Elasticsearch server. * - *

The process can be started by calling {@link #start(Terminal, ProcessInfo, ServerArgs)}. + *

The process can be started by calling {@link ServerProcessBuilder#start()}. * The process is controlled by internally sending arguments and control signals on stdin, * and receiving control signals on stderr. The start method does not return until the * server is ready to process requests and has exited the bootstrap thread. @@ -64,68 +47,6 @@ public class ServerProcess { this.errorPump = errorPump; } - // this allows mocking the process building by tests - interface OptionsBuilder { - List getJvmOptions(ServerArgs args, Path configDir, Path tmpDir, String envOptions) throws InterruptedException, - IOException, UserException; - } - - // this allows mocking the process building by tests - interface ProcessStarter { - Process start(ProcessBuilder pb) throws IOException; - } - - /** - * Start a server in a new process. - * - * @param terminal A terminal to connect the standard inputs and outputs to for the new process. - * @param processInfo Info about the current process, for passing through to the subprocess. - * @param args Arguments to the server process. - * @return A running server process that is ready for requests - * @throws UserException If the process failed during bootstrap - */ - public static ServerProcess start(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws UserException { - return start(terminal, processInfo, args, JvmOptionsParser::determineJvmOptions, ProcessBuilder::start); - } - - // package private so tests can mock options building and process starting - static ServerProcess start( - Terminal terminal, - ProcessInfo processInfo, - ServerArgs args, - OptionsBuilder optionsBuilder, - ProcessStarter processStarter - ) throws UserException { - Process jvmProcess = null; - ErrorPumpThread errorPump; - - boolean success = false; - try { - jvmProcess = createProcess(args, processInfo, args.configDir(), optionsBuilder, processStarter); - errorPump = new ErrorPumpThread(terminal.getErrorWriter(), jvmProcess.getErrorStream()); - errorPump.start(); - sendArgs(args, jvmProcess.getOutputStream()); - - String errorMsg = errorPump.waitUntilReady(); - if (errorMsg != null) { - // something bad happened, wait for the process to exit then rethrow - int exitCode = jvmProcess.waitFor(); - throw new UserException(exitCode, errorMsg); - } - success = true; - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } finally { - if (success == false && jvmProcess != null && jvmProcess.isAlive()) { - jvmProcess.destroyForcibly(); - } - } - - return new ServerProcess(jvmProcess, errorPump); - } - /** * Return the process id of the server. */ @@ -169,19 +90,6 @@ public synchronized void stop() { waitFor(); // ignore exit code, we are already shutting down } - private static void sendArgs(ServerArgs args, OutputStream processStdin) { - // DO NOT close the underlying process stdin, since we need to be able to write to it to signal exit - var out = new OutputStreamStreamOutput(processStdin); - try { - args.writeTo(out); - out.flush(); - } catch (IOException ignore) { - // A failure to write here means the process has problems, and it will die anyway. We let this fall through - // so the pump thread can complete, writing out the actual error. All we get here is the failure to write to - // the process pipe, which isn't helpful to print. - } - } - private void sendShutdownMarker() { try { OutputStream os = jvmProcess.getOutputStream(); @@ -191,80 +99,4 @@ private void sendShutdownMarker() { // process is already effectively dead, fall through to wait for it, or should we SIGKILL? } } - - private static Process createProcess( - ServerArgs args, - ProcessInfo processInfo, - Path configDir, - OptionsBuilder optionsBuilder, - ProcessStarter processStarter - ) throws InterruptedException, IOException, UserException { - Map envVars = new HashMap<>(processInfo.envVars()); - Path tempDir = setupTempDir(processInfo, envVars.remove("ES_TMPDIR")); - if (envVars.containsKey("LIBFFI_TMPDIR") == false) { - envVars.put("LIBFFI_TMPDIR", tempDir.toString()); - } - - List jvmOptions = optionsBuilder.getJvmOptions(args, configDir, tempDir, envVars.remove("ES_JAVA_OPTS")); - // also pass through distribution type - jvmOptions.add("-Des.distribution.type=" + processInfo.sysprops().get("es.distribution.type")); - - Path esHome = processInfo.workingDir(); - Path javaHome = PathUtils.get(processInfo.sysprops().get("java.home")); - List command = new ArrayList<>(); - boolean isWindows = processInfo.sysprops().get("os.name").startsWith("Windows"); - command.add(javaHome.resolve("bin").resolve("java" + (isWindows ? ".exe" : "")).toString()); - command.addAll(jvmOptions); - command.add("--module-path"); - command.add(esHome.resolve("lib").toString()); - // Special circumstances require some modules (not depended on by the main server module) to be explicitly added: - command.add("--add-modules=jdk.net"); // needed to reflectively set extended socket options - // we control the module path, which may have additional modules not required by server - command.add("--add-modules=ALL-MODULE-PATH"); - command.add("-m"); - command.add("org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch"); - - var builder = new ProcessBuilder(command); - builder.environment().putAll(envVars); - builder.redirectOutput(ProcessBuilder.Redirect.INHERIT); - - return processStarter.start(builder); - } - - /** - * Returns the java.io.tmpdir Elasticsearch should use, creating it if necessary. - * - *

On non-Windows OS, this will be created as a subdirectory of the default temporary directory. - * Note that this causes the created temporary directory to be a private temporary directory. - */ - private static Path setupTempDir(ProcessInfo processInfo, String tmpDirOverride) throws UserException, IOException { - final Path path; - if (tmpDirOverride != null) { - path = Paths.get(tmpDirOverride); - if (Files.exists(path) == false) { - throw new UserException(ExitCodes.CONFIG, "Temporary directory [" + path + "] does not exist or is not accessible"); - } - if (Files.isDirectory(path) == false) { - throw new UserException(ExitCodes.CONFIG, "Temporary directory [" + path + "] is not a directory"); - } - } else { - if (processInfo.sysprops().get("os.name").startsWith("Windows")) { - /* - * On Windows, we avoid creating a unique temporary directory per invocation lest - * we pollute the temporary directory. On other operating systems, temporary directories - * will be cleaned automatically via various mechanisms (e.g., systemd, or restarts). - */ - path = Paths.get(processInfo.sysprops().get("java.io.tmpdir"), "elasticsearch"); - Files.createDirectories(path); - } else { - path = createTempDirectory("elasticsearch-"); - } - } - return path; - } - - @SuppressForbidden(reason = "Files#createTempDirectory(String, FileAttribute...)") - private static Path createTempDirectory(final String prefix, final FileAttribute... attrs) throws IOException { - return Files.createTempDirectory(prefix, attrs); - } } diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java new file mode 100644 index 0000000000000..4ef1e2bfd4737 --- /dev/null +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessBuilder.java @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.server.cli; + +import org.elasticsearch.bootstrap.ServerArgs; +import org.elasticsearch.cli.ProcessInfo; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.core.PathUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +/** + * This class is used to create a {@link ServerProcess}. + * Each ServerProcessBuilder instance manages a collection of process attributes. The {@link ServerProcessBuilder#start()} method creates + * a new {@link ServerProcess} instance with those attributes. + * + * Each process builder manages these process attributes: + * - a temporary directory + * - process info to pass through to the new Java subprocess + * - the command line arguments to run Elasticsearch + * - a list of JVM options to be passed to the Elasticsearch Java process + * - a {@link Terminal} to read input and write output from/to the cli console + */ +public class ServerProcessBuilder { + private Path tempDir; + private ServerArgs serverArgs; + private ProcessInfo processInfo; + private List jvmOptions; + private Terminal terminal; + + // this allows mocking the process building by tests + interface ProcessStarter { + Process start(ProcessBuilder pb) throws IOException; + } + + /** + * Specifies the temporary directory to be used by the server process + */ + public ServerProcessBuilder withTempDir(Path tempDir) { + this.tempDir = tempDir; + return this; + } + + /** + * Specifies the process info to pass through to the new Java subprocess + */ + public ServerProcessBuilder withProcessInfo(ProcessInfo processInfo) { + this.processInfo = processInfo; + return this; + } + + /** + * Specifies the command line arguments to run Elasticsearch + */ + public ServerProcessBuilder withServerArgs(ServerArgs serverArgs) { + this.serverArgs = serverArgs; + return this; + } + + /** + * Specifies the JVM options to be passed to the Elasticsearch Java process + */ + public ServerProcessBuilder withJvmOptions(List jvmOptions) { + this.jvmOptions = jvmOptions; + return this; + } + + /** + * Specifies the {@link Terminal} to use for reading input and writing output from/to the cli console + */ + public ServerProcessBuilder withTerminal(Terminal terminal) { + this.terminal = terminal; + return this; + } + + private Map getEnvironment() { + Map envVars = new HashMap<>(processInfo.envVars()); + + envVars.remove("ES_TMPDIR"); + if (envVars.containsKey("LIBFFI_TMPDIR") == false) { + envVars.put("LIBFFI_TMPDIR", tempDir.toString()); + } + envVars.remove("ES_JAVA_OPTS"); + + return envVars; + } + + private List getJvmArgs() { + Path esHome = processInfo.workingDir(); + return List.of( + "--module-path", + esHome.resolve("lib").toString(), + // Special circumstances require some modules (not depended on by the main server module) to be explicitly added: + "--add-modules=jdk.net", // needed to reflectively set extended socket options + // we control the module path, which may have additional modules not required by server + "--add-modules=ALL-MODULE-PATH", + "-m", + "org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch" + ); + } + + private String getCommand() { + Path javaHome = PathUtils.get(processInfo.sysprops().get("java.home")); + + boolean isWindows = processInfo.sysprops().get("os.name").startsWith("Windows"); + return javaHome.resolve("bin").resolve("java" + (isWindows ? ".exe" : "")).toString(); + } + + /** + * Start a server in a new process. + * + * @return A running server process that is ready for requests + * @throws UserException If the process failed during bootstrap + */ + public ServerProcess start() throws UserException { + return start(ProcessBuilder::start); + } + + private static void checkRequiredArgument(Object argument, String argumentName) { + if (argument == null) { + throw new IllegalStateException( + Strings.format("'%s' is a required argument and needs to be specified before calling start()", argumentName) + ); + } + } + + // package private for testing + ServerProcess start(ProcessStarter processStarter) throws UserException { + checkRequiredArgument(tempDir, "tempDir"); + checkRequiredArgument(serverArgs, "serverArgs"); + checkRequiredArgument(processInfo, "processInfo"); + checkRequiredArgument(jvmOptions, "jvmOptions"); + checkRequiredArgument(terminal, "terminal"); + + Process jvmProcess = null; + ErrorPumpThread errorPump; + + boolean success = false; + try { + jvmProcess = createProcess(getCommand(), getJvmArgs(), jvmOptions, getEnvironment(), processStarter); + errorPump = new ErrorPumpThread(terminal.getErrorWriter(), jvmProcess.getErrorStream()); + errorPump.start(); + sendArgs(serverArgs, jvmProcess.getOutputStream()); + + String errorMsg = errorPump.waitUntilReady(); + if (errorMsg != null) { + // something bad happened, wait for the process to exit then rethrow + int exitCode = jvmProcess.waitFor(); + throw new UserException(exitCode, errorMsg); + } + success = true; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + if (success == false && jvmProcess != null && jvmProcess.isAlive()) { + jvmProcess.destroyForcibly(); + } + } + + return new ServerProcess(jvmProcess, errorPump); + } + + private static Process createProcess( + String command, + List jvmArgs, + List jvmOptions, + Map environment, + ProcessStarter processStarter + ) throws InterruptedException, IOException { + + var builder = new ProcessBuilder(Stream.concat(Stream.of(command), Stream.concat(jvmOptions.stream(), jvmArgs.stream())).toList()); + builder.environment().putAll(environment); + builder.redirectOutput(ProcessBuilder.Redirect.INHERIT); + + return processStarter.start(builder); + } + + private static void sendArgs(ServerArgs args, OutputStream processStdin) { + // DO NOT close the underlying process stdin, since we need to be able to write to it to signal exit + var out = new OutputStreamStreamOutput(processStdin); + try { + args.writeTo(out); + out.flush(); + } catch (IOException ignore) { + // A failure to write here means the process has problems, and it will die anyway. We let this fall through + // so the pump thread can complete, writing out the actual error. All we get here is the failure to write to + // the process pipe, which isn't helpful to print. + } + } +} diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java new file mode 100644 index 0000000000000..ebbc68b1be90b --- /dev/null +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcessUtils.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.server.cli; + +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.ProcessInfo; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.core.SuppressForbidden; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; + +public class ServerProcessUtils { + + /** + * Returns the java.io.tmpdir Elasticsearch should use, creating it if necessary. + * + *

On non-Windows OS, this will be created as a subdirectory of the default temporary directory. + * Note that this causes the created temporary directory to be a private temporary directory. + */ + public static Path setupTempDir(ProcessInfo processInfo) throws UserException { + final Path path; + String tmpDirOverride = processInfo.envVars().get("ES_TMPDIR"); + if (tmpDirOverride != null) { + path = Paths.get(tmpDirOverride); + if (Files.exists(path) == false) { + throw new UserException(ExitCodes.CONFIG, "Temporary directory [" + path + "] does not exist or is not accessible"); + } + if (Files.isDirectory(path) == false) { + throw new UserException(ExitCodes.CONFIG, "Temporary directory [" + path + "] is not a directory"); + } + } else { + try { + if (processInfo.sysprops().get("os.name").startsWith("Windows")) { + /* + * On Windows, we avoid creating a unique temporary directory per invocation lest + * we pollute the temporary directory. On other operating systems, temporary directories + * will be cleaned automatically via various mechanisms (e.g., systemd, or restarts). + */ + path = Paths.get(processInfo.sysprops().get("java.io.tmpdir"), "elasticsearch"); + Files.createDirectories(path); + } else { + path = createTempDirectory("elasticsearch-"); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return path; + } + + @SuppressForbidden(reason = "Files#createTempDirectory(String, FileAttribute...)") + private static Path createTempDirectory(final String prefix, final FileAttribute... attrs) throws IOException { + return Files.createTempDirectory(prefix, attrs); + } +} diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 6e250075f7747..4a8b3da4777a0 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -12,12 +12,13 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; final class SystemJvmOptions { - static List systemJvmOptions(Settings nodeSettings) { + static List systemJvmOptions(Settings nodeSettings, final Map sysprops) { return Stream.of( /* * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property networkaddress.cache.ttl; @@ -65,7 +66,9 @@ static List systemJvmOptions(Settings nodeSettings) { */ "--add-opens=java.base/java.io=org.elasticsearch.preallocate", maybeOverrideDockerCgroup(), - maybeSetActiveProcessorCount(nodeSettings) + maybeSetActiveProcessorCount(nodeSettings), + // Pass through distribution type + "-Des.distribution.type=" + sysprops.get("es.distribution.type") ).filter(e -> e.isEmpty() == false).collect(Collectors.toList()); } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java index 03856b1024992..101be4301b522 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java @@ -53,7 +53,6 @@ public void testUnversionedOptions() throws IOException { try (StringReader sr = new StringReader("-Xms1g\n-Xmx1g"); BufferedReader br = new BufferedReader(sr)) { assertExpectedJvmOptions(randomIntBetween(8, Integer.MAX_VALUE), br, Arrays.asList("-Xms1g", "-Xmx1g")); } - } public void testSingleVersionOption() throws IOException { @@ -351,25 +350,30 @@ public void accept(final int lineNumber, final String line) { public void testNodeProcessorsActiveCount() { { - final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, Map.of()); assertThat(jvmOptions, not(hasItem(containsString("-XX:ActiveProcessorCount=")))); } { Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1).build(); - final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of()); assertThat(jvmOptions, hasItem("-XX:ActiveProcessorCount=1")); } { // check rounding Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 0.2).build(); - final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings); + final List jvmOptions = SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of()); assertThat(jvmOptions, hasItem("-XX:ActiveProcessorCount=1")); } { // check validation Settings nodeSettings = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 10000).build(); - var e = expectThrows(IllegalArgumentException.class, () -> SystemJvmOptions.systemJvmOptions(nodeSettings)); + var e = expectThrows(IllegalArgumentException.class, () -> SystemJvmOptions.systemJvmOptions(nodeSettings, Map.of())); assertThat(e.getMessage(), containsString("setting [node.processors] must be <=")); } } + + public void testCommandLineDistributionType() { + final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, Map.of("es.distribution.type", "testdistro")); + assertThat(jvmOptions, hasItem("-Des.distribution.type=testdistro")); + } } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java index da2c0104dd08e..e469764590bd6 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java @@ -314,6 +314,21 @@ public void testIgnoreNullExceptionOutput() throws Exception { assertThat(terminal.getErrorOutput(), not(containsString("null"))); } + public void testOptionsBuildingInterrupted() throws IOException { + Command command = new TestServerCli() { + @Override + protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception { + throw new InterruptedException("interrupted while get jvm options"); + } + }; + var e = expectThrows( + InterruptedException.class, + () -> command.main(new String[0], terminal, new ProcessInfo(sysprops, envVars, esHomeDir)) + ); + assertThat(e.getMessage(), equalTo("interrupted while get jvm options")); + command.close(); + } + public void testServerExitsNonZero() throws Exception { mockServerExitCode = 140; int exitCode = executeMain(); @@ -480,63 +495,65 @@ void reset() { } } - @Override - protected Command newCommand() { - return new ServerCli() { - @Override - protected Command loadTool(String toolname, String libs) { - if (toolname.equals("auto-configure-node")) { - assertThat(libs, equalTo("modules/x-pack-core,modules/x-pack-security,lib/tools/security-cli")); - return AUTO_CONFIG_CLI; - } else if (toolname.equals("sync-plugins")) { - assertThat(libs, equalTo("lib/tools/plugin-cli")); - return SYNC_PLUGINS_CLI; - } - throw new AssertionError("Unknown tool: " + toolname); + private class TestServerCli extends ServerCli { + @Override + protected Command loadTool(String toolname, String libs) { + if (toolname.equals("auto-configure-node")) { + assertThat(libs, equalTo("modules/x-pack-core,modules/x-pack-security,lib/tools/security-cli")); + return AUTO_CONFIG_CLI; + } else if (toolname.equals("sync-plugins")) { + assertThat(libs, equalTo("lib/tools/plugin-cli")); + return SYNC_PLUGINS_CLI; } + throw new AssertionError("Unknown tool: " + toolname); + } - @Override - Environment autoConfigureSecurity( - Terminal terminal, - OptionSet options, - ProcessInfo processInfo, - Environment env, - SecureString keystorePassword - ) throws Exception { - if (mockSecureSettingsLoader != null && mockSecureSettingsLoader.supportsSecurityAutoConfiguration() == false) { - fail("We shouldn't be calling auto configure on loaders that don't support it"); - } - return super.autoConfigureSecurity(terminal, options, processInfo, env, keystorePassword); + @Override + Environment autoConfigureSecurity( + Terminal terminal, + OptionSet options, + ProcessInfo processInfo, + Environment env, + SecureString keystorePassword + ) throws Exception { + if (mockSecureSettingsLoader != null && mockSecureSettingsLoader.supportsSecurityAutoConfiguration() == false) { + fail("We shouldn't be calling auto configure on loaders that don't support it"); } + return super.autoConfigureSecurity(terminal, options, processInfo, env, keystorePassword); + } - @Override - protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) { - if (argsValidator != null) { - argsValidator.accept(args); - } - mockServer.reset(); - return mockServer; + @Override + void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception { + if (mockSecureSettingsLoader != null && mockSecureSettingsLoader instanceof MockSecureSettingsLoader mock) { + mock.verifiedEnv = true; + // equals as a pointer, environment shouldn't be changed if autoconfigure is not supported + assertFalse(mockSecureSettingsLoader.supportsSecurityAutoConfiguration()); + assertTrue(mock.environment == env); } - @Override - void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception { - if (mockSecureSettingsLoader != null && mockSecureSettingsLoader instanceof MockSecureSettingsLoader mock) { - mock.verifiedEnv = true; - // equals as a pointer, environment shouldn't be changed if autoconfigure is not supported - assertFalse(mockSecureSettingsLoader.supportsSecurityAutoConfiguration()); - assertTrue(mock.environment == env); - } + super.syncPlugins(terminal, env, processInfo); + } - super.syncPlugins(terminal, env, processInfo); + @Override + protected SecureSettingsLoader secureSettingsLoader(Environment env) { + if (mockSecureSettingsLoader != null) { + return mockSecureSettingsLoader; } + return new KeystoreSecureSettingsLoader(); + } + } + + @Override + protected Command newCommand() { + return new TestServerCli() { @Override - protected SecureSettingsLoader secureSettingsLoader(Environment env) { - if (mockSecureSettingsLoader != null) { - return mockSecureSettingsLoader; + protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) { + if (argsValidator != null) { + argsValidator.accept(args); } - - return new KeystoreSecureSettingsLoader(); + mockServer.reset(); + return mockServer; } }; } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java index 57993d40391ac..fa36007b40af7 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.ProcessInfo; -import org.elasticsearch.cli.UserException; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.SecureSettings; @@ -34,7 +33,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,7 +45,6 @@ import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.server.cli.ProcessUtil.nonInterruptibleVoid; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; @@ -56,7 +53,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.startsWith; public class ServerProcessTests extends ESTestCase { @@ -66,7 +62,6 @@ public class ServerProcessTests extends ESTestCase { protected final Map envVars = new HashMap<>(); Path esHomeDir; Settings.Builder nodeSettings; - ServerProcess.OptionsBuilder optionsBuilder; ProcessValidator processValidator; MainMethod mainCallback; MockElasticsearchProcess process; @@ -81,7 +76,7 @@ interface ProcessValidator { } int runForeground() throws Exception { - var server = startProcess(false, false, ""); + var server = startProcess(false, false); return server.waitFor(); } @@ -94,7 +89,6 @@ public void resetEnv() { envVars.clear(); esHomeDir = createTempDir(); nodeSettings = Settings.builder(); - optionsBuilder = (args, configDir, tmpDir, envOptions) -> new ArrayList<>(); processValidator = null; mainCallback = null; secrets = KeyStoreWrapper.create(); @@ -193,9 +187,12 @@ public Process destroyForcibly() { } } - ServerProcess startProcess(boolean daemonize, boolean quiet, String keystorePassword) throws Exception { - var pinfo = new ProcessInfo(Map.copyOf(sysprops), Map.copyOf(envVars), esHomeDir); - var args = new ServerArgs( + ProcessInfo createProcessInfo() { + return new ProcessInfo(Map.copyOf(sysprops), Map.copyOf(envVars), esHomeDir); + } + + ServerArgs createServerArgs(boolean daemonize, boolean quiet) { + return new ServerArgs( daemonize, quiet, null, @@ -204,14 +201,23 @@ ServerProcess startProcess(boolean daemonize, boolean quiet, String keystorePass esHomeDir.resolve("config"), esHomeDir.resolve("logs") ); - ServerProcess.ProcessStarter starter = pb -> { + } + + ServerProcess startProcess(boolean daemonize, boolean quiet) throws Exception { + var pinfo = createProcessInfo(); + ServerProcessBuilder.ProcessStarter starter = pb -> { if (processValidator != null) { processValidator.validate(pb); } process = new MockElasticsearchProcess(); return process; }; - return ServerProcess.start(terminal, pinfo, args, optionsBuilder, starter); + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(pinfo) + .withServerArgs(createServerArgs(daemonize, quiet)) + .withJvmOptions(List.of()) + .withTempDir(ServerProcessUtils.setupTempDir(pinfo)); + return serverProcessBuilder.start(starter); } public void testProcessBuilder() throws Exception { @@ -231,7 +237,7 @@ public void testProcessBuilder() throws Exception { } public void testPid() throws Exception { - var server = startProcess(true, false, ""); + var server = startProcess(true, false); assertThat(server.pid(), equalTo(12345L)); server.stop(); } @@ -246,18 +252,12 @@ public void testBootstrapError() throws Exception { assertThat(terminal.getErrorOutput(), containsString("a bootstrap exception")); } - public void testStartError() throws Exception { + public void testStartError() { processValidator = pb -> { throw new IOException("something went wrong"); }; - var e = expectThrows(UncheckedIOException.class, () -> runForeground()); + var e = expectThrows(UncheckedIOException.class, this::runForeground); assertThat(e.getCause().getMessage(), equalTo("something went wrong")); } - public void testOptionsBuildingInterrupted() throws Exception { - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { throw new InterruptedException("interrupted while get jvm options"); }; - var e = expectThrows(RuntimeException.class, () -> runForeground()); - assertThat(e.getCause().getMessage(), equalTo("interrupted while get jvm options")); - } - public void testEnvPassthrough() throws Exception { envVars.put("MY_ENV", "foo"); processValidator = pb -> { assertThat(pb.environment(), hasEntry(equalTo("MY_ENV"), equalTo("foo"))); }; @@ -276,83 +276,48 @@ public void testLibffiEnv() throws Exception { runForeground(); } - public void testTempDir() throws Exception { - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { - assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true)); - assertThat(tmpDir.getFileName().toString(), startsWith("elasticsearch-")); - return new ArrayList<>(); - }; - runForeground(); - } - - public void testTempDirWindows() throws Exception { - Path baseTmpDir = createTempDir(); - sysprops.put("os.name", "Windows 10"); - sysprops.put("java.io.tmpdir", baseTmpDir.toString()); - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { - assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true)); - assertThat(tmpDir.getFileName().toString(), equalTo("elasticsearch")); - assertThat(tmpDir.getParent().toString(), equalTo(baseTmpDir.toString())); - return new ArrayList<>(); - }; - runForeground(); - } - - public void testTempDirOverride() throws Exception { + public void testEnvCleared() throws Exception { Path customTmpDir = createTempDir(); envVars.put("ES_TMPDIR", customTmpDir.toString()); - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { - assertThat(tmpDir.toString(), equalTo(customTmpDir.toString())); - return new ArrayList<>(); - }; - processValidator = pb -> assertThat(pb.environment(), not(hasKey("ES_TMPDIR"))); - runForeground(); - } - - public void testTempDirOverrideMissing() throws Exception { - Path baseDir = createTempDir(); - envVars.put("ES_TMPDIR", baseDir.resolve("dne").toString()); - var e = expectThrows(UserException.class, () -> runForeground()); - assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); - assertThat(e.getMessage(), containsString("dne] does not exist")); - } - - public void testTempDirOverrideNotADirectory() throws Exception { - Path tmpFile = createTempFile(); - envVars.put("ES_TMPDIR", tmpFile.toString()); - var e = expectThrows(UserException.class, () -> runForeground()); - assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); - assertThat(e.getMessage(), containsString("is not a directory")); - } - - public void testCustomJvmOptions() throws Exception { envVars.put("ES_JAVA_OPTS", "-Dmyoption=foo"); - optionsBuilder = (args, configDir, tmpDir, envOptions) -> { - assertThat(envOptions, equalTo("-Dmyoption=foo")); - return new ArrayList<>(); + + processValidator = pb -> { + assertThat(pb.environment(), not(hasKey("ES_TMPDIR"))); + assertThat(pb.environment(), not(hasKey("ES_JAVA_OPTS"))); }; - processValidator = pb -> assertThat(pb.environment(), not(hasKey("ES_JAVA_OPTS"))); runForeground(); } public void testCommandLineSysprops() throws Exception { - optionsBuilder = (args, configDir, tmpDir, envOptions) -> List.of("-Dfoo1=bar", "-Dfoo2=baz"); - processValidator = pb -> { - assertThat(pb.command(), contains("-Dfoo1=bar")); - assertThat(pb.command(), contains("-Dfoo2=bar")); + ServerProcessBuilder.ProcessStarter starter = pb -> { + assertThat(pb.command(), hasItems("-Dfoo1=bar", "-Dfoo2=baz")); + process = new MockElasticsearchProcess(); + return process; }; + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(createProcessInfo()) + .withServerArgs(createServerArgs(false, false)) + .withJvmOptions(List.of("-Dfoo1=bar", "-Dfoo2=baz")) + .withTempDir(Path.of(".")); + serverProcessBuilder.start(starter).waitFor(); + } + + public void testServerProcessBuilderMissingArgumentError() throws Exception { + ServerProcessBuilder.ProcessStarter starter = pb -> new MockElasticsearchProcess(); + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(createProcessInfo()) + .withServerArgs(createServerArgs(false, false)) + .withTempDir(Path.of(".")); + var ex = expectThrows(IllegalStateException.class, () -> serverProcessBuilder.start(starter).waitFor()); + assertThat(ex.getMessage(), equalTo("'jvmOptions' is a required argument and needs to be specified before calling start()")); } public void testCommandLine() throws Exception { String mainClass = "org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch"; - String distroSysprop = "-Des.distribution.type=testdistro"; String modulePath = esHomeDir.resolve("lib").toString(); Path javaBin = Paths.get("javahome").resolve("bin"); - sysprops.put("es.distribution.type", "testdistro"); AtomicReference expectedJava = new AtomicReference<>(javaBin.resolve("java").toString()); - processValidator = pb -> { - assertThat(pb.command(), hasItems(expectedJava.get(), distroSysprop, "--module-path", modulePath, "-m", mainClass)); - }; + processValidator = pb -> { assertThat(pb.command(), hasItems(expectedJava.get(), "--module-path", modulePath, "-m", mainClass)); }; runForeground(); sysprops.put("os.name", "Windows 10"); @@ -370,7 +335,7 @@ public void testDetach() throws Exception { // will block until stdin closed manually after test assertThat(stdin.read(), equalTo(-1)); }; - var server = startProcess(true, false, ""); + var server = startProcess(true, false); server.detach(); assertThat(terminal.getErrorOutput(), containsString("final message")); server.stop(); // this should be a noop, and will fail the stdin read assert above if shutdown sent @@ -384,7 +349,7 @@ public void testStop() throws Exception { nonInterruptibleVoid(mainReady::await); stderr.println("final message"); }; - var server = startProcess(false, false, ""); + var server = startProcess(false, false); mainReady.countDown(); server.stop(); assertThat(process.main.isDone(), is(true)); // stop should have waited @@ -399,7 +364,7 @@ public void testWaitFor() throws Exception { assertThat(stdin.read(), equalTo((int) BootstrapInfo.SERVER_SHUTDOWN_MARKER)); stderr.println("final message"); }; - var server = startProcess(false, false, ""); + var server = startProcess(false, false); new Thread(() -> { // simulate stop run as shutdown hook in another thread, eg from Ctrl-C nonInterruptibleVoid(mainReady::await); @@ -420,7 +385,7 @@ public void testProcessDies() throws Exception { nonInterruptibleVoid(mainExit::await); exitCode.set(-9); }; - var server = startProcess(false, false, ""); + var server = startProcess(false, false); mainExit.countDown(); int exitCode = server.waitFor(); assertThat(exitCode, equalTo(-9)); diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java new file mode 100644 index 0000000000000..8cd1b63e41b03 --- /dev/null +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessUtilsTests.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.server.cli; + +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.ProcessInfo; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; + +public class ServerProcessUtilsTests extends ESTestCase { + + protected final Map sysprops = new HashMap<>(); + protected final Map envVars = new HashMap<>(); + + @Before + public void resetEnv() { + sysprops.clear(); + sysprops.put("os.name", "Linux"); + sysprops.put("java.home", "javahome"); + envVars.clear(); + } + + private ProcessInfo createProcessInfo() { + return new ProcessInfo(Map.copyOf(sysprops), Map.copyOf(envVars), Path.of(".")); + } + + public void testTempDir() throws Exception { + var tmpDir = ServerProcessUtils.setupTempDir(createProcessInfo()); + assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true)); + assertThat(tmpDir.getFileName().toString(), startsWith("elasticsearch-")); + } + + public void testTempDirWindows() throws Exception { + Path baseTmpDir = createTempDir(); + sysprops.put("os.name", "Windows 10"); + sysprops.put("java.io.tmpdir", baseTmpDir.toString()); + var tmpDir = ServerProcessUtils.setupTempDir(createProcessInfo()); + assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true)); + assertThat(tmpDir.getFileName().toString(), equalTo("elasticsearch")); + assertThat(tmpDir.getParent().toString(), equalTo(baseTmpDir.toString())); + } + + public void testTempDirOverride() throws Exception { + Path customTmpDir = createTempDir(); + envVars.put("ES_TMPDIR", customTmpDir.toString()); + var tmpDir = ServerProcessUtils.setupTempDir(createProcessInfo()); + assertThat(tmpDir.toString(), equalTo(customTmpDir.toString())); + } + + public void testTempDirOverrideMissing() { + Path baseDir = createTempDir(); + envVars.put("ES_TMPDIR", baseDir.resolve("dne").toString()); + var e = expectThrows(UserException.class, () -> ServerProcessUtils.setupTempDir(createProcessInfo())); + assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); + assertThat(e.getMessage(), containsString("dne] does not exist")); + } + + public void testTempDirOverrideNotADirectory() throws Exception { + Path tmpFile = createTempFile(); + envVars.put("ES_TMPDIR", tmpFile.toString()); + var e = expectThrows(UserException.class, () -> ServerProcessUtils.setupTempDir(createProcessInfo())); + assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); + assertThat(e.getMessage(), containsString("is not a directory")); + } +} diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java index 858787b361654..2c42dcf5cb2f5 100644 --- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java +++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java @@ -17,7 +17,10 @@ import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.env.Environment; +import org.elasticsearch.server.cli.JvmOptionsParser; import org.elasticsearch.server.cli.ServerProcess; +import org.elasticsearch.server.cli.ServerProcessBuilder; +import org.elasticsearch.server.cli.ServerProcessUtils; /** * Starts an Elasticsearch process, but does not wait for it to exit. @@ -38,7 +41,14 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce // the Windows service daemon doesn't support secure settings implementations other than the keystore try (var loadedSecrets = KeyStoreWrapper.bootstrap(env.configFile(), () -> new SecureString(new char[0]))) { var args = new ServerArgs(false, true, null, loadedSecrets, env.settings(), env.configFile(), env.logsFile()); - this.server = ServerProcess.start(terminal, processInfo, args); + var tempDir = ServerProcessUtils.setupTempDir(processInfo); + var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir); + var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal) + .withProcessInfo(processInfo) + .withServerArgs(args) + .withTempDir(tempDir) + .withJvmOptions(jvmOptions); + this.server = serverProcessBuilder.start(); // start does not return until the server is ready, and we do not wait for the process } } From c628d1256047d770291609bfbed1fdbfcf87bacb Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Thu, 14 Dec 2023 09:43:22 +0100 Subject: [PATCH 20/54] Fix test unmovable size (#103375) --- .../ReactiveStorageDeciderServiceTests.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java index c5f7ea1622178..12291799b430a 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java @@ -55,12 +55,12 @@ import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -491,7 +491,6 @@ static void addNode(ClusterState.Builder stateBuilder, DiscoveryNodeRole role) { ); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103357") public void testUnmovableSize() { Settings.Builder settingsBuilder = Settings.builder(); if (randomBoolean()) { @@ -521,25 +520,26 @@ public void testUnmovableSize() { stateBuilder.metadata(metaBuilder); ClusterState clusterState = stateBuilder.build(); - Set shards = IntStream.range(0, between(1, 10)) + var shards = IntStream.range(0, between(1, 10)) .mapToObj(i -> Tuple.tuple(new ShardId(indexMetadata.getIndex(), randomInt(10)), randomBoolean())) .distinct() .map(t -> TestShardRouting.newShardRouting(t.v1(), nodeId, t.v2(), ShardRoutingState.STARTED)) - .collect(Collectors.toSet()); + .toList(); long minShardSize = randomLongBetween(1, 10); - Map diskUsages = new HashMap<>(); - diskUsages.put(nodeId, new DiskUsage(nodeId, null, null, ByteSizeUnit.KB.toBytes(100), ByteSizeUnit.KB.toBytes(5))); - Map shardSize = new HashMap<>(); ShardRouting missingShard = randomBoolean() ? randomFrom(shards) : null; - Collection shardsWithSizes = shards.stream().filter(s -> s != missingShard).collect(Collectors.toSet()); - for (ShardRouting shard : shardsWithSizes) { - shardSize.put(shardIdentifier(shard), ByteSizeUnit.KB.toBytes(randomLongBetween(minShardSize, 100))); + Map shardSize = new HashMap<>(); + for (ShardRouting shard : shards) { + if (shard != missingShard) { + shardSize.put(shardIdentifier(shard), ByteSizeUnit.KB.toBytes(randomLongBetween(minShardSize, 100))); + } } - if (shardsWithSizes.isEmpty() == false) { - shardSize.put(shardIdentifier(randomFrom(shardsWithSizes)), ByteSizeUnit.KB.toBytes(minShardSize)); + if (shardSize.isEmpty() == false) { + shardSize.put(randomFrom(shardSize.keySet()), ByteSizeUnit.KB.toBytes(minShardSize)); } + + var diskUsages = Map.of(nodeId, new DiskUsage(nodeId, null, null, ByteSizeUnit.KB.toBytes(100), ByteSizeUnit.KB.toBytes(5))); ClusterInfo info = new ClusterInfo(diskUsages, diskUsages, shardSize, Map.of(), Map.of(), Map.of()); ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState( @@ -554,11 +554,12 @@ public void testUnmovableSize() { ); long result = allocationState.unmovableSize(nodeId, shards); - if (missingShard != null - && (missingShard.primary() - || clusterState.getRoutingNodes().activePrimary(missingShard.shardId()) == null - || info.getShardSize(clusterState.getRoutingNodes().activePrimary(missingShard.shardId())) == null) - || minShardSize < 5) { + + Predicate shardSizeKnown = shard -> shard.primary() + ? info.getShardSize(shard.shardId(), true) != null + : info.getShardSize(shard.shardId(), true) != null || info.getShardSize(shard.shardId(), false) != null; + + if ((missingShard != null && shardSizeKnown.test(missingShard) == false) || minShardSize < 5) { // the diff between used and high watermark is 5 KB. assertThat(result, equalTo(ByteSizeUnit.KB.toBytes(5))); } else { From 6a4000cec63c3481a3cca0b19832d6ff5343f0cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:20:08 +0100 Subject: [PATCH 21/54] Fix failing CI due to warning in Secure Settings Validation (#103307) * Fix failing CI due to warning in Secure Settings Validation * Validate settings in ReloadSecureSettings API (#103176) --- docs/changelog/103176.yaml | 5 ++ .../action/admin/ReloadSecureSettingsIT.java | 50 ++++++++++++++++++- ...nsportNodesReloadSecureSettingsAction.java | 2 + .../xpack/security/authc/jwt/JwtRestIT.java | 5 +- ...gsWithPasswordProtectedKeystoreRestIT.java | 2 +- 5 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/103176.yaml diff --git a/docs/changelog/103176.yaml b/docs/changelog/103176.yaml new file mode 100644 index 0000000000000..a0f46c1462f62 --- /dev/null +++ b/docs/changelog/103176.yaml @@ -0,0 +1,5 @@ +pr: 103176 +summary: Validate settings in `ReloadSecureSettings` API +area: Client +type: bug +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java index ad17e4f0d49dd..4aa3598608fb6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java @@ -11,10 +11,13 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.KeyStoreWrapper; +import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; @@ -43,6 +46,8 @@ @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class ReloadSecureSettingsIT extends ESIntegTestCase { + private static final String VALID_SECURE_SETTING_NAME = "some.setting.that.exists"; + @BeforeClass public static void disableInFips() { // Reload secure settings with a password protected keystore is tested in ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT @@ -350,9 +355,46 @@ public void testReloadWhileKeystoreChanged() throws Exception { } } + public void testInvalidKeyInSettings() throws Exception { + final Environment environment = internalCluster().getInstance(Environment.class); + + try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) { + keyStoreWrapper.setString(VALID_SECURE_SETTING_NAME, new char[0]); + keyStoreWrapper.save(environment.configFile(), new char[0], false); + } + + PlainActionFuture actionFuture = new PlainActionFuture<>(); + clusterAdmin().prepareReloadSecureSettings() + .setSecureStorePassword(new SecureString(new char[0])) + .setNodesIds(Strings.EMPTY_ARRAY) + .execute(actionFuture); + + actionFuture.get().getNodes().forEach(nodeResponse -> assertThat(nodeResponse.reloadException(), nullValue())); + + try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) { + assertThat(keyStoreWrapper, notNullValue()); + keyStoreWrapper.setString("some.setting.that.does.not.exist", new char[0]); + keyStoreWrapper.save(environment.configFile(), new char[0], false); + } + + actionFuture = new PlainActionFuture<>(); + clusterAdmin().prepareReloadSecureSettings() + .setSecureStorePassword(new SecureString(new char[0])) + .setNodesIds(Strings.EMPTY_ARRAY) + .execute(actionFuture); + + actionFuture.get() + .getNodes() + .forEach(nodeResponse -> assertThat(nodeResponse.reloadException(), instanceOf(IllegalArgumentException.class))); + } + @Override protected Collection> nodePlugins() { - final List> plugins = Arrays.asList(MockReloadablePlugin.class, MisbehavingReloadablePlugin.class); + final List> plugins = Arrays.asList( + MockWithSecureSettingPlugin.class, + MockReloadablePlugin.class, + MisbehavingReloadablePlugin.class + ); // shuffle as reload is called in order Collections.shuffle(plugins, random()); return plugins; @@ -455,4 +497,10 @@ public synchronized void setShouldThrow(boolean shouldThrow) { } } + public static class MockWithSecureSettingPlugin extends Plugin { + public List> getSettings() { + return List.of(SecureSetting.secureString(VALID_SECURE_SETTING_NAME, null)); + } + }; + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index 7fa97f1ee14b7..ed63e6d1b4474 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -122,6 +122,8 @@ protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation( keystore.decrypt(request.hasPassword() ? request.getSecureSettingsPassword().getChars() : new char[0]); // add the keystore to the original node settings object final Settings settingsWithKeystore = Settings.builder().put(environment.settings(), false).setSecureSettings(keystore).build(); + clusterService.getClusterSettings().validate(settingsWithKeystore, true); + final List exceptions = new ArrayList<>(); // broadcast the new settings object (with the open embedded keystore) to all reloadable plugins pluginsService.filterPlugins(ReloadablePlugin.class).forEach(p -> { diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java index 9c2fd118d59d9..d9fb77073b507 100644 --- a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java @@ -105,16 +105,15 @@ public class JwtRestIT extends ESRestTestCase { .setting("xpack.security.transport.ssl.enabled", "false") .setting("xpack.security.authc.token.enabled", "true") .setting("xpack.security.authc.api_key.enabled", "true") - .setting("xpack.security.http.ssl.enabled", "true") .setting("xpack.security.http.ssl.certificate", "http.crt") .setting("xpack.security.http.ssl.key", "http.key") - .setting("xpack.security.http.ssl.key_passphrase", "http-password") .setting("xpack.security.http.ssl.certificate_authorities", "ca.crt") .setting("xpack.security.http.ssl.client_authentication", "optional") .settings(JwtRestIT::realmSettings) .keystore("xpack.security.authc.realms.jwt.jwt2.hmac_key", HMAC_PASSPHRASE) .keystore("xpack.security.authc.realms.jwt.jwt3.hmac_jwkset", HMAC_JWKSET) + .keystore("xpack.security.http.ssl.secure_key_passphrase", "http-password") .keystore("xpack.security.authc.realms.jwt.jwt3.client_authentication.shared_secret", VALID_SHARED_SECRET) .keystore(keystoreSettings) .user("admin_user", "admin-password") @@ -508,8 +507,8 @@ public void testAuthenticationFailureIfDelegatedAuthorizationFails() throws Exce } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103308") public void testReloadClientSecret() throws Exception { + assumeFalse("Cannot run in FIPS mode because this test tampers with the keystore", inFipsJvm()); final String principal = SERVICE_SUBJECT.get(); final String username = getUsernameFromPrincipal(principal); final List roles = randomRoles(); diff --git a/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java b/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java index 49950b553bb20..0625ec166e32c 100644 --- a/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java +++ b/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java @@ -37,12 +37,12 @@ public class ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT extends ESR .nodes(NUM_NODES) .keystorePassword(KEYSTORE_PASSWORD) .name("javaRestTest") + .keystore(nodeSpec -> Map.of("xpack.security.transport.ssl.secure_key_passphrase", "transport-password")) .setting("xpack.security.enabled", "true") .setting("xpack.security.authc.anonymous.roles", "anonymous") .setting("xpack.security.transport.ssl.enabled", "true") .setting("xpack.security.transport.ssl.certificate", "transport.crt") .setting("xpack.security.transport.ssl.key", "transport.key") - .setting("xpack.security.transport.ssl.key_passphrase", "transport-password") .setting("xpack.security.transport.ssl.certificate_authorities", "ca.crt") .rolesFile(Resource.fromClasspath("roles.yml")) .configFile("transport.key", Resource.fromClasspath("ssl/transport.key")) From 9d27c2fc5a37b15eabaed8c9e6c77b90c8ec1251 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 14 Dec 2023 11:21:59 +0100 Subject: [PATCH 22/54] Enable MetricsAPMIT and change the assertion that logs on failure (#103405) this commit enables (temporarely) the test to gather some additional logging relates #103286 Co-authored-by: Moritz Mack --- .../org/elasticsearch/test/apmintegration/MetricsApmIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java index f6c5fdfe4db5c..93d08fbccd376 100644 --- a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java +++ b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java @@ -56,7 +56,6 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103286") @SuppressWarnings("unchecked") public void testApmIntegration() throws Exception { Map>> sampleAssertions = new HashMap<>( @@ -108,8 +107,8 @@ public void testApmIntegration() throws Exception { client().performRequest(new Request("GET", "/_use_apm_metrics")); - finished.await(30, TimeUnit.SECONDS); - assertThat(sampleAssertions, Matchers.anEmptyMap()); + assertTrue("Timeout when waiting for assertions to complete.", finished.await(30, TimeUnit.SECONDS)); + assertThat(sampleAssertions, Matchers.equalTo(Collections.emptyMap())); } private Map.Entry>> assertion( From c39a59503c0d855d0f803c522a39b9e59daacdc4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 14 Dec 2023 11:25:14 +0100 Subject: [PATCH 23/54] Add readme for APM metrics (#103400) this commit adds documentation about the APM metrics usage in Elasticsearch --- modules/apm/METERING.md | 171 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 modules/apm/METERING.md diff --git a/modules/apm/METERING.md b/modules/apm/METERING.md new file mode 100644 index 0000000000000..8c9280367f626 --- /dev/null +++ b/modules/apm/METERING.md @@ -0,0 +1,171 @@ +# Metrics in Elasticsearch + +Elasticsearch has the metrics API available in server's (perhaps we should move to lib?) package +`org.elasticsearch.telemetry.metric`. +This package contains base classes/interfaces for creating and working with metrics. +Please refer to the javadocs provided in these classes in that package for more details. +The entry point for working with metrics is `MeterRegistry`. + +## Implementation +We use elastic's apm-java-agent as an implementation of the API we expose. +the implementation can be found in `:modules:apm` +The apm-java-agent is responsible for buffering metrics and upon metrics_interval +send them over to apm server. +Metrics_interval is configured via a `tracing.apm.agent.metrics_interval` setting +The agent also collects a number of JVM metrics. +see https://www.elastic.co/guide/en/apm/agent/java/current/metrics.html#metrics-jvm + + +## How to choose an instrument + +We support various instruments and might be adding more as we go. +The choice of the right instrument is not always easy as often differences are subtle. +The simplified algorithm could be as follows: + +1. You want to measure something (absolute value) + 1. values are non-additive + 1. use a gauge + 2. Example: a cpu temperature + 2. values are additive + 1. use asynchronous counter + 2. Example: total number of requests +2. You want to count something + 1. values are monotonously increasing + 1. use a counter + 2. Example: Recording a failed authentication count + 2. values can be decreased + 1. use UpDownCounter + 2. Example: Number of orders in a queue +3. You want to record a statistics + 1. use a histogram + 1. Example: A statistics about how long it took to access a value from cache + +refer to https://opentelemetry.io/docs/specs/otel/metrics/supplementary-guidelines/#instrument-selection +for more details + +## How to name an instrument +See the naming guidelines for metrics: +https://docs.google.com/document/d/1jKxuaZi7QAMIRD_Eq3nonkYswVlQXW5TWllJEicoOtM/edit#heading=h.jxn90hx2ayic + +### Restarts and overflows +if the instrument is correctly chosen, the apm server will be able to determine if the metrics +were restarted (i.e. node was restarted) or there was a counter overflow +(the metric in ES might use an int internally, but apm backend might have a long ) + +## How to use an instrument +There are 2 types of usages of an instrument depending on a type. +- For synchronous instrument (counter/UpDownCounter) we need to register an instrument with + `MeterRegistry` and use the returned value to increment a value of that instrument +```java + MeterRegistry registry; + LongCounter longCounter = registry.registerLongCounter("es.test.requests.count", "a test counter", "count"); + longCounter.increment(); + longCounter.incrementBy(1, Map.of("name", "Alice")); + longCounter.incrementBy(1, Map.of("name", "Bob")); +``` + +- For asynchronous instrument (gauge/AsynchronousCounter) we register an instrument + and have to provide a callback that will report the absolute measured value. + This callback has to be provided upon registration and cannot be changed. +```java +MeterRegistry registry; +long someValue = 1; +registry.registerLongGauge("es.test.cpu.temperature", "a test gauge", "celcius", +() -> new LongWithAttributes(someValue, Map.of("cpuNumber", 1))); +``` + +If we don’t have access to ‘state’ that will be fetched on metric event (when callback is executed) +we can use a utility LongGaugeMetric or LongGaugeMetric +```java +MeterRegistry meterRegistry ; +LongGaugeMetric longGaugeMetric = LongGaugeMetric.create(meterRegistry, "es.test.gauge", "a test gauge", "total value"); +longGaugeMetric.set(123L); +``` +### The use of attributes aka dimensions +Each instrument can attach attributes to a reported value. This helps drilling down into the details +of value that was reported during the metric event + + +## Development + +### Fake http server + +The quickest way to verify that your metrics are working is to run `./gradlew run --with-apm-server`. +This will run ES node (or nodes in serverless) and also start a fake http server that will act +as an apm server. This fake http server will log all the http messages it receives from apm-agent + +### With APM server in cloud +You can also run local ES node with an apm server in cloud. +Create a new deployment in cloud, then click the 'hamburger :)' on the left, scroll to Observability and click APM under it. +At the upper right corner there is `Add data` link, then scroll down to `ApmAgents` section and pick Java +There you should be able to see `elastic.apm.secret_token` and `elastic.apm.server_url. You will use them in the next step. + +Next you should create a file `apm_server_ess.gradle` +in a different directory than your elasticsearch checkout (so that branch changes don't remove it) +The content of the file: +``` +rootProject { + if (project.name == 'elasticsearch') { + afterEvaluate { + testClusters.matching { it.name == "runTask" }.configureEach { + setting 'xpack.security.audit.enabled', 'true' + keystore 'tracing.apm.secret_token', 'REDACTED' + setting 'telemetry.metrics.enabled', 'true' + + setting 'tracing.apm.agent.server_url', 'https://REDACTED:443' + } + } + } +} +``` +Use the secret_token and server_url (REDACTED) from previous step. + +you can run your local ES node with APM in ESS with this command +`./gradlew run -I ../apm_enable_statefull.gradle` + +#### An init.d gradle setup + +Alternatively you can edit your `~/.gradle/init.d/apm.gradle` +```groovy +rootProject { + if (project.name == 'elasticsearch' && Boolean.getBoolean('metrics.enabled')) { + afterEvaluate { + testClusters.matching { it.name == "runTask" }.configureEach { + setting 'xpack.security.audit.enabled', 'true' + keystore 'tracing.apm.secret_token', 'TODO-REPLACE' + setting 'telemetry.metrics.enabled', 'true' + setting 'tracing.apm.agent.server_url', 'https://TODO-REPLACE-URL.apm.eastus2.staging.azure.foundit.no:443' + } + } + } +} +``` + +The example use: +``` +./gradlew :run -Dmetrics.enabled=true +``` + + + + +#### Logging +with any approach you took to run your ES with APM you will find apm-agent.json file +in ES's logs directory. If there are any problems with connecting to APM you will see WARN/ERROR messages. +We run apm-agent with logs at WARN level, so normally you should not see any logs there. + +When running ES in cloud, logs are being also indexed in a logging cluster, so you will be able to find them +in kibana. The `datastream.dataset` is `elasticsearch.apm_agent` + + +### Testing +We currently provide a base `TestTelemetryPlugin` which should help you write an integration test. +See an example `S3BlobStoreRepositoryTests` + + + + +# Links and further reading +https://opentelemetry.io/docs/specs/otel/metrics/supplementary-guidelines/ + +https://www.elastic.co/guide/en/apm/guide/current/data-model-metrics.html From 72cfa34d2f812d8abb1bbad21c1602a996e3ff22 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 Dec 2023 11:30:22 +0000 Subject: [PATCH 24/54] AwaitsFix for #102797 --- .../xpack/apmdata/APMIndexTemplateRegistryTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java index eb9d440106dea..9189cdff74547 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -137,6 +137,7 @@ public void testThatIndependentTemplatesAreAddedImmediatelyIfMissing() throws Ex assertThat(actualInstalledIndexTemplates.get(), equalTo(0)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102797") public void testIngestPipelines() { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); From 12f5f96345695bd2b0465cf50c680f1e3603ed83 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 14 Dec 2023 12:49:32 +0100 Subject: [PATCH 25/54] [fix] readme for APM metrics (#103428) in previous #103400 I forgot to commit a code review follow up --- modules/apm/METERING.md | 43 +++++++---------------------------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/modules/apm/METERING.md b/modules/apm/METERING.md index 8c9280367f626..0f5fcc977295d 100644 --- a/modules/apm/METERING.md +++ b/modules/apm/METERING.md @@ -1,6 +1,6 @@ # Metrics in Elasticsearch -Elasticsearch has the metrics API available in server's (perhaps we should move to lib?) package +Elasticsearch has the metrics API available in server's package `org.elasticsearch.telemetry.metric`. This package contains base classes/interfaces for creating and working with metrics. Please refer to the javadocs provided in these classes in that package for more details. @@ -18,7 +18,6 @@ see https://www.elastic.co/guide/en/apm/agent/java/current/metrics.html#metrics- ## How to choose an instrument -We support various instruments and might be adding more as we go. The choice of the right instrument is not always easy as often differences are subtle. The simplified algorithm could be as follows: @@ -45,7 +44,7 @@ for more details ## How to name an instrument See the naming guidelines for metrics: -https://docs.google.com/document/d/1jKxuaZi7QAMIRD_Eq3nonkYswVlQXW5TWllJEicoOtM/edit#heading=h.jxn90hx2ayic +[NAMING GUIDE](NAMING.md) ### Restarts and overflows if the instrument is correctly chosen, the apm server will be able to determine if the metrics @@ -70,7 +69,7 @@ There are 2 types of usages of an instrument depending on a type. ```java MeterRegistry registry; long someValue = 1; -registry.registerLongGauge("es.test.cpu.temperature", "a test gauge", "celcius", +registry.registerLongGauge("es.test.cpu.temperature", "the current CPU temperature as measured by psensor", "degrees Celsius", () -> new LongWithAttributes(someValue, Map.of("cpuNumber", 1))); ``` @@ -88,44 +87,19 @@ of value that was reported during the metric event ## Development -### Fake http server +### Mock http server The quickest way to verify that your metrics are working is to run `./gradlew run --with-apm-server`. -This will run ES node (or nodes in serverless) and also start a fake http server that will act +This will run ES node (or nodes in serverless) and also start a mock http server that will act as an apm server. This fake http server will log all the http messages it receives from apm-agent ### With APM server in cloud You can also run local ES node with an apm server in cloud. -Create a new deployment in cloud, then click the 'hamburger :)' on the left, scroll to Observability and click APM under it. +Create a new deployment in cloud, then click the 'hamburger' on the left, scroll to Observability and click APM under it. At the upper right corner there is `Add data` link, then scroll down to `ApmAgents` section and pick Java There you should be able to see `elastic.apm.secret_token` and `elastic.apm.server_url. You will use them in the next step. -Next you should create a file `apm_server_ess.gradle` -in a different directory than your elasticsearch checkout (so that branch changes don't remove it) -The content of the file: -``` -rootProject { - if (project.name == 'elasticsearch') { - afterEvaluate { - testClusters.matching { it.name == "runTask" }.configureEach { - setting 'xpack.security.audit.enabled', 'true' - keystore 'tracing.apm.secret_token', 'REDACTED' - setting 'telemetry.metrics.enabled', 'true' - - setting 'tracing.apm.agent.server_url', 'https://REDACTED:443' - } - } - } -} -``` -Use the secret_token and server_url (REDACTED) from previous step. - -you can run your local ES node with APM in ESS with this command -`./gradlew run -I ../apm_enable_statefull.gradle` - -#### An init.d gradle setup - -Alternatively you can edit your `~/.gradle/init.d/apm.gradle` +edit your `~/.gradle/init.d/apm.gradle` and replace the secret_token and the server_url. ```groovy rootProject { if (project.name == 'elasticsearch' && Boolean.getBoolean('metrics.enabled')) { @@ -146,9 +120,6 @@ The example use: ./gradlew :run -Dmetrics.enabled=true ``` - - - #### Logging with any approach you took to run your ES with APM you will find apm-agent.json file in ES's logs directory. If there are any problems with connecting to APM you will see WARN/ERROR messages. From 673cb49a4e69525b8c8014a0c4071ca7339a2909 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 14 Dec 2023 12:58:41 +0100 Subject: [PATCH 26/54] Avoid docker fixture failures on unsupported environments (#103420) Fixes #103417 --- .../testcontainers/DockerEnvironmentAwareTestContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java index e894c767d6217..ce4d6fda861cd 100644 --- a/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java +++ b/test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java @@ -58,13 +58,13 @@ private static boolean isDockerAvailable() { public DockerEnvironmentAwareTestContainer(ImageFromDockerfile imageFromDockerfile) { super(imageFromDockerfile); - withLogConsumer(new Slf4jLogConsumer(logger())); } @Override public void start() { Assume.assumeFalse("Docker support excluded on OS", EXCLUDED_OS); Assume.assumeTrue("Docker probing succesful", DOCKER_PROBING_SUCCESSFUL); + withLogConsumer(new Slf4jLogConsumer(logger())); super.start(); } From 133008bfc50f6af703e6bea7f9b8691666c3aee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= Date: Thu, 14 Dec 2023 13:45:41 +0100 Subject: [PATCH 27/54] [LTR] Enable by default on stateful only (#103333) --- .../test/cluster/FeatureFlag.java | 1 - .../ml/qa/basic-multi-node/build.gradle | 2 -- .../ml/qa/ml-with-security/build.gradle | 2 -- .../ml/DefaultMachineLearningExtension.java | 5 +++ .../xpack/ml/MachineLearning.java | 7 ++-- .../xpack/ml/MachineLearningExtension.java | 4 +++ .../ltr/LearningToRankRescorerFeature.java | 28 ---------------- .../ml/LocalStateMachineLearningAdOnly.java | 2 +- .../ml/LocalStateMachineLearningDfaOnly.java | 2 +- .../ml/LocalStateMachineLearningNlpOnly.java | 2 +- ...chineLearningInfoTransportActionTests.java | 9 ++++- .../xpack/ml/MachineLearningTests.java | 33 ++++++++++++------- .../CategorizeTextAggregatorTests.java | 3 +- ...nternalCategorizationAggregationTests.java | 4 +-- .../ChangePointAggregatorTests.java | 4 +-- .../FrequentItemSetsAggregatorTests.java | 4 +-- ...ernalItemSetMapReduceAggregationTests.java | 4 +-- .../ml/aggs/heuristic/PValueScoreTests.java | 11 +++---- ...erencePipelineAggregationBuilderTests.java | 5 ++- .../InternalInferenceAggregationTests.java | 4 +-- .../InternalKSTestAggregationTests.java | 4 +-- .../TextEmbeddingQueryVectorBuilderTests.java | 4 +-- .../xpack/test/rest/XPackRestIT.java | 1 - 23 files changed, 68 insertions(+), 77 deletions(-) delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerFeature.java diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index 2c313da69b42e..49fb38b518dce 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -16,7 +16,6 @@ */ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), - LEARNING_TO_RANK("es.learning_to_rank_feature_flag_enabled=true", Version.fromString("8.12.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null); public final String systemProperty; diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle index 3f2f85e3e09da..64970d18b5c82 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle +++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle @@ -1,4 +1,3 @@ -import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.legacy-java-rest-test' @@ -17,7 +16,6 @@ testClusters.configureEach { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' setting 'slm.history_index_enabled', 'false' - requiresFeature 'es.learning_to_rank_feature_flag_enabled', Version.fromString("8.12.0") } if (BuildParams.inFipsJvm){ diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index df2eb2c687fb5..f2ec17093bb93 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -1,4 +1,3 @@ -import org.elasticsearch.gradle.Version apply plugin: 'elasticsearch.legacy-yaml-rest-test' dependencies { @@ -258,5 +257,4 @@ testClusters.configureEach { user username: "no_ml", password: "x-pack-test-password", role: "minimal" setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' - requiresFeature 'es.learning_to_rank_feature_flag_enabled', Version.fromString("8.12.0") } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java index fa94bf96c1167..66f4797ef707c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java @@ -51,6 +51,11 @@ public boolean isNlpEnabled() { return true; } + @Override + public boolean isLearningToRankEnabled() { + return true; + } + @Override public String[] getAnalyticsDestIndexAllowedSettings() { return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 20efc5d681cc3..f9c483496445e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -327,7 +327,6 @@ import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.ltr.LearningToRankRescorerBuilder; -import org.elasticsearch.xpack.ml.inference.ltr.LearningToRankRescorerFeature; import org.elasticsearch.xpack.ml.inference.ltr.LearningToRankService; import org.elasticsearch.xpack.ml.inference.modelsize.MlModelSizeNamedXContentProvider; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -891,7 +890,7 @@ private static void reportClashingNodeAttribute(String attrName) { @Override public List> getRescorers() { - if (enabled && LearningToRankRescorerFeature.isEnabled()) { + if (enabled && machineLearningExtension.get().isLearningToRankEnabled()) { return List.of( new RescorerSpec<>( LearningToRankRescorerBuilder.NAME, @@ -1807,7 +1806,7 @@ public List getNamedXContent() { ); namedXContent.addAll(new CorrelationNamedContentProvider().getNamedXContentParsers()); // LTR Combine with Inference named content provider when feature flag is removed - if (LearningToRankRescorerFeature.isEnabled()) { + if (machineLearningExtension.get().isLearningToRankEnabled()) { namedXContent.addAll(new MlLTRNamedXContentProvider().getNamedXContentParsers()); } return namedXContent; @@ -1895,7 +1894,7 @@ public List getNamedWriteables() { namedWriteables.addAll(new CorrelationNamedContentProvider().getNamedWriteables()); namedWriteables.addAll(new ChangePointNamedContentProvider().getNamedWriteables()); // LTR Combine with Inference named content provider when feature flag is removed - if (LearningToRankRescorerFeature.isEnabled()) { + if (machineLearningExtension.get().isLearningToRankEnabled()) { namedWriteables.addAll(new MlLTRNamedXContentProvider().getNamedWriteables()); } return namedWriteables; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java index 552344b4ef10e..c27568c6e3b5c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java @@ -25,6 +25,10 @@ default void configure(Settings settings) {} boolean isNlpEnabled(); + default boolean isLearningToRankEnabled() { + return false; + } + String[] getAnalyticsDestIndexAllowedSettings(); AbstractNodeAvailabilityZoneMapper getNodeAvailabilityZoneMapper(Settings settings, ClusterSettings clusterSettings); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerFeature.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerFeature.java deleted file mode 100644 index 42598691beec2..0000000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerFeature.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.ml.inference.ltr; - -import org.elasticsearch.common.util.FeatureFlag; - -/** - * Learning to rank feature flag. When the feature is complete, this flag will be removed. - * - * Upon removal, ensure transport serialization is all corrected for future BWC. - * - * See {@link LearningToRankRescorerBuilder} - */ -public class LearningToRankRescorerFeature { - - private LearningToRankRescorerFeature() {} - - private static final FeatureFlag LEARNING_TO_RANK = new FeatureFlag("learning_to_rank"); - - public static boolean isEnabled() { - return LEARNING_TO_RANK.isEnabled(); - } -} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java index 3ff3a4a404f97..175a035a70f7e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java @@ -14,6 +14,6 @@ public class LocalStateMachineLearningAdOnly extends LocalStateMachineLearning { public LocalStateMachineLearningAdOnly(final Settings settings, final Path configPath) { - super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, true, false, false))); + super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, true, false, false, false))); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java index 1a72f27865d8a..f054e52dc29ec 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java @@ -14,6 +14,6 @@ public class LocalStateMachineLearningDfaOnly extends LocalStateMachineLearning { public LocalStateMachineLearningDfaOnly(final Settings settings, final Path configPath) { - super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, true, false))); + super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, true, false, false))); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java index 0f11e8033b83d..a3d684011e932 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java @@ -14,6 +14,6 @@ public class LocalStateMachineLearningNlpOnly extends LocalStateMachineLearning { public LocalStateMachineLearningNlpOnly(final Settings settings, final Path configPath) { - super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, false, true))); + super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, false, true, false))); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java index 08568bd3e0d1e..084a9d95939c5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java @@ -160,7 +160,14 @@ private MachineLearningUsageTransportAction newUsageAction( licenseState, jobManagerHolder, new MachineLearningExtensionHolder( - new MachineLearningTests.MlTestExtension(true, true, isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled) + new MachineLearningTests.MlTestExtension( + true, + true, + isAnomalyDetectionEnabled, + isDataFrameAnalyticsEnabled, + isNlpEnabled, + true + ) ) ); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java index 84cef907cd093..f5f81a5ca15f3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java @@ -221,9 +221,8 @@ public void testNoAttributes_givenClash() throws IOException { public void testAnomalyDetectionOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); - try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings)) { - MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, true, false, false)); - machineLearning.loadExtensions(loader); + MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, true, false, false, false)); + try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, hasItem(instanceOf(RestGetJobsAction.class))); @@ -242,9 +241,8 @@ public void testAnomalyDetectionOnly() throws IOException { public void testDataFrameAnalyticsOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); - try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings)) { - MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, true, false)); - machineLearning.loadExtensions(loader); + MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, true, false, false)); + try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, not(hasItem(instanceOf(RestGetJobsAction.class)))); @@ -263,9 +261,8 @@ public void testDataFrameAnalyticsOnly() throws IOException { public void testNlpOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); - try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings)) { - MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, false, true)); - machineLearning.loadExtensions(loader); + MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, false, true, false)); + try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, not(hasItem(instanceOf(RestGetJobsAction.class)))); @@ -291,19 +288,22 @@ public static class MlTestExtension implements MachineLearningExtension { private final boolean isAnomalyDetectionEnabled; private final boolean isDataFrameAnalyticsEnabled; private final boolean isNlpEnabled; + private final boolean isLearningToRankEnabled; MlTestExtension( boolean useIlm, boolean includeNodeInfo, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, - boolean isNlpEnabled + boolean isNlpEnabled, + boolean isLearningToRankEnabled ) { this.useIlm = useIlm; this.includeNodeInfo = includeNodeInfo; this.isAnomalyDetectionEnabled = isAnomalyDetectionEnabled; this.isDataFrameAnalyticsEnabled = isDataFrameAnalyticsEnabled; this.isNlpEnabled = isNlpEnabled; + this.isLearningToRankEnabled = isLearningToRankEnabled; } @Override @@ -331,6 +331,11 @@ public boolean isNlpEnabled() { return isNlpEnabled; } + @Override + public boolean isLearningToRankEnabled() { + return isLearningToRankEnabled; + } + @Override public String[] getAnalyticsDestIndexAllowedSettings() { return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS; @@ -377,6 +382,12 @@ protected XPackLicenseState getLicenseState() { } public static MachineLearning createTrialLicensedMachineLearning(Settings settings) { - return new TrialLicensedMachineLearning(settings); + return createTrialLicensedMachineLearning(settings, null); + } + + public static MachineLearning createTrialLicensedMachineLearning(Settings settings, MlTestExtensionLoader loader) { + MachineLearning mlPlugin = new TrialLicensedMachineLearning(settings); + mlPlugin.loadExtensions(loader); + return mlPlugin; } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java index 196584e4a7ce2..cb5b98af29d57 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.search.aggregations.metrics.Min; import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.io.IOException; import java.util.Arrays; @@ -53,7 +54,7 @@ protected AnalysisModule createAnalysisModule() throws Exception { @Override protected List getSearchPlugins() { - return List.of(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } private static final String TEXT_FIELD_NAME = "text"; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java index 1f9119463994d..be8b72d26fd71 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java @@ -18,7 +18,7 @@ import org.elasticsearch.test.InternalMultiBucketAggregationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import org.junit.After; import org.junit.Before; @@ -49,7 +49,7 @@ public void destroyHash() { @Override protected SearchPlugin registerPlugin() { - return new MachineLearning(Settings.EMPTY); + return MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregatorTests.java index 410fc474a503f..442641db8c4ed 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregatorTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.io.IOException; import java.util.Arrays; @@ -40,7 +40,7 @@ public class ChangePointAggregatorTests extends AggregatorTestCase { @Override protected List getSearchPlugins() { - return List.of(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } private static final DateHistogramInterval INTERVAL = DateHistogramInterval.minutes(1); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java index 5a3c8fcc5f7c7..93ef45285b60e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.EclatMapReducer.EclatResult; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.FrequentItemSetCollector.FrequentItemSet; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.InternalItemSetMapReduceAggregation; @@ -66,7 +66,7 @@ public class FrequentItemSetsAggregatorTests extends AggregatorTestCase { @Override protected List getSearchPlugins() { - return List.of(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java index 93ee8bec974a7..ccaa6b4f0f4ec 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.InternalItemSetMapReduceAggregationTests.WordCountMapReducer.WordCounts; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.ItemSetMapReduceValueSource.Field; import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.ItemSetMapReduceValueSource.ValueFormatter; @@ -247,7 +247,7 @@ protected void assertFromXContent( @Override protected SearchPlugin registerPlugin() { - return new MachineLearning(Settings.EMPTY); + return MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/heuristic/PValueScoreTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/heuristic/PValueScoreTests.java index 17898d7205b66..a6de69b684e12 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/heuristic/PValueScoreTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/heuristic/PValueScoreTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.util.List; import java.util.function.Function; @@ -56,16 +57,14 @@ public void testAssertions() { @Override protected NamedXContentRegistry xContentRegistry() { - return new NamedXContentRegistry( - new SearchModule(Settings.EMPTY, List.of(new MachineLearning(Settings.EMPTY))).getNamedXContents() - ); + MachineLearning mlPlugin = MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); + return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, List.of(mlPlugin)).getNamedXContents()); } @Override protected NamedWriteableRegistry writableRegistry() { - return new NamedWriteableRegistry( - new SearchModule(Settings.EMPTY, List.of(new MachineLearning(Settings.EMPTY))).getNamedWriteables() - ); + MachineLearning mlPlugin = MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); + return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, List.of(mlPlugin)).getNamedWriteables()); } public void testPValueScore_WhenAllDocsContainTerm() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilderTests.java index 4aed2261b89ef..8f1ef07b554da 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilderTests.java @@ -23,10 +23,9 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigUpdateTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ResultsFieldUpdate; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -43,7 +42,7 @@ public class InferencePipelineAggregationBuilderTests extends BasePipelineAggreg @Override protected List plugins() { - return Collections.singletonList(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java index 016a89fe4e4b7..fc1858167e7d7 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.xpack.core.ml.inference.results.TopClassEntry; import org.elasticsearch.xpack.core.ml.inference.results.WarningInferenceResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.util.Arrays; import java.util.Collections; @@ -40,7 +40,7 @@ public class InternalInferenceAggregationTests extends InternalAggregationTestCa @Override protected SearchPlugin registerPlugin() { - return new MachineLearning(Settings.EMPTY); + return MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java index 2085e44925cc9..f064b37c1fdec 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.test.InternalAggregationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.util.Arrays; import java.util.List; @@ -28,7 +28,7 @@ public class InternalKSTestAggregationTests extends InternalAggregationTestCase< @Override protected SearchPlugin registerPlugin() { - return new MachineLearning(Settings.EMPTY); + return MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY); } @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java index a44aa9404f4f9..8575c7e1f4bf3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/vectors/TextEmbeddingQueryVectorBuilderTests.java @@ -18,7 +18,7 @@ import org.elasticsearch.xpack.core.ml.action.InferModelAction; import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults; -import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.MachineLearningTests; import java.io.IOException; import java.util.List; @@ -30,7 +30,7 @@ public class TextEmbeddingQueryVectorBuilderTests extends AbstractQueryVectorBui @Override protected List additionalPlugins() { - return List.of(new MachineLearning(Settings.EMPTY)); + return List.of(MachineLearningTests.createTrialLicensedMachineLearning(Settings.EMPTY)); } @Override diff --git a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index 0efe2797c7f76..556a417fb5e79 100644 --- a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -43,7 +43,6 @@ public class XPackRestIT extends AbstractXPackRestTest { .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") .user("x_pack_rest_user", "x-pack-test-password") .feature(FeatureFlag.TIME_SERIES_MODE) - .feature(FeatureFlag.LEARNING_TO_RANK) .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) .configFile("service_tokens", Resource.fromClasspath("service_tokens")) From db7d5438dd6ddc18f0918227ef48c576905f3288 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 14 Dec 2023 13:47:10 +0100 Subject: [PATCH 28/54] [Connectors API] Unify update sync jobs API responses (#103403) Unify the update sync jobs API responses by using ConnectorUpdateActionResponse instead of AcknowledgedResponse. --- .../entsearch/420_connector_sync_job_check_in.yml | 2 +- .../entsearch/430_connector_sync_job_cancel.yml | 2 +- .../entsearch/450_connector_sync_job_error.yml | 2 +- .../460_connector_sync_job_update_stats.yml | 6 +++--- .../action/ConnectorUpdateActionResponse.java | 4 +++- .../action/CancelConnectorSyncJobAction.java | 6 +++--- .../action/CheckInConnectorSyncJobAction.java | 6 +++--- .../action/RestCancelConnectorSyncJobAction.java | 7 ++++++- .../action/RestCheckInConnectorSyncJobAction.java | 7 ++++++- .../RestUpdateConnectorSyncJobErrorAction.java | 3 ++- ...pdateConnectorSyncJobIngestionStatsAction.java | 4 ++-- .../TransportCancelConnectorSyncJobAction.java | 15 +++++++++++---- .../TransportCheckInConnectorSyncJobAction.java | 15 +++++++++++---- ...ransportUpdateConnectorSyncJobErrorAction.java | 12 ++++++++---- ...pdateConnectorSyncJobIngestionStatsAction.java | 11 +++++++---- .../action/UpdateConnectorSyncJobErrorAction.java | 6 +++--- ...pdateConnectorSyncJobIngestionStatsAction.java | 6 +++--- 17 files changed, 74 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/420_connector_sync_job_check_in.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/420_connector_sync_job_check_in.yml index d08f7f6a51c91..caa6543b6985a 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/420_connector_sync_job_check_in.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/420_connector_sync_job_check_in.yml @@ -34,7 +34,7 @@ setup: connector_sync_job.check_in: connector_sync_job_id: $sync-job-id-to-check-in - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/430_connector_sync_job_cancel.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/430_connector_sync_job_cancel.yml index d934b7c674f25..633c1a8cecb7b 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/430_connector_sync_job_cancel.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/430_connector_sync_job_cancel.yml @@ -27,7 +27,7 @@ setup: connector_sync_job.cancel: connector_sync_job_id: $sync-job-id-to-cancel - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/450_connector_sync_job_error.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/450_connector_sync_job_error.yml index 6f525a2ac2883..a565d28c3e788 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/450_connector_sync_job_error.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/450_connector_sync_job_error.yml @@ -29,7 +29,7 @@ setup: body: error: error - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/460_connector_sync_job_update_stats.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/460_connector_sync_job_update_stats.yml index 0c7300bd2b436..94572870f9164 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/460_connector_sync_job_update_stats.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/460_connector_sync_job_update_stats.yml @@ -32,7 +32,7 @@ setup: indexed_document_count: 20 indexed_document_volume: 1000 - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: @@ -143,7 +143,7 @@ setup: indexed_document_volume: 1000 total_document_count: 20 - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: @@ -173,7 +173,7 @@ setup: indexed_document_volume: 1000 last_seen: 2023-12-04T08:45:50.567149Z - - match: { acknowledged: true } + - match: { result: updated } - do: connector_sync_job.get: diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java index 3d8367dfdeee0..b77f04e7d9289 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorUpdateActionResponse.java @@ -14,12 +14,14 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import java.io.IOException; import java.util.Objects; /** - * Represents a response for update actions related to connectors and sync jobs. + * Represents a response for update actions related to {@link Connector} and {@link ConnectorSyncJob}. * The response encapsulates the result of the update action, represented by a {@link DocWriteResponse.Result}. */ public class ConnectorUpdateActionResponse extends ActionResponse implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java index 7179bbb3a62f2..111828680455c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -19,6 +18,7 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.Objects; @@ -27,13 +27,13 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE; -public class CancelConnectorSyncJobAction extends ActionType { +public class CancelConnectorSyncJobAction extends ActionType { public static final CancelConnectorSyncJobAction INSTANCE = new CancelConnectorSyncJobAction(); public static final String NAME = "cluster:admin/xpack/connector/sync_job/cancel"; private CancelConnectorSyncJobAction() { - super(NAME, AcknowledgedResponse::readFrom); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java index 3e5e1578cd54d..54ba26ec1533a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -19,6 +18,7 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; import java.io.IOException; @@ -27,13 +27,13 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class CheckInConnectorSyncJobAction extends ActionType { +public class CheckInConnectorSyncJobAction extends ActionType { public static final CheckInConnectorSyncJobAction INSTANCE = new CheckInConnectorSyncJobAction(); public static final String NAME = "cluster:admin/xpack/connector/sync_job/check_in"; private CheckInConnectorSyncJobAction() { - super(NAME, AcknowledgedResponse::readFrom); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCancelConnectorSyncJobAction.java index 82d679c6f0ad0..7cfce07aca48d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCancelConnectorSyncJobAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.List; @@ -42,6 +43,10 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient CancelConnectorSyncJobAction.Request request = new CancelConnectorSyncJobAction.Request( restRequest.param(CONNECTOR_SYNC_JOB_ID_PARAM) ); - return restChannel -> client.execute(CancelConnectorSyncJobAction.INSTANCE, request, new RestToXContentListener<>(restChannel)); + return restChannel -> client.execute( + CancelConnectorSyncJobAction.INSTANCE, + request, + new RestToXContentListener<>(restChannel, ConnectorUpdateActionResponse::status) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCheckInConnectorSyncJobAction.java index 86f97f4c5fdb4..882227e45169a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestCheckInConnectorSyncJobAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.List; @@ -41,6 +42,10 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest.param(CONNECTOR_SYNC_JOB_ID_PARAM) ); - return restChannel -> client.execute(CheckInConnectorSyncJobAction.INSTANCE, request, new RestToXContentListener<>(restChannel)); + return restChannel -> client.execute( + CheckInConnectorSyncJobAction.INSTANCE, + request, + new RestToXContentListener<>(restChannel, ConnectorUpdateActionResponse::status) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobErrorAction.java index e19a9675beebb..a05be4a92e6e3 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobErrorAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.List; @@ -46,7 +47,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return restChannel -> client.execute( UpdateConnectorSyncJobErrorAction.INSTANCE, request, - new RestToXContentListener<>(restChannel) + new RestToXContentListener<>(restChannel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobIngestionStatsAction.java index aedd1605b8bfb..57a362b55ee9b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestUpdateConnectorSyncJobIngestionStatsAction.java @@ -10,9 +10,9 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import java.io.IOException; import java.util.List; @@ -46,7 +46,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient return channel -> client.execute( UpdateConnectorSyncJobIngestionStatsAction.INSTANCE, request, - new RestToXContentListener<>(channel, r -> RestStatus.OK) + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java index ac61dcdf08a61..b0d4628e06202 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; public class TransportCancelConnectorSyncJobAction extends HandledTransportAction< CancelConnectorSyncJobAction.Request, - AcknowledgedResponse> { + ConnectorUpdateActionResponse> { protected ConnectorSyncJobIndexService connectorSyncJobIndexService; @@ -43,7 +43,14 @@ public TransportCancelConnectorSyncJobAction( } @Override - protected void doExecute(Task task, CancelConnectorSyncJobAction.Request request, ActionListener listener) { - connectorSyncJobIndexService.cancelConnectorSyncJob(request.getConnectorSyncJobId(), listener.map(r -> AcknowledgedResponse.TRUE)); + protected void doExecute( + Task task, + CancelConnectorSyncJobAction.Request request, + ActionListener listener + ) { + connectorSyncJobIndexService.cancelConnectorSyncJob( + request.getConnectorSyncJobId(), + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java index ebaadc80f4c27..9b57b090c3bd7 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; public class TransportCheckInConnectorSyncJobAction extends HandledTransportAction< CheckInConnectorSyncJobAction.Request, - AcknowledgedResponse> { + ConnectorUpdateActionResponse> { protected final ConnectorSyncJobIndexService connectorSyncJobIndexService; @@ -43,7 +43,14 @@ public TransportCheckInConnectorSyncJobAction( } @Override - protected void doExecute(Task task, CheckInConnectorSyncJobAction.Request request, ActionListener listener) { - connectorSyncJobIndexService.checkInConnectorSyncJob(request.getConnectorSyncJobId(), listener.map(r -> AcknowledgedResponse.TRUE)); + protected void doExecute( + Task task, + CheckInConnectorSyncJobAction.Request request, + ActionListener listener + ) { + connectorSyncJobIndexService.checkInConnectorSyncJob( + request.getConnectorSyncJobId(), + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java index c814092f2e7a2..d70d5cdd956fa 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; public class TransportUpdateConnectorSyncJobErrorAction extends HandledTransportAction< UpdateConnectorSyncJobErrorAction.Request, - AcknowledgedResponse> { + ConnectorUpdateActionResponse> { protected final ConnectorSyncJobIndexService connectorSyncJobIndexService; @@ -43,11 +43,15 @@ public TransportUpdateConnectorSyncJobErrorAction( } @Override - protected void doExecute(Task task, UpdateConnectorSyncJobErrorAction.Request request, ActionListener listener) { + protected void doExecute( + Task task, + UpdateConnectorSyncJobErrorAction.Request request, + ActionListener listener + ) { connectorSyncJobIndexService.updateConnectorSyncJobError( request.getConnectorSyncJobId(), request.getError(), - listener.map(r -> AcknowledgedResponse.TRUE) + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java index 864da6ca3095b..49c722b84e63b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java @@ -10,18 +10,18 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; public class TransportUpdateConnectorSyncJobIngestionStatsAction extends HandledTransportAction< UpdateConnectorSyncJobIngestionStatsAction.Request, - AcknowledgedResponse> { + ConnectorUpdateActionResponse> { protected final ConnectorSyncJobIndexService connectorSyncJobIndexService; @@ -46,8 +46,11 @@ public TransportUpdateConnectorSyncJobIngestionStatsAction( protected void doExecute( Task task, UpdateConnectorSyncJobIngestionStatsAction.Request request, - ActionListener listener + ActionListener listener ) { - connectorSyncJobIndexService.updateConnectorSyncJobIngestionStats(request, listener.map(r -> AcknowledgedResponse.TRUE)); + connectorSyncJobIndexService.updateConnectorSyncJobIngestionStats( + request, + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java index 820630bccee03..fe0893c82e27d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,6 +22,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; @@ -32,14 +32,14 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -public class UpdateConnectorSyncJobErrorAction extends ActionType { +public class UpdateConnectorSyncJobErrorAction extends ActionType { public static final UpdateConnectorSyncJobErrorAction INSTANCE = new UpdateConnectorSyncJobErrorAction(); public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_error"; public static final String ERROR_EMPTY_MESSAGE = "[error] of the connector sync job cannot be null or empty"; private UpdateConnectorSyncJobErrorAction() { - super(NAME, AcknowledgedResponse::readFrom); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java index 34d8be2af4881..b9c57cb6a0c61 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -25,6 +24,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import java.io.IOException; @@ -36,13 +36,13 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE; -public class UpdateConnectorSyncJobIngestionStatsAction extends ActionType { +public class UpdateConnectorSyncJobIngestionStatsAction extends ActionType { public static final UpdateConnectorSyncJobIngestionStatsAction INSTANCE = new UpdateConnectorSyncJobIngestionStatsAction(); public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_stats"; public UpdateConnectorSyncJobIngestionStatsAction() { - super(NAME, AcknowledgedResponse::readFrom); + super(NAME, ConnectorUpdateActionResponse::new); } public static class Request extends ActionRequest implements ToXContentObject { From 4e447c6eb76ce413ae2bf78a982342d8852c3bd3 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 14 Dec 2023 13:47:50 +0100 Subject: [PATCH 29/54] [Connectors API] Add docs for listing sync jobs (#103401) Add documentation for listing connector sync jobs API endpoint. --- .../connector/apis/connector-apis.asciidoc | 2 + .../list-connector-sync-jobs-api.asciidoc | 80 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index 0018f0e0aa158..e5e12ae2609c8 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -37,6 +37,7 @@ You can use these APIs to create, cancel, delete and update sync jobs. Use the following APIs to manage sync jobs: * <> +* <> include::create-connector-api.asciidoc[] @@ -44,3 +45,4 @@ include::create-connector-sync-job-api.asciidoc[] include::delete-connector-api.asciidoc[] include::get-connector-api.asciidoc[] include::list-connectors-api.asciidoc[] +include::list-connector-sync-jobs-api.asciidoc[] diff --git a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc new file mode 100644 index 0000000000000..8b88f318f5304 --- /dev/null +++ b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc @@ -0,0 +1,80 @@ +[role="xpack"] +[[list-connector-sync-jobs-api]] +=== List connector sync jobs API + +preview::[] + +++++ +List connector sync jobs +++++ + +Returns information about all stored connector sync jobs ordered by their creation date in ascending order. + + +[[list-connector-sync-jobs-api-request]] +==== {api-request-title} + +`GET _connector/_sync_job` + +[[list-connector-sync-jobs-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[list-connector-sync-jobs-api-path-params]] +==== {api-path-parms-title} + +`size`:: +(Optional, integer) Maximum number of results to retrieve. Defaults to `100`. + +`from`:: +(Optional, integer) The offset from the first result to fetch. Defaults to `0`. + +`status`:: +(Optional, job status) The job status the fetched sync jobs need to have. + +`connector_id`:: +(Optional, string) The connector id the fetched sync jobs need to have. + +[[list-connector-sync-jobs-api-example]] +==== {api-examples-title} + +The following example lists all connector sync jobs: + + +[source,console] +---- +GET _connector/_sync_job +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +The following example lists the first two connector sync jobs: + +[source,console] +---- +GET _connector/_sync_job?from=0&size=2 +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +The following example lists pending connector sync jobs (the first 100 per default): +[source,console] +---- +GET _connector/_sync_job?status=pending +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +The following example lists connector sync jobs (the first 100 per default) for the connector with id `connector-1`: +[source,console] +---- +GET _connector/_sync_job?connector_id=connector-1 +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +[[list-connector-sync-jobs-api-response-codes]] +==== {api-response-codes-title} + +`200`: +Indicates that results were successfully returned (results can also be empty). + +`400`: +Indicates that the request was malformed. From 36cc7ff0c1d77b555fcb41073a985849bc1a6311 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 Dec 2023 13:03:27 +0000 Subject: [PATCH 30/54] AwaitsFix for #103439 --- .../org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java index 2e61b6e978b61..6d3811fd66d9c 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java @@ -302,6 +302,7 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { }); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103439") public void testTsdbDataStreams() throws Exception { // Create the ILM policy DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); From 0d6d21c813521620124f87c0bb74c320323e8a2e Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 14 Dec 2023 14:14:57 +0100 Subject: [PATCH 31/54] [Connectors API] Add delete connector sync job docs (#103407) Add delete connector sync job API endpoint documentation. --- .../connector/apis/connector-apis.asciidoc | 3 +- .../delete-connector-sync-job-api.asciidoc | 54 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index e5e12ae2609c8..bc90f5bd4fc17 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -37,12 +37,13 @@ You can use these APIs to create, cancel, delete and update sync jobs. Use the following APIs to manage sync jobs: * <> +* < * <> - include::create-connector-api.asciidoc[] include::create-connector-sync-job-api.asciidoc[] include::delete-connector-api.asciidoc[] +include::delete-connector-sync-job-api.asciidoc[] include::get-connector-api.asciidoc[] include::list-connectors-api.asciidoc[] include::list-connector-sync-jobs-api.asciidoc[] diff --git a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..8641794576bf1 --- /dev/null +++ b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc @@ -0,0 +1,54 @@ +[[delete-connector-sync-job-api]] +=== Delete connector sync job API + +preview::[] + +++++ +Delete connector sync job +++++ + +Removes a connector sync job and its associated data. +This is a destructive action that is not recoverable. + +[[delete-connector-sync-job-api-request]] +==== {api-request-title} + +`DELETE _connector/_sync_job/` + +[[delete-connector-sync-job-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[delete-connector-sync-job-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[delete-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `connector_sync_job_id` was not provided. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[delete-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example deletes the connector sync job with ID `my-connector-sync-job-id`: + +[source,console] +---- +DELETE _connector/_sync_job/my-connector-sync-job-id +---- +// TEST[skip:there's no way to clean up after this code snippet, as we don't know the ids of sync jobs ahead of time] + +[source,console-result] +---- +{ + "acknowledged": true +} +---- From 92babbb217d1d79237cc597959f68bed6483216a Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 Dec 2023 13:19:16 +0000 Subject: [PATCH 32/54] Dispatch ClusterStateAction#buildResponse to executor (#103435) If waiting for a particular cluster state (e.g. on the CCR leader) we will compute the resulting cluster state and serialize it on the cluster applier thread, which can be too expensive in a large cluster for this thread. With this commit we dispatch the final action back to the original executor. --- docs/changelog/103435.yaml | 5 +++++ .../admin/cluster/state/TransportClusterStateAction.java | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/103435.yaml diff --git a/docs/changelog/103435.yaml b/docs/changelog/103435.yaml new file mode 100644 index 0000000000000..95e3c7169ada9 --- /dev/null +++ b/docs/changelog/103435.yaml @@ -0,0 +1,5 @@ +pr: 103435 +summary: Dispatch `ClusterStateAction#buildResponse` to executor +area: Distributed +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index f4e301e0748bb..29bffa3949258 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.ClusterState; @@ -112,7 +113,7 @@ public void onNewClusterState(ClusterState newState) { } if (acceptableClusterStatePredicate.test(newState)) { - ActionListener.completeWith(listener, () -> buildResponse(request, newState)); + executor.execute(ActionRunnable.supply(listener, () -> buildResponse(request, newState))); } else { listener.onFailure( new NotMasterException( @@ -150,6 +151,8 @@ private static Map> getClusterFeatures(ClusterState clusterS } private ClusterStateResponse buildResponse(final ClusterStateRequest request, final ClusterState currentState) { + ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // too heavy to construct & serialize cluster state without forking + logger.trace("Serving cluster state request using version {}", currentState.version()); ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName()); builder.version(currentState.version()); From 900d6e88ccba3ffb1da0f5033ac83064acc77220 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 14 Dec 2023 14:25:09 +0100 Subject: [PATCH 33/54] Fix FIPS support for JWT REST test (#103429) For FIPS support, the keystore needs to be password-protected. Tweaking the test suite to always set a keystore password on cluster setup and as part of the reload call. This requires TLS on transport. --- .../xpack/security/authc/jwt/JwtRestIT.java | 17 ++++++++--- .../resources/ssl/README.asciidoc | 14 +++++++++ .../resources/ssl/ca-transport.crt | 20 +++++++++++++ .../resources/ssl/ca-transport.key | 30 +++++++++++++++++++ .../javaRestTest/resources/ssl/transport.crt | 22 ++++++++++++++ .../javaRestTest/resources/ssl/transport.key | 30 +++++++++++++++++++ 6 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.crt create mode 100644 x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.key create mode 100644 x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.crt create mode 100644 x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.key diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java index d9fb77073b507..52d87c2e32c87 100644 --- a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java @@ -89,20 +89,27 @@ public class JwtRestIT extends ESRestTestCase { put("xpack.security.authc.realms.jwt.jwt2.client_authentication.shared_secret", VALID_SHARED_SECRET); } }; + private static final String KEYSTORE_PASSWORD = "keystore-password"; @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .nodes(2) .distribution(DistributionType.DEFAULT) + .keystorePassword(KEYSTORE_PASSWORD) .configFile("http.key", Resource.fromClasspath("ssl/http.key")) .configFile("http.crt", Resource.fromClasspath("ssl/http.crt")) .configFile("ca.crt", Resource.fromClasspath("ssl/ca.crt")) + .configFile("ca-transport.crt", Resource.fromClasspath("ssl/ca-transport.crt")) + .configFile("transport.key", Resource.fromClasspath("ssl/transport.key")) + .configFile("transport.crt", Resource.fromClasspath("ssl/transport.crt")) .configFile("rsa.jwkset", Resource.fromClasspath("jwk/rsa-public-jwkset.json")) .setting("xpack.ml.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") - .setting("xpack.security.http.ssl.enabled", "true") - .setting("xpack.security.transport.ssl.enabled", "false") + .setting("xpack.security.transport.ssl.enabled", "true") + .setting("xpack.security.transport.ssl.certificate", "transport.crt") + .setting("xpack.security.transport.ssl.key", "transport.key") + .setting("xpack.security.transport.ssl.certificate_authorities", "ca-transport.crt") .setting("xpack.security.authc.token.enabled", "true") .setting("xpack.security.authc.api_key.enabled", "true") .setting("xpack.security.http.ssl.enabled", "true") @@ -114,6 +121,7 @@ public class JwtRestIT extends ESRestTestCase { .keystore("xpack.security.authc.realms.jwt.jwt2.hmac_key", HMAC_PASSPHRASE) .keystore("xpack.security.authc.realms.jwt.jwt3.hmac_jwkset", HMAC_JWKSET) .keystore("xpack.security.http.ssl.secure_key_passphrase", "http-password") + .keystore("xpack.security.transport.ssl.secure_key_passphrase", "transport-password") .keystore("xpack.security.authc.realms.jwt.jwt3.client_authentication.shared_secret", VALID_SHARED_SECRET) .keystore(keystoreSettings) .user("admin_user", "admin-password") @@ -508,7 +516,6 @@ public void testAuthenticationFailureIfDelegatedAuthorizationFails() throws Exce } public void testReloadClientSecret() throws Exception { - assumeFalse("Cannot run in FIPS mode because this test tampers with the keystore", inFipsJvm()); final String principal = SERVICE_SUBJECT.get(); final String username = getUsernameFromPrincipal(principal); final List roles = randomRoles(); @@ -561,7 +568,9 @@ private void writeSettingToKeystoreThenReload(String setting, @Nullable String v keystoreSettings.put(setting, value); } cluster.updateStoredSecureSettings(); - assertOK(adminClient().performRequest(new Request("POST", "/_nodes/reload_secure_settings"))); + final var reloadRequest = new Request("POST", "/_nodes/reload_secure_settings"); + reloadRequest.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}"); + assertOK(adminClient().performRequest(reloadRequest)); } public void testFailureOnInvalidClientAuthentication() throws Exception { diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/README.asciidoc b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/README.asciidoc index 37185a996fbba..cb1aadc94e4ac 100644 --- a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/README.asciidoc +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/README.asciidoc @@ -28,6 +28,20 @@ rm http.zip rmdir http ----------------------------------------------------------------------------------------------------------- +[source,shell] +----------------------------------------------------------------------------------------------------------- +elasticsearch-certutil cert --pem --name=transport --out=transport.zip --pass="transport-password" --days=3500 \ + --ca-cert=ca-transport.crt --ca-key=ca-transport.key --ca-pass="ca-password" \ + --dns=localhost --dns=localhost.localdomain --dns=localhost4 --dns=localhost4.localdomain4 --dns=localhost6 --dns=localhost6.localdomain6 \ + --ip=127.0.0.1 --ip=0:0:0:0:0:0:0:1 + +unzip transport.zip +mv transport/transport.* ./ + +rm transport.zip +rmdir transport +----------------------------------------------------------------------------------------------------------- + [source,shell] ----------------------------------------------------------------------------------------------------------- elasticsearch-certutil cert --pem --name=pki --out=${PWD}/pki.zip --pass="pki-password" --days=3500 \ diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.crt b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.crt new file mode 100644 index 0000000000000..320f00ccde59b --- /dev/null +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUGuBmPtwyEv7WZ1H0Yy5vyEEYVR8wDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjAwNDA3MTEzMDA1WhcNMjkxMTA2MTEzMDA1WjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALeTNx0a6X+Fhf6IQj4ggN9U +1HGIzJKEHGIpATDgbfdIv88e0O0I6HN7pmLf5LuUPDGc2oLGnxqATgnFek5eJ4QW +sKgflGB4C0EgQH4JAooIG0EI6aj3IcdzBwH8bdymAdsGj0Zcvm6wjhLixgiN3yIM +8KJAtJrSCITI88gfXhXyU0XCSzgruFkdvHjFBCWpCaK3hnjoiO65186PcGbrZHB8 +Izs2soa6H1AHVDMhmJjlwJWYtibjok+sgrjkDWG7cBh6Al7yXGUBOs9SgMXUpI3Y +0r/dDdeISdI5VzwKZpX6qYcNJI+jtgZUD0alMKBxjq3+v8GlDE/QVNyDwp/7SA8C +AwEAAaNTMFEwHQYDVR0OBBYEFMdSVLWtAhqfDXRQj+5o80nK1XaQMB8GA1UdIwQY +MBaAFMdSVLWtAhqfDXRQj+5o80nK1XaQMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAEiaX+JtgDb6PuPqO9NtxuPEsg6Ghj0dVYLOYE9cmKS9xeyy +sIDJbPl8vy7gomVBDYe+3qM2UElKFdApKGjOctjNlH8npkQxg72sarBUzOfuFPx5 +Z6u5TuKMfqjx2dwpTr3Yp/iUFs9e9zcDMnQYt5B6SCjKpKKodKKh+CE6wOg4nbSF +43OYO9HFHfwIlEqvof7j0r5gnPf5fYSYybYFAqy4oAfpESPq9lJuEvA46TrGpmP6 +IpMYkJJ6O+98A7RHo5kstZJdnG5paAKobdPEYxbIZvRyMJ8IxW8kSAaTKsK7W34k +IYciDd/YY3R+nhnh8F5DjVcyc79Zkv9Cjig/OxQ= +-----END CERTIFICATE----- diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.key b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.key new file mode 100644 index 0000000000000..6bbdd44d274df --- /dev/null +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/ca-transport.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,8209E02F62E3909502FECF5E5E9CF7A7 + +EdOFZ6/z/e4elfeAKs2B++/Px/IpiZdmiseZPjfwa6jgpY+8sehmze5+34vrxYJT +cMBH3QafmhdQZ4/Eo7DVFONrjJ3OmD5//ZiTIujTPwMsgGAdeq0yMC0cDkzg8SQ7 +KvTh0PY0feC6bVsY+YjDprDfpqIWf89F8ikgat9cmucV9YO3RbYnxgxRIztbHLP3 +GenAtdG+v7DzdefAdRQktBSNldkadsY6d/kVBknOHcA4pB/UtDpz77ZF40CNB95z +1Tr37nNnuRBUNHbKklXuozkvYLah66tFxA5v7Rf6F37d2QGBkgDphg/QMbJrrB+q +MsfiXeXqRaCzBN/ZuzTQAdQ/67XpQ+Ax89UOiT6SkKBKN1uNDk7Juzv5zHrq7aWS +aj1qtHDG2vMB+UM5A1MngD1LtXzs21Q0+9a2UT83x+VIP0hVq2uKmO8wAQ9gbBe9 +wkBPca4gLYlbIMWzaAe4DV0rcmux+i7Ezk8oVYW1JcoGjoZ3f9KewIQynBUlXXuO +EzSl4R3yiF/birrK9Lo6c9hOcQKCW2qAX73BKq8PjKgWT3rnqzg97q9PPK/vaien +fwSrTXDgEoy1RCwsPsxjyRf0LGFYLUFRVqrbFPhhjg4aEiuzawcpvRxjorC5UX0L +dpImNssdALDd0BbiqAbChUryFSSxFhQ2yo6hfUXZevD236b09V0jUpnZeyQjeTTk +fhhAUUpnd1YzWuYneD2JZQKvGdgWgYRyEKParFeHLjp95rXNWPSOgoAM+w0fFEjq +zkYQMaDGSnUWbc6LVv2exyRIRTrLAWamKnne7z8VxzetqXXmuX0WJb2lFiYMUw4/ +wf31RA8ZsVSgb9werSyPD9aRe/+YZM+kM3/3MC4jJGc6OJuDqEOhhB06L2Df2AWU +UQwZ7y+2yUC1kcFzc8+oT1TNgBHixouY+oqWkhbdCkbUFUe4FwXNXrMyrY9gZs1/ +PEkhVxxYgpLwifkbfQRJPeJvXxh7NxeolXyISaVENdLkMMYUhdsKTa+GOQbO2yfa +4BhOwAqJvyDFfsRxLiDlbxjzvY5qnMl0e/q8wZ60onHJOFCTCfm2BNx7sW+Sk5Kx +zm0Rxsz4rIIxA5S6zbbdsHxjTC9XiUelKaq+W0XTg76USYneORQNN/Mk9sCXvTud +HUqmSf1wREA1PdEcoJ3tMoAOZWGY43/IrdoG3bTNT96AdToD+D+Or8M2VcOZorVf +c3IRNfxGv2/SwhxW/z4tSLSToSJlt4QKxU9Xzm4UundDy1cHmS1faN6+bBnI5+/F +OKwzPCCUJ6H02CAjx2P/P6YEjoLl8B+7h4whlOfT/+IQbzOcGMpPyGu4jSf1KffA +asAQeBvYTx0QPdv2E7e216RLOlp/ERMzkUvF1G7UYKF7Ao6cUpSH6nvGABPLKNXV +fqjpWq8O4R1UEUXi6dqF1HfAHllI+vMw7LzRJK/5zVrWlJPm4c/Rng5OkK7aAGee +J0eTSlCdNpyaZzjyk2ZAQ54kZVqAS90zS1zo6lg2v9yfAfz6eYlfl2OGfFVG40Jt +oYxEVcG9LeD3XOkPOnTblHdKMor8cQt+TEJPu9eM31ay1QSilixx2yfOOFTgJZOi +-----END RSA PRIVATE KEY----- diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.crt b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.crt new file mode 100644 index 0000000000000..db93ca6ac4750 --- /dev/null +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDuDCCAqCgAwIBAgIVAIdpYPATbRn96E+eVTG/s0byNh/FMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTIwMDQwNzExMzA1OFoXDTI5MTEwNjExMzA1OFowFDESMBAG +A1UEAxMJdHJhbnNwb3J0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +pvfY9x3F8xR2+noX0PWxsc6XrKCxTQcbSHkkEr2Qr6BqGVXwEj/d6ocqdgcC7IZy +0HEwewBbO69Pg8INQ/21stcwZzW+4ZnKYsBmaZx1yCxYcplndFm9t5Fj/jTBsoJ3 +PemvJwMrewuVSwsE4WHVkXz4KVETfUd8DZiWoVnlRgaXfvvudib1DNxtuGEra+Zh +d3JcC1Wsn51qjpHsj/6s/usT6hmlm4Bu5tjAMxXFVX6J0skfRSVhLmNWgr86WBKB +9/qTJU34FBQGh2Ok/knkiO5rae+UCPpEpCNCCV3rFcMdxP613WfemRRqhUL0V6g3 +n4ExJa0853SsfvPEyHOADQIDAQABo4HgMIHdMB0GA1UdDgQWBBSraIvkNPX2TQQg +h8Ee3mWCALYr/zAfBgNVHSMEGDAWgBTHUlS1rQIanw10UI/uaPNJytV2kDCBjwYD +VR0RBIGHMIGEgglsb2NhbGhvc3SCF2xvY2FsaG9zdDYubG9jYWxkb21haW42hwR/ +AAABhxAAAAAAAAAAAAAAAAAAAAABggpsb2NhbGhvc3Q0ggpsb2NhbGhvc3Q2ghVs +b2NhbGhvc3QubG9jYWxkb21haW6CF2xvY2FsaG9zdDQubG9jYWxkb21haW40MAkG +A1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBABRKISeKdMRRQAvZBan2ONUFUD3P +7Aq8JP+GttC9Y+1uI0CTIZsnih+tuZ9l2Gq9nyfBVMcDSutlnBoQZLg5lCYQPaK7 +SuFhPRHGauEGYw1TjhrMEPokxzoT/X0/Sz5Ter6/qWzPKQs9EuaVJ27XfZd+kySn +S+cXd95woi+S8JQzQbcpA0rXCnWJ3l2frwG/3Hg8f82x2c6vgOzTG0Hklp0sFkUt +UqaBHGXPLiitaB01jUX60HZbxt5HIEseLctUmQlDtAEWwA3X6cRUEjulwRx8s52T +1FT2ORbVJ7ybKARGBSs932Fv2rWGmg8pOBA4ulJTJNvT0T+ob/H/i40Qd04= +-----END CERTIFICATE----- diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.key b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.key new file mode 100644 index 0000000000000..601c665a48ce6 --- /dev/null +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/resources/ssl/transport.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,DAC0DDB93011ABD08161118074F353A7 + +hPjzr8y4t3omv6jItFSxF/UeirrdlMhFoxxsw+E5fl4hRjD2J6LuUpOl0XBuvrCO +2NN9Simlkfo57l2O8tZ3xwKU037x9qP2O3wo0FZ4OuRcLbXZtp5kIV30/wdo0kbp +GV+18PtGfReo75rszs/VAm9Hg1URqVw0La2r7DomYQB9FJY8N8mwSdSvF194kjGO +pBxiuzzECUwXEGuMRzmc1Cddbw7NsIdg43FRd1uoC4dqj9yBonYEYe5P8WgopL4N +obTi6PzH+kqDSCaJo7Fdr9CYo37f2YsSbtHmuEZP58J/aSB9nl5wdAmas3/dohrI +5GSM9zp+UocFuV6Uf+X9TTJMt97BlRgFdPODh88pTKGLVQyKeBPQbVjgwl9mttxO +i+c/dej/jHt0gwlt8cvZw0Ss50YdNnWtck91yYpXE7iz59CTY+QI24DEvsaP4bkR +QYdIhJHOYamGW0ttCSU8bw1h9RubIvSa+BoiuB+1TaCYU+azuaAYnFlyuR31z4rD +yniPMnb0+5uOkU/srwb4MxVVw/0iYkKAGTEwdLPKhyheuDU9ixkNOQ/k12zV0R7d +gzMFQOlrB4v8Y4LrsNPnAz/uCTvKgBrOS8p076qeGkSX+JIZVNHYyzLnSy7p6hjO +eD3tDx/SA1HaiLzD1VqujnYb6wshYjQGkSPSY3COq8dQgpCqMAlkOycUQO1SbuNt +HZFv9X0w2z5HjPJXtKLLXMLeluNNRQD+IVhvbZjIM1cAUQNqL3OQPGa7W5RYoFYK +rDffzQAzukD5dt6jH+uu3cwnEeJiW8hxZ0+DHJR1X5EJWpN544yTl8jgSPT8MPAU +kxq7OyE0F/JY+UWP1hPILimDrf3Ov8KRtTDGsSvV3IcX+92QKMcvnK21QBZqZjSs +zcmjp2jN1MLwieJyZ3un0MUT9kOyG/5vGoAJe9O/KDtv6rrhKQN5JHi5yKw0Uwi9 +CwrwwkxbRLSBbWugZGXyBHkR/RGIuEEysLKRFej2q4WBZrPOzZlgyvgBbd6/4Eg5 +twngo6JTmYALwVJNW/drok1H0JelanZ6jM/JjNRFWMZnS5o+4cwQURB5O7TIKHdV +7zEkaw82Ng0Nq8dPrU8N9G3LTmIC1E4t++L+D18C2lV0ZDd0Svh3NIA65FXSRvX9 +2g9GQGfGGbgydmjv9j5lx6VdhuTgYKRL4b5bS7VnH+F9m864g/MtSQpXQPR5B54g +YHFGiKCAzruZt1MmJ5m8Jvpg84i2lIZkGImwAstV7xVkmQoC3i77awmcQP6s7rJd +Lo7RKEysVPDbzdnZnWGK0PWJwtgsqrAvVcK7ghygi+vSQkDF0L7qunchOKa00oZR +LZa08b5BWuXeqw4lXZDQDT7hk3NUyW3H7Z1uxUlt1kvcGb6zrInW6Bin0hqsODvj +0drMOZp/5NTDSwcEzkW+LgjfKZw8Szmhlt3v+luNFr3KzbnFtEvewD1OVikNGzm9 +sfZ899zNkWfvNJaXL3bvzbTn9d8T15YKCwO9RqPpYKDqXBaC4+OjbNsy4AW/JHPr +H/i3D3rhMXR/CALhp4+Knq4o3vMA+3TsUeZ3lOTogobVloWfixIIiRXfaqT4LmEC +-----END RSA PRIVATE KEY----- From 33e60d516e046cf8b3583088fa416386e44a7745 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 14 Dec 2023 14:51:06 +0100 Subject: [PATCH 34/54] [Connectors API] Add get connector sync job docs (#103425) Add docs for the get connector sync job API endpoint. --- .../connector/apis/connector-apis.asciidoc | 2 + .../apis/get-connector-sync-job-api.asciidoc | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 docs/reference/connector/apis/get-connector-sync-job-api.asciidoc diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index bc90f5bd4fc17..3095d3a1520de 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -38,6 +38,7 @@ Use the following APIs to manage sync jobs: * <> * < +* <> * <> include::create-connector-api.asciidoc[] @@ -45,5 +46,6 @@ include::create-connector-sync-job-api.asciidoc[] include::delete-connector-api.asciidoc[] include::delete-connector-sync-job-api.asciidoc[] include::get-connector-api.asciidoc[] +include::get-connector-sync-job-api.asciidoc[] include::list-connectors-api.asciidoc[] include::list-connector-sync-jobs-api.asciidoc[] diff --git a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..b33aec8c55e60 --- /dev/null +++ b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc @@ -0,0 +1,44 @@ +[[get-connector-sync-job-api]] +=== Get connector sync job API +preview::[] +++++ +Get connector sync job +++++ + +Retrieves the details about a connector sync job. + +[[get-connector-sync-job-api-request]] +==== {api-request-title} + +`GET _connector/_sync_job/` + +[[get-connector-sync-job-api-prereq]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. + +[[get-connector-sync-job-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[get-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`400`:: +The `connector_sync_job_id` was not provided. + +`404` (Missing resources):: +No connector sync job matching `connector_sync_job_id` could be found. + +[[get-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example gets the connector sync job `my-connector-sync-job`: + +[source,console] +---- +GET _connector/_sync_job/my-connector-sync-job +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] From 2bec7dc55f0741d72b353f68bfe2fd26319afdd3 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 Dec 2023 14:02:11 +0000 Subject: [PATCH 35/54] Reduce mutable state in BlobStoreCacheMaintenanceService (#103383) Today the `PeriodicMaintenanceTask` includes several mutable fields whose values determine what stage of the computation is currently active, so that the task can re-invoke itself to run each stage. This is pretty confusing and doesn't play very nicely with refcounting the search responses. With this commit we rework the task to avoid these mutable fields and clarify how it executes. Relates #103245 --- .../BlobStoreCacheMaintenanceService.java | 463 ++++++++---------- 1 file changed, 208 insertions(+), 255 deletions(-) diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java index 89cab65765bf9..1713be9feac65 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheMaintenanceService.java @@ -10,10 +10,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.search.ClosePointInTimeRequest; @@ -25,6 +25,7 @@ import org.elasticsearch.action.search.TransportClosePointInTimeAction; import org.elasticsearch.action.search.TransportOpenPointInTimeAction; import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; @@ -37,16 +38,16 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Releasable; -import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; @@ -73,9 +74,8 @@ import java.util.Objects; import java.util.Queue; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -88,10 +88,10 @@ /** * A service that delete documents in the snapshot blob cache index when they are not required anymore. - * + *

* This service runs on the data node that contains the snapshot blob cache primary shard. It listens to cluster state updates to find * searchable snapshot indices that are deleted and checks if the index snapshot is still used by other searchable snapshot indices. If the - * index snapshot is not used anymore then i triggers the deletion of corresponding cached blobs in the snapshot blob cache index using a + * index snapshot is not used anymore then it triggers the deletion of corresponding cached blobs in the snapshot blob cache index using a * delete-by-query. */ public class BlobStoreCacheMaintenanceService implements ClusterStateListener { @@ -266,6 +266,10 @@ private static boolean hasSearchableSnapshotWith(final ClusterState state, final return false; } + private static Instant getExpirationTime(TimeValue retention, ThreadPool threadPool) { + return Instant.ofEpochMilli(threadPool.absoluteTimeInMillis()).minus(retention.duration(), retention.timeUnit().toChronoUnit()); + } + private static Map> listSearchableSnapshots(final ClusterState state) { Map> snapshots = null; for (IndexMetadata indexMetadata : state.metadata()) { @@ -396,315 +400,264 @@ public void onFailure(Exception e) { /** * A maintenance task that periodically cleans up unused cache entries from the blob store cache index. - * + *

* This task first opens a point-in-time context on the blob store cache system index and uses it to search all documents. For each * document found the task verifies if it belongs to an existing searchable snapshot index. If the doc does not belong to any * index then it is deleted as part of a bulk request. Once the bulk is executed the next batch of documents is searched for. Once * all documents from the PIT have been verified the task closes the PIT and completes itself. - * + *

* The task executes every step (PIT opening, searches, bulk deletes, PIT closing) using the generic thread pool. * The same task instance is used for all the steps and makes sure that a closed instance is not executed again. */ - private class PeriodicMaintenanceTask implements Runnable, Releasable { - + private class PeriodicMaintenanceTask implements Runnable { private final TimeValue keepAlive; private final int batchSize; - private final AtomicReference error = new AtomicReference<>(); - private final AtomicBoolean closed = new AtomicBoolean(); + private final ThrottledTaskRunner taskRunner; private final AtomicLong deletes = new AtomicLong(); private final AtomicLong total = new AtomicLong(); - private volatile Map> existingSnapshots; - private volatile Set existingRepositories; - private final AtomicReference searchResponse = new AtomicReference<>(); - private volatile Instant expirationTime; - private volatile String pointIntTimeId; - private volatile Object[] searchAfter; - PeriodicMaintenanceTask(TimeValue keepAlive, int batchSize) { this.keepAlive = keepAlive; this.batchSize = batchSize; + this.taskRunner = new ThrottledTaskRunner(this.getClass().getCanonicalName(), 2, threadPool.generic()); } @Override public void run() { - assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC); - try { - ensureOpen(); - if (pointIntTimeId == null) { - final OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest(SNAPSHOT_BLOB_CACHE_INDEX); - openRequest.keepAlive(keepAlive); - clientWithOrigin.execute(TransportOpenPointInTimeAction.TYPE, openRequest, new ActionListener<>() { - @Override - public void onResponse(OpenPointInTimeResponse response) { - logger.trace("periodic maintenance task initialized with point-in-time id [{}]", response.getPointInTimeId()); - PeriodicMaintenanceTask.this.pointIntTimeId = response.getPointInTimeId(); - executeNext(PeriodicMaintenanceTask.this); - } - - @Override - public void onFailure(Exception e) { - if (TransportActions.isShardNotAvailableException(e)) { - complete(null); - } else { - complete(e); - } - } - }); - return; + ActionListener.run(ActionListener.runAfter(new ActionListener() { + @Override + public void onResponse(Void unused) { + logger.info( + () -> format( + "periodic maintenance task completed (%s deleted documents out of a total of %s)", + deletes.get(), + total.get() + ) + ); } - final String pitId = pointIntTimeId; - assert Strings.hasLength(pitId); - - SearchResponse searchResponseRef; - do { - searchResponseRef = searchResponse.get(); - if (searchResponseRef == null) { - handleMissingSearchResponse(pitId); - return; + @Override + public void onFailure(Exception e) { + logger.warn( + () -> format( + "periodic maintenance task completed with failure (%s deleted documents out of a total of %s)", + deletes.get(), + total.get() + ), + e + ); + } + }, BlobStoreCacheMaintenanceService.this::startPeriodicTask), listener -> { + final OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest(SNAPSHOT_BLOB_CACHE_INDEX); + openRequest.keepAlive(keepAlive); + clientWithOrigin.execute(TransportOpenPointInTimeAction.TYPE, openRequest, new ActionListener<>() { + @Override + public void onResponse(OpenPointInTimeResponse response) { + logger.trace("periodic maintenance task initialized with point-in-time id [{}]", response.getPointInTimeId()); + threadPool.generic().execute(ActionRunnable.wrap(listener, l -> { + final ClusterState state = clusterService.state(); + new RunningPeriodicMaintenanceTask( + response.getPointInTimeId(), + closingPitBefore(clientWithOrigin, response.getPointInTimeId(), l), + getExpirationTime(periodicTaskRetention, threadPool), + // compute the list of existing searchable snapshots and repositories up-front + listSearchableSnapshots(state), + RepositoriesMetadata.get(state) + .repositories() + .stream() + .map(RepositoryMetadata::name) + .collect(Collectors.toSet()) + ).run(); + })); } - } while (searchResponseRef.tryIncRef() == false); - try { - var searchHits = searchResponseRef.getHits().getHits(); - if (searchHits != null && searchHits.length > 0) { - updateWithSearchHits(searchHits); - return; + + @Override + public void onFailure(Exception e) { + if (TransportActions.isShardNotAvailableException(e)) { + listener.onResponse(null); + } else { + listener.onFailure(e); + } } - } finally { - searchResponseRef.decRef(); + }); + }); + } + + private static ActionListener closingPitBefore(Client client, String pointInTimeId, ActionListener listener) { + return new ActionListener<>() { + @Override + public void onResponse(Void unused) { + closePit(client, pointInTimeId, () -> listener.onResponse(null)); } - // we're done, complete the task - complete(null); - } catch (Exception e) { - complete(e); - } + + @Override + public void onFailure(Exception e) { + closePit(client, pointInTimeId, () -> listener.onFailure(e)); + } + }; } - private void handleMissingSearchResponse(String pitId) { - final SearchSourceBuilder searchSource = new SearchSourceBuilder(); - searchSource.fetchField(new FieldAndFormat(CachedBlob.CREATION_TIME_FIELD, "epoch_millis")); - searchSource.fetchSource(false); - searchSource.trackScores(false); - searchSource.sort(ShardDocSortField.NAME); - searchSource.size(batchSize); - if (searchAfter != null) { - searchSource.searchAfter(searchAfter); - searchSource.trackTotalHits(false); - } else { - searchSource.trackTotalHits(true); - } - final PointInTimeBuilder pointInTime = new PointInTimeBuilder(pitId); - pointInTime.setKeepAlive(keepAlive); - searchSource.pointInTimeBuilder(pointInTime); - final SearchRequest searchRequest = new SearchRequest(); - searchRequest.source(searchSource); - clientWithOrigin.execute(TransportSearchAction.TYPE, searchRequest, new ActionListener<>() { + private static void closePit(Client client, String pointInTimeId, Runnable onCompletion) { + client.execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(pointInTimeId), new ActionListener<>() { @Override - public void onResponse(SearchResponse response) { - if (searchAfter == null) { - assert PeriodicMaintenanceTask.this.total.get() == 0L; - PeriodicMaintenanceTask.this.total.set(response.getHits().getTotalHits().value); + public void onResponse(ClosePointInTimeResponse response) { + if (response.isSucceeded()) { + logger.debug("periodic maintenance task successfully closed point-in-time id [{}]", pointInTimeId); + } else { + logger.debug("point-in-time id [{}] not found", pointInTimeId); } - PeriodicMaintenanceTask.this.setCurrentResponse(response); - PeriodicMaintenanceTask.this.searchAfter = null; - executeNext(PeriodicMaintenanceTask.this); + onCompletion.run(); } @Override public void onFailure(Exception e) { - complete(e); + logger.warn(() -> "failed to close point-in-time id [" + pointInTimeId + "]", e); + onCompletion.run(); } }); } - private void updateWithSearchHits(SearchHit[] searchHits) { - if (expirationTime == null) { - final TimeValue retention = periodicTaskRetention; - expirationTime = Instant.ofEpochMilli(threadPool.absoluteTimeInMillis()) - .minus(retention.duration(), retention.timeUnit().toChronoUnit()); - - final ClusterState state = clusterService.state(); - // compute the list of existing searchable snapshots and repositories once - existingSnapshots = listSearchableSnapshots(state); - existingRepositories = RepositoriesMetadata.get(state) - .repositories() - .stream() - .map(RepositoryMetadata::name) - .collect(Collectors.toSet()); + /** + * The maintenance task, once it has opened its PIT and started running so that it has all the state it needs to do its job. + */ + private class RunningPeriodicMaintenanceTask implements Runnable { + private final String pointInTimeId; + private final RefCountingListener listeners; + private final Instant expirationTime; + private final Map> existingSnapshots; + private final Set existingRepositories; + + RunningPeriodicMaintenanceTask( + String pointInTimeId, + ActionListener listener, + Instant expirationTime, + Map> existingSnapshots, + Set existingRepositories + ) { + this.pointInTimeId = pointInTimeId; + this.listeners = new RefCountingListener(listener); + this.expirationTime = expirationTime; + this.existingSnapshots = existingSnapshots; + this.existingRepositories = existingRepositories; } - final BulkRequest bulkRequest = new BulkRequest(); - final Map> knownSnapshots = existingSnapshots; - assert knownSnapshots != null; - final Set knownRepositories = existingRepositories; - assert knownRepositories != null; - final Instant expirationTimeCopy = this.expirationTime; - assert expirationTimeCopy != null; - - Object[] lastSortValues = null; - for (SearchHit searchHit : searchHits) { - lastSortValues = searchHit.getSortValues(); - assert searchHit.getId() != null; - try { - boolean delete = false; - - // See {@link BlobStoreCacheService#generateId} - // doc id = {repository name}/{snapshot id}/{snapshot index id}/{shard id}/{file name}/@{file offset} - final String[] parts = Objects.requireNonNull(searchHit.getId()).split("/"); - assert parts.length == 6 : Arrays.toString(parts) + " vs " + searchHit.getId(); - - final String repositoryName = parts[0]; - if (knownRepositories.contains(repositoryName) == false) { - logger.trace("deleting blob store cache entry with id [{}]: repository does not exist", searchHit.getId()); - delete = true; - } else { - final Set knownIndexIds = knownSnapshots.get(parts[1]); - if (knownIndexIds == null || knownIndexIds.contains(parts[2]) == false) { - logger.trace("deleting blob store cache entry with id [{}]: not used", searchHit.getId()); - delete = true; - } - } - if (delete) { - final Instant creationTime = getCreationTime(searchHit); - if (creationTime.isAfter(expirationTimeCopy)) { - logger.trace("blob store cache entry with id [{}] was created recently, skipping deletion", searchHit.getId()); - continue; - } - bulkRequest.add(new DeleteRequest().index(searchHit.getIndex()).id(searchHit.getId())); - } - } catch (Exception e) { - logger.warn(() -> format("exception when parsing blob store cache entry with id [%s], skipping", searchHit.getId()), e); + @Override + public void run() { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC); + try (listeners) { + executeSearch(new SearchRequest().source(getSearchSourceBuilder().trackTotalHits(true)), (searchResponse, refs) -> { + assert total.get() == 0L; + total.set(searchResponse.getHits().getTotalHits().value); + handleSearchResponse(searchResponse, refs); + }); } } - assert lastSortValues != null; - if (bulkRequest.numberOfActions() == 0) { - setCurrentResponse(null); - this.searchAfter = lastSortValues; - executeNext(this); - return; + private void executeSearch(SearchRequest searchRequest, BiConsumer responseConsumer) { + clientWithOrigin.execute(TransportSearchAction.TYPE, searchRequest, listeners.acquire(searchResponse -> { + searchResponse.mustIncRef(); + taskRunner.enqueueTask(ActionListener.runAfter(listeners.acquire(ref -> { + final var refs = AbstractRefCounted.of(ref::close); + try { + responseConsumer.accept(searchResponse, refs); + } finally { + refs.decRef(); + } + }), searchResponse::decRef)); + })); + } + + private SearchSourceBuilder getSearchSourceBuilder() { + return new SearchSourceBuilder().fetchField(new FieldAndFormat(CachedBlob.CREATION_TIME_FIELD, "epoch_millis")) + .fetchSource(false) + .trackScores(false) + .sort(ShardDocSortField.NAME) + .size(batchSize) + .pointInTimeBuilder(new PointInTimeBuilder(pointInTimeId).setKeepAlive(keepAlive)); } - final Object[] finalSearchAfter = lastSortValues; - clientWithOrigin.execute(BulkAction.INSTANCE, bulkRequest, new ActionListener<>() { - @Override - public void onResponse(BulkResponse response) { - for (BulkItemResponse itemResponse : response.getItems()) { - if (itemResponse.isFailed() == false) { - assert itemResponse.getResponse() instanceof DeleteResponse; - deletes.incrementAndGet(); - } - } - PeriodicMaintenanceTask.this.setCurrentResponse(null); - PeriodicMaintenanceTask.this.searchAfter = finalSearchAfter; - executeNext(PeriodicMaintenanceTask.this); + private void handleSearchResponse(SearchResponse searchResponse, RefCounted refs) { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC); + + if (listeners.isFailing()) { + return; } - @Override - public void onFailure(Exception e) { - complete(e); + final var searchHits = searchResponse.getHits().getHits(); + if (searchHits == null || searchHits.length == 0) { + return; } - }); - } - public boolean isClosed() { - return closed.get(); - } + final BulkRequest bulkRequest = new BulkRequest(); - private void ensureOpen() { - if (isClosed()) { - assert false : "should not use periodic task after close"; - throw new IllegalStateException("Periodic maintenance task is closed"); - } - } + Object[] lastSortValues = null; + for (SearchHit searchHit : searchHits) { + lastSortValues = searchHit.getSortValues(); + assert searchHit.getId() != null; + try { + boolean delete = false; - @Override - public void close() { - if (closed.compareAndSet(false, true)) { - setCurrentResponse(null); - final Exception e = error.get(); - if (e != null) { - logger.warn( - () -> format( - "periodic maintenance task completed with failure (%s deleted documents out of a total of %s)", - deletes.get(), - total.get() - ), - e - ); - } else { - logger.info( - () -> format( - "periodic maintenance task completed (%s deleted documents out of a total of %s)", - deletes.get(), - total.get() - ) - ); - } - } - } + // See {@link BlobStoreCacheService#generateId} + // doc id = {repository name}/{snapshot id}/{snapshot index id}/{shard id}/{file name}/@{file offset} + final String[] parts = Objects.requireNonNull(searchHit.getId()).split("/"); + assert parts.length == 6 : Arrays.toString(parts) + " vs " + searchHit.getId(); - private void complete(@Nullable Exception failure) { - assert isClosed() == false; - final Releasable releasable = () -> { - try { - final Exception previous = error.getAndSet(failure); - assert previous == null : "periodic maintenance task already failed: " + previous; - close(); - } finally { - startPeriodicTask(); + final String repositoryName = parts[0]; + if (existingRepositories.contains(repositoryName) == false) { + logger.trace("deleting blob store cache entry with id [{}]: repository does not exist", searchHit.getId()); + delete = true; + } else { + final Set knownIndexIds = existingSnapshots.get(parts[1]); + if (knownIndexIds == null || knownIndexIds.contains(parts[2]) == false) { + logger.trace("deleting blob store cache entry with id [{}]: not used", searchHit.getId()); + delete = true; + } + } + if (delete) { + final Instant creationTime = getCreationTime(searchHit); + if (creationTime.isAfter(expirationTime)) { + logger.trace( + "blob store cache entry with id [{}] was created recently, skipping deletion", + searchHit.getId() + ); + continue; + } + bulkRequest.add(new DeleteRequest().index(searchHit.getIndex()).id(searchHit.getId())); + } + } catch (Exception e) { + logger.warn( + () -> format("exception when parsing blob store cache entry with id [%s], skipping", searchHit.getId()), + e + ); + } } - }; - boolean waitForRelease = false; - try { - final String pitId = pointIntTimeId; - if (Strings.hasLength(pitId)) { - final ClosePointInTimeRequest closeRequest = new ClosePointInTimeRequest(pitId); + + if (bulkRequest.numberOfActions() > 0) { + refs.mustIncRef(); clientWithOrigin.execute( - TransportClosePointInTimeAction.TYPE, - closeRequest, - ActionListener.runAfter(new ActionListener<>() { - @Override - public void onResponse(ClosePointInTimeResponse response) { - if (response.isSucceeded()) { - logger.debug("periodic maintenance task successfully closed point-in-time id [{}]", pitId); - } else { - logger.debug("point-in-time id [{}] not found", pitId); + BulkAction.INSTANCE, + bulkRequest, + ActionListener.releaseAfter(listeners.acquire(bulkResponse -> { + for (BulkItemResponse itemResponse : bulkResponse.getItems()) { + if (itemResponse.isFailed() == false) { + assert itemResponse.getResponse() instanceof DeleteResponse; + deletes.incrementAndGet(); } } - - @Override - public void onFailure(Exception e) { - logger.warn(() -> "failed to close point-in-time id [" + pitId + "]", e); - } - }, () -> Releasables.close(releasable)) + }), refs::decRef) ); - waitForRelease = true; - } - } finally { - if (waitForRelease == false) { - Releasables.close(releasable); } - } - } - private void setCurrentResponse(SearchResponse response) { - if (response != null) { - response.mustIncRef(); - } - var previous = searchResponse.getAndSet(response); - if (previous != null) { - previous.decRef(); + assert lastSortValues != null; + executeSearch( + new SearchRequest().source(getSearchSourceBuilder().trackTotalHits(false).searchAfter(lastSortValues)), + this::handleSearchResponse + ); } } } - private void executeNext(PeriodicMaintenanceTask maintenanceTask) { - threadPool.generic().execute(maintenanceTask); - } - private static Instant getCreationTime(SearchHit searchHit) { final DocumentField creationTimeField = searchHit.field(CachedBlob.CREATION_TIME_FIELD); assert creationTimeField != null; From 46667db0b42773b54148b19dd05199f8ed3744ab Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 14 Dec 2023 06:27:25 -0800 Subject: [PATCH 36/54] Add IndexVersion constant for 8.13 (#103096) This commit adds a dummy IndexVersion constant for 8.13. It is needed for CCR tests which expect a different index version between current and previous releases. --- server/src/main/java/org/elasticsearch/index/IndexVersions.java | 1 + .../java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 125f9529c4165..f4edb4f79d760 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -92,6 +92,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion ES_VERSION_8_12 = def(8_500_004, Version.LUCENE_9_8_0); public static final IndexVersion NORMALIZED_VECTOR_COSINE = def(8_500_005, Version.LUCENE_9_8_0); public static final IndexVersion UPGRADE_LUCENE_9_9 = def(8_500_006, Version.LUCENE_9_9_0); + public static final IndexVersion ES_VERSION_8_13 = def(8_500_007, Version.LUCENE_9_9_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java index 703b9e608db17..27250dd4e3367 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java @@ -206,7 +206,6 @@ public void testAutoFollowing() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103094") public void testCannotFollowLeaderInUpgradedCluster() throws Exception { if (upgradeState != UpgradeState.ALL) { return; From b53f9d45b28998e774d7d7f34e5abe961e498aa1 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 14 Dec 2023 15:28:13 +0100 Subject: [PATCH 37/54] [Connectors API] Add cancel connector sync job docs (#103446) Add docs for the cancel connector sync job API endpoint. --- .../cancel-connector-sync-job-api.asciidoc | 50 +++++++++++++++++++ .../connector/apis/connector-apis.asciidoc | 3 ++ 2 files changed, 53 insertions(+) create mode 100644 docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc diff --git a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..6123b7eb5511d --- /dev/null +++ b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc @@ -0,0 +1,50 @@ +[[cancel-connector-sync-job-api]] +=== Cancel connector sync job API +++++ +Cancel connector sync job +++++ + +Cancels a connector sync job. + +[[cancel-connector-sync-job-api-request]] +==== {api-request-title} +`PUT _connector/_sync_job//_cancel` + +[[cancel-connector-sync-job-api-prereqs]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_sync_job_id` parameter should reference an existing connector sync job. + +[[cancel-connector-sync-job-api-desc]] +==== {api-description-title} + +Cancels a connector sync job, which sets the `status` to `cancelling` and updates `cancellation_requested_at` to the current time. +The connector service is then responsible for setting the `status` of connector sync jobs to `cancelled`. + +[[cancel-connector-sync-job-api-path-params]] +==== {api-path-parms-title} + +`connector_sync_job_id`:: +(Required, string) + +[[cancel-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Connector sync job cancellation was successfully requested. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[cancel-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example cancels the connector sync job with ID `my-connector-sync-job-id`: + +[source,console] +---- +PUT _connector/_sync_job/my-connector-sync-job-id/_cancel +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] + diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index 3095d3a1520de..fb3ccd4982405 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -36,11 +36,14 @@ You can use these APIs to create, cancel, delete and update sync jobs. Use the following APIs to manage sync jobs: +* <> * <> * < * <> * <> + +include::cancel-connector-sync-job-api.asciidoc[] include::create-connector-api.asciidoc[] include::create-connector-sync-job-api.asciidoc[] include::delete-connector-api.asciidoc[] From 47dc5b67ce792de133974c2b48222147c3e39503 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Thu, 14 Dec 2023 15:39:39 +0100 Subject: [PATCH 38/54] Fix disk computation when initializing new shards (#102879) Currently elasticsearch checks if there is enough space to initialize each shard individually. This makes it possible to initialize 2 shards on the node that have enough space for only one of them. This change takes into account all shards initialized within a given round of `BalancedShardsAllocator.Balancer#allocateUnassigned` in order to prevent that. --- docs/changelog/102879.yaml | 5 + .../decider/DiskThresholdDeciderIT.java | 175 +++++++++++++----- .../routing/ExpectedShardSizeEstimator.java | 29 ++- .../allocation/DiskThresholdMonitor.java | 2 + .../decider/DiskThresholdDecider.java | 41 ++-- .../ExpectedShardSizeEstimatorTests.java | 63 ++++++- .../DesiredBalanceComputerTests.java | 154 ++++++++++++++- .../DiskThresholdDeciderUnitTests.java | 124 +++++-------- .../cluster/routing/TestShardRouting.java | 12 +- 9 files changed, 443 insertions(+), 162 deletions(-) create mode 100644 docs/changelog/102879.yaml diff --git a/docs/changelog/102879.yaml b/docs/changelog/102879.yaml new file mode 100644 index 0000000000000..b35d36dd0a3a9 --- /dev/null +++ b/docs/changelog/102879.yaml @@ -0,0 +1,5 @@ +pr: 102879 +summary: Fix disk computation when initializing new shards +area: Allocation +type: bug +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index b1ac5b02f7dd2..c044fafe31efc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; -import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.DiskUsageIntegTestCase; @@ -39,13 +38,16 @@ import org.hamcrest.TypeSafeMatcher; import java.util.Arrays; +import java.util.Comparator; import java.util.HashSet; -import java.util.Locale; +import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; -import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; +import static org.elasticsearch.cluster.routing.RoutingNodesHelper.numberOfShardsWithState; +import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; import static org.elasticsearch.index.store.Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; @@ -74,26 +76,25 @@ public void testHighWatermarkNotExceeded() throws Exception { final String dataNodeName = internalCluster().startDataOnlyNode(); ensureStableCluster(3); - final InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance( - ClusterInfoService.class - ); - internalCluster().getCurrentMasterNodeInstance(ClusterService.class) - .addListener(event -> ClusterInfoServiceUtils.refresh(clusterInfoService)); + final InternalClusterInfoService clusterInfoService = getInternalClusterInfoService(); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> { + ClusterInfoServiceUtils.refresh(clusterInfoService); + }); final String dataNode0Id = internalCluster().getInstance(NodeEnvironment.class, dataNodeName).nodeId(); - final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String indexName = randomIdentifier(); createIndex(indexName, indexSettings(6, 0).put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms").build()); - var smallestShard = createReasonableSizedShards(indexName); + var shardSizes = createReasonableSizedShards(indexName); // reduce disk size of node 0 so that no shards fit below the high watermark, forcing all shards onto the other data node // (subtract the translog size since the disk threshold decider ignores this and may therefore move the shard back again) - getTestFileStore(dataNodeName).setTotalSpace(smallestShard.size + WATERMARK_BYTES - 1L); + getTestFileStore(dataNodeName).setTotalSpace(shardSizes.getSmallestShardSize() + WATERMARK_BYTES - 1L); assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, empty()); // increase disk size of node 0 to allow just enough room for one shard, and check that it's rebalanced back - getTestFileStore(dataNodeName).setTotalSpace(smallestShard.size + WATERMARK_BYTES); - assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(smallestShard.shardIds)); + getTestFileStore(dataNodeName).setTotalSpace(shardSizes.getSmallestShardSize() + WATERMARK_BYTES); + assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(shardSizes.getSmallestShardIds())); } public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Exception { @@ -108,17 +109,20 @@ public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Excepti .setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean())) ); - final InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance( - ClusterInfoService.class - ); - internalCluster().getCurrentMasterNodeInstance(ClusterService.class) - .addListener(event -> ClusterInfoServiceUtils.refresh(clusterInfoService)); + final AtomicBoolean allowRelocations = new AtomicBoolean(true); + final InternalClusterInfoService clusterInfoService = getInternalClusterInfoService(); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> { + ClusterInfoServiceUtils.refresh(clusterInfoService); + if (allowRelocations.get() == false) { + assertThat(numberOfShardsWithState(event.state().getRoutingNodes(), ShardRoutingState.RELOCATING), equalTo(0)); + } + }); final String dataNode0Id = internalCluster().getInstance(NodeEnvironment.class, dataNodeName).nodeId(); - final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String indexName = randomIdentifier(); createIndex(indexName, indexSettings(6, 0).put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms").build()); - var smallestShard = createReasonableSizedShards(indexName); + var shardSizes = createReasonableSizedShards(indexName); final CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("repo", "snap") .setWaitForCompletion(true) @@ -128,15 +132,13 @@ public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Excepti assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); assertAcked(indicesAdmin().prepareDelete(indexName).get()); + updateClusterSettings(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString())); + allowRelocations.set(false); // reduce disk size of node 0 so that no shards fit below the low watermark, forcing shards to be assigned to the other data node - getTestFileStore(dataNodeName).setTotalSpace(smallestShard.size + WATERMARK_BYTES - 1L); + getTestFileStore(dataNodeName).setTotalSpace(shardSizes.getSmallestShardSize() + WATERMARK_BYTES - 1L); refreshDiskUsage(); - updateClusterSettings( - Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString()) - ); - final RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("repo", "snap") .setWaitForCompletion(true) .get(); @@ -144,13 +146,71 @@ public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Excepti assertThat(restoreInfo.successfulShards(), is(snapshotInfo.totalShards())); assertThat(restoreInfo.failedShards(), is(0)); - assertBusy(() -> assertThat(getShardIds(dataNode0Id, indexName), empty())); + assertThat(getShardIds(dataNode0Id, indexName), empty()); - updateClusterSettings(Settings.builder().putNull(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey())); + allowRelocations.set(true); + updateClusterSettings(Settings.builder().putNull(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey())); // increase disk size of node 0 to allow just enough room for one shard, and check that it's rebalanced back - getTestFileStore(dataNodeName).setTotalSpace(smallestShard.size + WATERMARK_BYTES); - assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(smallestShard.shardIds)); + getTestFileStore(dataNodeName).setTotalSpace(shardSizes.getSmallestShardSize() + WATERMARK_BYTES); + assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, new ContainsExactlyOneOf<>(shardSizes.getSmallestShardIds())); + } + + public void testRestoreSnapshotAllocationDoesNotExceedWatermarkWithMultipleShards() throws Exception { + internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String dataNodeName = internalCluster().startDataOnlyNode(); + ensureStableCluster(3); + + assertAcked( + clusterAdmin().preparePutRepository("repo") + .setType(FsRepository.TYPE) + .setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean())) + ); + + final AtomicBoolean allowRelocations = new AtomicBoolean(true); + final InternalClusterInfoService clusterInfoService = getInternalClusterInfoService(); + internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> { + ClusterInfoServiceUtils.refresh(clusterInfoService); + if (allowRelocations.get() == false) { + assertThat(numberOfShardsWithState(event.state().getRoutingNodes(), ShardRoutingState.RELOCATING), equalTo(0)); + } + }); + + final String dataNode0Id = internalCluster().getInstance(NodeEnvironment.class, dataNodeName).nodeId(); + + final String indexName = randomIdentifier(); + createIndex(indexName, indexSettings(6, 0).put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms").build()); + var shardSizes = createReasonableSizedShards(indexName); + + final CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("repo", "snap") + .setWaitForCompletion(true) + .get(); + final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(snapshotInfo.successfulShards(), is(snapshotInfo.totalShards())); + assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); + + assertAcked(indicesAdmin().prepareDelete(indexName).get()); + updateClusterSettings(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString())); + allowRelocations.set(false); + + // reduce disk size of node 0 so that only 1 of 2 smallest shards can be allocated + var usableSpace = shardSizes.sizes().get(1).size(); + getTestFileStore(dataNodeName).setTotalSpace(usableSpace + WATERMARK_BYTES + 1L); + refreshDiskUsage(); + + final RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("repo", "snap") + .setWaitForCompletion(true) + .get(); + final RestoreInfo restoreInfo = restoreSnapshotResponse.getRestoreInfo(); + assertThat(restoreInfo.successfulShards(), is(snapshotInfo.totalShards())); + assertThat(restoreInfo.failedShards(), is(0)); + + assertBusyWithDiskUsageRefresh( + dataNode0Id, + indexName, + new ContainsExactlyOneOf<>(shardSizes.getShardIdsWithSizeSmallerOrEqual(usableSpace)) + ); } private Set getShardIds(final String nodeId, final String indexName) { @@ -178,13 +238,9 @@ private Set getShardIds(final String nodeId, final String indexName) { /** * Index documents until all the shards are at least WATERMARK_BYTES in size, and return the one with the smallest size */ - private SmallestShards createReasonableSizedShards(final String indexName) throws InterruptedException { + private ShardSizes createReasonableSizedShards(final String indexName) throws InterruptedException { while (true) { - final IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[scaledRandomIntBetween(100, 10000)]; - for (int i = 0; i < indexRequestBuilders.length; i++) { - indexRequestBuilders[i] = prepareIndex(indexName).setSource("field", randomAlphaOfLength(10)); - } - indexRandom(true, indexRequestBuilders); + indexRandom(true, indexName, scaledRandomIntBetween(100, 10000)); forceMerge(); refresh(); @@ -201,23 +257,36 @@ private SmallestShards createReasonableSizedShards(final String indexName) throw .orElseThrow(() -> new AssertionError("no shards")); if (smallestShardSize > WATERMARK_BYTES) { - var smallestShardIds = Arrays.stream(shardStates) - .filter(it -> it.getStats().getStore().sizeInBytes() == smallestShardSize) - .map(it -> removeIndexUUID(it.getShardRouting().shardId())) - .collect(toSet()); - - logger.info( - "Created shards with sizes {}", - Arrays.stream(shardStates) - .collect(toMap(it -> it.getShardRouting().shardId(), it -> it.getStats().getStore().sizeInBytes())) - ); - - return new SmallestShards(smallestShardSize, smallestShardIds); + var shardSizes = Arrays.stream(shardStates) + .map(it -> new ShardSize(removeIndexUUID(it.getShardRouting().shardId()), it.getStats().getStore().sizeInBytes())) + .sorted(Comparator.comparing(ShardSize::size)) + .toList(); + logger.info("Created shards with sizes {}", shardSizes); + return new ShardSizes(shardSizes); } } } - private record SmallestShards(long size, Set shardIds) {} + private record ShardSizes(List sizes) { + + public long getSmallestShardSize() { + return sizes.get(0).size(); + } + + public Set getShardIdsWithSizeSmallerOrEqual(long size) { + return sizes.stream().filter(entry -> entry.size <= size).map(ShardSize::shardId).collect(toSet()); + } + + public Set getSmallestShardIds() { + return getShardIdsWithSizeSmallerOrEqual(getSmallestShardSize()); + } + + public Set getAllShardIds() { + return sizes.stream().map(ShardSize::shardId).collect(toSet()); + } + } + + private record ShardSize(ShardId shardId, long size) {} private static ShardId removeIndexUUID(ShardId shardId) { return ShardId.fromString(shardId.toString()); @@ -246,16 +315,20 @@ private void refreshDiskUsage() { ); } - private void assertBusyWithDiskUsageRefresh(String nodeName, String indexName, Matcher> matcher) throws Exception { + private void assertBusyWithDiskUsageRefresh(String nodeId, String indexName, Matcher> matcher) throws Exception { assertBusy(() -> { // refresh the master's ClusterInfoService before checking the assigned shards because DiskThresholdMonitor might still // be processing a previous ClusterInfo update and will skip the new one (see DiskThresholdMonitor#onNewInfo(ClusterInfo) // and its internal checkInProgress flag) refreshDiskUsage(); - final Set shardRoutings = getShardIds(nodeName, indexName); + final Set shardRoutings = getShardIds(nodeId, indexName); assertThat("Mismatching shard routings: " + shardRoutings, shardRoutings, matcher); - }, 30L, TimeUnit.SECONDS); + }, 5L, TimeUnit.SECONDS); + } + + private InternalClusterInfoService getInternalClusterInfoService() { + return (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance(ClusterInfoService.class); } private static final class ContainsExactlyOneOf extends TypeSafeMatcher> { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java b/server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java index 85e201d52f03b..1f364e1ace6e4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java @@ -20,9 +20,13 @@ public class ExpectedShardSizeEstimator { - public static long getExpectedShardSize(ShardRouting shardRouting, long defaultSize, RoutingAllocation allocation) { + public static boolean shouldReserveSpaceForInitializingShard(ShardRouting shard, RoutingAllocation allocation) { + return shouldReserveSpaceForInitializingShard(shard, allocation.metadata()); + } + + public static long getExpectedShardSize(ShardRouting shard, long defaultSize, RoutingAllocation allocation) { return getExpectedShardSize( - shardRouting, + shard, defaultSize, allocation.clusterInfo(), allocation.snapshotShardSizeInfo(), @@ -31,6 +35,27 @@ public static long getExpectedShardSize(ShardRouting shardRouting, long defaultS ); } + public static boolean shouldReserveSpaceForInitializingShard(ShardRouting shard, Metadata metadata) { + assert shard.initializing() : "Expected initializing shard, got: " + shard; + return switch (shard.recoverySource().getType()) { + // No need to reserve disk space when initializing a new empty shard + case EMPTY_STORE -> false; + + // No need to reserve disk space if the shard is already allocated on the disk. Starting it is not going to use more. + case EXISTING_STORE -> false; + + // Peer recovery require downloading all segments locally to start the shard. Reserve disk space for this + case PEER -> true; + + // Snapshot restore (unless it is partial) require downloading all segments locally from the blobstore to start the shard. + case SNAPSHOT -> metadata.getIndexSafe(shard.index()).isPartialSearchableSnapshot() == false; + + // shrink/split/clone operation is going to clone existing locally placed shards using file system hard links + // so no additional space is going to be used until future merges + case LOCAL_SHARDS -> false; + }; + } + /** * Returns the expected shard size for the given shard or the default value provided if not enough information are available * to estimate the shards size. diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java index 5c216b9a5b308..6645fd7d0e895 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.Releasable; import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import java.util.ArrayList; import java.util.Collections; @@ -425,6 +426,7 @@ long sizeOfRelocatingShards(RoutingNode routingNode, DiskUsage diskUsage, Cluste true, diskUsage.getPath(), info, + SnapshotShardSizeInfo.EMPTY, reroutedClusterState.metadata(), reroutedClusterState.routingTable(), 0L diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index e92a6106a6e33..2fa1994f9f74b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -32,6 +32,7 @@ import java.util.Map; import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.getExpectedShardSize; +import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.shouldReserveSpaceForInitializingShard; /** * The {@link DiskThresholdDecider} checks that the node a shard is potentially @@ -117,6 +118,7 @@ public static long sizeOfUnaccountedShards( boolean subtractShardsMovingAway, String dataPath, ClusterInfo clusterInfo, + SnapshotShardSizeInfo snapshotShardSizeInfo, Metadata metadata, RoutingTable routingTable, long sizeOfUnaccountableSearchableSnapshotShards @@ -129,28 +131,18 @@ public static long sizeOfUnaccountedShards( // Where reserved space is unavailable (e.g. stats are out-of-sync) compute a conservative estimate for initialising shards for (ShardRouting routing : node.initializing()) { - if (routing.relocatingNodeId() == null && metadata.getIndexSafe(routing.index()).isSearchableSnapshot() == false) { - // in practice the only initializing-but-not-relocating non-searchable-snapshot shards with a nonzero expected shard size - // will be ones created - // by a resize (shrink/split/clone) operation which we expect to happen using hard links, so they shouldn't be taking - // any additional space and can be ignored here - continue; - } - if (reservedSpace.containsShardId(routing.shardId())) { - continue; - } - final String actualPath = clusterInfo.getDataPath(routing); - // if we don't yet know the actual path of the incoming shard then conservatively assume it's going to the path with the least - // free space - if (actualPath == null || actualPath.equals(dataPath)) { - totalSize += getExpectedShardSize( - routing, - Math.max(routing.getExpectedShardSize(), 0L), - clusterInfo, - SnapshotShardSizeInfo.EMPTY, - metadata, - routingTable - ); + // Space needs to be reserved only when initializing shards that are going to use additional space + // that is not yet accounted for by `reservedSpace` in case of lengthy recoveries + if (shouldReserveSpaceForInitializingShard(routing, metadata) && reservedSpace.containsShardId(routing.shardId()) == false) { + final String actualPath = clusterInfo.getDataPath(routing); + // if we don't yet know the actual path of the incoming shard then conservatively assume + // it's going to the path with the least free space + if (actualPath == null || actualPath.equals(dataPath)) { + totalSize += Math.max( + routing.getExpectedShardSize(), + getExpectedShardSize(routing, 0L, clusterInfo, snapshotShardSizeInfo, metadata, routingTable) + ); + } } } @@ -159,7 +151,7 @@ public static long sizeOfUnaccountedShards( if (subtractShardsMovingAway) { for (ShardRouting routing : node.relocating()) { if (dataPath.equals(clusterInfo.getDataPath(routing))) { - totalSize -= getExpectedShardSize(routing, 0L, clusterInfo, SnapshotShardSizeInfo.EMPTY, metadata, routingTable); + totalSize -= getExpectedShardSize(routing, 0L, clusterInfo, snapshotShardSizeInfo, metadata, routingTable); } } } @@ -204,6 +196,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing false, usage.getPath(), allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), allocation.metadata(), allocation.routingTable(), allocation.unaccountedSearchableSnapshotSize(node) @@ -412,6 +405,7 @@ public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting true, usage.getPath(), allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), allocation.metadata(), allocation.routingTable(), allocation.unaccountedSearchableSnapshotSize(node) @@ -491,6 +485,7 @@ private static DiskUsageWithRelocations getDiskUsage( subtractLeavingShards, usage.getPath(), allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), allocation.metadata(), allocation.routingTable(), allocation.unaccountedSearchableSnapshotSize(node) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java index 62fd21defa676..f81d99c55e84e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimatorTests.java @@ -30,33 +30,54 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_RESIZE_SOURCE_NAME_KEY; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_RESIZE_SOURCE_UUID_KEY; import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.getExpectedShardSize; +import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.shouldReserveSpaceForInitializingShard; import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting; +import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; +import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE; +import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING; import static org.hamcrest.Matchers.equalTo; public class ExpectedShardSizeEstimatorTests extends ESAllocationTestCase { private final long defaultValue = randomLongBetween(-1, 0); - public void testShouldFallbackToDefaultValue() { + public void testShouldFallbackToDefaultExpectedShardSize() { var state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata(index("my-index"))).build(); - var shard = newShardRouting("my-index", 0, randomIdentifier(), true, ShardRoutingState.INITIALIZING); + var shard = newShardRouting( + new ShardId("my-index", "_na_", 0), + randomIdentifier(), + true, + ShardRoutingState.INITIALIZING, + randomFrom(RecoverySource.EmptyStoreRecoverySource.INSTANCE, RecoverySource.ExistingStoreRecoverySource.INSTANCE) + ); var allocation = createRoutingAllocation(state, ClusterInfo.EMPTY, SnapshotShardSizeInfo.EMPTY); assertThat(getExpectedShardSize(shard, defaultValue, allocation), equalTo(defaultValue)); + assertFalse( + "Should NOT reserve space for locally initializing primaries", + shouldReserveSpaceForInitializingShard(shard, allocation) + ); } public void testShouldReadExpectedSizeFromClusterInfo() { var shardSize = randomLongBetween(100, 1000); var state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata(index("my-index"))).build(); - var shard = newShardRouting("my-index", 0, randomIdentifier(), true, ShardRoutingState.INITIALIZING); + var shard = newShardRouting( + new ShardId("my-index", "_na_", 0), + randomIdentifier(), + true, + ShardRoutingState.INITIALIZING, + RecoverySource.PeerRecoverySource.INSTANCE + ); var clusterInfo = createClusterInfo(shard, shardSize); var allocation = createRoutingAllocation(state, clusterInfo, SnapshotShardSizeInfo.EMPTY); assertThat(getExpectedShardSize(shard, defaultValue, allocation), equalTo(shardSize)); + assertTrue("Should reserve space for relocating shard", shouldReserveSpaceForInitializingShard(shard, allocation)); } public void testShouldReadExpectedSizeFromPrimaryWhenAddingNewReplica() { @@ -70,21 +91,39 @@ public void testShouldReadExpectedSizeFromPrimaryWhenAddingNewReplica() { var allocation = createRoutingAllocation(state, clusterInfo, SnapshotShardSizeInfo.EMPTY); assertThat(getExpectedShardSize(replica, defaultValue, allocation), equalTo(shardSize)); + assertTrue("Should reserve space for peer recovery", shouldReserveSpaceForInitializingShard(replica, allocation)); } public void testShouldReadExpectedSizeWhenInitializingFromSnapshot() { var snapshotShardSize = randomLongBetween(100, 1000); - var state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata(index("my-index"))).build(); + + var index = switch (randomIntBetween(0, 2)) { + // regular snapshot + case 0 -> index("my-index"); + // searchable snapshot + case 1 -> index("my-index").settings( + indexSettings(IndexVersion.current(), 1, 0) // + .put(INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE) // + ); + // partial searchable snapshot + case 2 -> index("my-index").settings( + indexSettings(IndexVersion.current(), 1, 0) // + .put(INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE) // + .put(SNAPSHOT_PARTIAL_SETTING.getKey(), true) // + ); + default -> throw new AssertionError("unexpected index type"); + }; + var state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata(index)).build(); var snapshot = new Snapshot("repository", new SnapshotId("snapshot-1", "na")); var indexId = new IndexId("my-index", "_na_"); var shard = newShardRouting( new ShardId("my-index", "_na_", 0), - null, + randomIdentifier(), true, - ShardRoutingState.UNASSIGNED, + ShardRoutingState.INITIALIZING, new RecoverySource.SnapshotRecoverySource(randomUUID(), snapshot, IndexVersion.current(), indexId) ); @@ -94,6 +133,14 @@ public void testShouldReadExpectedSizeWhenInitializingFromSnapshot() { var allocation = createRoutingAllocation(state, ClusterInfo.EMPTY, snapshotShardSizeInfo); assertThat(getExpectedShardSize(shard, defaultValue, allocation), equalTo(snapshotShardSize)); + if (state.metadata().index("my-index").isPartialSearchableSnapshot() == false) { + assertTrue("Should reserve space for snapshot restore", shouldReserveSpaceForInitializingShard(shard, allocation)); + } else { + assertFalse( + "Should NOT reserve space for partial searchable snapshot restore as they do not download all data during initialization", + shouldReserveSpaceForInitializingShard(shard, allocation) + ); + } } public void testShouldReadSizeFromClonedShard() { @@ -127,6 +174,10 @@ public void testShouldReadSizeFromClonedShard() { var allocation = createRoutingAllocation(state, clusterInfo, SnapshotShardSizeInfo.EMPTY); assertThat(getExpectedShardSize(target, defaultValue, allocation), equalTo(sourceShardSize)); + assertFalse( + "Should NOT reserve space when using fs hardlink for clone/shrink/split", + shouldReserveSpaceForInitializingShard(target, state.metadata()) + ); } private static RoutingAllocation createRoutingAllocation( diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java index b4eba769543b8..5e3b6cd02f830 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.DiskUsage; import org.elasticsearch.cluster.ESAllocationTestCase; +import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.TestShardRoutingRoleStrategies; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; @@ -40,9 +41,14 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.snapshots.InternalSnapshotsInfoService.SnapshotShard; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import org.elasticsearch.test.MockLogAppender; import org.elasticsearch.threadpool.ThreadPool; @@ -61,6 +67,7 @@ import java.util.function.Function; import static java.util.stream.Collectors.toMap; +import static org.elasticsearch.cluster.ClusterInfo.shardIdentifierFromRouting; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; @@ -70,6 +77,7 @@ import static org.elasticsearch.test.MockLogAppender.assertThatLogger; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.hasEntry; @@ -623,7 +631,7 @@ public void testDesiredBalanceShouldConvergeInABigCluster() { var thisShardSize = smallShardSizeDeviation(shardSize); var primaryNodeId = pickAndRemoveRandomValueFrom(remainingNodeIds); - shardSizes.put(ClusterInfo.shardIdentifierFromRouting(shardId, true), thisShardSize); + shardSizes.put(shardIdentifierFromRouting(shardId, true), thisShardSize); totalShardsSize += thisShardSize; if (primaryNodeId != null) { dataPath.put(new NodeAndShard(primaryNodeId, shardId), "/data"); @@ -642,7 +650,7 @@ public void testDesiredBalanceShouldConvergeInABigCluster() { ); for (int replica = 0; replica < replicas; replica++) { var replicaNodeId = primaryNodeId == null ? null : pickAndRemoveRandomValueFrom(remainingNodeIds); - shardSizes.put(ClusterInfo.shardIdentifierFromRouting(shardId, false), thisShardSize); + shardSizes.put(shardIdentifierFromRouting(shardId, false), thisShardSize); totalShardsSize += thisShardSize; if (replicaNodeId != null) { dataPath.put(new NodeAndShard(replicaNodeId, shardId), "/data"); @@ -862,6 +870,146 @@ private static ClusterInfo createClusterInfo(List diskUsages, MapnewHashMapWithExpectedSize(5); + var snapshotShardSizes = Maps.newHashMapWithExpectedSize(5); + + var routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY); + // index-1 is allocated according to the desired balance + var indexMetadata1 = IndexMetadata.builder("index-1").settings(indexSettings(IndexVersion.current(), 2, 0)).build(); + routingTableBuilder.add( + IndexRoutingTable.builder(indexMetadata1.getIndex()) + .addShard(newShardRouting(shardIdFrom(indexMetadata1, 0), "node-1", true, STARTED)) + .addShard(newShardRouting(shardIdFrom(indexMetadata1, 1), "node-2", true, STARTED)) + ); + shardSizeInfo.put(shardIdentifierFromRouting(shardIdFrom(indexMetadata1, 0), true), ByteSizeValue.ofGb(8).getBytes()); + shardSizeInfo.put(shardIdentifierFromRouting(shardIdFrom(indexMetadata1, 1), true), ByteSizeValue.ofGb(8).getBytes()); + + // index-2 & index-3 are restored as new from snapshot + var indexMetadata2 = IndexMetadata.builder("index-2") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 2)) + .build(); + routingTableBuilder.addAsNewRestore( + indexMetadata2, + new RecoverySource.SnapshotRecoverySource("restore", snapshot, IndexVersion.current(), indexIdFrom(indexMetadata2)), + Set.of() + ); + snapshotShardSizes.put( + new SnapshotShard(snapshot, indexIdFrom(indexMetadata2), shardIdFrom(indexMetadata2, 0)), + ByteSizeValue.ofGb(1).getBytes() + ); + + var indexMetadata3 = IndexMetadata.builder("index-3") + .settings(indexSettings(IndexVersion.current(), 2, 0).put(IndexMetadata.INDEX_PRIORITY_SETTING.getKey(), 1)) + .build(); + routingTableBuilder.addAsNewRestore( + indexMetadata3, + new RecoverySource.SnapshotRecoverySource("restore", snapshot, IndexVersion.current(), indexIdFrom(indexMetadata3)), + Set.of() + ); + snapshotShardSizes.put( + new SnapshotShard(snapshot, indexIdFrom(indexMetadata3), shardIdFrom(indexMetadata3, 0)), + ByteSizeValue.ofMb(512).getBytes() + ); + snapshotShardSizes.put( + new SnapshotShard(snapshot, indexIdFrom(indexMetadata3), shardIdFrom(indexMetadata3, 1)), + ByteSizeValue.ofMb(512).getBytes() + ); + + var clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(newNode("node-1")).add(newNode("node-2"))) + .metadata(Metadata.builder().put(indexMetadata1, false).put(indexMetadata2, false).put(indexMetadata3, false).build()) + .routingTable(routingTableBuilder) + .customs( + Map.of( + RestoreInProgress.TYPE, + new RestoreInProgress.Builder().add( + new RestoreInProgress.Entry( + "restore", + snapshot, + RestoreInProgress.State.STARTED, + randomBoolean(), + List.of(indexMetadata2.getIndex().getName(), indexMetadata3.getIndex().getName()), + Map.ofEntries( + Map.entry(shardIdFrom(indexMetadata2, 0), new RestoreInProgress.ShardRestoreStatus(randomUUID())), + Map.entry(shardIdFrom(indexMetadata3, 0), new RestoreInProgress.ShardRestoreStatus(randomUUID())), + Map.entry(shardIdFrom(indexMetadata3, 1), new RestoreInProgress.ShardRestoreStatus(randomUUID())) + ) + ) + ).build() + ) + ) + .build(); + + var clusterInfo = createClusterInfo( + List.of( + // node-1 has enough space to only allocate the only [index-2] shard + new DiskUsage("node-1", "data-1", "/data", ByteSizeValue.ofGb(10).getBytes(), ByteSizeValue.ofGb(2).getBytes()), + // node-2 has enough space to only allocate both shards of [index-3] + new DiskUsage("node-2", "data-2", "/data", ByteSizeValue.ofGb(10).getBytes(), ByteSizeValue.ofGb(2).getBytes()) + ), + shardSizeInfo + ); + var snapshotShardSizeInfo = new SnapshotShardSizeInfo(snapshotShardSizes); + + var settings = Settings.EMPTY; + var allocation = new RoutingAllocation( + randomAllocationDeciders(settings, createBuiltInClusterSettings(settings)), + clusterState, + clusterInfo, + snapshotShardSizeInfo, + 0L + ); + var initialDesiredBalance = new DesiredBalance( + 1, + Map.ofEntries( + Map.entry(shardIdFrom(indexMetadata1, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata1, 1), new ShardAssignment(Set.of("node-2"), 1, 0, 0)) + ) + ); + var nextDesiredBalance = createDesiredBalanceComputer(new BalancedShardsAllocator()).compute( + initialDesiredBalance, + new DesiredBalanceInput(2, allocation, List.of()), + queue(), + input -> true + ); + + assertThat( + nextDesiredBalance.assignments(), + anyOf( + equalTo( + Map.ofEntries( + Map.entry(shardIdFrom(indexMetadata1, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata1, 1), new ShardAssignment(Set.of("node-2"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata2, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata3, 0), new ShardAssignment(Set.of("node-2"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata3, 1), new ShardAssignment(Set.of("node-2"), 1, 0, 0)) + ) + ), + equalTo( + Map.ofEntries( + Map.entry(shardIdFrom(indexMetadata1, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata1, 1), new ShardAssignment(Set.of("node-2"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata2, 0), new ShardAssignment(Set.of("node-2"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata3, 0), new ShardAssignment(Set.of("node-1"), 1, 0, 0)), + Map.entry(shardIdFrom(indexMetadata3, 1), new ShardAssignment(Set.of("node-1"), 1, 0, 0)) + ) + ) + ) + ); + } + + private static IndexId indexIdFrom(IndexMetadata indexMetadata) { + return new IndexId(indexMetadata.getIndex().getName(), indexMetadata.getIndex().getUUID()); + } + + private static ShardId shardIdFrom(IndexMetadata indexMetadata, int shardId) { + return new ShardId(indexMetadata.getIndex(), shardId); + } + public void testShouldLogComputationIteration() { checkIterationLogging( 999, @@ -943,7 +1091,7 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing } private static Map.Entry indexSize(ClusterState clusterState, String name, long size, boolean primary) { - return Map.entry(ClusterInfo.shardIdentifierFromRouting(findShardId(clusterState, name), primary), size); + return Map.entry(shardIdentifierFromRouting(findShardId(clusterState, name), primary), size); } private static ShardId findShardId(ClusterState clusterState, String name) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index b54480cdc0856..74d8bc62ff203 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -43,15 +43,15 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import static java.util.Collections.emptySet; +import static org.elasticsearch.cluster.ClusterInfo.shardIdentifierFromRouting; import static org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator.getExpectedShardSize; +import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE; import static org.hamcrest.Matchers.containsString; @@ -512,93 +512,64 @@ public void testShardSizeAndRelocatingSize() { } public void testTakesIntoAccountExpectedSizeForInitializingSearchableSnapshots() { - String mainIndexName = "test"; - Index index = new Index(mainIndexName, "1234"); - String anotherIndexName = "another_index"; - Index anotherIndex = new Index(anotherIndexName, "5678"); - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder(mainIndexName) - .settings( - settings(IndexVersion.current()).put("index.uuid", "1234") - .put(INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE) - ) - .numberOfShards(3) - .numberOfReplicas(1) - ) - .put( - IndexMetadata.builder(anotherIndexName) - .settings(settings(IndexVersion.current()).put("index.uuid", "5678")) - .numberOfShards(1) - .numberOfReplicas(1) - ) + + var searchableSnapshotIndex = IndexMetadata.builder("searchable_snapshot") + .settings(indexSettings(IndexVersion.current(), 3, 0).put(INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE)) .build(); + var regularIndex = IndexMetadata.builder("regular_index").settings(indexSettings(IndexVersion.current(), 1, 0)).build(); + String nodeId = "node1"; - String anotherNodeId = "another_node"; - - List shards = new ArrayList<>(); - int anotherNodeShardCounter = 0; - int nodeShardCounter = 0; - Map initializingShardSizes = new HashMap<>(); - for (int i = 1; i <= 3; i++) { - int expectedSize = 10 * i; - shards.add(createShard(index, nodeId, nodeShardCounter++, expectedSize)); - if (randomBoolean()) { - ShardRouting initializingShard = ShardRoutingHelper.initialize( - ShardRouting.newUnassigned( - new ShardId(index, nodeShardCounter++), - true, - EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"), - ShardRouting.Role.DEFAULT - ), - nodeId - ); - initializingShardSizes.put(ClusterInfo.shardIdentifierFromRouting(initializingShard), randomLongBetween(10, 50)); - shards.add(initializingShard); - } - // randomly add shards for non-searchable snapshot index - if (randomBoolean()) { - for (int j = 0; j < randomIntBetween(1, 5); j++) { - shards.add(createShard(anotherIndex, anotherNodeId, anotherNodeShardCounter++, expectedSize)); - } - } + var knownShardSizes = new HashMap(); + long unaccountedSearchableSnapshotSizes = 0; + long relocatingShardsSizes = 0; + + var searchableSnapshotIndexRoutingTableBuilder = IndexRoutingTable.builder(searchableSnapshotIndex.getIndex()); + for (int i = 0; i < searchableSnapshotIndex.getNumberOfShards(); i++) { + long expectedSize = randomLongBetween(10, 50); + // a searchable snapshot shard without corresponding entry in cluster info + ShardRouting startedShardWithExpectedSize = newShardRouting( + new ShardId(searchableSnapshotIndex.getIndex(), i), + nodeId, + true, + ShardRoutingState.STARTED, + expectedSize + ); + searchableSnapshotIndexRoutingTableBuilder.addShard(startedShardWithExpectedSize); + unaccountedSearchableSnapshotSizes += expectedSize; + } + var regularIndexRoutingTableBuilder = IndexRoutingTable.builder(regularIndex.getIndex()); + for (int i = 0; i < searchableSnapshotIndex.getNumberOfShards(); i++) { + var shardSize = randomLongBetween(10, 50); + // a shard relocating to this node + ShardRouting initializingShard = newShardRouting( + new ShardId(regularIndex.getIndex(), i), + nodeId, + true, + ShardRoutingState.INITIALIZING, + PeerRecoverySource.INSTANCE + ); + regularIndexRoutingTableBuilder.addShard(initializingShard); + knownShardSizes.put(shardIdentifierFromRouting(initializingShard), shardSize); + relocatingShardsSizes += shardSize; } - DiscoveryNode node = DiscoveryNodeUtils.builder(nodeId).roles(emptySet()).build(); - DiscoveryNode anotherNode = DiscoveryNodeUtils.builder(anotherNodeId).roles(emptySet()).build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) - .metadata(metadata) - .routingTable( - RoutingTable.builder() - .add( - shards.stream() - .filter(s -> s.getIndexName().equals(mainIndexName)) - .reduce(IndexRoutingTable.builder(index), IndexRoutingTable.Builder::addShard, (a, b) -> a) - ) - .add( - shards.stream() - .filter(s -> s.getIndexName().equals(anotherIndexName)) - .reduce(IndexRoutingTable.builder(anotherIndex), IndexRoutingTable.Builder::addShard, (a, b) -> a) - ) - .build() - ) - .nodes(DiscoveryNodes.builder().add(node).add(anotherNode).build()) + .metadata(Metadata.builder().put(searchableSnapshotIndex, false).put(regularIndex, false)) + .routingTable(RoutingTable.builder().add(searchableSnapshotIndexRoutingTableBuilder).add(regularIndexRoutingTableBuilder)) + .nodes(DiscoveryNodes.builder().add(newNode(nodeId)).build()) .build(); + RoutingAllocation allocation = new RoutingAllocation( null, clusterState, - new DevNullClusterInfo(Map.of(), Map.of(), initializingShardSizes), + new DevNullClusterInfo(Map.of(), Map.of(), knownShardSizes), null, 0 ); - long sizeOfUnaccountedShards = sizeOfUnaccountedShards( - allocation, - RoutingNodesHelper.routingNode(nodeId, node, shards.toArray(ShardRouting[]::new)), - false, - "/dev/null" + assertEquals( + unaccountedSearchableSnapshotSizes + relocatingShardsSizes, + sizeOfUnaccountedShards(allocation, clusterState.getRoutingNodes().node(nodeId), false, "/dev/null") ); - assertEquals(60L + initializingShardSizes.values().stream().mapToLong(Long::longValue).sum(), sizeOfUnaccountedShards); } private ShardRouting createShard(Index index, String nodeId, int i, int expectedSize) { @@ -620,6 +591,7 @@ public long sizeOfUnaccountedShards(RoutingAllocation allocation, RoutingNode no subtractShardsMovingAway, dataPath, allocation.clusterInfo(), + allocation.snapshotShardSizeInfo(), allocation.metadata(), allocation.routingTable(), allocation.unaccountedSearchableSnapshotSize(node) diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java index 89c8546d6b7d2..1158e805ba3c1 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -36,6 +36,16 @@ public static ShardRouting newShardRouting(String index, int shardId, String cur } public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, ShardRoutingState state) { + return newShardRouting(shardId, currentNodeId, primary, state, -1); + } + + public static ShardRouting newShardRouting( + ShardId shardId, + String currentNodeId, + boolean primary, + ShardRoutingState state, + long expectedShardSize + ) { assertNotEquals(ShardRoutingState.RELOCATING, state); return new ShardRouting( shardId, @@ -47,7 +57,7 @@ public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId buildUnassignedInfo(state), buildRelocationFailureInfo(state), buildAllocationId(state), - -1, + expectedShardSize, ShardRouting.Role.DEFAULT ); } From 95c9c944a12e940517d2c232b342b685b0a00c24 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 14 Dec 2023 06:41:33 -0800 Subject: [PATCH 39/54] Cache component versions (#103408) Looking up component versions through SPI should not change. This commit captures the component versions of the running node once during startup, rather than every time node info is called. closes #102103 --- docs/changelog/103408.yaml | 6 ++++++ .../src/main/java/org/elasticsearch/node/NodeService.java | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/103408.yaml diff --git a/docs/changelog/103408.yaml b/docs/changelog/103408.yaml new file mode 100644 index 0000000000000..bf5081b854f08 --- /dev/null +++ b/docs/changelog/103408.yaml @@ -0,0 +1,6 @@ +pr: 103408 +summary: Cache component versions +area: Infra/Core +type: bug +issues: + - 102103 diff --git a/server/src/main/java/org/elasticsearch/node/NodeService.java b/server/src/main/java/org/elasticsearch/node/NodeService.java index e2283ea9851d7..4b9e5dc83c538 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeService.java +++ b/server/src/main/java/org/elasticsearch/node/NodeService.java @@ -62,6 +62,7 @@ public class NodeService implements Closeable { private final AggregationUsageService aggregationUsageService; private final Coordinator coordinator; private final RepositoriesService repositoriesService; + private final Map componentVersions; NodeService( Settings settings, @@ -100,6 +101,7 @@ public class NodeService implements Closeable { this.indexingPressure = indexingPressure; this.aggregationUsageService = aggregationUsageService; this.repositoriesService = repositoriesService; + this.componentVersions = findComponentVersions(pluginService); clusterService.addStateApplier(ingestService); } @@ -122,7 +124,7 @@ public NodeInfo info( Version.CURRENT.toString(), TransportVersion.current(), IndexVersion.current(), - findComponentVersions(), + componentVersions, Build.current(), transportService.getLocalNode(), settings ? settingsFilter.filter(this.settings) : null, @@ -140,7 +142,7 @@ public NodeInfo info( ); } - private Map findComponentVersions() { + private static Map findComponentVersions(PluginsService pluginService) { var versions = pluginService.loadServiceProviders(ComponentVersionNumber.class) .stream() .collect(Collectors.toUnmodifiableMap(ComponentVersionNumber::componentId, cvn -> cvn.versionNumber().id())); From e2e9eec44c7762db051ee4c748af4eebf5aa086f Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Thu, 14 Dec 2023 16:03:45 +0100 Subject: [PATCH 40/54] [Connectors API] Fix bug with missing TEXT DisplayType enum (#103430) --- docs/changelog/103430.yaml | 5 +++++ .../connector/configuration/ConfigurationDisplayType.java | 1 + 2 files changed, 6 insertions(+) create mode 100644 docs/changelog/103430.yaml diff --git a/docs/changelog/103430.yaml b/docs/changelog/103430.yaml new file mode 100644 index 0000000000000..cd2444270849d --- /dev/null +++ b/docs/changelog/103430.yaml @@ -0,0 +1,5 @@ +pr: 103430 +summary: "[Connectors API] Fix bug with missing TEXT `DisplayType` enum" +area: Application +type: bug +issues: [] diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/configuration/ConfigurationDisplayType.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/configuration/ConfigurationDisplayType.java index d6b3d83d705b9..df8dee04d61b9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/configuration/ConfigurationDisplayType.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/configuration/ConfigurationDisplayType.java @@ -10,6 +10,7 @@ import java.util.Locale; public enum ConfigurationDisplayType { + TEXT, TEXTBOX, TEXTAREA, NUMERIC, From 58a133ed78870fd8fb20871225f1b12647bafb06 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 14 Dec 2023 16:10:18 +0100 Subject: [PATCH 41/54] [Connectors API] Add check in connector sync job docs (#103426) Add docs for the check in connector sync job API endpoint. --- .../check-in-connector-sync-job-api.asciidoc | 48 +++++++++++++++++++ .../connector/apis/connector-apis.asciidoc | 3 ++ 2 files changed, 51 insertions(+) create mode 100644 docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc diff --git a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc new file mode 100644 index 0000000000000..04c8057e2c115 --- /dev/null +++ b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc @@ -0,0 +1,48 @@ +[[check-in-connector-sync-job-api]] +=== Check in connector sync job API +++++ +Check in connector sync job +++++ + +Checks in a connector sync job (updates `last_seen` to the current time). + +[[check-in-connector-sync-job-api-request]] +==== {api-request-title} +`PUT _connector/_sync_job//_check_in/` + +[[check-in-connector-sync-job-api-prereqs]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_sync_job_id` parameter should reference an existing connector sync job. + +[[check-in-connector-sync-job-api-desc]] +==== {api-description-title} + +Checks in a connector sync job and sets `last_seen` to the time right before updating it in the internal index. + +[[check-in-connector-sync-job-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[[check-in-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Connector sync job was successfully checked in. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[check-in-connector-sync-job-api-example]] +==== {api-examples-title} + +The following example checks in the connector sync job `my-connector-sync-job`: + +[source,console] +---- +PUT _connector/_sync_job/my-connector-sync-job/_check_in +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index fb3ccd4982405..0380682677340 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -36,7 +36,9 @@ You can use these APIs to create, cancel, delete and update sync jobs. Use the following APIs to manage sync jobs: + * <> +* <> * <> * < * <> @@ -44,6 +46,7 @@ Use the following APIs to manage sync jobs: include::cancel-connector-sync-job-api.asciidoc[] +include::check-in-connector-sync-job-api.asciidoc[] include::create-connector-api.asciidoc[] include::create-connector-sync-job-api.asciidoc[] include::delete-connector-api.asciidoc[] From 029d42b7d8aef612e81499c151f73108fd391925 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Thu, 14 Dec 2023 16:42:37 +0100 Subject: [PATCH 42/54] [Connector API] Fix bug with nullable tooltip field in parser (#103427) --- docs/changelog/103427.yaml | 5 +++ .../335_connector_update_configuration.yml | 35 +++++++++++++++++++ .../connector/ConnectorConfiguration.java | 3 +- .../application/connector/ConnectorTests.java | 25 +++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/103427.yaml diff --git a/docs/changelog/103427.yaml b/docs/changelog/103427.yaml new file mode 100644 index 0000000000000..57a27aa687ab7 --- /dev/null +++ b/docs/changelog/103427.yaml @@ -0,0 +1,5 @@ +pr: 103427 +summary: "[Connector API] Fix bug with nullable tooltip field in parser" +area: Application +type: bug +issues: [] diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml index 260e1784d29e2..5a012853b4bf9 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml @@ -85,6 +85,41 @@ setup: - match: { configuration.some_field.value: 456 } - match: { status: configured } +--- +"Update Connector Configuration with null tooltip": + - do: + connector.update_configuration: + connector_id: test-connector + body: + configuration: + some_field: + default_value: null + depends_on: + - field: some_field + value: 31 + display: numeric + label: Very important field + options: [ ] + order: 4 + required: true + sensitive: false + tooltip: null + type: str + ui_restrictions: [ ] + validations: + - constraint: 0 + type: greater_than + value: 123 + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { configuration.some_field.tooltip: null } + --- "Update Connector Configuration - Connector doesn't exist": - do: diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java index 103c647f180b4..8ed7c417a1af1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java @@ -54,6 +54,7 @@ public class ConnectorConfiguration implements Writeable, ToXContentObject { private final String placeholder; private final boolean required; private final boolean sensitive; + @Nullable private final String tooltip; private final ConfigurationFieldType type; private final List uiRestrictions; @@ -199,7 +200,7 @@ public ConnectorConfiguration(StreamInput in) throws IOException { PARSER.declareString(optionalConstructorArg(), PLACEHOLDER_FIELD); PARSER.declareBoolean(constructorArg(), REQUIRED_FIELD); PARSER.declareBoolean(constructorArg(), SENSITIVE_FIELD); - PARSER.declareStringOrNull(constructorArg(), TOOLTIP_FIELD); + PARSER.declareStringOrNull(optionalConstructorArg(), TOOLTIP_FIELD); PARSER.declareField( constructorArg(), (p, c) -> ConfigurationFieldType.fieldType(p.text()), diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java index 9401a2a58403e..cdfa3dea8a6fa 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java @@ -98,6 +98,31 @@ public void testToXContent() throws IOException { } ], "value":"" + }, + "field_with_null_tooltip":{ + "default_value":null, + "depends_on":[ + { + "field":"some_field", + "value":true + } + ], + "display":"textbox", + "label":"Very important field", + "options":[], + "order":4, + "required":true, + "sensitive":false, + "tooltip":null, + "type":"str", + "ui_restrictions":[], + "validations":[ + { + "constraint":0, + "type":"greater_than" + } + ], + "value":"" } }, "description":"test-connector", From a08219761241f71eee80427d8ed69038ccec8ffb Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Mon, 11 Dec 2023 11:04:23 -0800 Subject: [PATCH 43/54] Add release notes for 8.11.3 release (#103216) --- docs/reference/release-notes.asciidoc | 2 ++ docs/reference/release-notes/8.11.3.asciidoc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 340ef3a5c57c4..068cb3d2f127b 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. * <> * <> +* <> * <> * <> * <> @@ -59,6 +60,7 @@ This section summarizes the changes in each release. include::release-notes/8.13.0.asciidoc[] include::release-notes/8.12.0.asciidoc[] +include::release-notes/8.11.3.asciidoc[] include::release-notes/8.11.2.asciidoc[] include::release-notes/8.11.1.asciidoc[] include::release-notes/8.11.0.asciidoc[] diff --git a/docs/reference/release-notes/8.11.3.asciidoc b/docs/reference/release-notes/8.11.3.asciidoc index 48ab82d0d4391..ddeb50dad1f75 100644 --- a/docs/reference/release-notes/8.11.3.asciidoc +++ b/docs/reference/release-notes/8.11.3.asciidoc @@ -1,6 +1,8 @@ [[release-notes-8.11.3]] == {es} version 8.11.3 +coming[8.11.3] + Also see <>. [[bug-8.11.3]] From 8040b86d0e5ac7fd08e7f07cf00b7c4b6729cfd0 Mon Sep 17 00:00:00 2001 From: Pooya Salehi Date: Thu, 14 Dec 2023 17:13:02 +0100 Subject: [PATCH 44/54] Account for archive memory usage in LVM (#102752) Move memory accounting for VersionLookup into its class to make it easier to get the VersionLookup memory usage before/after aVersionLookup#merge. Relates ES-5921 --- .../index/engine/LiveVersionMap.java | 71 +++++++++++++------ .../index/engine/LiveVersionMapArchive.java | 8 +++ .../index/engine/LiveVersionMapTestUtils.java | 8 ++- .../index/engine/LiveVersionMapTests.java | 55 +++++++++++++- 4 files changed, 117 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java b/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java index ef0901bc17712..1cee2a90ec3f1 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java @@ -64,11 +64,18 @@ public static final class VersionLookup { // Modifies the map of this instance by merging with the given VersionLookup public void merge(VersionLookup versionLookup) { + long existingEntriesSize = 0; + for (var entry : versionLookup.map.entrySet()) { + var existingValue = map.get(entry.getKey()); + existingEntriesSize += existingValue == null ? 0 : mapEntryBytesUsed(entry.getKey(), existingValue); + } map.putAll(versionLookup.map); + adjustRam(versionLookup.ramBytesUsed() - existingEntriesSize); minDeleteTimestamp.accumulateAndGet(versionLookup.minDeleteTimestamp(), Math::min); } - private VersionLookup(Map map) { + // Visible for testing + VersionLookup(Map map) { this.map = map; } @@ -77,7 +84,11 @@ public VersionValue get(BytesRef key) { } VersionValue put(BytesRef key, VersionValue value) { - return map.put(key, value); + long ramAccounting = mapEntryBytesUsed(key, value); + VersionValue previousValue = map.put(key, value); + ramAccounting += previousValue == null ? 0 : -mapEntryBytesUsed(key, previousValue); + adjustRam(ramAccounting); + return previousValue; } public boolean isEmpty() { @@ -96,8 +107,12 @@ void markAsUnsafe() { unsafe = true; } - public VersionValue remove(BytesRef uid) { - return map.remove(uid); + VersionValue remove(BytesRef uid) { + VersionValue previousValue = map.remove(uid); + if (previousValue != null) { + adjustRam(-mapEntryBytesUsed(uid, previousValue)); + } + return previousValue; } public void updateMinDeletedTimestamp(DeleteVersionValue delete) { @@ -107,6 +122,26 @@ public void updateMinDeletedTimestamp(DeleteVersionValue delete) { public long minDeleteTimestamp() { return minDeleteTimestamp.get(); } + + void adjustRam(long value) { + if (value != 0) { + long v = ramBytesUsed.addAndGet(value); + assert v >= 0 : "bytes=" + v; + } + } + + public long ramBytesUsed() { + return ramBytesUsed.get(); + } + + public static long mapEntryBytesUsed(BytesRef key, VersionValue value) { + return (BASE_BYTES_PER_BYTESREF + key.bytes.length) + (BASE_BYTES_PER_CHM_ENTRY + value.ramBytesUsed()); + } + + // Used only for testing + Map getMap() { + return map; + } } private static final class Maps { @@ -170,27 +205,12 @@ Maps invalidateOldMap(LiveVersionMapArchive archive) { } void put(BytesRef uid, VersionValue version) { - long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; - long ramAccounting = BASE_BYTES_PER_CHM_ENTRY + version.ramBytesUsed() + uidRAMBytesUsed; - VersionValue previousValue = current.put(uid, version); - ramAccounting += previousValue == null ? 0 : -(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed); - adjustRam(ramAccounting); - } - - void adjustRam(long value) { - if (value != 0) { - long v = current.ramBytesUsed.addAndGet(value); - assert v >= 0 : "bytes=" + v; - } + current.put(uid, version); } void remove(BytesRef uid, DeleteVersionValue deleted) { - VersionValue previousValue = current.remove(uid); + current.remove(uid); current.updateMinDeletedTimestamp(deleted); - if (previousValue != null) { - long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; - adjustRam(-(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed)); - } if (old != VersionLookup.EMPTY) { // we also need to remove it from the old map here to make sure we don't read this stale value while // we are in the middle of a refresh. Most of the time the old map is an empty map so we can skip it there. @@ -452,7 +472,7 @@ synchronized void clear() { @Override public long ramBytesUsed() { - return maps.ramBytesUsed() + ramBytesUsedTombstones.get(); + return maps.ramBytesUsed() + ramBytesUsedTombstones.get() + ramBytesUsedForArchive(); } /** @@ -463,6 +483,13 @@ long ramBytesUsedForRefresh() { return maps.current.ramBytesUsed.get(); } + /** + * Returns how much RAM would be freed up by cleaning out the LiveVersionMapArchive. + */ + long ramBytesUsedForArchive() { + return archive.getMemoryBytesUsed(); + } + /** * Returns how much RAM is current being freed up by refreshing. This is the RAM usage of the previous version map that needs to stay * around until operations are safely recorded in the Lucene index. diff --git a/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMapArchive.java b/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMapArchive.java index a68a1cea368d4..9ccbf6ac16fed 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMapArchive.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LiveVersionMapArchive.java @@ -39,6 +39,14 @@ default boolean isUnsafe() { return false; } + /** + * Returns how much memory is currently being used by the archive and would be freed up after + * unpromotables are refreshed. + */ + default long getMemoryBytesUsed() { + return 0L; + } + LiveVersionMapArchive NOOP_ARCHIVE = new LiveVersionMapArchive() { @Override public void afterRefresh(LiveVersionMap.VersionLookup old) {} diff --git a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTestUtils.java b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTestUtils.java index 2c1daa09340d7..3185769bdab82 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTestUtils.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTestUtils.java @@ -61,11 +61,11 @@ public static void pruneTombstones(LiveVersionMap map, long maxTimestampToPrune, map.pruneTombstones(maxTimestampToPrune, maxSeqNoToPrune); } - static IndexVersionValue randomIndexVersionValue() { + public static IndexVersionValue randomIndexVersionValue() { return new IndexVersionValue(randomTranslogLocation(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); } - static Translog.Location randomTranslogLocation() { + public static Translog.Location randomTranslogLocation() { if (randomBoolean()) { return null; } else { @@ -93,6 +93,10 @@ public static boolean isSafeAccessRequired(LiveVersionMap map) { return map.isSafeAccessRequired(); } + public static void enforceSafeAccess(LiveVersionMap map) { + map.enforceSafeAccess(); + } + public static LiveVersionMapArchive getArchive(LiveVersionMap map) { return map.getArchive(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java index a9b42ccdef248..5ca7aadc35fa7 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java @@ -13,7 +13,9 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.Constants; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.test.ESTestCase; @@ -23,11 +25,16 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.stream.IntStream; +import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency; +import static org.elasticsearch.core.Tuple.tuple; import static org.elasticsearch.index.engine.LiveVersionMapTestUtils.randomIndexVersionValue; import static org.elasticsearch.index.engine.LiveVersionMapTestUtils.randomTranslogLocation; import static org.hamcrest.Matchers.empty; @@ -36,7 +43,6 @@ import static org.hamcrest.Matchers.nullValue; public class LiveVersionMapTests extends ESTestCase { - public void testRamBytesUsed() throws Exception { LiveVersionMap map = new LiveVersionMap(); for (int i = 0; i < 100000; ++i) { @@ -442,4 +448,51 @@ public void testRandomlyIndexDeleteAndRefresh() throws Exception { } } } + + public void testVersionLookupRamBytesUsed() { + var vl = new LiveVersionMap.VersionLookup(newConcurrentMapWithAggressiveConcurrency()); + assertEquals(0, vl.ramBytesUsed()); + Set existingKeys = new HashSet<>(); + Supplier> randomEntry = () -> { + var key = randomBoolean() || existingKeys.isEmpty() ? uid(randomIdentifier()) : randomFrom(existingKeys); + return tuple(key, randomIndexVersionValue()); + }; + IntStream.range(0, randomIntBetween(10, 100)).forEach(i -> { + switch (randomIntBetween(0, 2)) { + case 0: // put + var entry = randomEntry.get(); + var previousValue = vl.put(entry.v1(), entry.v2()); + if (existingKeys.contains(entry.v1())) { + assertNotNull(previousValue); + } else { + assertNull(previousValue); + existingKeys.add(entry.v1()); + } + break; + case 1: // remove + if (existingKeys.isEmpty() == false) { + var key = randomFrom(existingKeys); + assertNotNull(vl.remove(key)); + existingKeys.remove(key); + } + break; + case 2: // merge + var toMerge = new LiveVersionMap.VersionLookup(ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency()); + IntStream.range(0, randomIntBetween(1, 100)) + .mapToObj(n -> randomEntry.get()) + .forEach(kv -> toMerge.put(kv.v1(), kv.v2())); + vl.merge(toMerge); + existingKeys.addAll(toMerge.getMap().keySet()); + break; + default: + throw new IllegalStateException("branch value unexpected"); + } + }); + long actualRamBytesUsed = vl.getMap() + .entrySet() + .stream() + .mapToLong(entry -> LiveVersionMap.VersionLookup.mapEntryBytesUsed(entry.getKey(), entry.getValue())) + .sum(); + assertEquals(actualRamBytesUsed, vl.ramBytesUsed()); + } } From 7c7bf69ee43c16b90ea84041f492ba724d4350dc Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 Dec 2023 16:44:01 +0000 Subject: [PATCH 45/54] Avoid hot threads API in integ tests (#103431) The hot threads API fans out to all the nodes in the cluster, capturing all threads in each node's JVM. Since internal-cluster integ tests run many nodes in the same JVM, this means that the hot threads API will report the same information for each node. This is fairly pointless and _very_ noisy in the test failure logs. This commit removes this duplication by capturing and logging the hot threads once for the whole test JVM. --- .../transport/netty4/ESLoggingHandlerIT.java | 5 +- .../action/admin/HotThreadsIT.java | 81 +++++++++---------- .../recovery/IndexPrimaryRelocationIT.java | 25 +++--- .../snapshots/SnapshotStressTestsIT.java | 19 ++--- .../NodesHotThreadsRequestBuilder.java | 49 ----------- .../client/internal/ClusterAdminClient.java | 15 ---- .../internal/support/AbstractClient.java | 14 ---- .../cluster/RestNodesHotThreadsAction.java | 3 +- .../elasticsearch/test/ESIntegTestCase.java | 23 +++--- .../elasticsearch/xpack/CcrIntegTestCase.java | 24 +----- .../xpack/ml/integration/DatafeedJobsIT.java | 23 ++---- 11 files changed, 82 insertions(+), 199 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java index 61ef5f1973854..3c869a89cfaa9 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java @@ -11,6 +11,8 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.elasticsearch.ESNetty4IntegTestCase; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; +import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.MockLogAppender; @@ -19,6 +21,7 @@ import org.elasticsearch.transport.TransportLogger; import java.io.IOException; +import java.util.concurrent.TimeUnit; @ESIntegTestCase.ClusterScope(numDataNodes = 2, scope = ESIntegTestCase.Scope.TEST) public class ESLoggingHandlerIT extends ESNetty4IntegTestCase { @@ -84,7 +87,7 @@ public void testLoggingHandler() { appender.addExpectation(writeExpectation); appender.addExpectation(flushExpectation); appender.addExpectation(readExpectation); - clusterAdmin().prepareNodesHotThreads().get(); + client().execute(TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest()).actionGet(10, TimeUnit.SECONDS); appender.assertAllExpectationsMatched(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java index 45906abd29ff8..5fa63aaed0508 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java @@ -11,8 +11,9 @@ import org.apache.lucene.util.Constants; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; +import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.logging.ChunkedLoggingStreamTests; import org.elasticsearch.core.TimeValue; @@ -23,7 +24,7 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; @@ -41,38 +42,26 @@ public class HotThreadsIT extends ESIntegTestCase { - public void testHotThreadsDontFail() throws ExecutionException, InterruptedException { - /** - * This test just checks if nothing crashes or gets stuck etc. - */ + public void testHotThreadsDontFail() throws InterruptedException { + // This test just checks if nothing crashes or gets stuck etc. createIndex("test"); final int iters = scaledRandomIntBetween(2, 20); final AtomicBoolean hasErrors = new AtomicBoolean(false); for (int i = 0; i < iters; i++) { - final String type; - NodesHotThreadsRequestBuilder nodesHotThreadsRequestBuilder = clusterAdmin().prepareNodesHotThreads(); + final NodesHotThreadsRequest request = new NodesHotThreadsRequest(); if (randomBoolean()) { TimeValue timeValue = new TimeValue(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(20, 500)); - nodesHotThreadsRequestBuilder.setInterval(timeValue); + request.interval(timeValue); } if (randomBoolean()) { - nodesHotThreadsRequestBuilder.setThreads(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(1, 500)); + request.threads(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(1, 500)); } - nodesHotThreadsRequestBuilder.setIgnoreIdleThreads(randomBoolean()); + request.ignoreIdleThreads(randomBoolean()); if (randomBoolean()) { - type = switch (randomIntBetween(0, 3)) { - case 3 -> "mem"; - case 2 -> "cpu"; - case 1 -> "wait"; - default -> "block"; - }; - assertThat(type, notNullValue()); - nodesHotThreadsRequestBuilder.setType(HotThreads.ReportType.of(type)); - } else { - type = null; + request.type(HotThreads.ReportType.of(randomFrom("block", "mem", "cpu", "wait"))); } final CountDownLatch latch = new CountDownLatch(1); - nodesHotThreadsRequestBuilder.execute(new ActionListener() { + client().execute(TransportNodesHotThreadsAction.TYPE, request, new ActionListener<>() { @Override public void onResponse(NodesHotThreadsResponse nodeHotThreads) { boolean success = false; @@ -83,7 +72,6 @@ public void onResponse(NodesHotThreadsResponse nodeHotThreads) { assertThat(nodesMap.size(), equalTo(cluster().size())); for (NodeHotThreads ht : nodeHotThreads.getNodes()) { assertNotNull(ht.getHotThreads()); - // logger.info(ht.getHotThreads()); } success = true; } finally { @@ -120,40 +108,39 @@ public void onFailure(Exception e) { 3L ); } - latch.await(); + safeAwait(latch); assertThat(hasErrors.get(), is(false)); } } - public void testIgnoreIdleThreads() throws ExecutionException, InterruptedException { + public void testIgnoreIdleThreads() { assumeTrue("no support for hot_threads on FreeBSD", Constants.FREE_BSD == false); // First time, don't ignore idle threads: - NodesHotThreadsRequestBuilder builder = clusterAdmin().prepareNodesHotThreads(); - builder.setIgnoreIdleThreads(false); - builder.setThreads(Integer.MAX_VALUE); - NodesHotThreadsResponse response = builder.execute().get(); + final NodesHotThreadsResponse firstResponse = client().execute( + TransportNodesHotThreadsAction.TYPE, + new NodesHotThreadsRequest().ignoreIdleThreads(false).threads(Integer.MAX_VALUE) + ).actionGet(10, TimeUnit.SECONDS); final Matcher containsCachedTimeThreadRunMethod = containsString( "org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run" ); int totSizeAll = 0; - for (NodeHotThreads node : response.getNodesMap().values()) { + for (NodeHotThreads node : firstResponse.getNodesMap().values()) { totSizeAll += node.getHotThreads().length(); assertThat(node.getHotThreads(), containsCachedTimeThreadRunMethod); } // Second time, do ignore idle threads: - builder = clusterAdmin().prepareNodesHotThreads(); - builder.setThreads(Integer.MAX_VALUE); - + final var request = new NodesHotThreadsRequest().threads(Integer.MAX_VALUE); // Make sure default is true: - assertEquals(true, builder.request().ignoreIdleThreads()); - response = builder.execute().get(); + assertTrue(request.ignoreIdleThreads()); + final NodesHotThreadsResponse secondResponse = client().execute(TransportNodesHotThreadsAction.TYPE, request) + .actionGet(10, TimeUnit.SECONDS); int totSizeIgnoreIdle = 0; - for (NodeHotThreads node : response.getNodesMap().values()) { + for (NodeHotThreads node : secondResponse.getNodesMap().values()) { totSizeIgnoreIdle += node.getHotThreads().length(); assertThat(node.getHotThreads(), not(containsCachedTimeThreadRunMethod)); } @@ -162,22 +149,26 @@ public void testIgnoreIdleThreads() throws ExecutionException, InterruptedExcept assertThat(totSizeIgnoreIdle, lessThan(totSizeAll)); } - public void testTimestampAndParams() throws ExecutionException, InterruptedException { + public void testTimestampAndParams() { - NodesHotThreadsResponse response = clusterAdmin().prepareNodesHotThreads().execute().get(); + final NodesHotThreadsResponse response = client().execute(TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest()) + .actionGet(10, TimeUnit.SECONDS); if (Constants.FREE_BSD) { for (NodeHotThreads node : response.getNodesMap().values()) { - String result = node.getHotThreads(); - assertTrue(result.indexOf("hot_threads is not supported") != -1); + assertThat(node.getHotThreads(), containsString("hot_threads is not supported")); } } else { for (NodeHotThreads node : response.getNodesMap().values()) { - String result = node.getHotThreads(); - assertTrue(result.indexOf("Hot threads at") != -1); - assertTrue(result.indexOf("interval=500ms") != -1); - assertTrue(result.indexOf("busiestThreads=3") != -1); - assertTrue(result.indexOf("ignoreIdleThreads=true") != -1); + assertThat( + node.getHotThreads(), + allOf( + containsString("Hot threads at"), + containsString("interval=500ms"), + containsString("busiestThreads=3"), + containsString("ignoreIdleThreads=true") + ) + ); } } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java index 0fe5845e9ed32..779072272e59a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java @@ -8,22 +8,23 @@ package org.elasticsearch.indices.recovery; +import org.apache.logging.log4j.Level; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class IndexPrimaryRelocationIT extends ESIntegTestCase { @@ -71,20 +72,14 @@ public void run() { .setWaitForNoRelocatingShards(true) .get(); if (clusterHealthResponse.isTimedOut()) { - final String hotThreads = clusterAdmin().prepareNodesHotThreads() - .setIgnoreIdleThreads(false) - .get() - .getNodes() - .stream() - .map(NodeHotThreads::getHotThreads) - .collect(Collectors.joining("\n")); - final ClusterState clusterState = clusterAdmin().prepareState().get().getState(); - logger.info( - "timed out for waiting for relocation iteration [{}] \ncluster state {} \nhot threads {}", - i, - clusterState, - hotThreads + HotThreads.logLocalHotThreads( + logger, + Level.INFO, + "timed out waiting for relocation iteration [" + i + "]", + ReferenceDocs.LOGGING ); + final ClusterState clusterState = clusterAdmin().prepareState().get().getState(); + logger.info("timed out for waiting for relocation iteration [{}] \ncluster state {}", i, clusterState); finished.set(true); indexingThread.join(); throw new AssertionError("timed out waiting for relocation iteration [" + i + "] "); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java index 7eaa49b27007d..fa49dc26f2259 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.snapshots; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.tests.util.LuceneTestCase; @@ -16,7 +17,6 @@ import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder; @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -46,6 +47,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.repositories.RepositoryCleanupResult; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.test.InternalTestCluster; @@ -371,16 +373,11 @@ private void acquirePermitsAtEnd( "--> current cluster state:\n{}", Strings.toString(clusterAdmin().prepareState().get().getState(), true, true) ); - logger.info( - "--> hot threads:\n{}", - clusterAdmin().prepareNodesHotThreads() - .setThreads(99999) - .setIgnoreIdleThreads(false) - .get() - .getNodes() - .stream() - .map(NodeHotThreads::getHotThreads) - .collect(Collectors.joining("\n")) + HotThreads.logLocalHotThreads( + logger, + Level.INFO, + "hot threads while failing to acquire permit [" + label + "]", + ReferenceDocs.LOGGING ); failedPermitAcquisitions.add(label); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java deleted file mode 100644 index 6593b90fb7f65..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.node.hotthreads; - -import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; -import org.elasticsearch.client.internal.ElasticsearchClient; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.monitor.jvm.HotThreads; - -public class NodesHotThreadsRequestBuilder extends NodesOperationRequestBuilder< - NodesHotThreadsRequest, - NodesHotThreadsResponse, - NodesHotThreadsRequestBuilder> { - - public NodesHotThreadsRequestBuilder(ElasticsearchClient client) { - super(client, TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest()); - } - - public NodesHotThreadsRequestBuilder setThreads(int threads) { - request.threads(threads); - return this; - } - - public NodesHotThreadsRequestBuilder setIgnoreIdleThreads(boolean ignoreIdleThreads) { - request.ignoreIdleThreads(ignoreIdleThreads); - return this; - } - - public NodesHotThreadsRequestBuilder setType(HotThreads.ReportType type) { - request.type(type); - return this; - } - - public NodesHotThreadsRequestBuilder setSortOrder(HotThreads.SortOrder sortOrder) { - request.sortOrder(sortOrder); - return this; - } - - public NodesHotThreadsRequestBuilder setInterval(TimeValue interval) { - request.interval(interval); - return this; - } -} diff --git a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java index 17d712bdf5af4..4d27431dcd45d 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java @@ -16,9 +16,6 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -256,18 +253,6 @@ public interface ClusterAdminClient extends ElasticsearchClient { */ void nodesUsage(NodesUsageRequest request, ActionListener listener); - /** - * Returns top N hot-threads samples per node. The hot-threads are only sampled - * for the node ids specified in the request. - */ - void nodesHotThreads(NodesHotThreadsRequest request, ActionListener listener); - - /** - * Returns a request builder to fetch top N hot-threads samples per node. The hot-threads are only sampled - * for the node ids provided. Note: Use {@code *} to fetch samples for all nodes - */ - NodesHotThreadsRequestBuilder prepareNodesHotThreads(String... nodesIds); - /** * List tasks * diff --git a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java index 21c01abd52437..dd230f4d8e12d 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java @@ -24,10 +24,6 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -795,16 +791,6 @@ public ClusterStatsRequestBuilder prepareClusterStats() { return new ClusterStatsRequestBuilder(this); } - @Override - public void nodesHotThreads(NodesHotThreadsRequest request, ActionListener listener) { - execute(TransportNodesHotThreadsAction.TYPE, request, listener); - } - - @Override - public NodesHotThreadsRequestBuilder prepareNodesHotThreads(String... nodesIds) { - return new NodesHotThreadsRequestBuilder(this).setNodesIds(nodesIds); - } - @Override public ActionFuture listTasks(final ListTasksRequest request) { return execute(TransportListTasksAction.TYPE, request); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java index 095abcd14d355..76df8af1889a7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; +import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.core.RestApiVersion; @@ -112,7 +113,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC nodesHotThreadsRequest.interval(TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval(), "interval")); nodesHotThreadsRequest.snapshots(request.paramAsInt("snapshots", nodesHotThreadsRequest.snapshots())); nodesHotThreadsRequest.timeout(request.param("timeout")); - return channel -> client.admin().cluster().nodesHotThreads(nodesHotThreadsRequest, new RestResponseListener<>(channel) { + return channel -> client.execute(TransportNodesHotThreadsAction.TYPE, nodesHotThreadsRequest, new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(NodesHotThreadsResponse response) { return RestResponse.chunked(RestStatus.OK, fromTextChunks(TEXT_CONTENT_TYPE, response.getTextChunks(), null)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 23721de4aad9c..cf8d846c9c9e7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -26,8 +26,6 @@ import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -119,6 +117,7 @@ import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.store.IndicesStore; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.node.NodeMocksPlugin; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.plugins.NetworkPlugin; @@ -153,6 +152,7 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.io.StringWriter; import java.lang.annotation.Annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; @@ -971,17 +971,21 @@ private ClusterHealthStatus ensureColor( final var allocationExplainRef = new AtomicReference(); final var clusterStateRef = new AtomicReference(); final var pendingTasksRef = new AtomicReference(); - final var hotThreadsRef = new AtomicReference(); + final var hotThreadsRef = new AtomicReference(); final var detailsFuture = new PlainActionFuture(); try (var listeners = new RefCountingListener(detailsFuture)) { clusterAdmin().prepareAllocationExplain().execute(listeners.acquire(allocationExplainRef::set)); clusterAdmin().prepareState().execute(listeners.acquire(clusterStateRef::set)); clusterAdmin().preparePendingClusterTasks().execute(listeners.acquire(pendingTasksRef::set)); - clusterAdmin().prepareNodesHotThreads() - .setThreads(9999) - .setIgnoreIdleThreads(false) - .execute(listeners.acquire(hotThreadsRef::set)); + + try (var writer = new StringWriter()) { + new HotThreads().busiestThreads(9999).ignoreIdleThreads(false).detect(writer); + hotThreadsRef.set(writer.toString()); + } catch (Exception e) { + logger.error("exception capturing hot threads", e); + hotThreadsRef.set("exception capturing hot threads: " + e); + } } try { @@ -996,10 +1000,7 @@ private ClusterHealthStatus ensureColor( safeFormat(allocationExplainRef.get(), r -> Strings.toString(r.getExplanation(), true, true)), safeFormat(clusterStateRef.get(), r -> r.getState().toString()), safeFormat(pendingTasksRef.get(), r -> Strings.toString(r, true, true)), - safeFormat( - hotThreadsRef.get(), - r -> r.getNodes().stream().map(NodeHotThreads::getHotThreads).collect(Collectors.joining("\n")) - ) + hotThreadsRef.get() ); fail("timed out waiting for " + color + " state"); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 047a2d6225035..7d9880e4b755c 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -7,10 +7,10 @@ package org.elasticsearch.xpack; +import org.apache.logging.log4j.Level; import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.network.NetworkModule; @@ -59,6 +60,7 @@ import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.license.LicenseSettings; import org.elasticsearch.license.LicensesMetadata; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchResponseUtils; @@ -421,24 +423,19 @@ private ClusterHealthStatus ensureColor( {} timed out: leader cluster state: {} - leader cluster hot threads: - {} leader cluster tasks: {} follower cluster state: {} - follower cluster hot threads: - {} follower cluster tasks: {}""", method, leaderClient().admin().cluster().prepareState().get().getState(), - getHotThreads(leaderClient()), leaderClient().admin().cluster().preparePendingClusterTasks().get(), followerClient().admin().cluster().prepareState().get().getState(), - getHotThreads(followerClient()), followerClient().admin().cluster().preparePendingClusterTasks().get() ); + HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at timeout", ReferenceDocs.LOGGING); fail("timed out waiting for " + color + " state"); } assertThat( @@ -450,19 +447,6 @@ private ClusterHealthStatus ensureColor( return actionGet.getStatus(); } - static String getHotThreads(Client client) { - return client.admin() - .cluster() - .prepareNodesHotThreads() - .setThreads(99999) - .setIgnoreIdleThreads(false) - .get() - .getNodes() - .stream() - .map(NodeHotThreads::getHotThreads) - .collect(Collectors.joining("\n")); - } - protected final Index resolveLeaderIndex(String index) { GetIndexResponse getIndexResponse = leaderClient().admin().indices().prepareGetIndex().setIndices(index).get(); assertTrue("index " + index + " not found", getIndexResponse.getSettings().containsKey(index)); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java index 113ed9a5aa686..398ef5f2e743a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java @@ -6,19 +6,20 @@ */ package org.elasticsearch.xpack.ml.integration; +import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -468,11 +469,7 @@ public void testRealtime() throws Exception { StopDatafeedAction.Response stopJobResponse = stopDatafeed(datafeedId); assertTrue(stopJobResponse.isStopped()); } catch (Exception e) { - NodesHotThreadsResponse nodesHotThreadsResponse = clusterAdmin().prepareNodesHotThreads().get(); - int i = 0; - for (NodeHotThreads nodeHotThreads : nodesHotThreadsResponse.getNodes()) { - logger.info(i++ + ":\n" + nodeHotThreads.getHotThreads()); - } + HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at failure", ReferenceDocs.LOGGING); throw e; } assertBusy(() -> { @@ -491,11 +488,7 @@ public void testCloseJobStopsRealtimeDatafeed() throws Exception { CloseJobAction.Response closeJobResponse = closeJob(jobId); assertTrue(closeJobResponse.isClosed()); } catch (Exception e) { - NodesHotThreadsResponse nodesHotThreadsResponse = clusterAdmin().prepareNodesHotThreads().get(); - int i = 0; - for (NodeHotThreads nodeHotThreads : nodesHotThreadsResponse.getNodes()) { - logger.info(i++ + ":\n" + nodeHotThreads.getHotThreads()); - } + HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at failure", ReferenceDocs.LOGGING); throw e; } assertBusy(() -> { @@ -538,11 +531,7 @@ public void testCloseJobStopsLookbackOnlyDatafeed() throws Exception { CloseJobAction.Response closeJobResponse = closeJob(jobId, useForce); assertTrue(closeJobResponse.isClosed()); } catch (Exception e) { - NodesHotThreadsResponse nodesHotThreadsResponse = clusterAdmin().prepareNodesHotThreads().get(); - int i = 0; - for (NodeHotThreads nodeHotThreads : nodesHotThreadsResponse.getNodes()) { - logger.info(i++ + ":\n" + nodeHotThreads.getHotThreads()); - } + HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at failure", ReferenceDocs.LOGGING); throw e; } GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedId); From d0ed8e60803697ec8200cf3d2f63b90d457da94b Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 14 Dec 2023 12:11:09 -0500 Subject: [PATCH 46/54] [ci] Offset periodic pipeline triggers for each branch (#103415) --- .buildkite/scripts/periodic.trigger.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.buildkite/scripts/periodic.trigger.sh b/.buildkite/scripts/periodic.trigger.sh index 3571d112c5b6d..cc10a5ae41861 100755 --- a/.buildkite/scripts/periodic.trigger.sh +++ b/.buildkite/scripts/periodic.trigger.sh @@ -6,11 +6,26 @@ echo "steps:" source .buildkite/scripts/branches.sh +IS_FIRST=true +SKIP_DELAY="${SKIP_DELAY:-false}" + for BRANCH in "${BRANCHES[@]}"; do INTAKE_PIPELINE_SLUG="elasticsearch-intake" BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}') LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit') + # Put a delay between each branch's set of pipelines by prepending each non-first branch with a sleep + # This is to smooth out the spike in agent requests + if [[ "$IS_FIRST" != "true" && "$SKIP_DELAY" != "true" ]]; then + cat < Date: Thu, 14 Dec 2023 18:15:53 +0100 Subject: [PATCH 47/54] ESQL: Update ToDoubleTests's random string conversion checks (#103462) This updates the checks for warnings triggered by parsing random strings. Besides some UTF8 issues, there are more ways the parsing can fail than tested before. Fixes #103127 --- .../function/scalar/convert/ToDoubleTests.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java index ebcaf367b1226..0309bcce85581 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java @@ -54,17 +54,13 @@ public static Iterable parameters() { List.of() ); // random strings that don't look like a double - TestCaseSupplier.forUnaryStrings( - suppliers, - evaluatorName.apply("String"), - DataTypes.DOUBLE, - bytesRef -> null, - bytesRef -> List.of( + TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("String"), DataTypes.DOUBLE, bytesRef -> null, bytesRef -> { + var exception = expectThrows(NumberFormatException.class, () -> Double.parseDouble(bytesRef.utf8ToString())); + return List.of( "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", - "Line -1:-1: java.lang.NumberFormatException: " - + (bytesRef.utf8ToString().isEmpty() ? "empty String" : ("For input string: \"" + bytesRef.utf8ToString() + "\"")) - ) - ); + "Line -1:-1: " + exception + ); + }); TestCaseSupplier.forUnaryUnsignedLong( suppliers, evaluatorName.apply("UnsignedLong"), From 4217a934541fe63d1cd3bce0b06b5dac92f0bfa1 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 14 Dec 2023 18:24:38 +0100 Subject: [PATCH 48/54] [Connectors API] Add set connector sync job error API docs (#103456) Add set connector sync job error API endpoint docs. --- .../connector/apis/connector-apis.asciidoc | 4 +- .../set-connector-sync-job-error-api.asciidoc | 58 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index 0380682677340..e31eb7899f5d9 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -40,9 +40,10 @@ Use the following APIs to manage sync jobs: * <> * <> * <> -* < +* <> * <> * <> +* <> include::cancel-connector-sync-job-api.asciidoc[] @@ -55,3 +56,4 @@ include::get-connector-api.asciidoc[] include::get-connector-sync-job-api.asciidoc[] include::list-connectors-api.asciidoc[] include::list-connector-sync-jobs-api.asciidoc[] +include::set-connector-sync-job-error-api.asciidoc[] diff --git a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc new file mode 100644 index 0000000000000..935fcccc77fcf --- /dev/null +++ b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc @@ -0,0 +1,58 @@ +[[set-connector-sync-job-error-api]] +=== Set connector sync job error API +++++ +Set connector sync job error +++++ + +Sets a connector sync job error. + +[[set-connector-sync-job-error-api-request]] +==== {api-request-title} +`PUT _connector/_sync_job//_error` + +[[set-connector-sync-job-error-api-prereqs]] +==== {api-prereq-title} + +* To sync data using connectors, it's essential to have the Elastic connectors service running. +* The `connector_sync_job_id` parameter should reference an existing connector sync job. + +[[set-connector-sync-job-error-api-desc]] +==== {api-description-title} + +Sets the `error` field for the specified connector sync job and sets its `status` to `error`. + +[[set-connector-sync-job-error-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) + +[role="child_attributes"] +[[set-connector-sync-job-error-api-request-body]] +==== {api-request-body-title} + +`error`:: +(Required, string) The error to set the connector sync job `error` field to. + +[[set-connector-sync-job-api-response-codes]] +==== {api-response-codes-title} + +`200`:: +Indicates that the connector sync job error was set successfully. + +`404`:: +No connector sync job matching `connector_sync_job_id` could be found. + +[[set-connector-sync-job-error-api-example]] +==== {api-examples-title} + +The following example sets the error `some-error` in the connector sync job `my-connector-sync-job`: + +[source,console] +---- +PUT _connector/_sync_job/my-connector-sync-job/_error +{ + "error": "some-error" +} +---- +// TEST[skip:there's no way to clean up after creating a connector sync job, as we don't know the id ahead of time. Therefore, skip this test.] From 13cc2a883afdca82213f4d25b38cc4916ec47d59 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:59:50 -0500 Subject: [PATCH 49/54] [ML] Prevent attempts to access non-existent node information during rebalancing (#103361) * Preventing null pointer * Update docs/changelog/103361.yaml * adding identity to help debugging * Updating bug info * Adding test * Removing present node log message and fixing identity time * Make test more realistic --------- Co-authored-by: David Roberts --- docs/changelog/103361.yaml | 5 +++ .../StartTrainedModelDeploymentAction.java | 6 ++- .../TrainedModelAssignmentClusterService.java | 34 +++++++++++++++- ...nedModelAssignmentClusterServiceTests.java | 39 ++++++++++++++++++- 4 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/103361.yaml diff --git a/docs/changelog/103361.yaml b/docs/changelog/103361.yaml new file mode 100644 index 0000000000000..441acc09895ef --- /dev/null +++ b/docs/changelog/103361.yaml @@ -0,0 +1,5 @@ +pr: 103361 +summary: Prevent attempts to access non-existent node information during rebalancing +area: Machine Learning +type: bug +issues: [ ] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java index ad9ab7088fef5..c05c73bc31ddf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java @@ -374,8 +374,10 @@ public static class TaskParams implements MlTaskParams, Writeable, ToXContentObj // TODO add support for other roles? If so, it may have to be an instance method... // NOTE, whatever determines assignment should not be dynamically set on the node // Otherwise assignment logic might fail - public static boolean mayAssignToNode(DiscoveryNode node) { - return node.getRoles().contains(DiscoveryNodeRole.ML_ROLE) && MlConfigVersion.fromNode(node).onOrAfter(VERSION_INTRODUCED); + public static boolean mayAssignToNode(@Nullable DiscoveryNode node) { + return node != null + && node.getRoles().contains(DiscoveryNodeRole.ML_ROLE) + && MlConfigVersion.fromNode(node).onOrAfter(VERSION_INTRODUCED); } public static final MlConfigVersion VERSION_INTRODUCED = MlConfigVersion.V_8_0_0; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java index fe4462d6556ee..a1ac1aa55c320 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java @@ -1099,31 +1099,61 @@ static boolean haveMlNodesChanged(ClusterChangedEvent event, TrainedModelAssignm // it may get re-allocated to that node when another node is added/removed... boolean nodesShutdownChanged = event.changedCustomMetadataSet().contains(NodesShutdownMetadata.TYPE); if (event.nodesChanged() || nodesShutdownChanged) { + // This is just to track the various log messages that happen in this function to help with debugging in the future + // so that we can reasonably assume they're all related + // If the log messages printed from this method get interlaced across nodes it can make debugging difficult + var eventIdentity = Long.toHexString(System.nanoTime()); + Set shuttingDownNodes = nodesShuttingDown(event.state()); DiscoveryNodes.Delta nodesDelta = event.nodesDelta(); Set removedNodes = nodesDelta.removedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); Set addedNodes = nodesDelta.addedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); + logger.debug( + () -> format( + "Initial node change info; identity: %s; removed nodes: %s; added nodes: %s; shutting down nodes: %s", + eventIdentity, + removedNodes, + addedNodes, + shuttingDownNodes + ) + ); + Set exitingShutDownNodes; if (nodesShutdownChanged) { Set previousShuttingDownNodes = nodesShuttingDown(event.previousState()); + Set presentNodes = event.state().nodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); // Add nodes that where marked for shutdown in the previous state // but are no longer marked as shutdown in the current state. - Set returningShutDownNodes = Sets.difference(previousShuttingDownNodes, shuttingDownNodes); + // The intersection is to only include the nodes that actually exist + Set returningShutDownNodes = Sets.intersection( + presentNodes, + Sets.difference(previousShuttingDownNodes, shuttingDownNodes) + ); addedNodes.addAll(returningShutDownNodes); // and nodes that are marked for shutdown in this event only exitingShutDownNodes = Sets.difference(shuttingDownNodes, previousShuttingDownNodes); removedNodes.addAll(exitingShutDownNodes); + + logger.debug( + () -> format( + "Shutting down nodes were changed; identity: %s; previous shutting down nodes: %s; returning nodes: %s", + eventIdentity, + previousShuttingDownNodes, + returningShutDownNodes + ) + ); } else { exitingShutDownNodes = Collections.emptySet(); } logger.debug( () -> format( - "added nodes %s; removed nodes %s; shutting down nodes %s; exiting shutdown nodes %s", + "identity: %s; added nodes %s; removed nodes %s; shutting down nodes %s; exiting shutdown nodes %s", + eventIdentity, addedNodes, removedNodes, shuttingDownNodes, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java index 0a54b97cf2f2a..f966ac85c7a65 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterServiceTests.java @@ -114,7 +114,7 @@ public class TrainedModelAssignmentClusterServiceTests extends ESTestCase { private NodeAvailabilityZoneMapper nodeAvailabilityZoneMapper; private Client client; private static MockAppender appender; - private static Logger testLogger1 = LogManager.getLogger(TrainedModelAssignmentClusterService.class); + private static final Logger testLogger1 = LogManager.getLogger(TrainedModelAssignmentClusterService.class); @Before public void setupObjects() throws IllegalAccessException { @@ -546,6 +546,43 @@ public void testCreateAssignmentWhileResetModeIsTrue() throws InterruptedExcepti latch.await(); } + public void testHaveMlNodesChanged_ReturnsFalseWhenPreviouslyShuttingDownNode_IsMarkedAsReturning_ButIsNotAPresentNode() { + String model1 = "model-1"; + String shuttingDownNode = "ml-shutting-down-node"; + String mlNode1 = "ml-node-with-room"; + + ClusterState stateWithShuttingDownNodeAndMlNode1 = createClusterState( + List.of(shuttingDownNode, mlNode1), + Metadata.builder() + .putCustom( + TrainedModelAssignmentMetadata.NAME, + TrainedModelAssignmentMetadata.Builder.empty() + .addNewAssignment( + model1, + TrainedModelAssignment.Builder.empty(newParams(model1, 100)) + .addRoutingEntry(mlNode1, new RoutingInfo(1, 1, RoutingState.STARTING, "")) + ) + .build() + ) + .putCustom(NodesShutdownMetadata.TYPE, shutdownMetadata(shuttingDownNode)) + .build() + ); + + ClusterState stateWithMlNode1 = ClusterState.builder(stateWithShuttingDownNodeAndMlNode1) + .nodes(DiscoveryNodes.builder(stateWithShuttingDownNodeAndMlNode1.nodes()).remove(shuttingDownNode).build()) + .metadata( + Metadata.builder(stateWithShuttingDownNodeAndMlNode1.metadata()) + .putCustom(NodesShutdownMetadata.TYPE, NodesShutdownMetadata.EMPTY) + .build() + ) + .build(); + + var shutdownEvent = new ClusterChangedEvent("test", stateWithMlNode1, stateWithShuttingDownNodeAndMlNode1); + var metadata = TrainedModelAssignmentMetadata.fromState(shutdownEvent.state()); + + assertFalse(TrainedModelAssignmentClusterService.haveMlNodesChanged(shutdownEvent, metadata)); + } + public void testHaveMlNodesChanged_ReturnsTrueWhenNodeShutsDownAndWasRoutedTo() { String model1 = "model-1"; String mlNode1 = "ml-node-with-room"; From 08201860757acc8ddc21652e4120a4d16ffce1a1 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Thu, 14 Dec 2023 13:33:29 -0500 Subject: [PATCH 50/54] Fix text_expansion yaml flakey test (#103468) --- .../rest-api-spec/test/ml/text_expansion_search.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml index fe25d8957216c..5e29d3cdf2ae6 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/text_expansion_search.yml @@ -222,12 +222,12 @@ setup: query: weighted_tokens: ml.tokens: - tokens: [{"the": 1.0}, {"octopus":1.0}, {"comforter":1.0}] + tokens: [{"the": 1.0}, {"comforter":1.0}, {"smells":1.0}, {"bad": 1.0}] pruning_config: tokens_freq_ratio_threshold: 1 tokens_weight_threshold: 0.4 only_score_pruned_tokens: false - - match: { hits.total.value: 4 } + - match: { hits.total.value: 5 } - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } --- @@ -243,9 +243,9 @@ setup: query: weighted_tokens: ml.tokens: - tokens: [{"the": 1.0}, {"octopus":1.0}, {"comforter":1.0}] + tokens: [{"the": 1.0}, {"comforter":1.0}, {"smells":1.0}, {"bad": 1.0}] pruning_config: {} - - match: { hits.total.value: 4 } + - match: { hits.total.value: 5 } - match: { hits.hits.0._source.source_text: "the octopus comforter smells" } --- @@ -261,7 +261,7 @@ setup: query: weighted_tokens: ml.tokens: - tokens: [{"the": 1.0}, {"octopus":1.0}, {"comforter":1.0}] + tokens: [{"the": 1.0}, {"comforter":1.0}, {"smells":1.0}, {"bad": 1.0}] pruning_config: tokens_freq_ratio_threshold: 4 tokens_weight_threshold: 0.4 From a2dc45bc0213f8db65412f9d02e11d61a77d4e2b Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Thu, 14 Dec 2023 14:53:43 -0500 Subject: [PATCH 51/54] [Synonyms] Mark Synonyms as GA (#103223) * Remove beta from synonyms docs and json definitions --------- Co-authored-by: Elastic Machine --- docs/changelog/103223.yaml | 10 ++++++++++ .../synonyms/apis/delete-synonym-rule.asciidoc | 2 -- .../synonyms/apis/delete-synonyms-set.asciidoc | 2 -- docs/reference/synonyms/apis/get-synonym-rule.asciidoc | 2 -- docs/reference/synonyms/apis/get-synonyms-set.asciidoc | 2 -- .../synonyms/apis/list-synonyms-sets.asciidoc | 2 -- docs/reference/synonyms/apis/put-synonym-rule.asciidoc | 2 -- docs/reference/synonyms/apis/put-synonyms-set.asciidoc | 2 -- docs/reference/synonyms/apis/synonyms-apis.asciidoc | 2 -- .../rest-api-spec/api/synonyms.delete_synonym.json | 2 +- .../api/synonyms.delete_synonym_rule.json | 2 +- .../rest-api-spec/api/synonyms.get_synonym.json | 2 +- .../rest-api-spec/api/synonyms.get_synonym_rule.json | 2 +- .../rest-api-spec/api/synonyms.get_synonyms_sets.json | 2 +- .../rest-api-spec/api/synonyms.put_synonym.json | 2 +- .../rest-api-spec/api/synonyms.put_synonym_rule.json | 2 +- 16 files changed, 17 insertions(+), 23 deletions(-) create mode 100644 docs/changelog/103223.yaml diff --git a/docs/changelog/103223.yaml b/docs/changelog/103223.yaml new file mode 100644 index 0000000000000..c2f4c1b6a2cf4 --- /dev/null +++ b/docs/changelog/103223.yaml @@ -0,0 +1,10 @@ +pr: 103223 +summary: "[Synonyms] Mark Synonyms as GA" +area: "Search" +type: feature +issues: [] +highlight: + title: "GA Release of Synonyms API" + body: |- + Removes the beta label for the Synonyms API to make it GA. + notable: true diff --git a/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc b/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc index eaff47f5d7909..74cbab8c0b4a2 100644 --- a/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/delete-synonym-rule.asciidoc @@ -1,8 +1,6 @@ [[delete-synonym-rule]] === Delete synonym rule -beta::[] - ++++ Delete synonym rule ++++ diff --git a/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc b/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc index 6ba4dcdc8f7be..9ba33ff3a5c75 100644 --- a/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/delete-synonyms-set.asciidoc @@ -1,8 +1,6 @@ [[delete-synonyms-set]] === Delete synonyms set -beta::[] - ++++ Delete synonyms set ++++ diff --git a/docs/reference/synonyms/apis/get-synonym-rule.asciidoc b/docs/reference/synonyms/apis/get-synonym-rule.asciidoc index 6ce978ae68ac6..c6c35e0efecca 100644 --- a/docs/reference/synonyms/apis/get-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/get-synonym-rule.asciidoc @@ -1,8 +1,6 @@ [[get-synonym-rule]] === Get synonym rule -beta::[] - ++++ Get synonym rule ++++ diff --git a/docs/reference/synonyms/apis/get-synonyms-set.asciidoc b/docs/reference/synonyms/apis/get-synonyms-set.asciidoc index ddd7d2079dbf5..70bb5fb69526d 100644 --- a/docs/reference/synonyms/apis/get-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/get-synonyms-set.asciidoc @@ -1,8 +1,6 @@ [[get-synonyms-set]] === Get synonyms set -beta::[] - ++++ Get synonyms set ++++ diff --git a/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc b/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc index 2522542886d9e..705a24c809e99 100644 --- a/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc +++ b/docs/reference/synonyms/apis/list-synonyms-sets.asciidoc @@ -1,8 +1,6 @@ [[list-synonyms-sets]] === List synonyms sets -beta::[] - ++++ List synonyms sets ++++ diff --git a/docs/reference/synonyms/apis/put-synonym-rule.asciidoc b/docs/reference/synonyms/apis/put-synonym-rule.asciidoc index 95492c95d36fe..de2865632d55e 100644 --- a/docs/reference/synonyms/apis/put-synonym-rule.asciidoc +++ b/docs/reference/synonyms/apis/put-synonym-rule.asciidoc @@ -1,8 +1,6 @@ [[put-synonym-rule]] === Create or update synonym rule -beta::[] - ++++ Create or update synonym rule ++++ diff --git a/docs/reference/synonyms/apis/put-synonyms-set.asciidoc b/docs/reference/synonyms/apis/put-synonyms-set.asciidoc index a3c06c70db17b..5651c4c99adcd 100644 --- a/docs/reference/synonyms/apis/put-synonyms-set.asciidoc +++ b/docs/reference/synonyms/apis/put-synonyms-set.asciidoc @@ -1,8 +1,6 @@ [[put-synonyms-set]] === Create or update synonyms set -beta::[] - ++++ Create or update synonyms set ++++ diff --git a/docs/reference/synonyms/apis/synonyms-apis.asciidoc b/docs/reference/synonyms/apis/synonyms-apis.asciidoc index 6849477177dcf..9b92ba8e8579d 100644 --- a/docs/reference/synonyms/apis/synonyms-apis.asciidoc +++ b/docs/reference/synonyms/apis/synonyms-apis.asciidoc @@ -1,8 +1,6 @@ [[synonyms-apis]] == Synonyms APIs -beta::[] - ++++ Synonyms APIs ++++ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym.json index 00142ebcf00fc..9273a8dea87c3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-synonyms-set.html", "description": "Deletes a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym_rule.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym_rule.json index 11fb113d6b629..5a0de4ab94a7c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym_rule.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.delete_synonym_rule.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-synonym-rule.html", "description": "Deletes a synonym rule in a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym.json index 6cb4fcc46f26b..25c177cabbdf1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-synonyms-set.html", "description": "Retrieves a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym_rule.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym_rule.json index 5a718f1a48e46..ff9e7eb57b8a7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym_rule.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonym_rule.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-synonym-rule.html", "description": "Retrieves a synonym rule from a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonyms_sets.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonyms_sets.json index 66bd8df92e1e7..d94bef32cddcd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonyms_sets.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.get_synonyms_sets.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/list-synonyms-sets.html", "description": "Retrieves a summary of all defined synonym sets" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym.json index 6c412d174434b..e09bbb7e428a1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-synonyms-set.html", "description": "Creates or updates a synonyms set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym_rule.json b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym_rule.json index 082432ae662f0..51503b5819862 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym_rule.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/synonyms.put_synonym_rule.json @@ -4,7 +4,7 @@ "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-synonym-rule.html", "description": "Creates or updates a synonym rule in a synonym set" }, - "stability": "experimental", + "stability": "stable", "visibility": "public", "headers": { "accept": [ From b3c1637e454e0b5628fa80da8e7d4517420f17a1 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 Dec 2023 20:14:18 +0000 Subject: [PATCH 52/54] Remove pending tasks helpers from client API (#103451) There's no need for special helpers in `AdminClient` for this transport action, we only use it in a few integ tests. --- .../cluster/tasks/PendingTasksBlocksIT.java | 6 ++-- .../cluster/service/ClusterServiceIT.java | 4 +-- .../index/store/CorruptedFileIT.java | 12 ++----- .../PendingClusterTasksRequestBuilder.java | 22 ------------ .../client/internal/ClusterAdminClient.java | 15 -------- .../internal/support/AbstractClient.java | 14 -------- .../RestPendingClusterTasksAction.java | 9 +++-- .../cat/RestPendingClusterTasksAction.java | 11 +++--- .../elasticsearch/test/ESIntegTestCase.java | 34 ++++++++++++++----- .../test/ESSingleNodeTestCase.java | 2 +- .../elasticsearch/xpack/CcrIntegTestCase.java | 4 +-- ...StreamLifecycleDownsampleDisruptionIT.java | 6 +--- .../DownsampleClusterDisruptionIT.java | 6 ++-- .../downsample/ILMDownsampleDisruptionIT.java | 2 +- 14 files changed, 53 insertions(+), 94 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java index 595788b1eb9f5..eaf8948348684 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java @@ -37,7 +37,7 @@ public void testPendingTasksWithIndexBlocks() { )) { try { enableIndexBlock("test", blockSetting); - PendingClusterTasksResponse response = clusterAdmin().preparePendingClusterTasks().get(); + PendingClusterTasksResponse response = getClusterPendingTasks(); assertNotNull(response.pendingTasks()); } finally { disableIndexBlock("test", blockSetting); @@ -53,7 +53,7 @@ public void testPendingTasksWithClusterReadOnlyBlock() { try { setClusterReadOnly(true); - PendingClusterTasksResponse response = clusterAdmin().preparePendingClusterTasks().get(); + PendingClusterTasksResponse response = getClusterPendingTasks(); assertNotNull(response.pendingTasks()); } finally { setClusterReadOnly(false); @@ -80,7 +80,7 @@ public boolean validateClusterForming() { } }); - assertNotNull(clusterAdmin().preparePendingClusterTasks().get().pendingTasks()); + assertNotNull(getClusterPendingTasks().pendingTasks()); // starting one more node allows the cluster to recover internalCluster().startNode(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java index 873f8083f4e0c..fde465346d4be 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java @@ -357,7 +357,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) assertTrue(controlSources.isEmpty()); controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10")); - PendingClusterTasksResponse response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get(); + PendingClusterTasksResponse response = getClusterPendingTasks(internalCluster().coordOnlyNodeClient()); assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10)); assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1")); assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true)); @@ -419,7 +419,7 @@ public void onFailure(Exception e) { } assertTrue(controlSources.isEmpty()); - response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get(); + response = getClusterPendingTasks(internalCluster().coordOnlyNodeClient()); assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5)); controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5")); for (PendingClusterTask task : response.pendingTasks()) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java index ec79b53ccd174..c1da93140a0b0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -182,11 +182,7 @@ public void testCorruptFileAndRecover() throws InterruptedException, IOException .waitForNoRelocatingShards(true) ).actionGet(); if (health.isTimedOut()) { - logger.info( - "cluster state:\n{}\n{}", - clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() - ); + logger.info("cluster state:\n{}\n{}", clusterAdmin().prepareState().get().getState(), getClusterPendingTasks()); assertThat("timed out waiting for green state", health.isTimedOut(), equalTo(false)); } assertThat(health.getStatus(), equalTo(ClusterHealthStatus.GREEN)); @@ -295,11 +291,7 @@ public void testCorruptPrimaryNoReplica() throws ExecutionException, Interrupted if (response.getStatus() != ClusterHealthStatus.RED) { logger.info("Cluster turned red in busy loop: {}", didClusterTurnRed); - logger.info( - "cluster state:\n{}\n{}", - clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() - ); + logger.info("cluster state:\n{}\n{}", clusterAdmin().prepareState().get().getState(), getClusterPendingTasks()); } assertThat(response.getStatus(), is(ClusterHealthStatus.RED)); ClusterState state = clusterAdmin().prepareState().get().getState(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java deleted file mode 100644 index aa3f226d23c9d..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksRequestBuilder.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.action.admin.cluster.tasks; - -import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; -import org.elasticsearch.client.internal.ElasticsearchClient; - -public class PendingClusterTasksRequestBuilder extends MasterNodeReadOperationRequestBuilder< - PendingClusterTasksRequest, - PendingClusterTasksResponse, - PendingClusterTasksRequestBuilder> { - - public PendingClusterTasksRequestBuilder(ElasticsearchClient client) { - super(client, TransportPendingClusterTasksAction.TYPE, new PendingClusterTasksRequest()); - } -} diff --git a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java index 4d27431dcd45d..9e3bed8cef09a 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java @@ -85,9 +85,6 @@ import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; import org.elasticsearch.action.admin.indices.dangling.delete.DeleteDanglingIndexRequest; import org.elasticsearch.action.admin.indices.dangling.import_index.ImportDanglingIndexRequest; import org.elasticsearch.action.admin.indices.dangling.list.ListDanglingIndicesRequest; @@ -441,18 +438,6 @@ public interface ClusterAdminClient extends ElasticsearchClient { */ RestoreSnapshotRequestBuilder prepareRestoreSnapshot(String repository, String snapshot); - /** - * Returns a list of the pending cluster tasks, that are scheduled to be executed. This includes operations - * that update the cluster state (for example, a create index operation) - */ - void pendingClusterTasks(PendingClusterTasksRequest request, ActionListener listener); - - /** - * Returns a list of the pending cluster tasks, that are scheduled to be executed. This includes operations - * that update the cluster state (for example, a create index operation) - */ - PendingClusterTasksRequestBuilder preparePendingClusterTasks(); - /** * Get snapshot status. */ diff --git a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java index dd230f4d8e12d..075d1a4bb1e66 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java @@ -118,10 +118,6 @@ import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder; import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; -import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; -import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; @@ -851,16 +847,6 @@ public ClusterSearchShardsRequestBuilder prepareSearchShards(String... indices) return new ClusterSearchShardsRequestBuilder(this).setIndices(indices); } - @Override - public PendingClusterTasksRequestBuilder preparePendingClusterTasks() { - return new PendingClusterTasksRequestBuilder(this); - } - - @Override - public void pendingClusterTasks(PendingClusterTasksRequest request, ActionListener listener) { - execute(TransportPendingClusterTasksAction.TYPE, request, listener); - } - @Override public void putRepository(PutRepositoryRequest request, ActionListener listener) { execute(TransportPutRepositoryAction.TYPE, request, listener); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java index 8442507c36b1c..e9f9b9bf4327d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.rest.action.admin.cluster; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; +import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -39,8 +40,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest(); pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout())); pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local())); - return channel -> client.admin() - .cluster() - .pendingClusterTasks(pendingClusterTasksRequest, new RestChunkedToXContentListener<>(channel)); + return channel -> client.execute( + TransportPendingClusterTasksAction.TYPE, + pendingClusterTasksRequest, + new RestChunkedToXContentListener<>(channel) + ); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java index 7408bf3ab229e..19ebbd2f19df4 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.service.PendingClusterTask; import org.elasticsearch.common.Table; @@ -46,15 +47,17 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest(); pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout())); pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local())); - return channel -> client.admin() - .cluster() - .pendingClusterTasks(pendingClusterTasksRequest, new RestResponseListener(channel) { + return channel -> client.execute( + TransportPendingClusterTasksAction.TYPE, + pendingClusterTasksRequest, + new RestResponseListener<>(channel) { @Override public RestResponse buildResponse(PendingClusterTasksResponse pendingClusterTasks) throws Exception { Table tab = buildTable(request, pendingClusterTasks); return RestTable.buildResponse(tab, channel); } - }); + } + ); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index cf8d846c9c9e7..72fa522686632 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -29,7 +29,9 @@ import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; @@ -860,7 +862,10 @@ public void waitNoPendingTasksOnAll() throws Exception { for (Client client : clients()) { ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth().setLocal(true).get(); assertThat("client " + client + " still has in flight fetch", clusterHealth.getNumberOfInFlightFetch(), equalTo(0)); - PendingClusterTasksResponse pendingTasks = client.admin().cluster().preparePendingClusterTasks().setLocal(true).get(); + PendingClusterTasksResponse pendingTasks = client.execute( + TransportPendingClusterTasksAction.TYPE, + new PendingClusterTasksRequest().local(true) + ).get(); assertThat( "client " + client + " still has pending tasks " + pendingTasks, pendingTasks.pendingTasks(), @@ -977,8 +982,11 @@ private ClusterHealthStatus ensureColor( try (var listeners = new RefCountingListener(detailsFuture)) { clusterAdmin().prepareAllocationExplain().execute(listeners.acquire(allocationExplainRef::set)); clusterAdmin().prepareState().execute(listeners.acquire(clusterStateRef::set)); - clusterAdmin().preparePendingClusterTasks().execute(listeners.acquire(pendingTasksRef::set)); - + client().execute( + TransportPendingClusterTasksAction.TYPE, + new PendingClusterTasksRequest(), + listeners.acquire(pendingTasksRef::set) + ); try (var writer = new StringWriter()) { new HotThreads().busiestThreads(9999).ignoreIdleThreads(false).detect(writer); hotThreadsRef.set(writer.toString()); @@ -1040,7 +1048,7 @@ public ClusterHealthStatus waitForRelocation(ClusterHealthStatus status) { "waitForRelocation timed out (status={}), cluster state:\n{}\n{}", status, clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() + getClusterPendingTasks() ); assertThat("timed out waiting for relocation", actionGet.isTimedOut(), equalTo(false)); } @@ -1050,6 +1058,18 @@ public ClusterHealthStatus waitForRelocation(ClusterHealthStatus status) { return actionGet.getStatus(); } + public static PendingClusterTasksResponse getClusterPendingTasks() { + return getClusterPendingTasks(client()); + } + + public static PendingClusterTasksResponse getClusterPendingTasks(Client client) { + try { + return client.execute(TransportPendingClusterTasksAction.TYPE, new PendingClusterTasksRequest()).get(10, TimeUnit.SECONDS); + } catch (Exception e) { + return fail(e); + } + } + /** * Waits until at least a give number of document is visible for searchers * @@ -1146,11 +1166,7 @@ public static DiscoveryNode waitAndGetHealthNode(InternalTestCluster internalClu * Prints the current cluster state as debug logging. */ public void logClusterState() { - logger.debug( - "cluster state:\n{}\n{}", - clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() - ); + logger.debug("cluster state:\n{}\n{}", clusterAdmin().prepareState().get().getState(), getClusterPendingTasks()); } protected void ensureClusterSizeConsistency() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 1517571878fa2..b201d58ac0e23 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -421,7 +421,7 @@ public ClusterHealthStatus ensureGreen(TimeValue timeout, String... indices) { logger.info( "ensureGreen timed out, cluster state:\n{}\n{}", clusterAdmin().prepareState().get().getState(), - clusterAdmin().preparePendingClusterTasks().get() + ESIntegTestCase.getClusterPendingTasks(client()) ); assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false)); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 7d9880e4b755c..ea4bc8c92047a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -431,9 +431,9 @@ private ClusterHealthStatus ensureColor( {}""", method, leaderClient().admin().cluster().prepareState().get().getState(), - leaderClient().admin().cluster().preparePendingClusterTasks().get(), + ESIntegTestCase.getClusterPendingTasks(leaderClient()), followerClient().admin().cluster().prepareState().get().getState(), - followerClient().admin().cluster().preparePendingClusterTasks().get() + ESIntegTestCase.getClusterPendingTasks(followerClient()) ); HotThreads.logLocalHotThreads(logger, Level.INFO, "hot threads at timeout", ReferenceDocs.LOGGING); fail("timed out waiting for " + color + " state"); diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index 826c958de4c18..f248da8a7842a 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -126,11 +126,7 @@ public boolean validateClusterForming() { } })).start(); - waitUntil( - () -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty(), - 60, - TimeUnit.SECONDS - ); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty(), 60, TimeUnit.SECONDS); ensureStableCluster(cluster.numDataAndMasterNodes()); final String targetIndex = "downsample-5m-" + sourceIndex; diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java index d6549a9618d36..874b68a4bec55 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java @@ -203,7 +203,7 @@ public boolean validateClusterForming() { } })).start(); startDownsampleTaskDuringDisruption(sourceIndex, targetIndex, config, disruptionStart, disruptionEnd); - waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty()); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty()); ensureStableCluster(cluster.numDataAndMasterNodes()); assertTargetIndex(cluster, sourceIndex, targetIndex, indexedDocs); } @@ -265,7 +265,7 @@ public boolean validateClusterForming() { })).start(); startDownsampleTaskDuringDisruption(sourceIndex, targetIndex, config, disruptionStart, disruptionEnd); - waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty()); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty()); ensureStableCluster(cluster.numDataAndMasterNodes()); assertTargetIndex(cluster, sourceIndex, targetIndex, indexedDocs); } @@ -354,7 +354,7 @@ public boolean validateClusterForming() { })).start(); startDownsampleTaskDuringDisruption(sourceIndex, downsampleIndex, config, disruptionStart, disruptionEnd); - waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty()); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty()); ensureStableCluster(cluster.numDataAndMasterNodes()); assertTargetIndex(cluster, sourceIndex, downsampleIndex, indexedDocs); } diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java index a023f171ad209..0e84ed460cf5d 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java @@ -194,7 +194,7 @@ public boolean validateClusterForming() { final String targetIndex = "downsample-1h-" + sourceIndex; startDownsampleTaskViaIlm(sourceIndex, targetIndex, disruptionStart, disruptionEnd); - waitUntil(() -> cluster.client().admin().cluster().preparePendingClusterTasks().get().pendingTasks().isEmpty()); + waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty(), 60, TimeUnit.SECONDS); ensureStableCluster(cluster.numDataAndMasterNodes()); assertTargetIndex(cluster, targetIndex, indexedDocs); } From 2517d5c065a03d409f55806dc1c97bfb80146b66 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 14 Dec 2023 15:20:41 -0500 Subject: [PATCH 53/54] [ci] Add amazon-2023 to platform-support matrix (#103466) --- .../pipelines/periodic-platform-support.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index b52f8506885c9..faf904f2f8b04 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -80,3 +80,19 @@ steps: diskName: /dev/sda1 env: GRADLE_TASK: "{{matrix.GRADLE_TASK}}" + - group: platform-support-unix-aws + steps: + - label: "{{matrix.image}} / platform-support-aws" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests + timeout_in_minutes: 420 + matrix: + setup: + image: + - amazonlinux-2023 + agents: + provider: aws + imagePrefix: elasticsearch-{{matrix.image}} + instanceType: m6a.8xlarge + diskSizeGb: 350 + diskType: gp3 + diskName: /dev/sda1 From 90ae2151fde4f2db3a89e98a6685e8238d46b7ce Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 14 Dec 2023 13:25:34 -0800 Subject: [PATCH 54/54] Fix now in millis for ESQL search contexts (#103474) Currently, ESQL search contexts do not have the correct actual 'now' in milliseconds. Ideally, we should use a consistent 'now' in millis for all nodes participating in the execution of a search request. I will follow up with a PR to deserialize that value between nodes for version 8.13 and later. Closes #103455 --- docs/changelog/103474.yaml | 6 ++ .../xpack/esql/action/TimeBasedIndicesIT.java | 58 +++++++++++++++++++ .../xpack/esql/plugin/ComputeService.java | 15 +++-- .../xpack/esql/session/EsqlConfiguration.java | 9 +++ 4 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 docs/changelog/103474.yaml create mode 100644 x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java diff --git a/docs/changelog/103474.yaml b/docs/changelog/103474.yaml new file mode 100644 index 0000000000000..a1da15a6bfbe5 --- /dev/null +++ b/docs/changelog/103474.yaml @@ -0,0 +1,6 @@ +pr: 103474 +summary: Fix now in millis for ESQL search contexts +area: ES|QL +type: bug +issues: + - 103455 diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java new file mode 100644 index 0000000000000..a1fbee17ef8ec --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.RangeQueryBuilder; + +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.Matchers.hasSize; + +public class TimeBasedIndicesIT extends AbstractEsqlIntegTestCase { + + public void testFilter() { + long epoch = System.currentTimeMillis(); + assertAcked(client().admin().indices().prepareCreate("test").setMapping("@timestamp", "type=date", "value", "type=long")); + BulkRequestBuilder bulk = client().prepareBulk("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + int oldDocs = between(10, 100); + for (int i = 0; i < oldDocs; i++) { + long timestamp = epoch - TimeValue.timeValueHours(between(1, 2)).millis(); + bulk.add(new IndexRequest().source("@timestamp", timestamp, "value", -i)); + } + int newDocs = between(10, 100); + for (int i = 0; i < newDocs; i++) { + long timestamp = epoch + TimeValue.timeValueHours(between(1, 2)).millis(); + bulk.add(new IndexRequest().source("@timestamp", timestamp, "value", i)); + } + bulk.get(); + { + EsqlQueryRequest request = new EsqlQueryRequest(); + request.query("FROM test | limit 1000"); + request.filter(new RangeQueryBuilder("@timestamp").from(epoch - TimeValue.timeValueHours(3).millis()).to("now")); + try (var resp = run(request)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(oldDocs)); + } + } + { + EsqlQueryRequest request = new EsqlQueryRequest(); + request.query("FROM test | limit 1000"); + request.filter(new RangeQueryBuilder("@timestamp").from("now").to(epoch + TimeValue.timeValueHours(3).millis())); + try (var resp = run(request)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(newDocs)); + } + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index b7b31868d65e2..20fcc05e80440 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -328,6 +328,7 @@ void runCompute(CancellableTask task, ComputeContext context, PhysicalPlan plan, private void acquireSearchContexts( List shardIds, + EsqlConfiguration configuration, Map aliasFilters, ActionListener> listener ) { @@ -351,11 +352,12 @@ private void acquireSearchContexts( try { for (IndexShard shard : targetShards) { var aliasFilter = aliasFilters.getOrDefault(shard.shardId().getIndex(), AliasFilter.EMPTY); - ShardSearchRequest shardSearchLocalRequest = new ShardSearchRequest(shard.shardId(), 0, aliasFilter); - SearchContext context = searchService.createSearchContext( - shardSearchLocalRequest, - SearchService.NO_TIMEOUT + var shardRequest = new ShardSearchRequest( + shard.shardId(), + configuration.absoluteStartedTimeInMillis(), + aliasFilter ); + SearchContext context = searchService.createSearchContext(shardRequest, SearchService.NO_TIMEOUT); searchContexts.add(context); } for (SearchContext searchContext : searchContexts) { @@ -501,8 +503,9 @@ public void messageReceived(DataNodeRequest request, TransportChannel channel, T final var exchangeSink = exchangeService.getSinkHandler(sessionId); parentTask.addListener(() -> exchangeService.finishSinkHandler(sessionId, new TaskCancelledException("task cancelled"))); final ActionListener listener = new OwningChannelActionListener<>(channel); - acquireSearchContexts(request.shardIds(), request.aliasFilters(), ActionListener.wrap(searchContexts -> { - var computeContext = new ComputeContext(sessionId, searchContexts, request.configuration(), null, exchangeSink); + final EsqlConfiguration configuration = request.configuration(); + acquireSearchContexts(request.shardIds(), configuration, request.aliasFilters(), ActionListener.wrap(searchContexts -> { + var computeContext = new ComputeContext(sessionId, searchContexts, configuration, null, exchangeSink); runCompute(parentTask, computeContext, request.plan(), ActionListener.wrap(driverProfiles -> { // don't return until all pages are fetched exchangeSink.addCompletionListener( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java index ac13f25c2d2a9..ccec6554cb2cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java @@ -112,6 +112,15 @@ public String query() { return query; } + /** + * Returns the current time in milliseconds from the time epoch for the execution of this request. + * It ensures consistency by using the same value on all nodes involved in the search request. + * Note: Currently, it returns {@link System#currentTimeMillis()}, but this value will be serialized between nodes. + */ + public long absoluteStartedTimeInMillis() { + return System.currentTimeMillis(); + } + /** * Enable profiling, sacrificing performance to return information about * what operations are taking the most time.