diff --git a/.backportrc.json b/.backportrc.json index d2e92817c026b..03f3f892f9227 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,10 +1,10 @@ { "upstream" : "elastic/elasticsearch", - "targetBranchChoices" : [ "main", "8.x", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], + "targetBranchChoices" : [ "main", "8.x", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], "targetPRLabels" : [ "backport" ], "branchLabelMapping" : { "^v9.0.0$" : "main", - "^v8.16.0$" : "8.x", + "^v8.17.0$" : "8.x", "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2" } } diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml index f530f237113a9..57412bbe908bc 100644 --- a/.buildkite/pipelines/intake.template.yml +++ b/.buildkite/pipelines/intake.template.yml @@ -75,6 +75,7 @@ steps: - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow async: true + branches: "main 8.* 7.17" build: branch: "$BUILDKITE_BRANCH" commit: "$BUILDKITE_COMMIT" diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 1bb13c4c10966..1ddb3e82920cd 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -56,7 +56,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"] + BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index b29747c60617e..03368e7e4a9c0 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -304,6 +304,22 @@ steps: env: BWC_VERSION: 8.16.0 + - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.17.0 + - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index cbca7f820c7b7..d572dd104d215 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -325,6 +325,25 @@ steps: - signal_reason: agent_stop limit: 3 + - label: 8.17.0 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + preemptible: true + env: + BWC_VERSION: 8.17.0 + retry: + automatic: + - exit_status: "-1" + limit: 3 + signal_reason: none + - signal_reason: agent_stop + limit: 3 + - label: 9.0.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest timeout_in_minutes: 300 @@ -410,7 +429,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk21 - BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"] + BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -452,7 +471,7 @@ steps: ES_RUNTIME_JAVA: - openjdk21 - openjdk23 - BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"] + BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -554,7 +573,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - if: build.branch == "main" || build.branch == "7.17" + if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency timeout_in_minutes: 15 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index de0505c61a251..cd1f7d1ae269f 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -16,4 +16,5 @@ BWC_VERSION: - "8.14.3" - "8.15.3" - "8.16.0" + - "8.17.0" - "9.0.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 24f58abc72493..67ebf0c51ab1f 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,5 @@ BWC_VERSION: - "8.15.3" - "8.16.0" + - "8.17.0" - "9.0.0" diff --git a/branches.json b/branches.json index e464d6179f2ba..e81d511a88458 100644 --- a/branches.json +++ b/branches.json @@ -4,6 +4,9 @@ { "branch": "main" }, + { + "branch": "8.16" + }, { "branch": "8.x" }, diff --git a/docs/changelog/113237.yaml b/docs/changelog/113237.yaml new file mode 100644 index 0000000000000..45343dbf17114 --- /dev/null +++ b/docs/changelog/113237.yaml @@ -0,0 +1,5 @@ +pr: 113237 +summary: Retry throttled snapshot deletions +area: Snapshot/Restore +type: bug +issues: [] diff --git a/docs/changelog/114271.yaml b/docs/changelog/114271.yaml new file mode 100644 index 0000000000000..7b47b922ff811 --- /dev/null +++ b/docs/changelog/114271.yaml @@ -0,0 +1,5 @@ +pr: 114271 +summary: "[ES|QL] Skip validating remote cluster index names in parser" +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/114636.yaml b/docs/changelog/114636.yaml deleted file mode 100644 index c63876fda67f7..0000000000000 --- a/docs/changelog/114636.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114636 -summary: Dynamically get of num allocations -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/114784.yaml b/docs/changelog/114784.yaml new file mode 100644 index 0000000000000..24ebe8b5fc09a --- /dev/null +++ b/docs/changelog/114784.yaml @@ -0,0 +1,5 @@ +pr: 114784 +summary: "[ES|QL] make named parameter for identifier and pattern snapshot" +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/114813.yaml b/docs/changelog/114813.yaml new file mode 100644 index 0000000000000..1595b004178c4 --- /dev/null +++ b/docs/changelog/114813.yaml @@ -0,0 +1,5 @@ +pr: 114813 +summary: Retry `S3BlobContainer#getRegister` on all exceptions +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/114836.yaml b/docs/changelog/114836.yaml new file mode 100644 index 0000000000000..6f21d3bfb9327 --- /dev/null +++ b/docs/changelog/114836.yaml @@ -0,0 +1,6 @@ +pr: 114836 +summary: Support multi-valued fields in compute engine for ST_DISTANCE +area: ES|QL +type: enhancement +issues: + - 112910 diff --git a/docs/changelog/114837.yaml b/docs/changelog/114837.yaml new file mode 100644 index 0000000000000..313d88f92282c --- /dev/null +++ b/docs/changelog/114837.yaml @@ -0,0 +1,5 @@ +pr: 114837 +summary: Add warning headers for ingest pipelines containing special characters +area: Ingest Node +type: bug +issues: [ 104411 ] diff --git a/docs/changelog/114854.yaml b/docs/changelog/114854.yaml new file mode 100644 index 0000000000000..144a10ba85043 --- /dev/null +++ b/docs/changelog/114854.yaml @@ -0,0 +1,10 @@ +pr: 114854 +summary: Adding deprecation warnings for rrf using rank and `sub_searches` +area: Search +type: deprecation +issues: [] +deprecation: + title: Adding deprecation warnings for rrf using rank and `sub_searches` + area: REST API + details: Search API parameter `sub_searches` will no longer be a supported and will be removed in future releases. Similarly, `rrf` can only be used through the specified `retriever` and no longer though the `rank` parameter + impact: Requests specifying rrf through `rank` and/or `sub_searches` elements will be disallowed in a future version. Users should instead utilize the new `retriever` parameter. diff --git a/docs/changelog/114856.yaml b/docs/changelog/114856.yaml new file mode 100644 index 0000000000000..da7fae3ee18ea --- /dev/null +++ b/docs/changelog/114856.yaml @@ -0,0 +1,5 @@ +pr: 114856 +summary: "OTel mappings: avoid metrics to be rejected when attributes are malformed" +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/114869.yaml b/docs/changelog/114869.yaml new file mode 100644 index 0000000000000..755418e7ce4d9 --- /dev/null +++ b/docs/changelog/114869.yaml @@ -0,0 +1,5 @@ +pr: 114869 +summary: Standardize error code when bulk body is invalid +area: CRUD +type: bug +issues: [] diff --git a/docs/changelog/114888.yaml b/docs/changelog/114888.yaml new file mode 100644 index 0000000000000..6b99eb82d10f3 --- /dev/null +++ b/docs/changelog/114888.yaml @@ -0,0 +1,6 @@ +pr: 114888 +summary: Fix ST_CENTROID_AGG when no records are aggregated +area: ES|QL +type: bug +issues: + - 106025 diff --git a/docs/changelog/114924.yaml b/docs/changelog/114924.yaml new file mode 100644 index 0000000000000..536f446ef790d --- /dev/null +++ b/docs/changelog/114924.yaml @@ -0,0 +1,5 @@ +pr: 114924 +summary: Reducing error-level stack trace logging for normal events in `GeoIpDownloader` +area: Ingest Node +type: bug +issues: [] diff --git a/docs/plugins/mapper-annotated-text.asciidoc b/docs/plugins/mapper-annotated-text.asciidoc index e4141e98a2285..9b6eccd136696 100644 --- a/docs/plugins/mapper-annotated-text.asciidoc +++ b/docs/plugins/mapper-annotated-text.asciidoc @@ -167,8 +167,16 @@ duplicates removed. So: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "text": { "type": "annotated_text", @@ -215,8 +223,16 @@ are preserved. ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "text": { "type": "annotated_text", "store": true } } diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc index cf1cc9f825cb2..b8dda01c2eae0 100644 --- a/docs/reference/cat/indices.asciidoc +++ b/docs/reference/cat/indices.asciidoc @@ -6,8 +6,8 @@ [IMPORTANT] ==== -cat APIs are only intended for human consumption using the command line or {kib} -console. They are _not_ intended for use by applications. For application +cat APIs are only intended for human consumption using the command line or {kib} +console. They are _not_ intended for use by applications. For application consumption, use the <>. ==== @@ -50,6 +50,10 @@ indexing and search. As a result, all document counts include hidden To get an accurate count of {es} documents, use the <> or <> APIs. +Note that information such as document count, deleted document count and store size are not shown for +indices restored from <> since these indices +do not contain the relevant data structures to retrieve this information from. + [[cat-indices-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc index 5f67014d5bb4a..8770ec373bb18 100644 --- a/docs/reference/how-to/size-your-shards.asciidoc +++ b/docs/reference/how-to/size-your-shards.asciidoc @@ -208,6 +208,7 @@ index can be <>. You may then consider setting <> against the destination index for the source index's name to point to it for continuity. +See this https://www.youtube.com/watch?v=sHyNYnwbYro[fixing shard sizes video] for an example troubleshooting walkthrough. [discrete] [[shard-count-recommendation]] @@ -571,6 +572,8 @@ PUT _cluster/settings } ---- +For more information, see <>. + [discrete] [[troubleshooting-max-docs-limit]] ==== Number of documents in the shard cannot exceed [2147483519] diff --git a/docs/reference/ilm/error-handling.asciidoc b/docs/reference/ilm/error-handling.asciidoc index f810afc6c2b5f..e8df44653e9c5 100644 --- a/docs/reference/ilm/error-handling.asciidoc +++ b/docs/reference/ilm/error-handling.asciidoc @@ -8,6 +8,9 @@ When this happens, {ilm-init} moves the index to an `ERROR` step. If {ilm-init} cannot resolve the error automatically, execution is halted until you resolve the underlying issues with the policy, index, or cluster. +See this https://www.youtube.com/watch?v=VCIqkji3IwY[{ilm-init} health video] +for example troubleshooting walkthrough. + For example, you might have a `shrink-index` policy that shrinks an index to four shards once it is at least five days old: @@ -183,6 +186,8 @@ The rollover action then manages setting and updating the alias to Do not explicitly configure this same alias in the aliases section of an index template. +See this https://www.youtube.com/watch?v=Ww5POq4zZtY[resolving `duplicate alias` video] for an example troubleshooting walkthrough. + [discrete] ==== index.lifecycle.rollover_alias [x] does not point to index [y] @@ -191,6 +196,8 @@ Either the index is using the wrong alias or the alias does not exist. Check the `index.lifecycle.rollover_alias` <>. To see what aliases are configured, use <>. +See this https://www.youtube.com/watch?v=NKSe67x7aw8[resolving `not point to index` video] for an example troubleshooting walkthrough. + [discrete] ==== Setting [index.lifecycle.rollover_alias] for index [y] is empty or not defined @@ -198,6 +205,8 @@ The `index.lifecycle.rollover_alias` setting must be configured for the rollover Update the index settings to set `index.lifecycle.rollover_alias`. +See this https://www.youtube.com/watch?v=LRpMC2GS_FQ[resolving `empty or not defined` video] for an example troubleshooting walkthrough. + [discrete] ==== Alias [x] has more than one write index [y,z] @@ -205,6 +214,8 @@ Only one index can be designated as the write index for a particular alias. Use the <> API to set `is_write_index:false` for all but one index. +See this https://www.youtube.com/watch?v=jCUvZCT5Hm4[resolving `more than one write index` video] for an example troubleshooting walkthrough. + [discrete] ==== index name [x] does not match pattern ^.*-\d+ @@ -214,6 +225,8 @@ For example, `my-index` does not match the pattern requirement. Append a numeric value to the index name, for example `my-index-000001`. +See this https://www.youtube.com/watch?v=9sp1zF6iL00[resolving `does not match pattern` video] for an example troubleshooting walkthrough. + [discrete] ==== CircuitBreakingException: [x] data too large, data for [y] @@ -227,8 +240,7 @@ For more information, see <>. This indicates that the cluster is running out of disk space. This can happen when you don't have {ilm} set up to roll over from hot to warm nodes. - -Consider adding nodes, upgrading your hardware, or deleting unneeded indices. +For more information, see <>. [discrete] ==== security_exception: action [] is unauthorized for user [] with roles [], this action is granted by the index privileges [manage_follow_index,manage,all] diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc index c7217f38d459b..6afc2a2e3ef65 100644 --- a/docs/reference/inference/service-elser.asciidoc +++ b/docs/reference/inference/service-elser.asciidoc @@ -80,12 +80,13 @@ Must be a power of 2. Max allowed value is 32. [[inference-example-elser]] ==== ELSER service example -The following example shows how to create an {infer} endpoint called -`my-elser-model` to perform a `sparse_embedding` task type. +The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type. Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info. -The request below will automatically download the ELSER model if it isn't -already downloaded and then deploy the model. +NOTE: If you want to optimize your ELSER endpoint for ingest, set the number of threads to `1` (`"num_threads": 1`). +If you want to optimize your ELSER endpoint for search, set the number of threads to greater than `1`. + +The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model. [source,console] ------------------------------------------------------------ @@ -100,7 +101,6 @@ PUT _inference/sparse_embedding/my-elser-model ------------------------------------------------------------ // TEST[skip:TBD] - Example response: [source,console-result] @@ -130,12 +130,12 @@ If using the Python client, you can set the `timeout` parameter to a higher valu [[inference-example-elser-adaptive-allocation]] ==== Setting adaptive allocation for the ELSER service -The following example shows how to create an {infer} endpoint called -`my-elser-model` to perform a `sparse_embedding` task type and configure -adaptive allocations. +NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation. +To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page. + +The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations. -The request below will automatically download the ELSER model if it isn't -already downloaded and then deploy the model. +The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model. [source,console] ------------------------------------------------------------ diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index 902b6c26611e5..f8666e2993d6a 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -2,7 +2,7 @@ ==== Synthetic `_source` IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices -(indices that have `index.mode` set to `time_series`). For other indices +(indices that have `index.mode` set to `time_series`). For other indices, synthetic `_source` is in technical preview. Features in technical preview may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA @@ -11,15 +11,19 @@ of official GA features. Though very handy to have around, the source field takes up a significant amount of space on disk. Instead of storing source documents on disk exactly as you send them, Elasticsearch can reconstruct source content on the fly upon retrieval. -Enable this by setting `mode: synthetic` in `_source`: +Enable this by using the value `synthetic` for the index setting `index.mapping.source.mode`: [source,console,id=enable-synthetic-source-example] ---- PUT idx { - "mappings": { - "_source": { - "mode": "synthetic" + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } } } } @@ -38,7 +42,7 @@ properties when used with synthetic `_source`. <> construct synthetic `_source` using existing data, most commonly <> and <>. For these field types, no additional space is needed to store the contents of `_source` field. Due to the storage layout of <>, the -generated `_source` field undergoes <> compared to original document. +generated `_source` field undergoes <> compared to the original document. For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original @@ -227,10 +231,16 @@ For instance: ---- PUT idx_keep { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { - "mode": "synthetic" - }, "properties": { "path": { "type": "object", diff --git a/docs/reference/mapping/types/aggregate-metric-double.asciidoc b/docs/reference/mapping/types/aggregate-metric-double.asciidoc index 8e14fba976360..8a4ddffc30bbd 100644 --- a/docs/reference/mapping/types/aggregate-metric-double.asciidoc +++ b/docs/reference/mapping/types/aggregate-metric-double.asciidoc @@ -267,8 +267,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "agg_metric": { "type": "aggregate_metric_double", diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc index 32f3d13edf581..494c41021dd2a 100644 --- a/docs/reference/mapping/types/boolean.asciidoc +++ b/docs/reference/mapping/types/boolean.asciidoc @@ -249,8 +249,16 @@ Synthetic source always sorts `boolean` fields. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "bool": { "type": "boolean" } } diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index ca2c23f932fc3..53b17a669ae75 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -130,7 +130,7 @@ The following parameters are accepted by `date` fields: <>:: If `true`, malformed numbers are ignored. If `false` (default), malformed - numbers throw an exception and reject the whole document. Note that this + numbers throw an exception and reject the whole document. Note that this cannot be set if the `script` parameter is used. <>:: @@ -248,8 +248,16 @@ Synthetic source always sorts `date` fields. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "date": { "type": "date" } } diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc index 1a3b390b1690c..e9ec85c470ecf 100644 --- a/docs/reference/mapping/types/date_nanos.asciidoc +++ b/docs/reference/mapping/types/date_nanos.asciidoc @@ -160,8 +160,16 @@ Synthetic source always sorts `date_nanos` fields. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "date": { "type": "date_nanos" } } diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc index 0a72ebc98ecef..af6ef3e739d0f 100644 --- a/docs/reference/mapping/types/flattened.asciidoc +++ b/docs/reference/mapping/types/flattened.asciidoc @@ -334,8 +334,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "flattened": { "type": "flattened" } } @@ -367,8 +375,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "flattened": { "type": "flattened" } } @@ -407,8 +423,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "flattened": { "type": "flattened" } } diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc index 6db05188dfb98..9ba8ea6e46782 100644 --- a/docs/reference/mapping/types/geo-point.asciidoc +++ b/docs/reference/mapping/types/geo-point.asciidoc @@ -229,8 +229,16 @@ longitude) and reduces them to their stored precision. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "point": { "type": "geo_point" } } diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc index f068916478a78..f85dd78ecbd4a 100644 --- a/docs/reference/mapping/types/ip.asciidoc +++ b/docs/reference/mapping/types/ip.asciidoc @@ -170,8 +170,16 @@ Synthetic source always sorts `ip` fields and removes duplicates. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "ip": { "type": "ip" } } diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index a4be7026dffcd..b94216042427f 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -188,8 +188,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "kwd": { "type": "keyword" } } @@ -218,8 +226,16 @@ are preserved. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "kwd": { "type": "keyword", "store": true } } @@ -248,8 +264,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "kwd": { "type": "keyword", "ignore_above": 3 } } diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index d1e1c037e571e..5bfa1bc7c1240 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -259,8 +259,16 @@ Synthetic source always sorts numeric fields. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "long": { "type": "long" } } @@ -287,8 +295,16 @@ Scaled floats will always apply their scaling factor so: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "f": { "type": "scaled_float", "scaling_factor": 0.01 } } diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc index 14c5b6098acbe..04341f68c630a 100644 --- a/docs/reference/mapping/types/range.asciidoc +++ b/docs/reference/mapping/types/range.asciidoc @@ -249,13 +249,21 @@ of official GA features. `range` fields support <> in their default configuration. Synthetic `_source` cannot be used with <> disabled. -Synthetic source always sorts values and removes duplicates for all `range` fields except `ip_range` . Ranges are sorted by their lower bound and then by upper bound. For example: +Synthetic source always sorts values and removes duplicates for all `range` fields except `ip_range`. Ranges are sorted by their lower bound and then by upper bound. For example: [source,console,id=synthetic-source-range-sorting-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "long_range" } } @@ -316,8 +324,16 @@ For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "ip_range" } } @@ -352,13 +368,21 @@ Will become: // TEST[s/^/{"_source":/ s/\n$/}/] [[range-synthetic-source-inclusive]] -Range field vales are always represented as inclusive on both sides with bounds adjusted accordingly. Default values for range bounds are represented as `null`. This is true even if range bound was explicitly provided. For example: +Range field values are always represented as inclusive on both sides with bounds adjusted accordingly. Default values for range bounds are represented as `null`. This is true even if range bound was explicitly provided. For example: [source,console,id=synthetic-source-range-normalization-example] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "long_range" } } @@ -394,8 +418,16 @@ Default values for range bounds are represented as `null` in synthetic source. T ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "integer_range" } } @@ -429,8 +461,16 @@ Will become: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "my_range": { "type": "date_range" } } diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc index c33af69df5607..ca69c93e8f1a8 100644 --- a/docs/reference/mapping/types/text.asciidoc +++ b/docs/reference/mapping/types/text.asciidoc @@ -177,15 +177,23 @@ a <> sub-field that supports synthetic `_source` or if the `text` field sets `store` to `true`. Either way, it may not have <>. -If using a sub-`keyword` field then the values are sorted in the same way as +If using a sub-`keyword` field, then the values are sorted in the same way as a `keyword` field's values are sorted. By default, that means sorted with duplicates removed. So: [source,console,id=synthetic-source-text-example-default] ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "text": { "type": "text", @@ -233,8 +241,16 @@ are preserved. ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "text": { "type": "text", "store": true } } diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc index 8da0fcae80fcd..1600451432bd8 100644 --- a/docs/reference/mapping/types/version.asciidoc +++ b/docs/reference/mapping/types/version.asciidoc @@ -63,16 +63,15 @@ The following parameters are accepted by `version` fields: [discrete] ==== Limitations -This field type isn't optimized for heavy wildcard, regex or fuzzy searches. While those -type of queries work in this field, you should consider using a regular `keyword` field if -you strongly rely on these kind of queries. - +This field type isn't optimized for heavy wildcard, regex, or fuzzy searches. While those +types of queries work in this field, you should consider using a regular `keyword` field if +you strongly rely on these kinds of queries. [[version-synthetic-source]] ==== Synthetic `_source` IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices -(indices that have `index.mode` set to `time_series`). For other indices +(indices that have `index.mode` set to `time_series`). For other indices, synthetic `_source` is in technical preview. Features in technical preview may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA @@ -86,8 +85,16 @@ Synthetic source always sorts `version` fields and removes duplicates. For examp ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "versions": { "type": "version" } } diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc index 79fc953051d54..89a3109a37164 100644 --- a/docs/reference/mapping/types/wildcard.asciidoc +++ b/docs/reference/mapping/types/wildcard.asciidoc @@ -141,8 +141,16 @@ Synthetic source always sorts `wildcard` fields. For example: ---- PUT idx { + "settings": { + "index": { + "mapping": { + "source": { + "mode": "synthetic" + } + } + } + }, "mappings": { - "_source": { "mode": "synthetic" }, "properties": { "card": { "type": "wildcard" } } diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index 0690f60495c97..719588cb4b0d0 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -1,5 +1,6 @@ include::migration_intro.asciidoc[] +* <> * <> * <> * <> @@ -18,6 +19,7 @@ include::migration_intro.asciidoc[] * <> * <> +include::migrate_8_17.asciidoc[] include::migrate_8_16.asciidoc[] include::migrate_8_15.asciidoc[] include::migrate_8_14.asciidoc[] diff --git a/docs/reference/migration/migrate_8_17.asciidoc b/docs/reference/migration/migrate_8_17.asciidoc new file mode 100644 index 0000000000000..15bc6431c60ba --- /dev/null +++ b/docs/reference/migration/migrate_8_17.asciidoc @@ -0,0 +1,20 @@ +[[migrating-8.17]] +== Migrating to 8.17 +++++ +8.17 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.17. + +See also <> and <>. + +coming::[8.17.0] + + +[discrete] +[[breaking-changes-8.17]] +=== Breaking changes + +There are no breaking changes in {es} 8.17. + diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 6f32b55c49af8..c912b0e62b94d 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,6 +6,7 @@ This section summarizes the changes in each release. +* <> * <> * <> * <> @@ -72,6 +73,7 @@ This section summarizes the changes in each release. -- +include::release-notes/8.17.0.asciidoc[] include::release-notes/8.16.0.asciidoc[] include::release-notes/8.15.1.asciidoc[] include::release-notes/8.15.0.asciidoc[] diff --git a/docs/reference/release-notes/8.17.0.asciidoc b/docs/reference/release-notes/8.17.0.asciidoc new file mode 100644 index 0000000000000..59962fd83e9b7 --- /dev/null +++ b/docs/reference/release-notes/8.17.0.asciidoc @@ -0,0 +1,8 @@ +[[release-notes-8.17.0]] +== {es} version 8.17.0 + +coming[8.17.0] + +Also see <>. + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 1e0018f590ac0..81d46b5773877 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -11,7 +11,8 @@ For detailed information about this release, see the <> and // Add previous release to the list Other versions: -{ref-bare}/8.15/release-highlights.html[8.15] +{ref-bare}/8.16/release-highlights.html[8.16] +| {ref-bare}/8.15/release-highlights.html[8.15] | {ref-bare}/8.14/release-highlights.html[8.14] | {ref-bare}/8.13/release-highlights.html[8.13] | {ref-bare}/8.12/release-highlights.html[8.12] @@ -30,6 +31,8 @@ Other versions: endif::[] +// The notable-highlights tag marks entries that +// should be featured in the Stack Installation and Upgrade Guide: // tag::notable-highlights[] [discrete] @@ -97,29 +100,3 @@ ZStandard offers ~12% lower storage usage and a ~14% higher indexing throughput // end::notable-highlights[] -[discrete] -[[esql_multi_value_fields_supported_in_geospatial_predicates]] -=== ESQL: Multi-value fields supported in Geospatial predicates -Supporting multi-value fields in `WHERE` predicates is a challenge due to not knowing whether `ALL` or `ANY` -of the values in the field should pass the predicate. -For example, should the field `age:[10,30]` pass the predicate `WHERE age>20` or not? -This ambiguity does not exist with the spatial predicates -`ST_INTERSECTS` and `ST_DISJOINT`, because the choice between `ANY` or `ALL` -is implied by the predicate itself. -Consider a predicate checking a field named `location` against a test geometry named `shape`: - -* `ST_INTERSECTS(field, shape)` - true if `ANY` value can intersect the shape -* `ST_DISJOINT(field, shape)` - true only if `ALL` values are disjoint from the shape - -This works even if the shape argument is itself a complex or compound geometry. - -Similar logic exists for `ST_CONTAINS` and `ST_WITHIN` predicates, but these are not as easily solved -with `ANY` or `ALL`, because a collection of geometries contains another collection if each of the contained -geometries is within at least one of the containing geometries. Evaluating this requires that the multi-value -field is first combined into a single geometry before performing the predicate check. - -* `ST_CONTAINS(field, shape)` - true if the combined geometry contains the shape -* `ST_WITHIN(field, shape)` - true if the combined geometry is within the shape - -{es-pull}112063[#112063] - diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index 54836ac33762d..9306d83c79136 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -1,8 +1,6 @@ [[retriever]] === Retriever -preview::["This functionality is in technical preview and may be changed or removed in a future release. The syntax will likely change before GA. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] - A retriever is a specification to describe top documents returned from a search. A retriever replaces other elements of the <> that also return top documents such as <> and @@ -75,7 +73,7 @@ Collapses the top documents by a specified key into a single top document per ke ===== Restrictions When a retriever tree contains a compound retriever (a retriever with two or more child -retrievers) *only* the query element is allowed. +retrievers) the <> parameter is not supported. [discrete] [[standard-retriever-example]] @@ -245,12 +243,6 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-window-size] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-filter] -===== Restrictions - -An RRF retriever is a compound retriever. Child retrievers may not use -elements that are restricted by having a compound retriever as part of -the retriever tree. - [discrete] [[rrf-retriever-example-hybrid]] ==== Example: Hybrid search @@ -413,10 +405,6 @@ Applies the specified <> to the chil If the child retriever already specifies any filters, then this top-level filter is applied in conjuction with the filter defined in the child retriever. -===== Restrictions - -A text similarity re-ranker retriever is a compound retriever. Child retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree. - [discrete] [[text-similarity-reranker-retriever-example-cohere]] ==== Example: Cohere Rerank @@ -555,4 +543,3 @@ at the top-level and instead are only allowed as elements of specific retrievers * <> * <> * <> -* <> diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc index 2a676e5fba336..edd3b67e3de04 100644 --- a/docs/reference/search/rrf.asciidoc +++ b/docs/reference/search/rrf.asciidoc @@ -1,8 +1,6 @@ [[rrf]] === Reciprocal rank fusion -preview::["This functionality is in technical preview and may be changed or removed in a future release. The syntax will likely change before GA. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] - https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf[Reciprocal rank fusion (RRF)] is a method for combining multiple result sets with different relevance indicators into a single result set. RRF requires no tuning, and the different relevance indicators do not have to be related to each other to achieve high-quality results. @@ -95,19 +93,21 @@ The `rrf` retriever supports: * <> * <> +* <> +* <> +* <> +* <> The `rrf` retriever does not currently support: * <> -* <> * <> * <> -* <> -* <> -* <> -* <> Using unsupported features as part of a search with an `rrf` retriever results in an exception. ++ +IMPORTANT: It is best to avoid providing a <> as part of the request, as +RRF creates one internally that is shared by all sub-retrievers to ensure consistent results. [[rrf-using-multiple-standard-retrievers]] ==== Reciprocal rank fusion using multiple standard retrievers diff --git a/docs/reference/search/search-your-data/retrievers-overview.asciidoc b/docs/reference/search/search-your-data/retrievers-overview.asciidoc index c0fe7471946f3..9df4026fc6445 100644 --- a/docs/reference/search/search-your-data/retrievers-overview.asciidoc +++ b/docs/reference/search/search-your-data/retrievers-overview.asciidoc @@ -1,9 +1,7 @@ [[retrievers-overview]] === Retrievers -preview::[] - -A retriever is an abstraction that was added to the Search API in *8.14.0*. +A retriever is an abstraction that was added to the Search API in *8.14.0* and was made generally available in *8.16.0*. This abstraction enables the configuration of multi-stage retrieval pipelines within a single `_search` call. This simplifies your search application logic, because you no longer need to configure complex searches via multiple {es} calls or implement additional client-side logic to combine results from different queries. @@ -32,8 +30,7 @@ with different relevance indicators into a single result set. An RRF retriever is a *compound retriever*, where its `filter` element is propagated to its sub retrievers. + -Sub retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree. -See the <> for detailed examples and information on how to use the RRF retriever. + * <>. Used for <>. Requires first creating a `rerank` task using the <>. @@ -72,82 +69,56 @@ When using compound retrievers, only the query element is allowed, which enforce [[retrievers-overview-example]] ==== Example -The following example demonstrates how using retrievers simplify the composability of queries for RRF ranking. +The following example demonstrates the powerful queries that we can now compose, and how retrievers simplify this process. We can use any combination of retrievers we want, propagating the +results of a nested retriever to its parent. In this scenario, we'll make use of all 4 (currently) available retrievers, i.e. `standard`, `knn`, `text_similarity_reranker` and `rrf`. +We'll first combine the results of a `semantic` query using the `standard` retriever, and that of a `knn` search on a dense vector field, using `rrf` to get the top 100 results. +Finally, we'll then rerank the top-50 results of `rrf` using the `text_similarity_reranker` [source,js] ---- GET example-index/_search { "retriever": { - "rrf": { - "retrievers": [ - { - "standard": { - "query": { - "sparse_vector": { - "field": "vector.tokens", - "inference_id": "my-elser-endpoint", - "query": "What blue shoes are on sale?" + "text_similarity_reranker": { + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "semantic": { + "field": "inference_field", + "query": "state of the art vector database" + } + } + } + }, + { + "knn": { + "query_vector": [ + 0.54, + ..., + 0.245 + ], + "field": "embedding", + "k": 10, + "num_candidates": 15 } } - } - }, - { - "standard": { - "query": { - "match": { - "text": "blue shoes sale" - } - } - } - } - ] - } - } -} ----- -//NOTCONSOLE - -This example demonstrates how you can combine different retrieval strategies into a single `retriever` pipeline. - -Compare to `RRF` with `sub_searches` approach: - -.*Expand* for example -[%collapsible] -============== - -[source,js] ----- -GET example-index/_search -{ - "sub_searches":[ - { - "query":{ - "match":{ - "text":"blue shoes sale" - } - } - }, - { - "query":{ - "sparse_vector": { - "field": "vector.tokens", - "inference_id": "my-elser-endoint", - "query": "What blue shoes are on sale?" - } + ], + "rank_window_size": 100, + "rank_constant": 10 } - } - ], - "rank":{ - "rrf":{ - "rank_window_size":50, - "rank_constant":20 + }, + "rank_window_size": 50, + "field": "description", + "inference_text": "what's the best way to create complex pipelines and retrieve documents?", + "inference_id": "my-awesome-rerank-model" } } } ---- //NOTCONSOLE -============== [discrete] [[retrievers-overview-glossary]] diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index dbcfbb1b615f9..60692c19c184a 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -50,7 +50,7 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1> be used and ELSER creates sparse vectors. The `inference_id` is `my-elser-endpoint`. <2> The `elser` service is used in this example. -<3> This setting enables and configures adaptive allocations. +<3> This setting enables and configures {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations]. Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process. [NOTE] @@ -284,6 +284,8 @@ query from the `semantic-embedding` index: [discrete] [[semantic-text-further-examples]] -==== Further examples +==== Further examples and reading -If you want to use `semantic_text` in hybrid search, refer to https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/09-semantic-text.ipynb[this notebook] for a step-by-step guide. \ No newline at end of file +* If you want to use `semantic_text` in hybrid search, refer to https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/09-semantic-text.ipynb[this notebook] for a step-by-step guide. +* For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation. +* To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page. \ No newline at end of file diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc index 71a9fd8b87c96..36f311b1cdd97 100644 --- a/docs/reference/snapshot-restore/repository-s3.asciidoc +++ b/docs/reference/snapshot-restore/repository-s3.asciidoc @@ -329,6 +329,25 @@ include::repository-shared-settings.asciidoc[] `1000` which is the maximum number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up dangling multipart uploads. +`throttled_delete_retry.delay_increment`:: + + (<>) This value is used as the delay before the first retry and the amount the delay is incremented by on each subsequent retry. Default is 50ms, minimum is 0ms. + +`throttled_delete_retry.maximum_delay`:: + + (<>) This is the upper bound on how long the delays between retries will grow to. Default is 5s, minimum is 0ms. + +`throttled_delete_retry.maximum_number_of_retries`:: + + (integer) Sets the number times to retry a throttled snapshot deletion. Defaults to `10`, minimum value is `0` which + will disable retries altogether. Note that if retries are enabled in the Azure client, each of these retries + comprises that many client-level retries. + +`get_register_retry_delay` + + (<>) Sets the time to wait before trying again if an attempt to read a + <> fails. Defaults to `5s`. + NOTE: The option of defining client settings in the repository settings as documented below is considered deprecated, and will be removed in a future version. diff --git a/docs/reference/snapshot-restore/repository-source-only.asciidoc b/docs/reference/snapshot-restore/repository-source-only.asciidoc index 04e53c42aff9d..3c11d6ca6e59c 100644 --- a/docs/reference/snapshot-restore/repository-source-only.asciidoc +++ b/docs/reference/snapshot-restore/repository-source-only.asciidoc @@ -27,6 +27,9 @@ As a result, indices adopting synthetic source cannot be restored. When you rest * The mapping of the restored index is empty, but the original mapping is available from the types top level `meta` element. + * Information such as document count, deleted document count and store size are not available for such indices + since these indices do not contain the relevant data structures to retrieve this information from. Therefore, + this information is not shown for such indices in APIs such as the <>. ================================================== Before registering a source-only repository, use {kib} or the diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index 638e4d813a79a..212b869c6d933 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -138,9 +138,7 @@ public void setup() throws Exception { // Initialize the failure store. RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null); rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(b -> b.includeRegularIndices(false).includeFailureIndices(true)) - .build() + IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES).build() ); response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get(); assertTrue(response.isAcknowledged()); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java index b5d06dc33e035..679ad5b000c8f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java @@ -195,9 +195,7 @@ public void testRejectionFromFailureStore() throws IOException { // Initialize failure store. var rolloverRequest = new RolloverRequest(dataStream, null); rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false)) - .build() + IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES).build() ); var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet(); var failureStoreIndex = rolloverResponse.getNewIndex(); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java index 8b29b1609711f..d07e29c6b6b31 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java @@ -10,23 +10,30 @@ package org.elasticsearch.datastreams.logsdb.qa; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.logsdb.datageneration.DataGenerator; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; -import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.DocumentGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; +import org.elasticsearch.logsdb.datageneration.Mapping; +import org.elasticsearch.logsdb.datageneration.MappingGenerator; +import org.elasticsearch.logsdb.datageneration.Template; +import org.elasticsearch.logsdb.datageneration.TemplateGenerator; import org.elasticsearch.logsdb.datageneration.fields.PredefinedField; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.function.Consumer; public class DataGenerationHelper { private final boolean keepArraySource; - private final DataGenerator dataGenerator; + private final DocumentGenerator documentGenerator; + + private final Template template; + private final Mapping mapping; public DataGenerationHelper() { this(b -> {}); @@ -40,62 +47,47 @@ public DataGenerationHelper(Consumer builder .withPredefinedFields( List.of( // Customized because it always needs doc_values for aggregations. - new PredefinedField.WithGenerator("host.name", new FieldDataGenerator() { - @Override - public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "keyword").endObject(); - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(ESTestCase.randomAlphaOfLength(5)); - } - }), + new PredefinedField.WithGenerator( + "host.name", + FieldType.KEYWORD, + Map.of("type", "keyword"), + () -> ESTestCase.randomAlphaOfLength(5) + ), // Needed for terms query - new PredefinedField.WithGenerator("method", new FieldDataGenerator() { - @Override - public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "keyword").endObject(); - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(ESTestCase.randomFrom("put", "post", "get")); - } - }), + new PredefinedField.WithGenerator( + "method", + FieldType.KEYWORD, + Map.of("type", "keyword"), + () -> ESTestCase.randomFrom("put", "post", "get") + ), // Needed for histogram aggregation - new PredefinedField.WithGenerator("memory_usage_bytes", new FieldDataGenerator() { - @Override - public CheckedConsumer mappingWriter() { - return b -> b.startObject().field("type", "long").endObject(); - } - - @Override - public CheckedConsumer fieldValueGenerator() { - // We can generate this using standard long field but we would get "too many buckets" - return b -> b.value(ESTestCase.randomLongBetween(1000, 2000)); - } - }) + new PredefinedField.WithGenerator( + "memory_usage_bytes", + FieldType.LONG, + Map.of("type", "long"), + () -> ESTestCase.randomLongBetween(1000, 2000) + ) ) ); // Customize builder if necessary builderConfigurator.accept(specificationBuilder); - this.dataGenerator = new DataGenerator(specificationBuilder.build()); - } + var specification = specificationBuilder.build(); + + this.documentGenerator = new DocumentGenerator(specification); - DataGenerator getDataGenerator() { - return dataGenerator; + this.template = new TemplateGenerator(specification).generate(); + this.mapping = new MappingGenerator(specification).generate(template); } void logsDbMapping(XContentBuilder builder) throws IOException { - dataGenerator.writeMapping(builder); + builder.map(mapping.raw()); } void standardMapping(XContentBuilder builder) throws IOException { - dataGenerator.writeMapping(builder); + builder.map(mapping.raw()); } void logsDbSettings(Settings.Builder builder) { @@ -103,4 +95,11 @@ void logsDbSettings(Settings.Builder builder) { builder.put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), "arrays"); } } + + void generateDocument(XContentBuilder document, Map additionalFields) throws IOException { + var generated = documentGenerator.generate(template, mapping); + generated.putAll(additionalFields); + + document.map(generated); + } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java index 43efdbdcf8b1c..4c896e1f262b2 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -53,7 +53,7 @@ * This test uses simple mapping and document structure in order to allow easier debugging of the test itself. */ public class StandardVersusLogsIndexModeChallengeRestIT extends AbstractChallengeRestTest { - private final int numShards = randomBoolean() ? randomIntBetween(2, 5) : 0; + private final int numShards = randomBoolean() ? randomIntBetween(2, 4) : 0; private final int numReplicas = randomBoolean() ? randomIntBetween(1, 3) : 0; private final boolean fullyDynamicMapping = randomBoolean(); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index 751336cc1f646..6a20626634499 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.Map; /** * Challenge test (see {@link StandardVersusLogsIndexModeChallengeRestIT}) that uses randomly generated @@ -53,10 +54,10 @@ public void contenderSettings(Settings.Builder builder) { @Override protected XContentBuilder generateDocument(final Instant timestamp) throws IOException { var document = XContentFactory.jsonBuilder(); - dataGenerationHelper.getDataGenerator().generateDocument(document, doc -> { - doc.field("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp)); - }); - + dataGenerationHelper.generateDocument( + document, + Map.of("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp)) + ); return document; } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 878583de4861f..7bbf7137d290e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -946,7 +946,7 @@ private Set maybeExecuteForceMerge(ClusterState state, List indice UpdateSettingsRequest updateMergePolicySettingsRequest = new UpdateSettingsRequest(); updateMergePolicySettingsRequest.indicesOptions( IndicesOptions.builder(updateMergePolicySettingsRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(true, true)) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .build() ); updateMergePolicySettingsRequest.indices(indexName); @@ -1409,7 +1409,7 @@ static RolloverRequest getDefaultRolloverRequest( if (rolloverFailureStore) { rolloverRequest.setIndicesOptions( IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build() ); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index 3456f4b679474..b61e38297397d 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -43,7 +43,7 @@ public class RestGetDataStreamsAction extends BaseRestHandler { IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, "verbose" ), - DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FailureStoreOptions.FAILURE_STORE) : Set.of() + DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FAILURE_STORE_QUERY_PARAM) : Set.of() ) ); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 0d5ce54c44b56..d6bf80798764d 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -225,17 +225,11 @@ public void testOperationsExecutedOnce() { assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0); assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat( - rolloverBackingIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(new IndicesOptions.FailureStoreOptions(true, false)) - ); + assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_DATA)); assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class)); RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1); assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat( - rolloverFailureIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(new IndicesOptions.FailureStoreOptions(false, true)) - ); + assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_FAILURES)); List deleteRequests = clientSeenRequests.subList(2, 5) .stream() .map(transportRequest -> (DeleteIndexRequest) transportRequest) @@ -1552,17 +1546,11 @@ public void testFailureStoreIsManagedEvenWhenDisabled() { assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0); assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat( - rolloverBackingIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(new IndicesOptions.FailureStoreOptions(true, false)) - ); + assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_DATA)); assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class)); RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1); assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat( - rolloverFailureIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(new IndicesOptions.FailureStoreOptions(false, true)) - ); + assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_FAILURES)); assertThat( ((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0], is(dataStream.getFailureIndices().getIndices().get(0).getName()) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java index 0bc7114f626c4..ad1fce0ca689a 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; @@ -40,18 +41,24 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.Map; import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; +import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING; import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING; import static org.hamcrest.Matchers.equalTo; public class EnterpriseGeoIpDownloaderIT extends ESIntegTestCase { - private static final String DATABASE_TYPE = "GeoIP2-City"; + private static final String MAXMIND_DATABASE_TYPE = "GeoIP2-City"; + private static final String IPINFO_DATABASE_TYPE = "asn"; @ClassRule - public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture(DATABASE_TYPE); + public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture( + List.of(MAXMIND_DATABASE_TYPE), + List.of(IPINFO_DATABASE_TYPE) + ); protected String getEndpoint() { return fixture.getAddress(); @@ -61,6 +68,7 @@ protected String getEndpoint() { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString(MAXMIND_LICENSE_KEY_SETTING.getKey(), "license_key"); + secureSettings.setString(IPINFO_TOKEN_SETTING.getKey(), "token"); Settings.Builder builder = Settings.builder(); builder.setSecureSettings(secureSettings) .put(super.nodeSettings(nodeOrdinal, otherSettings)) @@ -87,22 +95,27 @@ public void testEnterpriseDownloaderTask() throws Exception { * Note that the "enterprise database" is actually just a geolite database being loaded by the GeoIpHttpFixture. */ EnterpriseGeoIpDownloader.DEFAULT_MAXMIND_ENDPOINT = getEndpoint(); - final String pipelineName = "enterprise_geoip_pipeline"; + EnterpriseGeoIpDownloader.DEFAULT_IPINFO_ENDPOINT = getEndpoint(); final String indexName = "enterprise_geoip_test_index"; + final String geoipPipelineName = "enterprise_geoip_pipeline"; + final String iplocationPipelineName = "enterprise_iplocation_pipeline"; final String sourceField = "ip"; - final String targetField = "ip-city"; + final String targetField = "ip-result"; startEnterpriseGeoIpDownloaderTask(); - configureDatabase(DATABASE_TYPE); - createGeoIpPipeline(pipelineName, DATABASE_TYPE, sourceField, targetField); + configureMaxmindDatabase(MAXMIND_DATABASE_TYPE); + configureIpinfoDatabase(IPINFO_DATABASE_TYPE); + waitAround(); + createPipeline(geoipPipelineName, "geoip", MAXMIND_DATABASE_TYPE, sourceField, targetField); + createPipeline(iplocationPipelineName, "ip_location", IPINFO_DATABASE_TYPE, sourceField, targetField); + /* + * We know that the databases index has been populated (because we waited around, :wink:), but we don't know for sure that + * the databases have been pulled down and made available on all nodes. So we run these ingest-and-check steps in assertBusy blocks. + */ assertBusy(() -> { - /* - * We know that the .geoip_databases index has been populated, but we don't know for sure that the database has been pulled - * down and made available on all nodes. So we run this ingest-and-check step in an assertBusy. - */ logger.info("Ingesting a test document"); - String documentId = ingestDocument(indexName, pipelineName, sourceField); + String documentId = ingestDocument(indexName, geoipPipelineName, sourceField, "89.160.20.128"); GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet(); Map returnedSource = getResponse.getSource(); assertNotNull(returnedSource); @@ -110,6 +123,16 @@ public void testEnterpriseDownloaderTask() throws Exception { assertNotNull(targetFieldValue); assertThat(((Map) targetFieldValue).get("organization_name"), equalTo("Bredband2 AB")); }); + assertBusy(() -> { + logger.info("Ingesting another test document"); + String documentId = ingestDocument(indexName, iplocationPipelineName, sourceField, "12.10.66.1"); + GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet(); + Map returnedSource = getResponse.getSource(); + assertNotNull(returnedSource); + Object targetFieldValue = returnedSource.get(targetField); + assertNotNull(targetFieldValue); + assertThat(((Map) targetFieldValue).get("organization_name"), equalTo("OAKLAWN JOCKEY CLUB, INC.")); + }); } private void startEnterpriseGeoIpDownloaderTask() { @@ -128,36 +151,53 @@ private void startEnterpriseGeoIpDownloaderTask() { ); } - private void configureDatabase(String databaseType) throws Exception { + private void configureMaxmindDatabase(String databaseType) { admin().cluster() .execute( PutDatabaseConfigurationAction.INSTANCE, new PutDatabaseConfigurationAction.Request( TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, - new DatabaseConfiguration("test", databaseType, new DatabaseConfiguration.Maxmind("test_account")) + new DatabaseConfiguration("test-1", databaseType, new DatabaseConfiguration.Maxmind("test_account")) ) ) .actionGet(); + } + + private void configureIpinfoDatabase(String databaseType) { + admin().cluster() + .execute( + PutDatabaseConfigurationAction.INSTANCE, + new PutDatabaseConfigurationAction.Request( + TimeValue.MAX_VALUE, + TimeValue.MAX_VALUE, + new DatabaseConfiguration("test-2", databaseType, new DatabaseConfiguration.Ipinfo()) + ) + ) + .actionGet(); + } + + private void waitAround() throws Exception { ensureGreen(GeoIpDownloader.DATABASES_INDEX); assertBusy(() -> { SearchResponse searchResponse = client().search(new SearchRequest(GeoIpDownloader.DATABASES_INDEX)).actionGet(); try { - assertThat(searchResponse.getHits().getHits().length, equalTo(1)); + assertThat(searchResponse.getHits().getHits().length, equalTo(2)); } finally { searchResponse.decRef(); } }); } - private void createGeoIpPipeline(String pipelineName, String databaseType, String sourceField, String targetField) throws IOException { + private void createPipeline(String pipelineName, String processorType, String databaseType, String sourceField, String targetField) + throws IOException { putJsonPipeline(pipelineName, (builder, params) -> { builder.field("description", "test"); builder.startArray("processors"); { builder.startObject(); { - builder.startObject("geoip"); + builder.startObject(processorType); { builder.field("field", sourceField); builder.field("target_field", targetField); @@ -171,11 +211,11 @@ private void createGeoIpPipeline(String pipelineName, String databaseType, Strin }); } - private String ingestDocument(String indexName, String pipelineName, String sourceField) { + private String ingestDocument(String indexName, String pipelineName, String sourceField, String value) { BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add( - new IndexRequest(indexName).source("{\"" + sourceField + "\": \"89.160.20.128\"}", XContentType.JSON).setPipeline(pipelineName) - ); + bulkRequest.add(new IndexRequest(indexName).source(Strings.format(""" + { "%s": "%s"} + """, sourceField, value), XContentType.JSON).setPipeline(pipelineName)); BulkResponse response = client().bulk(bulkRequest).actionGet(); BulkItemResponse[] bulkItemResponses = response.getItems(); assertThat(bulkItemResponses.length, equalTo(1)); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java index 3bbb0539f193a..e04014ff693be 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -39,6 +38,8 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import java.io.Closeable; @@ -57,6 +58,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_SETTINGS_PREFIX; import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_SETTINGS_PREFIX; /** @@ -72,6 +74,9 @@ public class EnterpriseGeoIpDownloader extends AllocatedPersistentTask { // a sha256 checksum followed by two spaces followed by an (ignored) file name private static final Pattern SHA256_CHECKSUM_PATTERN = Pattern.compile("(\\w{64})\\s\\s(.*)"); + // an md5 checksum + private static final Pattern MD5_CHECKSUM_PATTERN = Pattern.compile("(\\w{32})"); + // for overriding in tests static String DEFAULT_MAXMIND_ENDPOINT = System.getProperty( MAXMIND_SETTINGS_PREFIX + "endpoint.default", // @@ -80,6 +85,14 @@ public class EnterpriseGeoIpDownloader extends AllocatedPersistentTask { // n.b. a future enhancement might be to allow for a MAXMIND_ENDPOINT_SETTING, but // at the moment this is an unsupported system property for use in tests (only) + // for overriding in tests + static String DEFAULT_IPINFO_ENDPOINT = System.getProperty( + IPINFO_SETTINGS_PREFIX + "endpoint.default", // + "https://ipinfo.io/data" + ); + // n.b. a future enhancement might be to allow for an IPINFO_ENDPOINT_SETTING, but + // at the moment this is an unsupported system property for use in tests (only) + static final String DATABASES_INDEX = ".geoip_databases"; static final int MAX_CHUNK_SIZE = 1024 * 1024; @@ -444,16 +457,15 @@ private void scheduleNextRun(TimeValue time) { } } - @Nullable private ProviderDownload downloaderFor(DatabaseConfiguration database) { - if (database.provider() instanceof DatabaseConfiguration.Maxmind) { - return new MaxmindDownload(database.name(), (DatabaseConfiguration.Maxmind) database.provider()); - } else if (database.provider() instanceof DatabaseConfiguration.Ipinfo) { - // as a temporary implementation detail, null here means 'not actually supported *just yet*' - return null; + if (database.provider() instanceof DatabaseConfiguration.Maxmind maxmind) { + return new MaxmindDownload(database.name(), maxmind); + } else if (database.provider() instanceof DatabaseConfiguration.Ipinfo ipinfo) { + return new IpinfoDownload(database.name(), ipinfo); } else { - assert false : "Attempted to use database downloader with unsupported provider type [" + database.provider().getClass() + "]"; - return null; + throw new IllegalArgumentException( + Strings.format("Unexpected provider [%s] for configuration [%s]", database.provider().getClass(), database.id()) + ); } } @@ -488,7 +500,7 @@ public HttpClient.PasswordAuthenticationHolder buildCredentials() { @Override public boolean validCredentials() { - return auth.get() != null; + return auth != null && auth.get() != null; } @Override @@ -529,7 +541,101 @@ public CheckedSupplier download() { @Override public void close() throws IOException { - auth.close(); + if (auth != null) auth.close(); + } + } + + class IpinfoDownload implements ProviderDownload { + + final String name; + final DatabaseConfiguration.Ipinfo ipinfo; + HttpClient.PasswordAuthenticationHolder auth; + + IpinfoDownload(String name, DatabaseConfiguration.Ipinfo ipinfo) { + this.name = name; + this.ipinfo = ipinfo; + this.auth = buildCredentials(); + } + + @Override + public HttpClient.PasswordAuthenticationHolder buildCredentials() { + final char[] tokenChars = tokenProvider.apply("ipinfo"); + + // if the token is missing or empty, return null as 'no auth' + if (tokenChars == null || tokenChars.length == 0) { + return null; + } + + // ipinfo uses the token as the username component of basic auth, see https://ipinfo.io/developers#authentication + return new HttpClient.PasswordAuthenticationHolder(new String(tokenChars), new char[] {}); + } + + @Override + public boolean validCredentials() { + return auth != null && auth.get() != null; + } + + private static final Set FREE_DATABASES = Set.of("asn", "country", "country_asn"); + + @Override + public String url(String suffix) { + // note: the 'free' databases are in the sub-path 'free/' in terms of the download endpoint + final String internalName; + if (FREE_DATABASES.contains(name)) { + internalName = "free/" + name; + } else { + internalName = name; + } + + // reminder, we're passing the ipinfo token as the username part of http basic auth, + // see https://ipinfo.io/developers#authentication + + String endpointPattern = DEFAULT_IPINFO_ENDPOINT; + if (endpointPattern.contains("%")) { + throw new IllegalArgumentException("Invalid endpoint [" + endpointPattern + "]"); + } + if (endpointPattern.endsWith("/") == false) { + endpointPattern += "/"; + } + endpointPattern += "%s.%s"; + + // at this point the pattern looks like this (in the default case): + // https://ipinfo.io/data/%s.%s + // also see https://ipinfo.io/developers/database-download, + // and https://ipinfo.io/developers/database-filename-reference for more + + return Strings.format(endpointPattern, internalName, suffix); + } + + @Override + public Checksum checksum() throws IOException { + final String checksumJsonUrl = this.url("mmdb/checksums"); // a minor abuse of the idea of a 'suffix', :shrug: + byte[] data = httpClient.getBytes(auth.get(), checksumJsonUrl); // this throws if the auth is bad + Map checksums; + try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, data)) { + checksums = parser.map(); + } + @SuppressWarnings("unchecked") + String md5 = ((Map) checksums.get("checksums")).get("md5"); + logger.trace("checksum was [{}]", md5); + + var matcher = MD5_CHECKSUM_PATTERN.matcher(md5); + boolean match = matcher.matches(); + if (match == false) { + throw new RuntimeException("Unexpected md5 response from [" + checksumJsonUrl + "]"); + } + return Checksum.md5(md5); + } + + @Override + public CheckedSupplier download() { + final String mmdbUrl = this.url("mmdb"); + return () -> httpClient.get(auth.get(), mmdbUrl); + } + + @Override + public void close() throws IOException { + if (auth != null) auth.close(); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java index 5214c0e4a6a51..ae9bb109a3bf8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTaskExecutor.java @@ -54,11 +54,15 @@ public class EnterpriseGeoIpDownloaderTaskExecutor extends PersistentTasksExecut static final String MAXMIND_SETTINGS_PREFIX = "ingest.geoip.downloader.maxmind."; + static final String IPINFO_SETTINGS_PREFIX = "ingest.ip_location.downloader.ipinfo."; + public static final Setting MAXMIND_LICENSE_KEY_SETTING = SecureSetting.secureString( MAXMIND_SETTINGS_PREFIX + "license_key", null ); + public static final Setting IPINFO_TOKEN_SETTING = SecureSetting.secureString(IPINFO_SETTINGS_PREFIX + "token", null); + private final Client client; private final HttpClient httpClient; private final ClusterService clusterService; @@ -106,6 +110,10 @@ private char[] getSecureToken(final String type) { if (cachedSecureSettings.getSettingNames().contains(MAXMIND_LICENSE_KEY_SETTING.getKey())) { token = cachedSecureSettings.getString(MAXMIND_LICENSE_KEY_SETTING.getKey()).getChars(); } + } else if (type.equals("ipinfo")) { + if (cachedSecureSettings.getSettingNames().contains(IPINFO_TOKEN_SETTING.getKey())) { + token = cachedSecureSettings.getString(IPINFO_TOKEN_SETTING.getKey()).getChars(); + } } return token; } @@ -166,7 +174,7 @@ public synchronized void reload(Settings settings) { // `SecureSettings` are available here! cache them as they will be needed // whenever dynamic cluster settings change and we have to rebuild the accounts try { - this.cachedSecureSettings = extractSecureSettings(settings, List.of(MAXMIND_LICENSE_KEY_SETTING)); + this.cachedSecureSettings = extractSecureSettings(settings, List.of(MAXMIND_LICENSE_KEY_SETTING, IPINFO_TOKEN_SETTING)); } catch (GeneralSecurityException e) { // rethrow as a runtime exception, there's logging higher up the call chain around ReloadablePlugin throw new ElasticsearchException("Exception while reloading enterprise geoip download task executor", e); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java index dcaa8f6f2fb03..ae562d3c7359a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -139,11 +138,18 @@ void updateDatabases() throws IOException { if (geoipIndex != null) { logger.trace("The {} index is not null", GeoIpDownloader.DATABASES_INDEX); if (clusterState.getRoutingTable().index(geoipIndex.getWriteIndex()).allPrimaryShardsActive() == false) { - throw new ElasticsearchException("not all primary shards of [" + DATABASES_INDEX + "] index are active"); + logger.debug( + "Not updating geoip database because not all primary shards of the [" + DATABASES_INDEX + "] index are active." + ); + return; } var blockException = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, geoipIndex.getWriteIndex().getName()); if (blockException != null) { - throw blockException; + logger.debug( + "Not updating geoip database because there is a write block on the " + geoipIndex.getWriteIndex().getName() + " index", + blockException + ); + return; } } if (eagerDownloadSupplier.get() || atLeastOneGeoipProcessorSupplier.get()) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index cc0bec583483e..3107f0bed55e8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -112,7 +112,8 @@ public List> getSettings() { GeoIpDownloaderTaskExecutor.ENABLED_SETTING, GeoIpDownloader.ENDPOINT_SETTING, GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING, - EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING + EnterpriseGeoIpDownloaderTaskExecutor.MAXMIND_LICENSE_KEY_SETTING, + EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING ); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java index 19a98fb1b5746..5a13ea93ff032 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java @@ -114,10 +114,10 @@ static Database getIpinfoDatabase(final String databaseType) { @Nullable static Function, IpDataLookup> getIpinfoLookup(final Database database) { return switch (database) { - case Database.AsnV2 -> IpinfoIpDataLookups.Asn::new; - case Database.CountryV2 -> IpinfoIpDataLookups.Country::new; - case Database.CityV2 -> IpinfoIpDataLookups.Geolocation::new; - case Database.PrivacyDetection -> IpinfoIpDataLookups.PrivacyDetection::new; + case AsnV2 -> IpinfoIpDataLookups.Asn::new; + case CountryV2 -> IpinfoIpDataLookups.Country::new; + case CityV2 -> IpinfoIpDataLookups.Geolocation::new; + case PrivacyDetection -> IpinfoIpDataLookups.PrivacyDetection::new; default -> null; }; } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java index 88c37409713ac..e1cd127be9c87 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java @@ -488,6 +488,36 @@ public void testMaxmindUrls() { } } + public void testIpinfoUrls() { + // a 'free' database like 'asn' has 'free/' in the url (automatically) + final EnterpriseGeoIpDownloader.IpinfoDownload download = geoIpDownloader.new IpinfoDownload( + "asn", new DatabaseConfiguration.Ipinfo() + ); + + { + String url = "https://ipinfo.io/data/free/asn.mmdb"; + assertThat(download.url("mmdb"), equalTo(url)); + } + { + String url = "https://ipinfo.io/data/free/asn.mmdb/checksums"; + assertThat(download.url("mmdb/checksums"), equalTo(url)); + } + + // but a non-'free' database like 'standard_asn' does not + final EnterpriseGeoIpDownloader.IpinfoDownload download2 = geoIpDownloader.new IpinfoDownload( + "standard_asn", new DatabaseConfiguration.Ipinfo() + ); + + { + String url = "https://ipinfo.io/data/standard_asn.mmdb"; + assertThat(download2.url("mmdb"), equalTo(url)); + } + { + String url = "https://ipinfo.io/data/standard_asn.mmdb/checksums"; + assertThat(download2.url("mmdb/checksums"), equalTo(url)); + } + } + private static class MockClient extends NoOpClient { private final Map, BiConsumer>> handlers = new HashMap<>(); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java index e73f0a36cc632..5698328792787 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.ingest.geoip; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -25,11 +24,9 @@ import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -583,37 +580,28 @@ void processDatabase(Map databaseInfo) { assertFalse(it.hasNext()); } - public void testUpdateDatabasesWriteBlock() { + public void testUpdateDatabasesWriteBlock() throws IOException { + /* + * Here we make sure that we bail out before making an httpClient request if there is write block on the .geoip_databases index + */ ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of())); var geoIpIndex = state.getMetadata().getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX).getWriteIndex().getName(); state = ClusterState.builder(state) .blocks(new ClusterBlocks.Builder().addIndexBlock(geoIpIndex, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK)) .build(); when(clusterService.state()).thenReturn(state); - var e = expectThrows(ClusterBlockException.class, () -> geoIpDownloader.updateDatabases()); - assertThat( - e.getMessage(), - equalTo( - "index [" - + geoIpIndex - + "] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, " - + "index has read-only-allow-delete block; for more information, see " - + ReferenceDocs.FLOOD_STAGE_WATERMARK - + "];" - ) - ); + geoIpDownloader.updateDatabases(); verifyNoInteractions(httpClient); } - public void testUpdateDatabasesIndexNotReady() { + public void testUpdateDatabasesIndexNotReady() throws IOException { + /* + * Here we make sure that we bail out before making an httpClient request if there are unallocated shards on the .geoip_databases + * index + */ ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of()), true); - var geoIpIndex = state.getMetadata().getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX).getWriteIndex().getName(); - state = ClusterState.builder(state) - .blocks(new ClusterBlocks.Builder().addIndexBlock(geoIpIndex, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK)) - .build(); when(clusterService.state()).thenReturn(state); - var e = expectThrows(ElasticsearchException.class, () -> geoIpDownloader.updateDatabases()); - assertThat(e.getMessage(), equalTo("not all primary shards of [.geoip_databases] index are active")); + geoIpDownloader.updateDatabases(); verifyNoInteractions(httpClient); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index eec6e003f3556..b748a0ced8be9 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -65,10 +64,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client * Parses a {@link RestRequest} body and returns a {@link MultiSearchTemplateRequest} */ public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException { - if (restRequest.getRestApiVersion() == RestApiVersion.V_7 && restRequest.hasParam("type")) { - restRequest.param("type"); - } - MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest(); if (restRequest.hasParam("max_concurrent_searches")) { multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0)); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 0ee93474234ec..6b37b02a945b2 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -54,7 +54,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; @@ -312,9 +311,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep if (indexedDocumentIndex != null) { builder.field(INDEXED_DOCUMENT_FIELD_INDEX.getPreferredName(), indexedDocumentIndex); } - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(INDEXED_DOCUMENT_FIELD_TYPE.getPreferredName(), MapperService.SINGLE_MAPPING_NAME); - } if (indexedDocumentId != null) { builder.field(INDEXED_DOCUMENT_FIELD_ID.getPreferredName(), indexedDocumentId); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java index 48a892f796f92..095d119bf2719 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -28,7 +27,6 @@ import java.io.IOException; import java.util.Map; import java.util.function.Consumer; -import java.util.function.IntConsumer; import java.util.function.Predicate; /** @@ -54,10 +52,7 @@ protected void parseInternalRequest( SearchRequest searchRequest = internal.getSearchRequest(); try (XContentParser parser = extractRequestSpecificFields(restRequest, bodyConsumers)) { - IntConsumer sizeConsumer = restRequest.getRestApiVersion() == RestApiVersion.V_7 - ? size -> setMaxDocsFromSearchSize(internal, size) - : size -> failOnSizeSpecified(); - RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, clusterSupportsFeature, sizeConsumer); + RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, clusterSupportsFeature, size -> failOnSizeSpecified()); } searchRequest.source().size(restRequest.paramAsInt("scroll_size", searchRequest.source().size())); diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java index 67ea34f504790..632ed73b9b2fa 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/RestUpdateByQueryAction.java @@ -10,7 +10,6 @@ package org.elasticsearch.reindex; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.UpdateByQueryAction; import org.elasticsearch.index.reindex.UpdateByQueryRequest; @@ -55,9 +54,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client @Override protected UpdateByQueryRequest buildRequest(RestRequest request) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - } /* * Passing the search request through UpdateByQueryRequest first allows * it to set its own defaults which differ from SearchRequest's diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java index a6e28477f8582..ebb4471566fbd 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java @@ -15,8 +15,8 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Collections; -import static java.util.Collections.emptyList; import static org.elasticsearch.reindex.BulkByScrollParallelizationHelper.sliceIntoSubRequests; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder; @@ -24,7 +24,7 @@ public class BulkByScrollParallelizationHelperTests extends ESTestCase { public void testSliceIntoSubRequests() throws IOException { SearchRequest searchRequest = randomSearchRequest( - () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, () -> null, () -> emptyList(), () -> null, () -> null) + () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, Collections::emptyList, () -> null, () -> null) ); if (searchRequest.source() != null) { // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is. diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java index e55668adea101..21f42bf9eb99c 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; @@ -31,6 +32,8 @@ import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -38,6 +41,7 @@ import java.util.Queue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; import static org.elasticsearch.repositories.RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM; @@ -48,9 +52,11 @@ import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_THROTTLES_HISTOGRAM; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_THROTTLES_TOTAL; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL; +import static org.elasticsearch.repositories.s3.S3RepositoriesMetrics.METRIC_DELETE_RETRIES_HISTOGRAM; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.elasticsearch.rest.RestStatus.NOT_FOUND; import static org.elasticsearch.rest.RestStatus.REQUESTED_RANGE_NOT_SATISFIED; +import static org.elasticsearch.rest.RestStatus.SERVICE_UNAVAILABLE; import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -61,14 +67,22 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class S3BlobStoreRepositoryMetricsTests extends S3BlobStoreRepositoryTests { - private final Queue errorStatusQueue = new LinkedBlockingQueue<>(); + private static final S3ErrorResponse S3_SLOW_DOWN_RESPONSE = new S3ErrorResponse(SERVICE_UNAVAILABLE, """ + + + SlowDown + This is a throttling message + /bucket/ + 4442587FB7D0A2F9 + """); + private final Queue errorResponseQueue = new LinkedBlockingQueue<>(); // Always create erroneous handler @Override protected Map createHttpHandlers() { return Collections.singletonMap( "/bucket", - new S3StatsCollectorHttpHandler(new S3MetricErroneousHttpHandler(new S3BlobStoreHttpHandler("bucket"), errorStatusQueue)) + new S3StatsCollectorHttpHandler(new S3MetricErroneousHttpHandler(new S3BlobStoreHttpHandler("bucket"), errorResponseQueue)) ); } @@ -244,8 +258,74 @@ public void testMetricsForRequestRangeNotSatisfied() { } } + public void testRetrySnapshotDeleteMetricsOnEventualSuccess() throws IOException { + final int maxRetries = 5; + final String repositoryName = randomRepositoryName(); + // Disable retries in the client for this repo + createRepository( + repositoryName, + Settings.builder() + .put(repositorySettings(repositoryName)) + .put(S3ClientSettings.MAX_RETRIES_SETTING.getConcreteSettingForNamespace("placeholder").getKey(), 0) + .put(S3Repository.RETRY_THROTTLED_DELETE_DELAY_INCREMENT.getKey(), TimeValue.timeValueMillis(10)) + .put(S3Repository.RETRY_THROTTLED_DELETE_MAX_NUMBER_OF_RETRIES.getKey(), maxRetries) + .build(), + false + ); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repositoryName); + final TestTelemetryPlugin plugin = getPlugin(dataNodeName); + final int numberOfDeletes = randomIntBetween(1, 3); + final List numberOfRetriesPerAttempt = new ArrayList<>(); + for (int i = 0; i < numberOfDeletes; i++) { + int numFailures = randomIntBetween(1, maxRetries); + numberOfRetriesPerAttempt.add((long) numFailures); + IntStream.range(0, numFailures).forEach(ignored -> addErrorStatus(S3_SLOW_DOWN_RESPONSE)); + blobContainer.deleteBlobsIgnoringIfNotExists( + randomFrom(OperationPurpose.SNAPSHOT_DATA, OperationPurpose.SNAPSHOT_METADATA), + List.of(randomIdentifier()).iterator() + ); + } + List longHistogramMeasurement = plugin.getLongHistogramMeasurement(METRIC_DELETE_RETRIES_HISTOGRAM); + assertThat(longHistogramMeasurement.stream().map(Measurement::getLong).toList(), equalTo(numberOfRetriesPerAttempt)); + } + + public void testRetrySnapshotDeleteMetricsWhenRetriesExhausted() { + final String repositoryName = randomRepositoryName(); + // Disable retries in the client for this repo + int maxRetries = 3; + createRepository( + repositoryName, + Settings.builder() + .put(repositorySettings(repositoryName)) + .put(S3ClientSettings.MAX_RETRIES_SETTING.getConcreteSettingForNamespace("placeholder").getKey(), 0) + .put(S3Repository.RETRY_THROTTLED_DELETE_DELAY_INCREMENT.getKey(), TimeValue.timeValueMillis(10)) + .put(S3Repository.RETRY_THROTTLED_DELETE_MAX_NUMBER_OF_RETRIES.getKey(), maxRetries) + .build(), + false + ); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repositoryName); + final TestTelemetryPlugin plugin = getPlugin(dataNodeName); + // Keep throttling past the max number of retries + IntStream.range(0, maxRetries + 1).forEach(ignored -> addErrorStatus(S3_SLOW_DOWN_RESPONSE)); + assertThrows( + IOException.class, + () -> blobContainer.deleteBlobsIgnoringIfNotExists( + randomFrom(OperationPurpose.SNAPSHOT_DATA, OperationPurpose.SNAPSHOT_METADATA), + List.of(randomIdentifier()).iterator() + ) + ); + List longHistogramMeasurement = plugin.getLongHistogramMeasurement(METRIC_DELETE_RETRIES_HISTOGRAM); + assertThat(longHistogramMeasurement.get(0).getLong(), equalTo(3L)); + } + private void addErrorStatus(RestStatus... statuses) { - errorStatusQueue.addAll(Arrays.asList(statuses)); + errorResponseQueue.addAll(Arrays.stream(statuses).map(S3ErrorResponse::new).toList()); + } + + private void addErrorStatus(S3ErrorResponse... responses) { + errorResponseQueue.addAll(Arrays.asList(responses)); } private long getLongCounterValue(TestTelemetryPlugin plugin, String instrumentName, Operation operation) { @@ -275,25 +355,25 @@ private long getLongHistogramValue(TestTelemetryPlugin plugin, String instrument private static class S3MetricErroneousHttpHandler implements DelegatingHttpHandler { private final HttpHandler delegate; - private final Queue errorStatusQueue; + private final Queue errorResponseQueue; - S3MetricErroneousHttpHandler(HttpHandler delegate, Queue errorStatusQueue) { + S3MetricErroneousHttpHandler(HttpHandler delegate, Queue errorResponseQueue) { this.delegate = delegate; - this.errorStatusQueue = errorStatusQueue; + this.errorResponseQueue = errorResponseQueue; } @Override public void handle(HttpExchange exchange) throws IOException { - final RestStatus status = errorStatusQueue.poll(); - if (status == null) { + final S3ErrorResponse errorResponse = errorResponseQueue.poll(); + if (errorResponse == null) { delegate.handle(exchange); - } else if (status == INTERNAL_SERVER_ERROR) { + } else if (errorResponse.status == INTERNAL_SERVER_ERROR) { // Simulate an retryable exception throw new IOException("ouch"); } else { try (exchange) { drainInputStream(exchange.getRequestBody()); - exchange.sendResponseHeaders(status.getStatus(), -1); + errorResponse.writeResponse(exchange); } } } @@ -302,4 +382,22 @@ public HttpHandler getDelegate() { return delegate; } } + + record S3ErrorResponse(RestStatus status, String responseBody) { + + S3ErrorResponse(RestStatus status) { + this(status, null); + } + + @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint") + public void writeResponse(HttpExchange exchange) throws IOException { + if (responseBody != null) { + byte[] responseBytes = responseBody.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(status.getStatus(), responseBytes.length); + exchange.getResponseBody().write(responseBytes); + } else { + exchange.sendResponseHeaders(status.getStatus(), -1); + } + } + } } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index 49df078453327..902dcb42fc0cb 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -40,6 +40,7 @@ import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.cluster.service.MasterService; +import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobContainer; @@ -910,21 +911,44 @@ public void compareAndExchangeRegister( @Override public void getRegister(OperationPurpose purpose, String key, ActionListener listener) { ActionListener.completeWith(listener, () -> { - final var getObjectRequest = new GetObjectRequest(blobStore.bucket(), buildKey(key)); - S3BlobStore.configureRequestForMetrics(getObjectRequest, blobStore, Operation.GET_OBJECT, purpose); - try ( - var clientReference = blobStore.clientReference(); - var s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest)); - var stream = s3Object.getObjectContent() - ) { - return OptionalBytesReference.of(getRegisterUsingConsistentRead(stream, keyPath, key)); - } catch (AmazonS3Exception e) { - logger.trace(() -> Strings.format("[%s]: getRegister failed", key), e); - if (e.getStatusCode() == 404) { - return OptionalBytesReference.EMPTY; - } else { - throw e; + final var backoffPolicy = purpose == OperationPurpose.REPOSITORY_ANALYSIS + ? BackoffPolicy.noBackoff() + : BackoffPolicy.constantBackoff(blobStore.getGetRegisterRetryDelay(), blobStore.getMaxRetries()); + final var retryDelayIterator = backoffPolicy.iterator(); + + Exception finalException = null; + while (true) { + final var getObjectRequest = new GetObjectRequest(blobStore.bucket(), buildKey(key)); + S3BlobStore.configureRequestForMetrics(getObjectRequest, blobStore, Operation.GET_OBJECT, purpose); + try ( + var clientReference = blobStore.clientReference(); + var s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest)); + var stream = s3Object.getObjectContent() + ) { + return OptionalBytesReference.of(getRegisterUsingConsistentRead(stream, keyPath, key)); + } catch (Exception attemptException) { + logger.trace(() -> Strings.format("[%s]: getRegister failed", key), attemptException); + if (attemptException instanceof AmazonS3Exception amazonS3Exception && amazonS3Exception.getStatusCode() == 404) { + return OptionalBytesReference.EMPTY; + } else if (finalException == null) { + finalException = attemptException; + } else if (finalException != attemptException) { + finalException.addSuppressed(attemptException); + } } + if (retryDelayIterator.hasNext()) { + try { + // noinspection BusyWait + Thread.sleep(retryDelayIterator.next().millis()); + continue; + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + finalException.addSuppressed(interruptedException); + // fall through and throw the exception + } + } + + throw finalException; } }); } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 3e6b7c356cb11..5fb3254df819b 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -14,6 +14,7 @@ import com.amazonaws.Request; import com.amazonaws.Response; import com.amazonaws.metrics.RequestMetricCollector; +import com.amazonaws.retry.RetryUtils; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.MultiObjectDeleteException; @@ -25,6 +26,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; @@ -91,6 +93,9 @@ class S3BlobStore implements BlobStore { private final StatsCollectors statsCollectors = new StatsCollectors(); private final int bulkDeletionBatchSize; + private final BackoffPolicy retryThrottledDeleteBackoffPolicy; + + private final TimeValue getRegisterRetryDelay; S3BlobStore( S3Service service, @@ -102,7 +107,8 @@ class S3BlobStore implements BlobStore { RepositoryMetadata repositoryMetadata, BigArrays bigArrays, ThreadPool threadPool, - S3RepositoriesMetrics s3RepositoriesMetrics + S3RepositoriesMetrics s3RepositoriesMetrics, + BackoffPolicy retryThrottledDeleteBackoffPolicy ) { this.service = service; this.bigArrays = bigArrays; @@ -116,7 +122,8 @@ class S3BlobStore implements BlobStore { this.snapshotExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT); this.s3RepositoriesMetrics = s3RepositoriesMetrics; this.bulkDeletionBatchSize = S3Repository.DELETION_BATCH_SIZE_SETTING.get(repositoryMetadata.settings()); - + this.retryThrottledDeleteBackoffPolicy = retryThrottledDeleteBackoffPolicy; + this.getRegisterRetryDelay = S3Repository.GET_REGISTER_RETRY_DELAY.get(repositoryMetadata.settings()); } RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) { @@ -255,7 +262,8 @@ private boolean assertConsistencyBetweenHttpRequestAndOperation(Request reque private static long getCountForMetric(TimingInfo info, AWSRequestMetrics.Field field) { var count = info.getCounter(field.name()); if (count == null) { - if (field == AWSRequestMetrics.Field.RequestCount) { + // This can be null if the thread was interrupted + if (field == AWSRequestMetrics.Field.RequestCount && Thread.currentThread().isInterrupted() == false) { final String message = "Expected request count to be tracked but found not count."; assert false : message; logger.warn(message); @@ -331,18 +339,18 @@ public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator partition = new ArrayList<>(); - try (AmazonS3Reference clientReference = clientReference()) { + try { // S3 API only allows 1k blobs per delete so we split up the given blobs into requests of max. 1k deletes final AtomicReference aex = new AtomicReference<>(); blobNames.forEachRemaining(key -> { partition.add(key); if (partition.size() == bulkDeletionBatchSize) { - deletePartition(purpose, clientReference, partition, aex); + deletePartition(purpose, partition, aex); partition.clear(); } }); if (partition.isEmpty() == false) { - deletePartition(purpose, clientReference, partition, aex); + deletePartition(purpose, partition, aex); } if (aex.get() != null) { throw aex.get(); @@ -352,30 +360,84 @@ public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator partition, - AtomicReference aex - ) { - try { - SocketAccess.doPrivilegedVoid(() -> clientReference.client().deleteObjects(bulkDelete(purpose, this, partition))); - } catch (MultiObjectDeleteException e) { - // We are sending quiet mode requests so we can't use the deleted keys entry on the exception and instead - // first remove all keys that were sent in the request and then add back those that ran into an exception. + /** + * Delete one partition of a batch of blobs + * + * @param purpose The {@link OperationPurpose} of the deletion + * @param partition The list of blobs to delete + * @param aex A holder for any exception(s) thrown during the deletion + */ + private void deletePartition(OperationPurpose purpose, List partition, AtomicReference aex) { + final Iterator retries = retryThrottledDeleteBackoffPolicy.iterator(); + int retryCounter = 0; + while (true) { + try (AmazonS3Reference clientReference = clientReference()) { + SocketAccess.doPrivilegedVoid(() -> clientReference.client().deleteObjects(bulkDelete(purpose, this, partition))); + s3RepositoriesMetrics.retryDeletesHistogram().record(retryCounter); + return; + } catch (MultiObjectDeleteException e) { + // We are sending quiet mode requests so we can't use the deleted keys entry on the exception and instead + // first remove all keys that were sent in the request and then add back those that ran into an exception. + logger.warn( + () -> format( + "Failed to delete some blobs %s", + e.getErrors() + .stream() + .map(err -> "[" + err.getKey() + "][" + err.getCode() + "][" + err.getMessage() + "]") + .toList() + ), + e + ); + aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e)); + return; + } catch (AmazonClientException e) { + if (shouldRetryDelete(purpose) && RetryUtils.isThrottlingException(e)) { + // S3 is asking us to slow down. Pause for a bit and retry + if (maybeDelayAndRetryDelete(retries)) { + retryCounter++; + } else { + s3RepositoriesMetrics.retryDeletesHistogram().record(retryCounter); + aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e)); + return; + } + } else { + // The AWS client threw any unexpected exception and did not execute the request at all so we do not + // remove any keys from the outstanding deletes set. + aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e)); + return; + } + } + } + } + + /** + * If there are remaining retries, pause for the configured interval then return true + * + * @param retries The retries iterator + * @return true to try the deletion again, false otherwise + */ + private boolean maybeDelayAndRetryDelete(Iterator retries) { + if (retries.hasNext()) { + try { + Thread.sleep(retries.next().millis()); + return true; + } catch (InterruptedException iex) { + Thread.currentThread().interrupt(); + // If we're interrupted, record the exception and abort retries + logger.warn("Aborting tenacious snapshot delete retries due to interrupt"); + } + } else { logger.warn( - () -> format( - "Failed to delete some blobs %s", - e.getErrors().stream().map(err -> "[" + err.getKey() + "][" + err.getCode() + "][" + err.getMessage() + "]").toList() - ), - e + "Exceeded maximum tenacious snapshot delete retries, aborting. Using back-off policy " + + retryThrottledDeleteBackoffPolicy + + ", see the throttled_delete_retry.* S3 repository properties to configure the back-off parameters" ); - aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e)); - } catch (AmazonClientException e) { - // The AWS client threw any unexpected exception and did not execute the request at all so we do not - // remove any keys from the outstanding deletes set. - aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e)); } + return false; + } + + private boolean shouldRetryDelete(OperationPurpose operationPurpose) { + return operationPurpose == OperationPurpose.SNAPSHOT_DATA || operationPurpose == OperationPurpose.SNAPSHOT_METADATA; } private static DeleteObjectsRequest bulkDelete(OperationPurpose purpose, S3BlobStore blobStore, List blobs) { @@ -409,6 +471,10 @@ public StorageClass getStorageClass() { return storageClass; } + public TimeValue getGetRegisterRetryDelay() { + return getRegisterRetryDelay; + } + public static StorageClass initStorageClass(String storageClass) { if ((storageClass == null) || storageClass.equals("")) { return StorageClass.Standard; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java index 74682ca190a0c..03106c26c9a29 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoriesMetrics.java @@ -17,7 +17,8 @@ public record S3RepositoriesMetrics( RepositoriesMetrics common, LongCounter retryStartedCounter, LongCounter retryCompletedCounter, - LongHistogram retryHistogram + LongHistogram retryHistogram, + LongHistogram retryDeletesHistogram ) { public static S3RepositoriesMetrics NOOP = new S3RepositoriesMetrics(RepositoriesMetrics.NOOP); @@ -25,6 +26,7 @@ public record S3RepositoriesMetrics( public static final String METRIC_RETRY_EVENT_TOTAL = "es.repositories.s3.input_stream.retry.event.total"; public static final String METRIC_RETRY_SUCCESS_TOTAL = "es.repositories.s3.input_stream.retry.success.total"; public static final String METRIC_RETRY_ATTEMPTS_HISTOGRAM = "es.repositories.s3.input_stream.retry.attempts.histogram"; + public static final String METRIC_DELETE_RETRIES_HISTOGRAM = "es.repositories.s3.delete.retry.attempts.histogram"; public S3RepositoriesMetrics(RepositoriesMetrics common) { this( @@ -32,7 +34,8 @@ public S3RepositoriesMetrics(RepositoriesMetrics common) { common.meterRegistry().registerLongCounter(METRIC_RETRY_EVENT_TOTAL, "s3 input stream retry event count", "unit"), common.meterRegistry().registerLongCounter(METRIC_RETRY_SUCCESS_TOTAL, "s3 input stream retry success count", "unit"), common.meterRegistry() - .registerLongHistogram(METRIC_RETRY_ATTEMPTS_HISTOGRAM, "s3 input stream retry attempts histogram", "unit") + .registerLongHistogram(METRIC_RETRY_ATTEMPTS_HISTOGRAM, "s3 input stream retry attempts histogram", "unit"), + common.meterRegistry().registerLongHistogram(METRIC_DELETE_RETRIES_HISTOGRAM, "s3 delete retry attempts histogram", "unit") ); } } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index af385eeac6a5b..fde15d5d6e6bc 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobPath; @@ -202,6 +203,36 @@ class S3Repository extends MeteredBlobStoreRepository { Setting.Property.Dynamic ); + /** + * We will retry deletes that fail due to throttling. We use an {@link BackoffPolicy#linearBackoff(TimeValue, int, TimeValue)} + * with the following parameters + */ + static final Setting RETRY_THROTTLED_DELETE_DELAY_INCREMENT = Setting.timeSetting( + "throttled_delete_retry.delay_increment", + TimeValue.timeValueMillis(50), + TimeValue.ZERO + ); + static final Setting RETRY_THROTTLED_DELETE_MAXIMUM_DELAY = Setting.timeSetting( + "throttled_delete_retry.maximum_delay", + TimeValue.timeValueSeconds(5), + TimeValue.ZERO + ); + static final Setting RETRY_THROTTLED_DELETE_MAX_NUMBER_OF_RETRIES = Setting.intSetting( + "throttled_delete_retry.maximum_number_of_retries", + 10, + 0 + ); + + /** + * Time to wait before trying again if getRegister fails. + */ + static final Setting GET_REGISTER_RETRY_DELAY = Setting.timeSetting( + "get_register_retry_delay", + new TimeValue(5, TimeUnit.SECONDS), + new TimeValue(0, TimeUnit.MILLISECONDS), + Setting.Property.Dynamic + ); + private final S3Service service; private final String bucket; @@ -424,7 +455,12 @@ protected S3BlobStore createBlobStore() { metadata, bigArrays, threadPool, - s3RepositoriesMetrics + s3RepositoriesMetrics, + BackoffPolicy.linearBackoff( + RETRY_THROTTLED_DELETE_DELAY_INCREMENT.get(metadata.settings()), + RETRY_THROTTLED_DELETE_MAX_NUMBER_OF_RETRIES.get(metadata.settings()), + RETRY_THROTTLED_DELETE_MAXIMUM_DELAY.get(metadata.settings()) + ) ); } diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index 1443ff704efd1..2eb2ed26153f9 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -10,10 +10,12 @@ import fixture.s3.S3HttpHandler; +import com.amazonaws.AbortedException; import com.amazonaws.DnsResolver; import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.internal.MD5DigestCalculatingInputStream; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.util.Base16; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; @@ -21,9 +23,12 @@ import org.apache.http.HttpStatus; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.common.BackoffPolicy; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.OperationPurpose; +import org.elasticsearch.common.blobstore.OptionalBytesReference; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.lucene.store.ByteArrayIndexInput; @@ -62,15 +67,18 @@ import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.nio.file.NoSuchFileException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalInt; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntConsumer; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; @@ -98,6 +106,7 @@ @SuppressForbidden(reason = "use a http server") public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTestCase { + private static final int MAX_NUMBER_SNAPSHOT_DELETE_RETRIES = 10; private S3Service service; private AtomicBoolean shouldErrorOnDns; private RecordingMeterRegistry recordingMeterRegistry; @@ -183,7 +192,10 @@ protected BlobContainer createBlobContainer( final RepositoryMetadata repositoryMetadata = new RepositoryMetadata( "repository", S3Repository.TYPE, - Settings.builder().put(S3Repository.CLIENT_NAME.getKey(), clientName).build() + Settings.builder() + .put(S3Repository.CLIENT_NAME.getKey(), clientName) + .put(S3Repository.GET_REGISTER_RETRY_DELAY.getKey(), TimeValue.ZERO) + .build() ); final S3BlobStore s3BlobStore = new S3BlobStore( @@ -196,7 +208,8 @@ protected BlobContainer createBlobContainer( repositoryMetadata, BigArrays.NON_RECYCLING_INSTANCE, new DeterministicTaskQueue().getThreadPool(), - new S3RepositoriesMetrics(new RepositoriesMetrics(recordingMeterRegistry)) + new S3RepositoriesMetrics(new RepositoriesMetrics(recordingMeterRegistry)), + BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(1), MAX_NUMBER_SNAPSHOT_DELETE_RETRIES) ); return new S3BlobContainer(randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo"), s3BlobStore) { @Override @@ -771,6 +784,240 @@ public void handle(HttpExchange exchange) throws IOException { assertThat(getRetryHistogramMeasurements(), empty()); } + public void testSnapshotDeletesRetryOnThrottlingError() throws IOException { + // disable AWS-client retries + final BlobContainer blobContainer = createBlobContainer(0, null, true, null); + + int numBlobsToDelete = randomIntBetween(500, 3000); + List blobsToDelete = new ArrayList<>(); + for (int i = 0; i < numBlobsToDelete; i++) { + blobsToDelete.add(randomIdentifier()); + } + int throttleTimesBeforeSuccess = randomIntBetween(1, MAX_NUMBER_SNAPSHOT_DELETE_RETRIES); + logger.info("--> Throttling {} times before success", throttleTimesBeforeSuccess); + ThrottlingDeleteHandler handler = new ThrottlingDeleteHandler(throttleTimesBeforeSuccess, attempt -> {}); + httpServer.createContext("/", handler); + blobContainer.deleteBlobsIgnoringIfNotExists(randomFrom(operationPurposesThatRetryOnDelete()), blobsToDelete.iterator()); + + int expectedNumberOfBatches = expectedNumberOfBatches(numBlobsToDelete); + assertThat(handler.numberOfDeleteAttempts.get(), equalTo(throttleTimesBeforeSuccess + expectedNumberOfBatches)); + assertThat(handler.numberOfSuccessfulDeletes.get(), equalTo(expectedNumberOfBatches)); + } + + public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() { + // disable AWS-client retries + final BlobContainer blobContainer = createBlobContainer(0, null, true, null); + + int numBlobsToDelete = randomIntBetween(500, 3000); + List blobsToDelete = new ArrayList<>(); + for (int i = 0; i < numBlobsToDelete; i++) { + blobsToDelete.add(randomIdentifier()); + } + + final Thread clientThread = Thread.currentThread(); + int interruptBeforeAttempt = randomIntBetween(0, randomIntBetween(1, 10)); + logger.info("--> Deleting {} blobs, interrupting before attempt {}", numBlobsToDelete, interruptBeforeAttempt); + ThrottlingDeleteHandler handler = new ThrottlingDeleteHandler(Integer.MAX_VALUE, attempt -> { + if (attempt == interruptBeforeAttempt) { + clientThread.interrupt(); + } + }); + httpServer.createContext("/", handler); + + try { + IOException exception = assertThrows( + IOException.class, + () -> blobContainer.deleteBlobsIgnoringIfNotExists( + randomFrom(operationPurposesThatRetryOnDelete()), + blobsToDelete.iterator() + ) + ); + assertThat(exception.getCause(), instanceOf(AbortedException.class)); + assertThat(handler.numberOfDeleteAttempts.get(), equalTo(interruptBeforeAttempt + 1)); + assertThat(handler.numberOfSuccessfulDeletes.get(), equalTo(0)); + } finally { + // interrupt should be preserved, clear it to prevent it leaking between tests + assertTrue(Thread.interrupted()); + } + } + + public void testNonSnapshotDeletesAreNotRetried() { + // disable AWS-client retries + final BlobContainer blobContainer = createBlobContainer(0, null, true, null); + + int numBlobsToDelete = randomIntBetween(500, 3000); + List blobsToDelete = new ArrayList<>(); + for (int i = 0; i < numBlobsToDelete; i++) { + blobsToDelete.add(randomIdentifier()); + } + ThrottlingDeleteHandler handler = new ThrottlingDeleteHandler(Integer.MAX_VALUE, attempt -> {}); + httpServer.createContext("/", handler); + IOException exception = assertThrows( + IOException.class, + () -> blobContainer.deleteBlobsIgnoringIfNotExists( + randomValueOtherThanMany( + op -> operationPurposesThatRetryOnDelete().contains(op), + () -> randomFrom(OperationPurpose.values()) + ), + blobsToDelete.iterator() + ) + ); + assertEquals( + ThrottlingDeleteHandler.THROTTLING_ERROR_CODE, + asInstanceOf(AmazonS3Exception.class, exception.getCause()).getErrorCode() + ); + assertThat(handler.numberOfDeleteAttempts.get(), equalTo(expectedNumberOfBatches(numBlobsToDelete))); + assertThat(handler.numberOfSuccessfulDeletes.get(), equalTo(0)); + } + + public void testNonThrottlingErrorsAreNotRetried() { + // disable AWS-client retries + final BlobContainer blobContainer = createBlobContainer(0, null, true, null); + + int numBlobsToDelete = randomIntBetween(500, 3000); + List blobsToDelete = new ArrayList<>(); + for (int i = 0; i < numBlobsToDelete; i++) { + blobsToDelete.add(randomIdentifier()); + } + ThrottlingDeleteHandler handler = new ThrottlingDeleteHandler(Integer.MAX_VALUE, attempt -> {}, "NotThrottling"); + httpServer.createContext("/", handler); + assertThrows( + IOException.class, + () -> blobContainer.deleteBlobsIgnoringIfNotExists(randomFrom(operationPurposesThatRetryOnDelete()), blobsToDelete.iterator()) + ); + assertThat(handler.numberOfDeleteAttempts.get(), equalTo(expectedNumberOfBatches(numBlobsToDelete))); + assertThat(handler.numberOfSuccessfulDeletes.get(), equalTo(0)); + } + + private int expectedNumberOfBatches(int blobsToDelete) { + return (blobsToDelete / 1_000) + (blobsToDelete % 1_000 == 0 ? 0 : 1); + } + + @SuppressForbidden(reason = "use a http server") + private class ThrottlingDeleteHandler extends S3HttpHandler { + + private static final String THROTTLING_ERROR_CODE = "SlowDown"; + + private final AtomicInteger throttleTimesBeforeSuccess; + private final AtomicInteger numberOfDeleteAttempts; + private final AtomicInteger numberOfSuccessfulDeletes; + private final IntConsumer onAttemptCallback; + private final String errorCode; + + ThrottlingDeleteHandler(int throttleTimesBeforeSuccess, IntConsumer onAttemptCallback) { + this(throttleTimesBeforeSuccess, onAttemptCallback, THROTTLING_ERROR_CODE); + } + + ThrottlingDeleteHandler(int throttleTimesBeforeSuccess, IntConsumer onAttemptCallback, String errorCode) { + super("bucket"); + this.numberOfDeleteAttempts = new AtomicInteger(); + this.numberOfSuccessfulDeletes = new AtomicInteger(); + this.throttleTimesBeforeSuccess = new AtomicInteger(throttleTimesBeforeSuccess); + this.onAttemptCallback = onAttemptCallback; + this.errorCode = errorCode; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + if (exchange.getRequestMethod().equals("POST") && exchange.getRequestURI().toString().startsWith("/bucket/?delete")) { + onAttemptCallback.accept(numberOfDeleteAttempts.get()); + numberOfDeleteAttempts.incrementAndGet(); + if (throttleTimesBeforeSuccess.getAndDecrement() > 0) { + final byte[] responseBytes = Strings.format(""" + + + %s + This is a throttling message + /bucket/ + 4442587FB7D0A2F9 + """, errorCode).getBytes(StandardCharsets.UTF_8); + + exchange.sendResponseHeaders(HttpStatus.SC_SERVICE_UNAVAILABLE, responseBytes.length); + exchange.getResponseBody().write(responseBytes); + exchange.close(); + } else { + numberOfSuccessfulDeletes.incrementAndGet(); + super.handle(exchange); + } + } else { + super.handle(exchange); + } + } + } + + private Set operationPurposesThatRetryOnDelete() { + return Set.of(OperationPurpose.SNAPSHOT_DATA, OperationPurpose.SNAPSHOT_METADATA); + } + + public void testGetRegisterRetries() { + final var maxRetries = between(0, 3); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null); + + interface FailingHandlerFactory { + void addHandler(String blobName, Integer... responseCodes); + } + + final var requestCounter = new AtomicInteger(); + final FailingHandlerFactory countingFailingHandlerFactory = (blobName, responseCodes) -> httpServer.createContext( + downloadStorageEndpoint(blobContainer, blobName), + exchange -> { + requestCounter.incrementAndGet(); + try (exchange) { + exchange.sendResponseHeaders(randomFrom(responseCodes), -1); + } + } + ); + + countingFailingHandlerFactory.addHandler("test_register_no_internal_retries", HttpStatus.SC_UNPROCESSABLE_ENTITY); + countingFailingHandlerFactory.addHandler( + "test_register_internal_retries", + HttpStatus.SC_INTERNAL_SERVER_ERROR, + HttpStatus.SC_SERVICE_UNAVAILABLE + ); + countingFailingHandlerFactory.addHandler("test_register_not_found", HttpStatus.SC_NOT_FOUND); + + { + final var exceptionWithInternalRetries = safeAwaitFailure( + OptionalBytesReference.class, + l -> blobContainer.getRegister(randomRetryingPurpose(), "test_register_internal_retries", l) + ); + assertThat(exceptionWithInternalRetries, instanceOf(AmazonS3Exception.class)); + assertEquals((maxRetries + 1) * (maxRetries + 1), requestCounter.get()); + assertEquals(maxRetries, exceptionWithInternalRetries.getSuppressed().length); + } + + { + requestCounter.set(0); + final var exceptionWithoutInternalRetries = safeAwaitFailure( + OptionalBytesReference.class, + l -> blobContainer.getRegister(randomRetryingPurpose(), "test_register_no_internal_retries", l) + ); + assertThat(exceptionWithoutInternalRetries, instanceOf(AmazonS3Exception.class)); + assertEquals(maxRetries + 1, requestCounter.get()); + assertEquals(maxRetries, exceptionWithoutInternalRetries.getSuppressed().length); + } + + { + requestCounter.set(0); + final var repoAnalysisException = safeAwaitFailure( + OptionalBytesReference.class, + l -> blobContainer.getRegister(OperationPurpose.REPOSITORY_ANALYSIS, "test_register_no_internal_retries", l) + ); + assertThat(repoAnalysisException, instanceOf(AmazonS3Exception.class)); + assertEquals(1, requestCounter.get()); + assertEquals(0, repoAnalysisException.getSuppressed().length); + } + + { + requestCounter.set(0); + final OptionalBytesReference expectEmpty = safeAwait( + l -> blobContainer.getRegister(randomPurpose(), "test_register_not_found", l) + ); + assertEquals(OptionalBytesReference.EMPTY, expectEmpty); + assertEquals(1, requestCounter.get()); + } + } + @Override protected Matcher getMaxRetriesMatcher(int maxRetries) { // some attempts make meaningful progress and do not count towards the max retry limit diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java index b55f4eccafca8..cad839bed9555 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java @@ -103,7 +103,7 @@ public void testDefaultKeepAliveSettings() throws IOException { MockTransportService serviceD = buildService("TS_D", VersionInformation.CURRENT, TransportVersion.current(), Settings.EMPTY) ) { - try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), TestProfiles.LIGHT_PROFILE)) { + try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalNode(), TestProfiles.LIGHT_PROFILE)) { assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class)); Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection(); assertThat(conn, instanceOf(TcpTransport.NodeChannels.class)); @@ -147,7 +147,7 @@ public void testTransportProfile() { MockTransportService serviceD = buildService("TS_D", VersionInformation.CURRENT, TransportVersion.current(), Settings.EMPTY) ) { - try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), connectionProfile)) { + try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalNode(), connectionProfile)) { assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class)); Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection(); assertThat(conn, instanceOf(TcpTransport.NodeChannels.class)); diff --git a/muted-tests.yml b/muted-tests.yml index 1c7ee862d6453..03825168fe1b0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -80,12 +80,6 @@ tests: - class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT method: testSnapshotRestore {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/111799 -- class: org.elasticsearch.xpack.inference.InferenceRestIT - method: test {p0=inference/80_random_rerank_retriever/Random rerank retriever predictably shuffles results} - issue: https://github.com/elastic/elasticsearch/issues/111999 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testDeleteJobAfterMissingIndex - issue: https://github.com/elastic/elasticsearch/issues/112088 - class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/112147 - class: org.elasticsearch.smoketest.WatcherYamlRestIT @@ -94,21 +88,9 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=ml/inference_processor/Test create processor with missing mandatory fields} issue: https://github.com/elastic/elasticsearch/issues/112191 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testDeleteJobAsync - issue: https://github.com/elastic/elasticsearch/issues/112212 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/rest-api/watcher/put-watch/line_120} issue: https://github.com/elastic/elasticsearch/issues/99517 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testMultiIndexDelete - issue: https://github.com/elastic/elasticsearch/issues/112381 -- class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests - method: "testAggregateIntermediate {TestCase= #2}" - issue: https://github.com/elastic/elasticsearch/issues/112461 -- class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests - method: testAggregateIntermediate {TestCase=} - issue: https://github.com/elastic/elasticsearch/issues/112463 - class: org.elasticsearch.xpack.esql.action.ManyShardsIT method: testRejection issue: https://github.com/elastic/elasticsearch/issues/112406 @@ -129,9 +111,6 @@ tests: - class: org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT method: testIndexPatternErrorMessageComparison_ESQL_SearchDSL issue: https://github.com/elastic/elasticsearch/issues/112630 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testPutJob_GivenFarequoteConfig - issue: https://github.com/elastic/elasticsearch/issues/112382 - class: org.elasticsearch.packaging.test.PackagesSecurityAutoConfigurationTests method: test20SecurityNotAutoConfiguredOnReInstallation issue: https://github.com/elastic/elasticsearch/issues/112635 @@ -147,26 +126,11 @@ tests: - class: org.elasticsearch.xpack.sql.qa.single_node.JdbcSqlSpecIT method: test {case-functions.testUcaseInline3} issue: https://github.com/elastic/elasticsearch/issues/112643 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testDelete_multipleRequest - issue: https://github.com/elastic/elasticsearch/issues/112701 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testCreateJobInSharedIndexUpdatesMapping - issue: https://github.com/elastic/elasticsearch/issues/112729 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testGetJob_GivenNoSuchJob - issue: https://github.com/elastic/elasticsearch/issues/112730 - class: org.elasticsearch.script.StatsSummaryTests method: testEqualsAndHashCode issue: https://github.com/elastic/elasticsearch/issues/112439 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testDeleteJobAfterMissingAliases - issue: https://github.com/elastic/elasticsearch/issues/112823 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT issue: https://github.com/elastic/elasticsearch/issues/112889 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testCreateJob_WithClashingFieldMappingsFails - issue: https://github.com/elastic/elasticsearch/issues/113046 - class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT method: test {case-functions.testUcaseInline1} issue: https://github.com/elastic/elasticsearch/issues/112641 @@ -218,9 +182,6 @@ tests: - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/ccr/apis/follow/post-resume-follow/line_84} issue: https://github.com/elastic/elasticsearch/issues/113343 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testDeleteJob_TimingStatsDocumentIsDeleted - issue: https://github.com/elastic/elasticsearch/issues/113370 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search/500_date_range/from, to, include_lower, include_upper deprecated} issue: https://github.com/elastic/elasticsearch/pull/113286 @@ -233,15 +194,6 @@ tests: - class: org.elasticsearch.xpack.inference.InferenceCrudIT method: testSupportedStream issue: https://github.com/elastic/elasticsearch/issues/113430 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testOutOfOrderData - issue: https://github.com/elastic/elasticsearch/issues/113477 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testCreateJobsWithIndexNameOption - issue: https://github.com/elastic/elasticsearch/issues/113528 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testCantCreateJobWithSameID - issue: https://github.com/elastic/elasticsearch/issues/113581 - class: org.elasticsearch.integration.KibanaUserRoleIntegTests method: testFieldMappings issue: https://github.com/elastic/elasticsearch/issues/113592 @@ -254,12 +206,6 @@ tests: - class: org.elasticsearch.smoketest.MlWithSecurityIT method: test {yaml=ml/3rd_party_deployment/Test start and stop multiple deployments} issue: https://github.com/elastic/elasticsearch/issues/101458 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testGetJobs_GivenMultipleJobs - issue: https://github.com/elastic/elasticsearch/issues/113654 -- class: org.elasticsearch.xpack.ml.integration.MlJobIT - method: testGetJobs_GivenSingleJob - issue: https://github.com/elastic/elasticsearch/issues/113655 - class: org.elasticsearch.xpack.security.authz.interceptor.SearchRequestCacheDisablingInterceptorTests method: testHasRemoteIndices issue: https://github.com/elastic/elasticsearch/issues/113660 @@ -310,12 +256,6 @@ tests: - class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests method: "testFold {TestCase= #7}" issue: https://github.com/elastic/elasticsearch/issues/114175 -- class: org.elasticsearch.action.bulk.IncrementalBulkIT - method: testMultipleBulkPartsWithBackoff - issue: https://github.com/elastic/elasticsearch/issues/114181 -- class: org.elasticsearch.action.bulk.IncrementalBulkIT - method: testIncrementalBulkLowWatermarkBackOff - issue: https://github.com/elastic/elasticsearch/issues/114182 - class: org.elasticsearch.xpack.ilm.ExplainLifecycleIT method: testStepInfoPreservedOnAutoRetry issue: https://github.com/elastic/elasticsearch/issues/114220 @@ -325,9 +265,6 @@ tests: - class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests method: testInfer_StreamRequest_ErrorResponse issue: https://github.com/elastic/elasticsearch/issues/114327 -- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT - method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} - issue: https://github.com/elastic/elasticsearch/issues/114331 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search} issue: https://github.com/elastic/elasticsearch/issues/114371 @@ -425,9 +362,6 @@ tests: - class: org.elasticsearch.xpack.eql.EqlRestIT method: testUnicodeChars issue: https://github.com/elastic/elasticsearch/issues/114791 -- class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderNestedDocsIT - method: testRRFExplainWithNamedRetrievers - issue: https://github.com/elastic/elasticsearch/issues/114820 - class: org.elasticsearch.ingest.geoip.HttpClientTests issue: https://github.com/elastic/elasticsearch/issues/112618 - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT @@ -436,6 +370,23 @@ tests: - class: org.elasticsearch.xpack.enrich.EnrichIT method: testImmutablePolicy issue: https://github.com/elastic/elasticsearch/issues/114839 +- class: org.elasticsearch.license.LicensingTests + issue: https://github.com/elastic/elasticsearch/issues/114865 +- class: org.elasticsearch.xpack.enrich.EnrichIT + method: testDeleteIsCaseSensitive + issue: https://github.com/elastic/elasticsearch/issues/114840 +- class: org.elasticsearch.packaging.test.EnrollmentProcessTests + method: test20DockerAutoFormCluster + issue: https://github.com/elastic/elasticsearch/issues/114885 +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT + method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search} + issue: https://github.com/elastic/elasticsearch/issues/114902 +- class: org.elasticsearch.xpack.enrich.EnrichRestIT + method: test {p0=enrich/40_synthetic_source/enrich documents over _bulk} + issue: https://github.com/elastic/elasticsearch/issues/114825 +- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT + method: testInferDeploysDefaultElser + issue: https://github.com/elastic/elasticsearch/issues/114913 # Examples: # diff --git a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java index 7b42292848395..780f3994ce627 100644 --- a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -154,7 +154,7 @@ public void testSearchSkipUnavailable() throws IOException { threadPool ) ) { - DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode(); + DiscoveryNode remoteNode = remoteTransport.getLocalNode(); updateRemoteClusterSettings(Collections.singletonMap("seeds", remoteNode.getAddress().toString())); @@ -307,7 +307,7 @@ public void testSkipUnavailableDependsOnSeeds() throws IOException { threadPool ) ) { - DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode(); + DiscoveryNode remoteNode = remoteTransport.getLocalNode(); { // check that skip_unavailable alone cannot be set diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java deleted file mode 100644 index a391ee5a3bd7b..0000000000000 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package org.elasticsearch.backwards; - -import org.apache.http.HttpHost; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.core.Strings; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.ObjectPath; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class SearchWithMinCompatibleSearchNodeIT extends ESRestTestCase { - - private static final String BWC_NODES_VERSION = System.getProperty("tests.bwc_nodes_version"); - private static final String NEW_NODES_VERSION = System.getProperty("tests.new_nodes_version"); - - private static String index = "test_min_version"; - private static int numShards; - private static int numReplicas = 1; - private static int numDocs; - private static MixedClusterTestNodes nodes; - private static List allNodes; - - @Before - public void prepareTestData() throws IOException { - nodes = MixedClusterTestNodes.buildNodes(client(), BWC_NODES_VERSION); - numShards = nodes.size(); - numDocs = randomIntBetween(numShards, 16); - allNodes = new ArrayList<>(); - allNodes.addAll(nodes.getBWCNodes()); - allNodes.addAll(nodes.getNewNodes()); - - if (client().performRequest(new Request("HEAD", "/" + index)).getStatusLine().getStatusCode() == 404) { - createIndex(index, indexSettings(numShards, numReplicas).build()); - for (int i = 0; i < numDocs; i++) { - Request request = new Request("PUT", index + "/_doc/" + i); - request.setJsonEntity("{\"test\": \"test_" + randomAlphaOfLength(2) + "\"}"); - assertOK(client().performRequest(request)); - } - ensureGreen(index); - } - } - - public void testMinVersionAsNewVersion() throws Exception { - try ( - RestClient client = buildClient( - restClientSettings(), - allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) - ) - ) { - Request newVersionRequest = new Request( - "POST", - index + "/_search?min_compatible_shard_node=" + NEW_NODES_VERSION + "&ccs_minimize_roundtrips=false" - ); - assertBusy(() -> { - ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(newVersionRequest)); - assertThat( - responseException.getResponse().getStatusLine().getStatusCode(), - equalTo(RestStatus.INTERNAL_SERVER_ERROR.getStatus()) - ); - assertThat(responseException.getMessage(), containsString(""" - {"error":{"root_cause":[],"type":"search_phase_execution_exception\"""")); - assertThat(responseException.getMessage(), containsString(Strings.format(""" - caused_by":{"type":"version_mismatch_exception",\ - "reason":"One of the shards is incompatible with the required minimum version [%s]\"""", NEW_NODES_VERSION))); - }); - } - } - - public void testMinVersionAsOldVersion() throws Exception { - try ( - RestClient client = buildClient( - restClientSettings(), - allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) - ) - ) { - Request oldVersionRequest = new Request( - "POST", - index + "/_search?min_compatible_shard_node=" + BWC_NODES_VERSION + "&ccs_minimize_roundtrips=false" - ); - oldVersionRequest.setJsonEntity(""" - {"query":{"match_all":{}},"_source":false}"""); - assertBusy(() -> { - Response response = client.performRequest(oldVersionRequest); - ObjectPath responseObject = ObjectPath.createFromResponse(response); - Map shardsResult = responseObject.evaluate("_shards"); - assertThat(shardsResult.get("total"), equalTo(numShards)); - assertThat(shardsResult.get("successful"), equalTo(numShards)); - assertThat(shardsResult.get("failed"), equalTo(0)); - Map hitsResult = responseObject.evaluate("hits.total"); - assertThat(hitsResult.get("value"), equalTo(numDocs)); - assertThat(hitsResult.get("relation"), equalTo("eq")); - }); - } - } - - public void testCcsMinimizeRoundtripsIsFalse() throws Exception { - try ( - RestClient client = buildClient( - restClientSettings(), - allNodes.stream().map(MixedClusterTestNode::publishAddress).toArray(HttpHost[]::new) - ) - ) { - String version = randomBoolean() ? NEW_NODES_VERSION : BWC_NODES_VERSION; - - Request request = new Request( - "POST", - index + "/_search?min_compatible_shard_node=" + version + "&ccs_minimize_roundtrips=true" - ); - assertBusy(() -> { - ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(request)); - assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); - assertThat(responseException.getMessage(), containsString(""" - {"error":{"root_cause":[{"type":"action_request_validation_exception"\ - """)); - assertThat( - responseException.getMessage(), - containsString( - "\"reason\":\"Validation Failed: 1: " - + "[ccs_minimize_roundtrips] cannot be [true] when setting a minimum compatible shard version;\"" - ) - ); - }); - } - } -} diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java similarity index 81% rename from qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java rename to qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java index da05011696274..369d0824bdb28 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BulkRestIT.java @@ -9,6 +9,8 @@ package org.elasticsearch.http; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -19,24 +21,30 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.OK; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0) -public class IncrementalBulkRestIT extends HttpSmokeTestCase { +public class BulkRestIT extends HttpSmokeTestCase { @Override protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal, otherSettings)) - .put(IncrementalBulkService.INCREMENTAL_BULK.getKey(), true) + .put(IncrementalBulkService.INCREMENTAL_BULK.getKey(), seventyFivePercentOfTheTime()) .build(); } + private static boolean seventyFivePercentOfTheTime() { + return (randomBoolean() && randomBoolean()) == false; + } + public void testBulkUriMatchingDoesNotMatchBulkCapabilitiesApi() throws IOException { Request request = new Request("GET", "/_capabilities?method=GET&path=%2F_bulk&capabilities=failure_store_status&pretty"); Response response = getRestClient().performRequest(request); @@ -51,6 +59,26 @@ public void testBulkMissingBody() throws IOException { assertThat(responseException.getMessage(), containsString("request body is required")); } + public void testBulkInvalidIndexNameString() throws IOException { + Request request = new Request("POST", "/_bulk"); + + byte[] bytes1 = "{\"create\":{\"_index\":\"".getBytes(StandardCharsets.UTF_8); + byte[] bytes2 = new byte[] { (byte) 0xfe, (byte) 0xfe, (byte) 0xff, (byte) 0xff }; + byte[] bytes3 = "\",\"_id\":\"1\"}}\n{\"field\":1}\n\r\n".getBytes(StandardCharsets.UTF_8); + byte[] bulkBody = new byte[bytes1.length + bytes2.length + bytes3.length]; + System.arraycopy(bytes1, 0, bulkBody, 0, bytes1.length); + System.arraycopy(bytes2, 0, bulkBody, bytes1.length, bytes2.length); + System.arraycopy(bytes3, 0, bulkBody, bytes1.length + bytes2.length, bytes3.length); + + request.setEntity(new ByteArrayEntity(bulkBody, ContentType.APPLICATION_JSON)); + + ResponseException responseException = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); + assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(BAD_REQUEST.getStatus())); + assertThat(responseException.getMessage(), containsString("could not parse bulk request body")); + assertThat(responseException.getMessage(), containsString("json_parse_exception")); + assertThat(responseException.getMessage(), containsString("Invalid UTF-8")); + } + public void testBulkRequestBodyImproperlyTerminated() throws IOException { Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk"); // missing final line of the bulk body. cannot process @@ -61,10 +89,10 @@ public void testBulkRequestBodyImproperlyTerminated() throws IOException { ); ResponseException responseException = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request)); assertEquals(400, responseException.getResponse().getStatusLine().getStatusCode()); - assertThat(responseException.getMessage(), containsString("could not parse bulk request body")); + assertThat(responseException.getMessage(), containsString("The bulk request must be terminated by a newline")); } - public void testIncrementalBulk() throws IOException { + public void testBulkRequest() throws IOException { Request createRequest = new Request("PUT", "/index_name"); createRequest.setJsonEntity(""" { @@ -81,7 +109,6 @@ public void testIncrementalBulk() throws IOException { Request firstBulkRequest = new Request("POST", "/index_name/_bulk"); - // index documents for the rollup job String bulkBody = "{\"index\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n" + "{\"field\":1}\n" + "{\"index\":{\"_index\":\"index_name\",\"_id\":\"2\"}}\n" @@ -113,7 +140,6 @@ public void testBulkWithIncrementalDisabled() throws IOException { Request firstBulkRequest = new Request("POST", "/index_name/_bulk"); - // index documents for the rollup job String bulkBody = "{\"index\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n" + "{\"field\":1}\n" + "{\"index\":{\"_index\":\"index_name\",\"_id\":\"2\"}}\n" @@ -137,7 +163,7 @@ public void testBulkWithIncrementalDisabled() throws IOException { } } - public void testIncrementalMalformed() throws IOException { + public void testMalformedActionLineBulk() throws IOException { Request createRequest = new Request("PUT", "/index_name"); createRequest.setJsonEntity(""" { @@ -154,7 +180,6 @@ public void testIncrementalMalformed() throws IOException { Request bulkRequest = new Request("POST", "/index_name/_bulk"); - // index documents for the rollup job final StringBuilder bulk = new StringBuilder(); bulk.append("{\"index\":{\"_index\":\"index_name\"}}\n"); bulk.append("{\"field\":1}\n"); @@ -170,7 +195,6 @@ public void testIncrementalMalformed() throws IOException { private static void sendLargeBulk() throws IOException { Request bulkRequest = new Request("POST", "/index_name/_bulk"); - // index documents for the rollup job final StringBuilder bulk = new StringBuilder(); bulk.append("{\"delete\":{\"_index\":\"index_name\",\"_id\":\"1\"}}\n"); int updates = 0; diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml index a8f7e1e5877c8..cc767dfa56597 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yml @@ -214,7 +214,7 @@ "Test rolling up json object arrays": - do: ingest.put_pipeline: - id: "_id" + id: "pipeline-id" body: > { "processors": [ @@ -237,7 +237,7 @@ index: index: test id: "1" - pipeline: "_id" + pipeline: "pipeline-id" body: { values_flat : [], values: [ diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml index 9a7444c4ffc6c..ef790843b7bfb 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_combine_processors.yml @@ -2,7 +2,7 @@ "Test with date processor": - do: ingest.put_pipeline: - id: "_id" + id: "pipeline-id" body: > { "processors": [ @@ -44,7 +44,7 @@ index: index: test id: "1" - pipeline: "_id" + pipeline: "pipeline-id" body: { log: "89.160.20.128 - - [08/Sep/2014:02:54:42 +0000] \"GET /presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png HTTP/1.1\" 200 175208 \"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\" \"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\"" } @@ -71,7 +71,7 @@ "Test with date processor and ECS-v1": - do: ingest.put_pipeline: - id: "_id" + id: "pipeline-id" body: > { "processors": [ @@ -102,7 +102,7 @@ index: index: test id: "1" - pipeline: "_id" + pipeline: "pipeline-id" body: { log: "89.160.20.128 - - [08/Sep/2014:02:54:42 +0000] \"GET /presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png HTTP/1.1\" 200 175208 \"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\" \"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\"" } @@ -128,7 +128,7 @@ "Test mutate": - do: ingest.put_pipeline: - id: "_id" + id: "pipeline-id" body: > { "processors": [ @@ -188,7 +188,7 @@ index: index: test id: "1" - pipeline: "_id" + pipeline: "pipeline-id" body: { "age" : 33, "eyeColor" : "brown", diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_deprecation_warnings_on_invalid_names_ingest.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_deprecation_warnings_on_invalid_names_ingest.yml new file mode 100644 index 0000000000000..64f5ccc4609ac --- /dev/null +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_deprecation_warnings_on_invalid_names_ingest.yml @@ -0,0 +1,28 @@ +--- +"Test invalid name warnings": + - requires: + cluster_features: [ "ingest.pipeline_name_special_chars_warning" ] + test_runner_features: [ "warnings" ] + reason: verifying deprecation warnings from 9.0 onwards for invalid pipeline names + + - do: + cluster.health: + wait_for_status: green + + - do: + ingest.put_pipeline: + id: "Invalid*-pipeline:id" + body: > + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field1", + "value": "_value" + } + }] + } + warnings: + - "Invalid pipeline id: Invalid*-pipeline:id" + - match: { acknowledged: true } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index b5dc4d62a2f0f..25b4efd9c4c37 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -237,10 +237,6 @@ "description":"Indicates whether hits.total should be rendered as an integer or an object in the rest search response", "default":false }, - "min_compatible_shard_node":{ - "type":"string", - "description":"The minimum compatible version that all shards involved in search should have for this request to be successful" - }, "include_named_queries_score":{ "type": "boolean", "description":"Indicates whether hit.matched_queries should be rendered as a map that includes the name of the matched query associated with its score (true) or as an array containing the name of the matched queries (false)", diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index 5d04e31069b1c..4119e12d45f6c 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -1819,12 +1819,6 @@ private enum ElasticsearchExceptionHandle { 160, TransportVersions.V_7_10_0 ), - VERSION_MISMATCH_EXCEPTION( - org.elasticsearch.action.search.VersionMismatchException.class, - org.elasticsearch.action.search.VersionMismatchException::new, - 161, - TransportVersions.V_7_12_0 - ), AUTHENTICATION_PROCESSING_ERROR( org.elasticsearch.ElasticsearchAuthenticationProcessingError.class, org.elasticsearch.ElasticsearchAuthenticationProcessingError::new, diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 4038d5a224850..d1d423dcc5405 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -245,6 +245,8 @@ static TransportVersion def(int id) { public static final TransportVersion QUERY_RULE_TEST_API = def(8_769_00_0); public static final TransportVersion ESQL_PER_AGGREGATE_FILTER = def(8_770_00_0); public static final TransportVersion ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT = def(8_771_00_0); + public static final TransportVersion CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY = def(8_772_00_0); + public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 4b19d4b428526..48bf08ddfc028 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -187,6 +187,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_15_2 = new Version(8_15_02_99); public static final Version V_8_15_3 = new Version(8_15_03_99); public static final Version V_8_16_0 = new Version(8_16_00_99); + public static final Version V_8_17_0 = new Version(8_17_00_99); public static final Version V_9_0_0 = new Version(9_00_00_99); public static final Version CURRENT = V_9_0_0; diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java index 095ccd71fa266..d47469ddf10d9 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -298,9 +297,6 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t builder.field(_SEQ_NO, getSeqNo()); builder.field(_PRIMARY_TERM, getPrimaryTerm()); } - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME); - } return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java index 7ff7066a15fc2..4c5ee08beb192 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java @@ -98,9 +98,7 @@ public GetIndexRequest() { super( DataStream.isFailureStoreFeatureFlagEnabled() ? IndicesOptions.builder(IndicesOptions.strictExpandOpen()) - .failureStoreOptions( - IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true) - ) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .build() : IndicesOptions.strictExpandOpen() ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index c3ed0c675c3c7..23a6b9c8c61a8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ToXContent; @@ -30,9 +29,6 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; - /** * A response for a get index action. */ @@ -200,14 +196,7 @@ public Iterator toXContentChunked(ToXContent.Params ignore if (indexMappings == null) { builder.startObject("mappings").endObject(); } else { - if (builder.getRestApiVersion() == RestApiVersion.V_7 - && params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { - builder.startObject("mappings"); - builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap()); - builder.endObject(); - } else { - builder.field("mappings", indexMappings.sourceAsMap()); - } + builder.field("mappings", indexMappings.sourceAsMap()); } builder.startObject("settings"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 4398b33cd798f..01e8fe9787014 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -15,10 +15,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.ToXContentObject; @@ -31,8 +29,6 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; - /** * Response object for {@link GetFieldMappingsRequest} API * @@ -91,16 +87,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(indexEntry.getKey()); builder.startObject(MAPPINGS.getPreferredName()); if (indexEntry.getValue() != null) { - if (builder.getRestApiVersion() == RestApiVersion.V_7 - && params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { - if (indexEntry.getValue().size() > 0) { - builder.startObject(MapperService.SINGLE_MAPPING_NAME); - addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); - builder.endObject(); - } - } else { - addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); - } + addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index 37edae05c22af..cff2cf9ec8c78 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -26,9 +25,6 @@ import java.util.Iterator; import java.util.Map; -import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; - public class GetMappingsResponse extends ActionResponse implements ChunkedToXContentObject { private static final ParseField MAPPINGS = new ParseField("mappings"); @@ -73,16 +69,7 @@ public Iterator toXContentChunked(ToXContent.Params outerParams) { Iterators.single((b, p) -> b.startObject()), Iterators.map(getMappings().entrySet().iterator(), indexEntry -> (builder, params) -> { builder.startObject(indexEntry.getKey()); - boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName && indexEntry.getValue() != null) { - builder.startObject(MAPPINGS.getPreferredName()); - - if (indexEntry.getValue() != MappingMetadata.EMPTY_MAPPINGS) { - builder.field(MapperService.SINGLE_MAPPING_NAME, indexEntry.getValue().sourceAsMap()); - } - builder.endObject(); - - } else if (indexEntry.getValue() != null) { + if (indexEntry.getValue() != null) { builder.field(MAPPINGS.getPreferredName(), indexEntry.getValue().sourceAsMap()); } else { builder.startObject(MAPPINGS.getPreferredName()).endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index b6356e92ad856..fefc41317591b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -42,7 +42,7 @@ */ public class RolloverRequest extends AcknowledgedRequest implements IndicesRequest { - private static final ObjectParser PARSER = new ObjectParser<>("rollover"); + private static final ObjectParser PARSER = new ObjectParser<>("rollover"); private static final ParseField CONDITIONS = new ParseField("conditions"); @@ -57,27 +57,18 @@ public class RolloverRequest extends AcknowledgedRequest implem CreateIndexRequest.SETTINGS, ObjectParser.ValueType.OBJECT ); - PARSER.declareField((parser, request, includeTypeName) -> { - if (includeTypeName) { - // expecting one type only - for (Map.Entry mappingsEntry : parser.map().entrySet()) { - @SuppressWarnings("unchecked") - final Map value = (Map) mappingsEntry.getValue(); - request.createIndexRequest.mapping(value); - } - } else { - // a type is not included, add a dummy _doc type - Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { - throw new IllegalArgumentException( - "The mapping definition cannot be nested under a type " - + "[" - + MapperService.SINGLE_MAPPING_NAME - + "] unless include_type_name is set to true." - ); - } - request.createIndexRequest.mapping(mappings); + PARSER.declareField((parser, request, context) -> { + // a type is not included, add a dummy _doc type + Map mappings = parser.map(); + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + throw new IllegalArgumentException( + "The mapping definition cannot be nested under a type " + + "[" + + MapperService.SINGLE_MAPPING_NAME + + "] unless include_type_name is set to true." + ); } + request.createIndexRequest.mapping(mappings); }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type @@ -147,8 +138,8 @@ public ActionRequestValidationException validate() { ); } - var failureStoreOptions = indicesOptions.failureStoreOptions(); - if (failureStoreOptions.includeRegularIndices() && failureStoreOptions.includeFailureIndices()) { + var selectors = indicesOptions.selectorOptions().defaultSelectors(); + if (selectors.size() > 1) { validationException = addValidationError( "rollover cannot be applied to both regular and failure indices at the same time", validationException @@ -188,7 +179,7 @@ public IndicesOptions indicesOptions() { * @return true of the rollover request targets the failure store, false otherwise. */ public boolean targetsFailureStore() { - return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.failureStoreOptions().includeFailureIndices(); + return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.includeFailureIndices(); } public void setIndicesOptions(IndicesOptions indicesOptions) { @@ -290,8 +281,8 @@ public CreateIndexRequest getCreateIndexRequest() { } // param isTypeIncluded decides how mappings should be parsed from XContent - public void fromXContent(boolean isTypeIncluded, XContentParser parser) throws IOException { - PARSER.parse(parser, this, isTypeIncluded); + public void fromXContent(XContentParser parser) throws IOException { + PARSER.parse(parser, this, null); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index d65a66dcc47fb..c5c874f9bcddf 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -150,7 +150,7 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState .matchClosed(request.indicesOptions().expandWildcardsClosed()) .build(), IndicesOptions.GatekeeperOptions.DEFAULT, - request.indicesOptions().failureStoreOptions() + request.indicesOptions().selectorOptions() ); return state.blocks() @@ -247,7 +247,7 @@ protected void masterOperation( IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS, IndicesOptions.WildcardOptions.builder().matchClosed(true).allowEmptyExpressions(false).build(), IndicesOptions.GatekeeperOptions.DEFAULT, - rolloverRequest.indicesOptions().failureStoreOptions() + rolloverRequest.indicesOptions().selectorOptions() ); IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getRolloverTarget()) .clear() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index 1071f120f929e..45d784d301bf1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -16,10 +16,8 @@ import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; @@ -120,9 +118,6 @@ protected Iterator customXContentChunks(ToXContent.Params params) { builder.field(Fields.NUM_DOCS, segment.getNumDocs()); builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs()); builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize()); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO); - } builder.field(Fields.COMMITTED, segment.isCommitted()); builder.field(Fields.SEARCH, segment.isSearch()); if (segment.getVersion() != null) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index fa3ac8ae720c2..2d854d2c6fa45 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -12,7 +12,6 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -22,7 +21,6 @@ import java.util.Objects; import static java.util.Collections.singletonMap; -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject { @@ -65,11 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); for (IndexTemplateMetadata indexTemplateMetadata : getIndexTemplates()) { - if (builder.getRestApiVersion() == RestApiVersion.V_7 && params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, false)) { - IndexTemplateMetadata.Builder.toXContentWithTypes(indexTemplateMetadata, builder, params); - } else { - IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); - } + IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java index c0ceab139ff1b..d5931c85bb2e1 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; @@ -57,10 +56,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(STATUS, response.status().getStatus()); } else { builder.field(_INDEX, failure.getIndex()); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME); - } - builder.field(_ID, failure.getId()); builder.field(STATUS, failure.getStatus().getStatus()); failure.getFailureStoreStatus().toXContent(builder, params); @@ -301,9 +296,6 @@ public void setFailureStoreStatus(IndexDocFailureStoreStatus failureStoreStatus) @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(INDEX_FIELD, index); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field("type", MapperService.SINGLE_MAPPING_NAME); - } if (id != null) { builder.field(ID_FIELD, id); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index f04d07fb690c4..007f274d7f493 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -212,7 +212,7 @@ private void rollOverFailureStores(Runnable runnable) { RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null); rolloverRequest.setIndicesOptions( IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build() ); // We are executing a lazy rollover because it is an action specialised for this situation, when we want an diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 61adf41a9a276..a3a73415ec4f6 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -418,7 +418,7 @@ private void rollOverDataStreams( if (targetFailureStore) { rolloverRequest.setIndicesOptions( IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build() ); } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java index fbb084e8cd121..1c30303915c8e 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java @@ -61,9 +61,7 @@ public Request() { .allowFailureIndices(true) .build() ) - .failureStoreOptions( - IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true).build() - ) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .build() ); } diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java index caedec6a563d4..b759baad2024c 100644 --- a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java +++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.RestStatus; @@ -138,10 +137,6 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(_INDEX.getPreferredName(), index); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME); - } - builder.field(_ID.getPreferredName(), id); builder.field(MATCHED.getPreferredName(), isMatch()); if (hasExplanation()) { diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java index cc1d0497bd51a..22537f1f51216 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java @@ -26,11 +26,9 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceLoader; -import org.elasticsearch.rest.action.document.RestMultiGetAction; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; @@ -454,47 +452,41 @@ private static void parseDocuments( index = parser.text(); } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) { id = parser.text(); - } else if (parser.getRestApiVersion() == RestApiVersion.V_7 - && TYPE.match(currentFieldName, parser.getDeprecationHandler())) { - deprecationLogger.compatibleCritical("mget_with_types", RestMultiGetAction.TYPES_DEPRECATION_MESSAGE); - } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { - routing = parser.text(); - } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { - throw new ParsingException( - parser.getTokenLocation(), - "Unsupported field [fields] used, expected [stored_fields] instead" - ); - } else if (STORED_FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { - storedFields = new ArrayList<>(); - storedFields.add(parser.text()); - } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) { - version = parser.longValue(); - } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) { - versionType = VersionType.fromString(parser.text()); - } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) { - if (parser.isBooleanValue()) { - fetchSourceContext = fetchSourceContext == null - ? FetchSourceContext.of(parser.booleanValue()) - : FetchSourceContext.of( - parser.booleanValue(), - fetchSourceContext.includes(), - fetchSourceContext.excludes() - ); - } else if (token == Token.VALUE_STRING) { - fetchSourceContext = FetchSourceContext.of( - fetchSourceContext == null || fetchSourceContext.fetchSource(), - new String[] { parser.text() }, - fetchSourceContext == null ? Strings.EMPTY_ARRAY : fetchSourceContext.excludes() + } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { + routing = parser.text(); + } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { + throw new ParsingException( + parser.getTokenLocation(), + "Unsupported field [fields] used, expected [stored_fields] instead" + ); + } else if (STORED_FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { + storedFields = new ArrayList<>(); + storedFields.add(parser.text()); + } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) { + version = parser.longValue(); + } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) { + versionType = VersionType.fromString(parser.text()); + } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) { + if (parser.isBooleanValue()) { + fetchSourceContext = fetchSourceContext == null + ? FetchSourceContext.of(parser.booleanValue()) + : FetchSourceContext.of( + parser.booleanValue(), + fetchSourceContext.includes(), + fetchSourceContext.excludes() ); - } else { - throw new ElasticsearchParseException("illegal type for _source: [{}]", token); - } - } else { - throw new ElasticsearchParseException( - "failed to parse multi get request. unknown field [{}]", - currentFieldName + } else if (token == Token.VALUE_STRING) { + fetchSourceContext = FetchSourceContext.of( + fetchSourceContext == null || fetchSourceContext.fetchSource(), + new String[] { parser.text() }, + fetchSourceContext == null ? Strings.EMPTY_ARRAY : fetchSourceContext.excludes() ); + } else { + throw new ElasticsearchParseException("illegal type for _source: [{}]", token); } + } else { + throw new ElasticsearchParseException("failed to parse multi get request. unknown field [{}]", currentFieldName); + } } else if (token == Token.START_ARRAY) { if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { throw new ParsingException( diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java index fc126b29b3265..08db6dee8e543 100644 --- a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java +++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; @@ -90,9 +89,6 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(INDEX.getPreferredName(), index); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME); - } builder.field(ID.getPreferredName(), id); ElasticsearchException.generateFailureXContent(builder, params, exception, true); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java index 6570343452afe..12f542aec71c1 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java @@ -13,8 +13,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.Maps; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.IngestDocument.Metadata; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -127,9 +125,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(key, value.toString()); } } - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME); - } builder.field(SOURCE_FIELD, ingestDocument.getSource()); builder.field(INGEST_FIELD, ingestDocument.getIngestMetadata()); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index caa7453185575..0c585c705dcd0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.OriginalIndices; @@ -234,15 +233,6 @@ public final void run() { } if (shardsIts.size() > 0) { doCheckNoMissingShards(getName(), request, shardsIts); - Version version = request.minCompatibleShardNode(); - if (version != null && Version.CURRENT.minimumCompatibilityVersion().equals(version) == false) { - if (checkMinimumVersion(shardsIts) == false) { - throw new VersionMismatchException( - "One of the shards is incompatible with the required minimum version [{}]", - request.minCompatibleShardNode() - ); - } - } for (int i = 0; i < shardsIts.size(); i++) { final SearchShardIterator shardRoutings = shardsIts.get(i); assert shardRoutings.skip() == false; @@ -260,21 +250,6 @@ void skipShard(SearchShardIterator iterator) { successfulShardExecution(iterator); } - private boolean checkMinimumVersion(GroupShardsIterator shardsIts) { - for (SearchShardIterator it : shardsIts) { - if (it.getTargetNodeIds().isEmpty() == false) { - boolean isCompatible = it.getTargetNodeIds().stream().anyMatch(nodeId -> { - Transport.Connection conn = getConnection(it.getClusterAlias(), nodeId); - return conn == null || conn.getNode().getVersion().onOrAfter(request.minCompatibleShardNode()); - }); - if (isCompatible == false) { - return false; - } - } - } - return true; - } - private static boolean assertExecuteOnStartThread() { // Ensure that the current code has the following stacktrace: // AbstractSearchAsyncAction#start -> AbstractSearchAsyncAction#executePhase -> AbstractSearchAsyncAction#performPhaseOnShard @@ -761,12 +736,7 @@ final void onPhaseDone() { // as a tribute to @kimchy aka. finishHim() @Override public final Transport.Connection getConnection(String clusterAlias, String nodeId) { - Transport.Connection conn = nodeIdToConnection.apply(clusterAlias, nodeId); - Version minVersion = request.minCompatibleShardNode(); - if (minVersion != null && conn != null && conn.getNode().getVersion().before(minVersion)) { - throw new VersionMismatchException("One of the shards is incompatible with the required minimum version [{}]", minVersion); - } - return conn; + return nodeIdToConnection.apply(clusterAlias, nodeId); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java index 8ce2cc7b6b19e..8dcfbf5f070a1 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.FixedBitSet; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.common.util.Maps; @@ -133,15 +132,6 @@ private static boolean assertSearchCoordinationThread() { public void run() { assert assertSearchCoordinationThread(); checkNoMissingShards(); - Version version = request.minCompatibleShardNode(); - if (version != null && Version.CURRENT.minimumCompatibilityVersion().equals(version) == false) { - if (checkMinimumVersion(shardsIts) == false) { - throw new VersionMismatchException( - "One of the shards is incompatible with the required minimum version [{}]", - request.minCompatibleShardNode() - ); - } - } runCoordinatorRewritePhase(); } @@ -378,21 +368,6 @@ public CanMatchNodeRequest.Shard buildShardLevelRequest(SearchShardIterator shar ); } - private boolean checkMinimumVersion(GroupShardsIterator shardsIts) { - for (SearchShardIterator it : shardsIts) { - if (it.getTargetNodeIds().isEmpty() == false) { - boolean isCompatible = it.getTargetNodeIds().stream().anyMatch(nodeId -> { - Transport.Connection conn = getConnection(new SendingTarget(it.getClusterAlias(), nodeId)); - return conn == null || conn.getNode().getVersion().onOrAfter(request.minCompatibleShardNode()); - }); - if (isCompatible == false) { - return false; - } - } - } - return true; - } - @Override public void start() { if (getNumShards() == 0) { @@ -421,12 +396,7 @@ public void onPhaseFailure(String msg, Exception cause) { } public Transport.Connection getConnection(SendingTarget sendingTarget) { - Transport.Connection conn = nodeIdToConnection.apply(sendingTarget.clusterAlias, sendingTarget.nodeId); - Version minVersion = request.minCompatibleShardNode(); - if (minVersion != null && conn != null && conn.getNode().getVersion().before(minVersion)) { - throw new VersionMismatchException("One of the shards is incompatible with the required minimum version [{}]", minVersion); - } - return conn; + return nodeIdToConnection.apply(sendingTarget.clusterAlias, sendingTarget.nodeId); } private int getNumShards() { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 9961c3770fa86..5aec2bcd04b26 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -92,9 +92,6 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla private boolean ccsMinimizeRoundtrips; - @Nullable - private final Version minCompatibleShardNode; - public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; @@ -112,15 +109,10 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla private boolean forceSyntheticSource = false; public SearchRequest() { - this((Version) null); - } - - public SearchRequest(Version minCompatibleShardNode) { this.localClusterAlias = null; this.absoluteStartMillis = DEFAULT_ABSOLUTE_START_MILLIS; this.finalReduce = true; - this.minCompatibleShardNode = minCompatibleShardNode; - this.ccsMinimizeRoundtrips = minCompatibleShardNode == null; + this.ccsMinimizeRoundtrips = true; } /** @@ -219,7 +211,6 @@ private SearchRequest( this.localClusterAlias = localClusterAlias; this.absoluteStartMillis = absoluteStartMillis; this.finalReduce = finalReduce; - this.minCompatibleShardNode = searchRequest.minCompatibleShardNode; this.waitForCheckpoints = searchRequest.waitForCheckpoints; this.waitForCheckpointsTimeout = searchRequest.waitForCheckpointsTimeout; this.forceSyntheticSource = searchRequest.forceSyntheticSource; @@ -263,10 +254,8 @@ public SearchRequest(StreamInput in) throws IOException { finalReduce = true; } ccsMinimizeRoundtrips = in.readBoolean(); - if (in.readBoolean()) { - minCompatibleShardNode = Version.readVersion(in); - } else { - minCompatibleShardNode = null; + if (in.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE) && in.readBoolean()) { + Version.readVersion(in); // and drop on the floor } waitForCheckpoints = in.readMap(StreamInput::readLongArray); waitForCheckpointsTimeout = in.readTimeValue(); @@ -302,9 +291,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(finalReduce); } out.writeBoolean(ccsMinimizeRoundtrips); - out.writeBoolean(minCompatibleShardNode != null); - if (minCompatibleShardNode != null) { - Version.writeVersion(minCompatibleShardNode, out); + if (out.getTransportVersion().before(TransportVersions.REMOVE_MIN_COMPATIBLE_SHARD_NODE)) { + out.writeBoolean(false); } out.writeMap(waitForCheckpoints, StreamOutput::writeLongArray); out.writeTimeValue(waitForCheckpointsTimeout); @@ -351,14 +339,6 @@ public ActionRequestValidationException validate() { validationException = addValidationError("[preference] cannot be used with point in time", validationException); } } - if (minCompatibleShardNode() != null) { - if (isCcsMinimizeRoundtrips()) { - validationException = addValidationError( - "[ccs_minimize_roundtrips] cannot be [true] when setting a minimum compatible " + "shard version", - validationException - ); - } - } if (pointInTimeBuilder() != null && waitForCheckpoints.isEmpty() == false) { validationException = addValidationError("using [point in time] is not allowed with wait_for_checkpoints", validationException); @@ -401,15 +381,6 @@ long getAbsoluteStartMillis() { return absoluteStartMillis; } - /** - * Returns the minimum compatible shard version the search request needs to run on. If the version is null, then there are no - * restrictions imposed on shards versions part of this search. - */ - @Nullable - public Version minCompatibleShardNode() { - return minCompatibleShardNode; - } - /** * Sets the indices the search will be executed on. */ @@ -818,7 +789,6 @@ public boolean equals(Object o) { && Objects.equals(localClusterAlias, that.localClusterAlias) && absoluteStartMillis == that.absoluteStartMillis && ccsMinimizeRoundtrips == that.ccsMinimizeRoundtrips - && Objects.equals(minCompatibleShardNode, that.minCompatibleShardNode) && forceSyntheticSource == that.forceSyntheticSource; } @@ -840,7 +810,6 @@ public int hashCode() { localClusterAlias, absoluteStartMillis, ccsMinimizeRoundtrips, - minCompatibleShardNode, forceSyntheticSource ); } diff --git a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java new file mode 100644 index 0000000000000..65b48db8f5cf3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.support; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * We define as index components the two different sets of indices a data stream could consist of: + * - DATA: represents the backing indices + * - FAILURES: represent the failing indices + * Note: An index is its own DATA component, but it cannot have a FAILURE component. + */ +public enum IndexComponentSelector { + DATA("data"), + FAILURES("failures"); + + private final String key; + + IndexComponentSelector(String key) { + this.key = key; + } + + public String getKey() { + return key; + } + + private static final Map REGISTRY; + + static { + Map registry = new HashMap<>(IndexComponentSelector.values().length); + for (IndexComponentSelector value : IndexComponentSelector.values()) { + registry.put(value.getKey(), value); + } + REGISTRY = Collections.unmodifiableMap(registry); + } + + public static IndexComponentSelector getByKey(String key) { + return REGISTRY.get(key); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index d3ea063247704..22d019f80837d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -31,7 +31,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.function.Consumer; import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; @@ -48,24 +47,36 @@ * @param gatekeeperOptions, applies to all the resolved indices and defines if throttled will be included and if certain type of * aliases or indices are allowed, or they will throw an error. It acts as a gatekeeper when an action * does not support certain options. - * @param failureStoreOptions, applies to all indices already matched and controls the type of indices that will be returned. Currently, - * there are two types, data stream failure indices (only certain data streams have them) and data stream - * backing indices or stand-alone indices. + * @param selectorOptions, applies to all resolved expressions, and it specifies the index component that should be included, if there + * is no index component defined on the expression level. */ public record IndicesOptions( ConcreteTargetOptions concreteTargetOptions, WildcardOptions wildcardOptions, GatekeeperOptions gatekeeperOptions, - FailureStoreOptions failureStoreOptions + SelectorOptions selectorOptions ) implements ToXContentFragment { - public IndicesOptions( - ConcreteTargetOptions concreteTargetOptions, - WildcardOptions wildcardOptions, - GatekeeperOptions gatekeeperOptions - ) { - this(concreteTargetOptions, wildcardOptions, gatekeeperOptions, FailureStoreOptions.DEFAULT); - } + /** + * @deprecated this query param will be replaced by the selector `::` on the expression level + */ + @Deprecated + public static final String FAILURE_STORE_QUERY_PARAM = "failure_store"; + /** + * @deprecated this value will be replaced by the selector `::*` on the expression level + */ + @Deprecated + public static final String INCLUDE_ALL = "include"; + /** + * @deprecated this value will be replaced by the selector `::data` on the expression level + */ + @Deprecated + public static final String INCLUDE_ONLY_REGULAR_INDICES = "exclude"; + /** + * @deprecated this value will be replaced by the selector `::failures` on the expression level + */ + @Deprecated + public static final String INCLUDE_ONLY_FAILURE_INDICES = "only"; public static IndicesOptions.Builder builder() { return new Builder(); @@ -310,7 +321,7 @@ public static Builder builder(WildcardOptions wildcardOptions) { * - The "allow*" flags, which purpose is to enable actions to define certain conditions that need to apply on the concrete indices * they accept. For example, single-index actions will set allowAliasToMultipleIndices to false, while search will not accept a * closed index etc. These options are not configurable by the end-user. - * - The ignoreThrottled flag, which is a depricared flag that will filter out frozen indices. + * - The ignoreThrottled flag, which is a deprecated flag that will filter out frozen indices. * @param allowAliasToMultipleIndices, allow aliases to multiple indices, true by default. * @param allowClosedIndices, allow closed indices, true by default. * @param allowFailureIndices, allow failure indices in the response, true by default @@ -408,87 +419,53 @@ public static Builder builder(GatekeeperOptions gatekeeperOptions) { } /** - * Applies to all indices already matched and controls the type of indices that will be returned. There are two types, data stream - * failure indices (only certain data streams have them) and data stream backing indices or stand-alone indices. - * @param includeRegularIndices, when true regular or data stream backing indices will be retrieved. - * @param includeFailureIndices, when true data stream failure indices will be included. + * Defines which selectors should be used by default for an index operation in the event that no selectors are provided. */ - public record FailureStoreOptions(boolean includeRegularIndices, boolean includeFailureIndices) - implements - Writeable, - ToXContentFragment { + public record SelectorOptions(EnumSet defaultSelectors) implements Writeable { - public static final String FAILURE_STORE = "failure_store"; - public static final String INCLUDE_ALL = "include"; - public static final String INCLUDE_ONLY_REGULAR_INDICES = "exclude"; - public static final String INCLUDE_ONLY_FAILURE_INDICES = "only"; - - public static final FailureStoreOptions DEFAULT = new FailureStoreOptions(true, false); - - public static FailureStoreOptions read(StreamInput in) throws IOException { - return new FailureStoreOptions(in.readBoolean(), in.readBoolean()); - } - - public static FailureStoreOptions parseParameters(Object failureStoreValue, FailureStoreOptions defaultOptions) { - if (failureStoreValue == null) { - return defaultOptions; - } - FailureStoreOptions.Builder builder = defaultOptions == null - ? new FailureStoreOptions.Builder() - : new FailureStoreOptions.Builder(defaultOptions); - return switch (failureStoreValue.toString()) { - case INCLUDE_ALL -> builder.includeRegularIndices(true).includeFailureIndices(true).build(); - case INCLUDE_ONLY_REGULAR_INDICES -> builder.includeRegularIndices(true).includeFailureIndices(false).build(); - case INCLUDE_ONLY_FAILURE_INDICES -> builder.includeRegularIndices(false).includeFailureIndices(true).build(); - default -> throw new IllegalArgumentException("No valid " + FAILURE_STORE + " value [" + failureStoreValue + "]"); - }; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.field(FAILURE_STORE, displayValue()); - } + public static final SelectorOptions DATA_AND_FAILURE = new SelectorOptions( + EnumSet.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES) + ); + public static final SelectorOptions ONLY_DATA = new SelectorOptions(EnumSet.of(IndexComponentSelector.DATA)); + public static final SelectorOptions ONLY_FAILURES = new SelectorOptions(EnumSet.of(IndexComponentSelector.FAILURES)); + /** + * Default instance. Uses 
::data
as the default selector if none are present in an index expression. + */ + public static final SelectorOptions DEFAULT = ONLY_DATA; - public String displayValue() { - if (includeRegularIndices && includeFailureIndices) { - return INCLUDE_ALL; - } else if (includeRegularIndices) { - return INCLUDE_ONLY_REGULAR_INDICES; - } - return INCLUDE_ONLY_FAILURE_INDICES; + public static SelectorOptions read(StreamInput in) throws IOException { + return new SelectorOptions(in.readEnumSet(IndexComponentSelector.class)); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(includeRegularIndices); - out.writeBoolean(includeFailureIndices); + out.writeEnumSet(defaultSelectors); } public static class Builder { - private boolean includeRegularIndices; - private boolean includeFailureIndices; + private EnumSet defaultSelectors; public Builder() { this(DEFAULT); } - Builder(FailureStoreOptions options) { - includeRegularIndices = options.includeRegularIndices; - includeFailureIndices = options.includeFailureIndices; + Builder(SelectorOptions options) { + defaultSelectors = EnumSet.copyOf(options.defaultSelectors); } - public Builder includeRegularIndices(boolean includeRegularIndices) { - this.includeRegularIndices = includeRegularIndices; + public Builder setDefaultSelectors(IndexComponentSelector first, IndexComponentSelector... remaining) { + defaultSelectors = EnumSet.of(first, remaining); return this; } - public Builder includeFailureIndices(boolean includeFailureIndices) { - this.includeFailureIndices = includeFailureIndices; + public Builder setDefaultSelectors(EnumSet defaultSelectors) { + this.defaultSelectors = EnumSet.copyOf(defaultSelectors); return this; } - public FailureStoreOptions build() { - return new FailureStoreOptions(includeRegularIndices, includeFailureIndices); + public SelectorOptions build() { + assert defaultSelectors.isEmpty() != true : "Default selectors cannot be an empty set"; + return new SelectorOptions(EnumSet.copyOf(defaultSelectors)); } } @@ -496,8 +473,8 @@ public static Builder builder() { return new Builder(); } - public static Builder builder(FailureStoreOptions failureStoreOptions) { - return new Builder(failureStoreOptions); + public static Builder builder(SelectorOptions selectorOptions) { + return new Builder(selectorOptions); } } @@ -550,7 +527,7 @@ private enum Option { ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS, WildcardOptions.DEFAULT, GatekeeperOptions.DEFAULT, - FailureStoreOptions.DEFAULT + SelectorOptions.DEFAULT ); public static final IndicesOptions STRICT_EXPAND_OPEN = IndicesOptions.builder() @@ -570,7 +547,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -589,7 +566,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(SelectorOptions.DATA_AND_FAILURE) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -608,7 +585,25 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) + .build(); + public static final IndicesOptions LENIENT_EXPAND_OPEN_NO_SELECTORS = IndicesOptions.builder() + .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) + .wildcardOptions( + WildcardOptions.builder() + .matchOpen(true) + .matchClosed(false) + .includeHidden(false) + .allowEmptyExpressions(true) + .resolveAliases(true) + ) + .gatekeeperOptions( + GatekeeperOptions.builder() + .allowAliasToMultipleIndices(true) + .allowClosedIndices(true) + .allowFailureIndices(false) + .ignoreThrottled(false) + ) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_HIDDEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -627,7 +622,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -646,7 +641,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -660,7 +655,20 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) + .build(); + public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR = IndicesOptions.builder() + .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) + .wildcardOptions( + WildcardOptions.builder().matchOpen(true).matchClosed(true).includeHidden(true).allowEmptyExpressions(true).resolveAliases(true) + ) + .gatekeeperOptions( + GatekeeperOptions.builder() + .allowAliasToMultipleIndices(true) + .allowClosedIndices(true) + .allowFailureIndices(false) + .ignoreThrottled(false) + ) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -679,7 +687,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -693,7 +701,20 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) + .build(); + public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTORS = IndicesOptions.builder() + .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + WildcardOptions.builder().matchOpen(true).matchClosed(true).includeHidden(true).allowEmptyExpressions(true).resolveAliases(true) + ) + .gatekeeperOptions( + GatekeeperOptions.builder() + .allowAliasToMultipleIndices(true) + .allowClosedIndices(true) + .allowFailureIndices(false) + .ignoreThrottled(false) + ) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -712,7 +733,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(SelectorOptions.DATA_AND_FAILURE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -726,7 +747,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(SelectorOptions.DATA_AND_FAILURE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -745,7 +766,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(SelectorOptions.DATA_AND_FAILURE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -764,7 +785,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -783,7 +804,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -802,7 +823,7 @@ private enum Option { .allowFailureIndices(true) .allowAliasToMultipleIndices(true) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -821,7 +842,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); public static final IndicesOptions STRICT_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -840,7 +861,7 @@ private enum Option { .allowFailureIndices(true) .ignoreThrottled(false) ) - .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false)) + .selectorOptions(SelectorOptions.ONLY_DATA) .build(); /** @@ -929,14 +950,14 @@ public boolean ignoreThrottled() { * @return whether regular indices (stand-alone or backing indices) will be included in the response */ public boolean includeRegularIndices() { - return failureStoreOptions().includeRegularIndices(); + return selectorOptions().defaultSelectors().contains(IndexComponentSelector.DATA); } /** * @return whether failure indices (only supported by certain data streams) will be included in the response */ public boolean includeFailureIndices() { - return failureStoreOptions().includeFailureIndices(); + return selectorOptions().defaultSelectors().contains(IndexComponentSelector.FAILURES); } public void writeIndicesOptions(StreamOutput out) throws IOException { @@ -977,8 +998,13 @@ public void writeIndicesOptions(StreamOutput out) throws IOException { states.add(WildcardStates.HIDDEN); } out.writeEnumSet(states); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) { - failureStoreOptions.writeTo(out); + if (out.getTransportVersion() + .between(TransportVersions.V_8_14_0, TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) { + out.writeBoolean(includeRegularIndices()); + out.writeBoolean(includeFailureIndices()); + } + if (out.getTransportVersion().onOrAfter(TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) { + out.writeEnumSet(selectorOptions.defaultSelectors); } } @@ -999,16 +1025,30 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti .allowFailureIndices(allowFailureIndices) .ignoreThrottled(options.contains(Option.IGNORE_THROTTLED)) .build(); - FailureStoreOptions failureStoreOptions = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) - ? FailureStoreOptions.read(in) - : FailureStoreOptions.DEFAULT; + SelectorOptions selectorOptions = SelectorOptions.DEFAULT; + if (in.getTransportVersion() + .between(TransportVersions.V_8_14_0, TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) { + // Reading from an older node, which will be sending two booleans that we must read out and ignore. + var includeData = in.readBoolean(); + var includeFailures = in.readBoolean(); + if (includeData && includeFailures) { + selectorOptions = SelectorOptions.DATA_AND_FAILURE; + } else if (includeData) { + selectorOptions = SelectorOptions.ONLY_DATA; + } else { + selectorOptions = SelectorOptions.ONLY_FAILURES; + } + } + if (in.getTransportVersion().onOrAfter(TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) { + selectorOptions = new SelectorOptions(in.readEnumSet(IndexComponentSelector.class)); + } return new IndicesOptions( options.contains(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS) ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS, wildcardOptions, gatekeeperOptions, - failureStoreOptions + selectorOptions ); } @@ -1016,7 +1056,7 @@ public static class Builder { private ConcreteTargetOptions concreteTargetOptions; private WildcardOptions wildcardOptions; private GatekeeperOptions gatekeeperOptions; - private FailureStoreOptions failureStoreOptions; + private SelectorOptions selectorOptions; Builder() { this(DEFAULT); @@ -1026,7 +1066,7 @@ public static class Builder { concreteTargetOptions = indicesOptions.concreteTargetOptions; wildcardOptions = indicesOptions.wildcardOptions; gatekeeperOptions = indicesOptions.gatekeeperOptions; - failureStoreOptions = indicesOptions.failureStoreOptions; + selectorOptions = indicesOptions.selectorOptions; } public Builder concreteTargetOptions(ConcreteTargetOptions concreteTargetOptions) { @@ -1054,25 +1094,18 @@ public Builder gatekeeperOptions(GatekeeperOptions.Builder generalOptions) { return this; } - public Builder failureStoreOptions(FailureStoreOptions failureStoreOptions) { - this.failureStoreOptions = failureStoreOptions; + public Builder selectorOptions(SelectorOptions selectorOptions) { + this.selectorOptions = selectorOptions; return this; } - public Builder failureStoreOptions(FailureStoreOptions.Builder failureStoreOptions) { - this.failureStoreOptions = failureStoreOptions.build(); - return this; - } - - public Builder failureStoreOptions(Consumer failureStoreOptionsConfig) { - FailureStoreOptions.Builder failureStoreOptionsBuilder = FailureStoreOptions.builder(failureStoreOptions); - failureStoreOptionsConfig.accept(failureStoreOptionsBuilder); - this.failureStoreOptions = failureStoreOptionsBuilder.build(); + public Builder selectorOptions(SelectorOptions.Builder selectorOptions) { + this.selectorOptions = selectorOptions.build(); return this; } public IndicesOptions build() { - return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, failureStoreOptions); + return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, selectorOptions); } } @@ -1171,11 +1204,12 @@ public static IndicesOptions fromOptions( .allowClosedIndices(forbidClosedIndices == false) .ignoreThrottled(ignoreThrottled) .build(); + final SelectorOptions selectorOptions = SelectorOptions.DEFAULT; return new IndicesOptions( ignoreUnavailable ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS, wildcards, gatekeeperOptions, - FailureStoreOptions.DEFAULT + selectorOptions ); } @@ -1189,9 +1223,7 @@ public static IndicesOptions fromRequest(RestRequest request, IndicesOptions def request.param(ConcreteTargetOptions.IGNORE_UNAVAILABLE), request.param(WildcardOptions.ALLOW_NO_INDICES), request.param(GatekeeperOptions.IGNORE_THROTTLED), - DataStream.isFailureStoreFeatureFlagEnabled() - ? request.param(FailureStoreOptions.FAILURE_STORE) - : FailureStoreOptions.INCLUDE_ONLY_REGULAR_INDICES, + DataStream.isFailureStoreFeatureFlagEnabled() ? request.param(FAILURE_STORE_QUERY_PARAM) : INCLUDE_ONLY_REGULAR_INDICES, defaultSettings ); } @@ -1207,7 +1239,7 @@ public static IndicesOptions fromMap(Map map, IndicesOptions def map.containsKey(GatekeeperOptions.IGNORE_THROTTLED) ? map.get(GatekeeperOptions.IGNORE_THROTTLED) : map.get("ignoreThrottled"), - map.containsKey(FailureStoreOptions.FAILURE_STORE) ? map.get(FailureStoreOptions.FAILURE_STORE) : map.get("failureStore"), + map.containsKey(FAILURE_STORE_QUERY_PARAM) ? map.get(FAILURE_STORE_QUERY_PARAM) : map.get("failureStore"), defaultSettings ); } @@ -1235,7 +1267,7 @@ public static boolean isIndicesOptions(String name) { || "ignoreThrottled".equals(name) || WildcardOptions.ALLOW_NO_INDICES.equals(name) || "allowNoIndices".equals(name) - || (DataStream.isFailureStoreFeatureFlagEnabled() && FailureStoreOptions.FAILURE_STORE.equals(name)) + || (DataStream.isFailureStoreFeatureFlagEnabled() && FAILURE_STORE_QUERY_PARAM.equals(name)) || (DataStream.isFailureStoreFeatureFlagEnabled() && "failureStore".equals(name)); } @@ -1267,26 +1299,51 @@ public static IndicesOptions fromParameters( WildcardOptions wildcards = WildcardOptions.parseParameters(wildcardsString, allowNoIndicesString, defaultSettings.wildcardOptions); GatekeeperOptions gatekeeperOptions = GatekeeperOptions.parseParameter(ignoreThrottled, defaultSettings.gatekeeperOptions); - FailureStoreOptions failureStoreOptions = DataStream.isFailureStoreFeatureFlagEnabled() - ? FailureStoreOptions.parseParameters(failureStoreString, defaultSettings.failureStoreOptions) - : FailureStoreOptions.DEFAULT; + SelectorOptions selectorOptions = DataStream.isFailureStoreFeatureFlagEnabled() + ? parseFailureStoreParameters(failureStoreString, defaultSettings.selectorOptions) + : SelectorOptions.DEFAULT; // note that allowAliasesToMultipleIndices is not exposed, always true (only for internal use) return IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.fromParameter(ignoreUnavailableString, defaultSettings.concreteTargetOptions)) .wildcardOptions(wildcards) .gatekeeperOptions(gatekeeperOptions) - .failureStoreOptions(failureStoreOptions) + .selectorOptions(selectorOptions) .build(); } + /** + * @deprecated This method parses the query parameter failure_store. This is a deprecated param, and it will be replaced + * the selector suffix, for example `my-data-stream::data` or `my-data-stream::failures` + */ + @Deprecated + private static SelectorOptions parseFailureStoreParameters(Object failureStoreValue, SelectorOptions defaultOptions) { + if (failureStoreValue == null) { + return defaultOptions; + } + return switch (failureStoreValue.toString()) { + case INCLUDE_ALL -> SelectorOptions.DATA_AND_FAILURE; + case INCLUDE_ONLY_REGULAR_INDICES -> SelectorOptions.ONLY_DATA; + case INCLUDE_ONLY_FAILURE_INDICES -> SelectorOptions.ONLY_FAILURES; + default -> throw new IllegalArgumentException("No valid " + FAILURE_STORE_QUERY_PARAM + " value [" + failureStoreValue + "]"); + }; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { concreteTargetOptions.toXContent(builder, params); wildcardOptions.toXContent(builder, params); gatekeeperOptions.toXContent(builder, params); if (DataStream.isFailureStoreFeatureFlagEnabled()) { - failureStoreOptions.toXContent(builder, params); + String displayValue; + if (SelectorOptions.DATA_AND_FAILURE.equals(selectorOptions())) { + displayValue = INCLUDE_ALL; + } else if (SelectorOptions.ONLY_DATA.equals(selectorOptions())) { + displayValue = INCLUDE_ONLY_REGULAR_INDICES; + } else { + displayValue = INCLUDE_ONLY_FAILURE_INDICES; + } + builder.field(FAILURE_STORE_QUERY_PARAM, displayValue); } return builder; } @@ -1295,7 +1352,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par private static final ParseField IGNORE_UNAVAILABLE_FIELD = new ParseField(ConcreteTargetOptions.IGNORE_UNAVAILABLE); private static final ParseField IGNORE_THROTTLED_FIELD = new ParseField(GatekeeperOptions.IGNORE_THROTTLED).withAllDeprecated(); private static final ParseField ALLOW_NO_INDICES_FIELD = new ParseField(WildcardOptions.ALLOW_NO_INDICES); - private static final ParseField FAILURE_STORE_FIELD = new ParseField(FailureStoreOptions.FAILURE_STORE); + private static final ParseField FAILURE_STORE_FIELD = new ParseField(FAILURE_STORE_QUERY_PARAM); public static IndicesOptions fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, null); @@ -1306,7 +1363,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic WildcardOptions.Builder wildcards = defaults == null ? null : WildcardOptions.builder(defaults.wildcardOptions()); GatekeeperOptions.Builder generalOptions = GatekeeperOptions.builder() .ignoreThrottled(defaults != null && defaults.gatekeeperOptions().ignoreThrottled()); - FailureStoreOptions failureStoreOptions = defaults == null ? FailureStoreOptions.DEFAULT : defaults.failureStoreOptions(); + SelectorOptions selectorOptions = defaults == null ? SelectorOptions.DEFAULT : defaults.selectorOptions(); Boolean allowNoIndices = defaults == null ? null : defaults.allowNoIndices(); Boolean ignoreUnavailable = defaults == null ? null : defaults.ignoreUnavailable(); Token token = parser.currentToken() == Token.START_OBJECT ? parser.currentToken() : parser.nextToken(); @@ -1358,7 +1415,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic generalOptions.ignoreThrottled(parser.booleanValue()); } else if (DataStream.isFailureStoreFeatureFlagEnabled() && FAILURE_STORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - failureStoreOptions = FailureStoreOptions.parseParameters(parser.text(), failureStoreOptions); + selectorOptions = parseFailureStoreParameters(parser.text(), selectorOptions); } else { throw new ElasticsearchParseException( "could not read indices options. Unexpected index option [" + currentFieldName + "]" @@ -1389,7 +1446,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic .concreteTargetOptions(new ConcreteTargetOptions(ignoreUnavailable)) .wildcardOptions(wildcards) .gatekeeperOptions(generalOptions) - .failureStoreOptions(failureStoreOptions) + .selectorOptions(selectorOptions) .build(); } diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java index 2beeaf1a26f0f..42196fac28528 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java @@ -16,8 +16,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -110,9 +108,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); Failure failure = response.getFailure(); builder.field(Fields._INDEX, failure.getIndex()); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(Fields._TYPE, MapperService.SINGLE_MAPPING_NAME); - } builder.field(Fields._ID, failure.getId()); ElasticsearchException.generateFailureXContent(builder, params, failure.getCause(), true); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java index 2d70af94d53d1..4c4aa7de46f6b 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ToXContentObject; @@ -164,9 +163,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (isArtificial() == false) { builder.field(FieldStrings._ID, id); } - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME); - } builder.field(FieldStrings._VERSION, docVersion); builder.field(FieldStrings.FOUND, isExists()); builder.field(FieldStrings.TOOK, tookInMillis); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 7f2c076281735..29720e98a6e7b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1373,7 +1373,7 @@ private static void updateIndexMappingsAndBuildSortOrder( MapperService mapperService = indexService.mapperService(); IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD; List allMappings = new ArrayList<>(); - final CompressedXContent defaultMapping = indexMode.getDefaultMapping(); + final CompressedXContent defaultMapping = indexMode.getDefaultMapping(indexService.getIndexSettings()); if (defaultMapping != null) { allMappings.add(defaultMapping); } diff --git a/server/src/main/java/org/elasticsearch/common/BackoffPolicy.java b/server/src/main/java/org/elasticsearch/common/BackoffPolicy.java index 27d98f9ade203..cacad64ab1d4f 100644 --- a/server/src/main/java/org/elasticsearch/common/BackoffPolicy.java +++ b/server/src/main/java/org/elasticsearch/common/BackoffPolicy.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.common; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import java.util.Collections; @@ -81,6 +82,18 @@ public static BackoffPolicy exponentialBackoff(TimeValue initialDelay, int maxNu return new ExponentialBackoff((int) checkDelay(initialDelay).millis(), maxNumberOfRetries); } + /** + * Creates a new linear backoff policy with the provided configuration + * + * @param delayIncrement The amount by which to increment the delay on each retry + * @param maxNumberOfRetries The maximum number of retries + * @param maximumDelay The maximum delay + * @return A backoff policy with linear increase in wait time for retries. + */ + public static BackoffPolicy linearBackoff(TimeValue delayIncrement, int maxNumberOfRetries, TimeValue maximumDelay) { + return new LinearBackoff(delayIncrement, maxNumberOfRetries, maximumDelay); + } + /** * Wraps the backoff policy in one that calls a method every time a new backoff is taken from the policy. */ @@ -100,6 +113,11 @@ private static class NoBackoff extends BackoffPolicy { public Iterator iterator() { return Collections.emptyIterator(); } + + @Override + public String toString() { + return "NoBackoff"; + } } private static class ExponentialBackoff extends BackoffPolicy { @@ -118,6 +136,11 @@ private ExponentialBackoff(int start, int numberOfElements) { public Iterator iterator() { return new ExponentialBackoffIterator(start, numberOfElements); } + + @Override + public String toString() { + return "ExponentialBackoff{start=" + start + ", numberOfElements=" + numberOfElements + '}'; + } } private static class ExponentialBackoffIterator implements Iterator { @@ -163,6 +186,11 @@ private static final class ConstantBackoff extends BackoffPolicy { public Iterator iterator() { return new ConstantBackoffIterator(delay, numberOfElements); } + + @Override + public String toString() { + return "ConstantBackoff{delay=" + delay + ", numberOfElements=" + numberOfElements + '}'; + } } private static final class ConstantBackoffIterator implements Iterator { @@ -203,6 +231,11 @@ private static final class WrappedBackoffPolicy extends BackoffPolicy { public Iterator iterator() { return new WrappedBackoffIterator(delegate.iterator(), onBackoff); } + + @Override + public String toString() { + return "WrappedBackoffPolicy{delegate=" + delegate + ", onBackoff=" + onBackoff + '}'; + } } private static final class WrappedBackoffIterator implements Iterator { @@ -228,4 +261,60 @@ public TimeValue next() { return delegate.next(); } } + + private static final class LinearBackoff extends BackoffPolicy { + + private final TimeValue delayIncrement; + private final int maxNumberOfRetries; + private final TimeValue maximumDelay; + + private LinearBackoff(TimeValue delayIncrement, int maxNumberOfRetries, @Nullable TimeValue maximumDelay) { + this.delayIncrement = delayIncrement; + this.maxNumberOfRetries = maxNumberOfRetries; + this.maximumDelay = maximumDelay; + } + + @Override + public Iterator iterator() { + return new LinearBackoffIterator(delayIncrement, maxNumberOfRetries, maximumDelay); + } + + @Override + public String toString() { + return "LinearBackoff{" + + "delayIncrement=" + + delayIncrement + + ", maxNumberOfRetries=" + + maxNumberOfRetries + + ", maximumDelay=" + + maximumDelay + + '}'; + } + } + + private static final class LinearBackoffIterator implements Iterator { + + private final TimeValue delayIncrement; + private final int maxNumberOfRetries; + private final TimeValue maximumDelay; + private int curr; + + private LinearBackoffIterator(TimeValue delayIncrement, int maxNumberOfRetries, @Nullable TimeValue maximumDelay) { + this.delayIncrement = delayIncrement; + this.maxNumberOfRetries = maxNumberOfRetries; + this.maximumDelay = maximumDelay; + } + + @Override + public boolean hasNext() { + return curr < maxNumberOfRetries; + } + + @Override + public TimeValue next() { + curr++; + TimeValue timeValue = TimeValue.timeValueMillis(curr * delayIncrement.millis()); + return maximumDelay == null ? timeValue : timeValue.compareTo(maximumDelay) < 0 ? timeValue : maximumDelay; + } + } } diff --git a/server/src/main/java/org/elasticsearch/common/Strings.java b/server/src/main/java/org/elasticsearch/common/Strings.java index 4314d2e16799a..82504b5840792 100644 --- a/server/src/main/java/org/elasticsearch/common/Strings.java +++ b/server/src/main/java/org/elasticsearch/common/Strings.java @@ -285,6 +285,7 @@ private static String changeFirstCharacterCase(String str, boolean capitalize) { static final Set INVALID_CHARS = Set.of('\\', '/', '*', '?', '"', '<', '>', '|', ' ', ','); public static final String INVALID_FILENAME_CHARS = INVALID_CHARS.stream() + .sorted() .map(c -> "'" + c + "'") .collect(Collectors.joining(",", "[", "]")); diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 2d9e89223d7a6..5908bc22e21e2 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -75,7 +75,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma } @Override - public CompressedXContent getDefaultMapping() { + public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) { return null; } @@ -171,7 +171,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma } @Override - public CompressedXContent getDefaultMapping() { + public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) { return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; } @@ -249,8 +249,10 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma } @Override - public CompressedXContent getDefaultMapping() { - return DEFAULT_LOGS_TIMESTAMP_MAPPING; + public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) { + return indexSettings != null && indexSettings.getIndexSortConfig().hasPrimarySortOnField(HOST_NAME) + ? DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME + : DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; } @Override @@ -308,6 +310,8 @@ public String getDefaultCodec() { } }; + private static final String HOST_NAME = "host.name"; + private static void validateTimeSeriesSettings(Map, Object> settings) { settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH); settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME); @@ -324,48 +328,33 @@ protected static String tsdbMode() { return "[" + IndexSettings.MODE.getKey() + "=time_series]"; } - public static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; + private static CompressedXContent createDefaultMapping(boolean includeHostName) throws IOException { + return new CompressedXContent((builder, params) -> { + builder.startObject(MapperService.SINGLE_MAPPING_NAME) + .startObject(DataStreamTimestampFieldMapper.NAME) + .field("enabled", true) + .endObject() + .startObject("properties") + .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH) + .field("type", DateFieldMapper.CONTENT_TYPE) + .endObject(); + + if (includeHostName) { + builder.startObject(HOST_NAME).field("type", KeywordFieldMapper.CONTENT_TYPE).field("ignore_above", 1024).endObject(); + } - static { - try { - DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = new CompressedXContent( - ((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME) - .startObject(DataStreamTimestampFieldMapper.NAME) - .field("enabled", true) - .endObject() - .startObject("properties") - .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH) - .field("type", DateFieldMapper.CONTENT_TYPE) - .field("ignore_malformed", "false") - .endObject() - .endObject() - .endObject()) - ); - } catch (IOException e) { - throw new AssertionError(e); - } + return builder.endObject().endObject(); + }); } - public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING; + private static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; + + private static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME; static { try { - DEFAULT_LOGS_TIMESTAMP_MAPPING = new CompressedXContent( - ((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME) - .startObject(DataStreamTimestampFieldMapper.NAME) - .field("enabled", true) - .endObject() - .startObject("properties") - .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH) - .field("type", DateFieldMapper.CONTENT_TYPE) - .endObject() - .startObject("host.name") - .field("type", KeywordFieldMapper.CONTENT_TYPE) - .field("ignore_above", 1024) - .endObject() - .endObject() - .endObject()) - ); + DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = createDefaultMapping(false); + DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME = createDefaultMapping(true); } catch (IOException e) { throw new AssertionError(e); } @@ -421,7 +410,7 @@ public String getName() { * Get default mapping for this index or {@code null} if there is none. */ @Nullable - public abstract CompressedXContent getDefaultMapping(); + public abstract CompressedXContent getDefaultMapping(IndexSettings indexSettings); /** * Build the {@link FieldMapper} for {@code _id}. diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java index 402f455d69bc2..109f645f24caf 100644 --- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -20,11 +20,9 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.rest.action.document.RestMultiGetAction; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -276,9 +274,6 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(_INDEX, index); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME); - } builder.field(_ID, id); if (isExists()) { if (version != -1) { @@ -316,8 +311,6 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index } else if (token.isValue()) { if (_INDEX.equals(currentFieldName)) { index = parser.text(); - } else if (parser.getRestApiVersion() == RestApiVersion.V_7 && MapperService.TYPE_FIELD_NAME.equals(currentFieldName)) { - deprecationLogger.compatibleCritical("mget_with_types", RestMultiGetAction.TYPES_DEPRECATION_MESSAGE); } else if (_ID.equals(currentFieldName)) { id = parser.text(); } else if (_VERSION.equals(currentFieldName)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 2eec14bd1a8d6..ef87ce52fbabf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -111,7 +111,7 @@ public int get() { private final Set ignoredFields; private final List ignoredFieldValues; private final List ignoredFieldsMissingValues; - private final boolean inArrayScopeEnabled; + private boolean inArrayScopeEnabled; private boolean inArrayScope; private final Map> dynamicMappers; @@ -376,13 +376,14 @@ public final Collection getIgnoredFieldsMiss * Applies to synthetic source only. */ public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException { - if (canAddIgnoredField() && mapper instanceof ObjectMapper && inArrayScopeEnabled) { - boolean isNested = mapper instanceof NestedObjectMapper; - if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) { - DocumentParserContext subcontext = switchParser(parser()); - subcontext.inArrayScope = inArrayScope == false; - return subcontext; - } + if (canAddIgnoredField() + && mapper instanceof ObjectMapper + && mapper instanceof NestedObjectMapper == false + && inArrayScope == false + && inArrayScopeEnabled) { + DocumentParserContext subcontext = switchParser(parser()); + subcontext.inArrayScope = true; + return subcontext; } return this; } @@ -709,12 +710,18 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested * Return a new context that has the provided document as the current document. */ public final DocumentParserContext switchDoc(final LuceneDocument document) { - return new Wrapper(this.parent, this) { + DocumentParserContext cloned = new Wrapper(this.parent, this) { @Override public LuceneDocument doc() { return document; } }; + // Disable tracking array scopes for ignored source, as it would be added to the parent doc. + // Nested documents are added to preserve object structure within arrays of objects, so the use + // of ignored source for arrays inside them should be mostly redundant. + cloned.inArrayScope = false; + cloned.inArrayScopeEnabled = false; + return cloned; } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java index 7549873a10bc1..033151da362ef 100644 --- a/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; import org.elasticsearch.geometry.ShapeType; @@ -429,9 +428,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep GeoJson.toXContent(shape, builder, params); } else { builder.startObject(INDEXED_SHAPE_FIELD.getPreferredName()).field(SHAPE_ID_FIELD.getPreferredName(), indexedShapeId); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(SHAPE_TYPE_FIELD.getPreferredName(), MapperService.SINGLE_MAPPING_NAME); - } if (indexedShapeIndex != null) { builder.field(SHAPE_INDEX_FIELD.getPreferredName(), indexedShapeIndex); } @@ -555,16 +551,13 @@ public static ParsedGeometryQueryParams parsedParamsFromXContent(XContentParser } else if (token.isValue()) { if (SHAPE_ID_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { params.id = parser.text(); - } else if (parser.getRestApiVersion() == RestApiVersion.V_7 - && SHAPE_TYPE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - deprecationLogger.compatibleCritical("geo_share_query_with_types", TYPES_DEPRECATION_MESSAGE); - } else if (SHAPE_INDEX_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - params.index = parser.text(); - } else if (SHAPE_PATH_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - params.shapePath = parser.text(); - } else if (SHAPE_ROUTING_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - params.shapeRouting = parser.text(); - } + } else if (SHAPE_INDEX_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + params.index = parser.text(); + } else if (SHAPE_PATH_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + params.shapePath = parser.text(); + } else if (SHAPE_ROUTING_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + params.shapeRouting = parser.text(); + } } else { throw new ParsingException( parser.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index a7b3b9145d2ca..2401719caaa87 100644 --- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; @@ -238,9 +237,7 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep doXArrayContent(FILTER, filterClauses, builder, params); doXArrayContent(MUST_NOT, mustNotClauses, builder, params); doXArrayContent(SHOULD, shouldClauses, builder, params); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(ADJUST_PURE_NEGATIVE.getPreferredName(), adjustPureNegative); - } else if (adjustPureNegative != ADJUST_PURE_NEGATIVE_DEFAULT) { + if (adjustPureNegative != ADJUST_PURE_NEGATIVE_DEFAULT) { builder.field(ADJUST_PURE_NEGATIVE.getPreferredName(), adjustPureNegative); } if (minimumShouldMatch != null) { diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java index 86413404d9571..22cc68fb3a2f2 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java @@ -358,18 +358,14 @@ public static GeoBoundingBoxQueryBuilder fromXContent(XContentParser parser) thr validationMethod = GeoValidationMethod.fromString(parser.text()); } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { ignoreUnmapped = parser.booleanValue(); - } else if (parser.getRestApiVersion() == RestApiVersion.V_7 - && TYPE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - deprecationLogger.compatibleCritical("geo_bounding_box_type", TYPE_PARAMETER_DEPRECATION_MESSAGE); - parser.text(); // ignore value - } else { - throw new ParsingException( - parser.getTokenLocation(), - "failed to parse [{}] query. unexpected field [{}]", - NAME, - currentFieldName - ); - } + } else { + throw new ParsingException( + parser.getTokenLocation(), + "failed to parse [{}] query. unexpected field [{}]", + NAME, + currentFieldName + ); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java index a4a76c078cb55..d72b755e7e77a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java @@ -528,15 +528,12 @@ public static MatchQueryBuilder fromXContent(XContentParser parser) throws IOExc queryName = parser.text(); } else if (GENERATE_SYNONYMS_PHRASE_QUERY.match(currentFieldName, parser.getDeprecationHandler())) { autoGenerateSynonymsPhraseQuery = parser.booleanValue(); - } else if (parser.getRestApiVersion() == RestApiVersion.V_7 - && CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - throw new ParsingException(parser.getTokenLocation(), CUTOFF_FREQUENCY_DEPRECATION_MSG); - } else { - throw new ParsingException( - parser.getTokenLocation(), - "[" + NAME + "] query does not support [" + currentFieldName + "]" - ); - } + } else { + throw new ParsingException( + parser.getTokenLocation(), + "[" + NAME + "] query does not support [" + currentFieldName + "]" + ); + } } else { throw new ParsingException( parser.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index c1e97e7429643..7e644a8800bbd 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -37,12 +37,10 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; @@ -352,40 +350,32 @@ public static Item parse(XContentParser parser, Item item) throws IOException { } else if (currentFieldName != null) { if (INDEX.match(currentFieldName, parser.getDeprecationHandler())) { item.index = parser.text(); - } else if (parser.getRestApiVersion() == RestApiVersion.V_7 - && TYPE.match(currentFieldName, parser.getDeprecationHandler())) { - deprecationLogger.compatibleCritical("more_like_this_query_with_types", TYPES_DEPRECATION_MESSAGE); - } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) { - item.id = parser.text(); - } else if (DOC.match(currentFieldName, parser.getDeprecationHandler())) { - item.doc = BytesReference.bytes(jsonBuilder().copyCurrentStructure(parser)); - item.xContentType = XContentType.JSON; - } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { - if (token == XContentParser.Token.START_ARRAY) { - List fields = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - fields.add(parser.text()); - } - item.fields(fields.toArray(new String[fields.size()])); - } else { - throw new ElasticsearchParseException( - "failed to parse More Like This item. field [fields] must be an array" - ); + } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) { + item.id = parser.text(); + } else if (DOC.match(currentFieldName, parser.getDeprecationHandler())) { + item.doc = BytesReference.bytes(jsonBuilder().copyCurrentStructure(parser)); + item.xContentType = XContentType.JSON; + } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { + if (token == XContentParser.Token.START_ARRAY) { + List fields = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + fields.add(parser.text()); } - } else if (PER_FIELD_ANALYZER.match(currentFieldName, parser.getDeprecationHandler())) { - item.perFieldAnalyzer(TermVectorsRequest.readPerFieldAnalyzer(parser.map())); - } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { - item.routing = parser.text(); - } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) { - item.version = parser.longValue(); - } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) { - item.versionType = VersionType.fromString(parser.text()); + item.fields(fields.toArray(new String[fields.size()])); } else { - throw new ElasticsearchParseException( - "failed to parse More Like This item. unknown field [{}]", - currentFieldName - ); + throw new ElasticsearchParseException("failed to parse More Like This item. field [fields] must be an array"); } + } else if (PER_FIELD_ANALYZER.match(currentFieldName, parser.getDeprecationHandler())) { + item.perFieldAnalyzer(TermVectorsRequest.readPerFieldAnalyzer(parser.map())); + } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) { + item.routing = parser.text(); + } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) { + item.version = parser.longValue(); + } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) { + item.versionType = VersionType.fromString(parser.text()); + } else { + throw new ElasticsearchParseException("failed to parse More Like This item. unknown field [{}]", currentFieldName); + } } } if (item.id != null && item.doc != null) { @@ -405,9 +395,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (this.index != null) { builder.field(INDEX.getPreferredName(), this.index); } - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(TYPE.getPreferredName(), MapperService.SINGLE_MAPPING_NAME); - } if (this.id != null) { builder.field(ID.getPreferredName(), this.id); } diff --git a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java index 1deba84cce355..a83fb8d1fd419 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java @@ -654,15 +654,12 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws autoGenerateSynonymsPhraseQuery = parser.booleanValue(); } else if (FUZZY_TRANSPOSITIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { fuzzyTranspositions = parser.booleanValue(); - } else if (parser.getRestApiVersion() == RestApiVersion.V_7 - && CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - throw new ParsingException(parser.getTokenLocation(), CUTOFF_FREQUENCY_DEPRECATION_MSG); - } else { - throw new ParsingException( - parser.getTokenLocation(), - "[" + NAME + "] query does not support [" + currentFieldName + "]" - ); - } + } else { + throw new ParsingException( + parser.getTokenLocation(), + "[" + NAME + "] query does not support [" + currentFieldName + "]" + ); + } } else { throw new ParsingException( parser.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java index b4a8610ded04d..b0a4d333ba77f 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xcontent.ToXContent; @@ -305,13 +304,6 @@ public Stats getTotal() { public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(Fields.INDEXING); totalStats.toXContent(builder, params); - if (builder.getRestApiVersion() == RestApiVersion.V_7 && params.param("types") != null) { - builder.startObject(Fields.TYPES); - builder.startObject(MapperService.SINGLE_MAPPING_NAME); - totalStats.toXContent(builder, params); - builder.endObject(); - builder.endObject(); - } builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index 2c99563955746..d437533a8603d 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -210,8 +210,4 @@ default List defaultConfigIds() { default void defaultConfigs(ActionListener> defaultsListener) { defaultsListener.onResponse(List.of()); } - - default void updateModelsWithDynamicFields(List model, ActionListener> listener) { - listener.onResponse(model); - } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 0f63d2a8dcc1b..99ff44a3cd135 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -39,6 +39,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; @@ -46,6 +47,8 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; @@ -55,7 +58,9 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.env.Environment; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.grok.MatcherWatchdog; import org.elasticsearch.index.IndexSettings; @@ -97,6 +102,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.core.UpdateForV10.Owner.DATA_MANAGEMENT; /** * Holder class for several ingest related services. @@ -107,7 +113,10 @@ public class IngestService implements ClusterStateApplier, ReportingService taskQueue; private final ClusterService clusterService; @@ -652,12 +661,24 @@ public IngestMetadata execute(IngestMetadata currentIngestMetadata, Collection ingestInfos, String pipelineId, Map pipelineConfig) throws Exception { if (ingestInfos.isEmpty()) { throw new IllegalStateException("Ingest info is empty"); } + try { + MetadataCreateIndexService.validateIndexOrAliasName( + pipelineId, + (pipelineName, error) -> new IllegalArgumentException( + "Pipeline name [" + pipelineName + "] will be disallowed in a future version for the following reason: " + error + ) + ); + } catch (IllegalArgumentException e) { + deprecationLogger.critical(DeprecationCategory.API, "pipeline_name_special_chars", e.getMessage()); + } + Pipeline pipeline = Pipeline.create(pipelineId, pipelineConfig, processorFactories, scriptService); List exceptions = new ArrayList<>(); for (Processor processor : pipeline.flattenAllProcessors()) { diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 2f7bb80a8d46a..0a99ee777bb76 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -47,13 +47,6 @@ */ public abstract class BaseRestHandler implements RestHandler { - /** - * Parameter that controls whether certain REST apis should include type names in their requests or responses. - * Note: This parameter is only available through compatible rest api for {@link RestApiVersion#V_7}. - */ - public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name"; - public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false; - public static final Setting MULTI_ALLOW_EXPLICIT_INDEX = Setting.boolSetting( "rest.action.multi.allow_explicit_index", true, diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java index 933dd4d966fdf..c2848781bc7a7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -72,11 +71,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("types")) { - deprecationLogger.compatibleCritical("nodes_stats_types", TYPES_DEPRECATION_MESSAGE); - request.param("types"); - } - String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); Set metricNames = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 8e1a122f98a3a..e30d2f8d5c733 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -12,11 +12,8 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -25,7 +22,6 @@ import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -39,10 +35,6 @@ @ServerlessScope(Scope.PUBLIC) public class RestCreateIndexAction extends BaseRestHandler { - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestCreateIndexAction.class); - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in create " - + "index requests is deprecated. The parameter will be removed in the next major version."; - @Override public List routes() { return List.of(new Route(PUT, "/{index}")); @@ -55,61 +47,10 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - CreateIndexRequest createIndexRequest; - if (request.getRestApiVersion() == RestApiVersion.V_7) { - createIndexRequest = prepareRequestV7(request); - } else { - createIndexRequest = prepareRequest(request); - } + final var createIndexRequest = prepareRequest(request); return channel -> client.admin().indices().create(createIndexRequest, new RestToXContentListener<>(channel)); } - // default scope for testing types in mapping - static CreateIndexRequest prepareRequestV7(RestRequest request) { - CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.param(INCLUDE_TYPE_NAME_PARAMETER);// just consume, it is always replaced with _doc - deprecationLogger.compatibleCritical("create_index_with_types", TYPES_DEPRECATION_MESSAGE); - } - - if (request.hasContent()) { - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - - sourceAsMap = prepareMappingsV7(sourceAsMap, request); - - createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); - } - - createIndexRequest.ackTimeout(getAckTimeout(request)); - createIndexRequest.masterNodeTimeout(getMasterNodeTimeout(request)); - createIndexRequest.waitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); - return createIndexRequest; - } - - static Map prepareMappingsV7(Map source, RestRequest request) { - final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, false); - - @SuppressWarnings("unchecked") - Map mappings = (Map) source.get("mappings"); - - if (includeTypeName && mappings != null && mappings.size() == 1) { - Map newSource = new HashMap<>(); - newSource.putAll(source); // mappings will be overridden. Aliases, settings stay the same - String typeName = mappings.keySet().iterator().next(); - if (Strings.hasText(typeName) == false) { - throw new IllegalArgumentException("name cannot be empty string"); - } - @SuppressWarnings("unchecked") - Map typedMappings = (Map) mappings.get(typeName); - - // no matter what the type was, replace it with _doc, because the internal representation still uses single type `_doc`. - newSource.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings)); - return newSource; - } else { - return prepareMappings(source); - } - } - static CreateIndexRequest prepareRequest(RestRequest request) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 37391028dbd6e..5f648ca8e77e5 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -58,25 +57,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); - if (request.getRestApiVersion() == RestApiVersion.V_7) { - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - deprecationLogger.compatibleCritical("get_field_mapping_with_types", INCLUDE_TYPE_DEPRECATION_MESSAGE); - } - boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - final String[] types = request.paramAsStringArrayOrEmptyIfAll("type"); - if (includeTypeName == false && types.length > 0) { - throw new IllegalArgumentException("Types cannot be specified unless include_type_name is set to true."); - } - - if (request.hasParam("local")) { - request.param("local"); - deprecationLogger.compatibleCritical( - "get_field_mapping_local", - "Use [local] in get field mapping requests is deprecated. The parameter will be removed in the next major version" - ); - } - } - GetFieldMappingsRequest getMappingsRequest = new GetFieldMappingsRequest(); getMappingsRequest.indices(indices).fields(fields).includeDefaults(request.paramAsBoolean("include_defaults", false)); getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index 25f471183d805..b0b879a6d787c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -12,9 +12,7 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -23,7 +21,6 @@ import java.util.List; import java.util.Set; -import static org.elasticsearch.common.util.set.Sets.addToCopy; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.HEAD; import static org.elasticsearch.rest.RestStatus.NOT_FOUND; @@ -34,13 +31,6 @@ * The REST handler for get template and head template APIs. */ public class RestGetIndexTemplateAction extends BaseRestHandler { - - private static final Set COMPATIBLE_RESPONSE_PARAMS = addToCopy(Settings.FORMAT_PARAMS, INCLUDE_TYPE_NAME_PARAMETER); - - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetIndexTemplateAction.class); - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " - + "index template requests is deprecated. The parameter will be removed in the next major version."; - @Override public List routes() { return List.of(new Route(GET, "/_template"), new Route(GET, "/_template/{name}"), new Route(HEAD, "/_template/{name}")); @@ -53,9 +43,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - deprecationLogger.compatibleCritical("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); - } final String[] names = Strings.splitStringByCommaToArray(request.param("name")); final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names); @@ -75,13 +62,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC protected Set responseParams() { return Settings.FORMAT_PARAMS; } - - @Override - protected Set responseParams(RestApiVersion restApiVersion) { - if (restApiVersion == RestApiVersion.V_7) { - return COMPATIBLE_RESPONSE_PARAMS; - } else { - return responseParams(); - } - } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java index 3d78fa538bf04..9ca890eaff65b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -27,7 +26,6 @@ import java.util.List; import java.util.Set; -import static org.elasticsearch.common.util.set.Sets.addToCopy; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.HEAD; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; @@ -41,8 +39,6 @@ public class RestGetIndicesAction extends BaseRestHandler { public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using `include_type_name` in get indices requests" + " is deprecated. The parameter will be removed in the next major version."; - private static final Set COMPATIBLE_RESPONSE_PARAMS = addToCopy(Settings.FORMAT_PARAMS, INCLUDE_TYPE_NAME_PARAMETER); - @Override public List routes() { return List.of(new Route(GET, "/{index}"), new Route(HEAD, "/{index}")); @@ -55,13 +51,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - // starting with 7.0 we don't include types by default in the response to GET requests - if (request.getRestApiVersion() == RestApiVersion.V_7 - && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER) - && request.method().equals(GET)) { - deprecationLogger.compatibleCritical("get_indices_with_types", TYPES_DEPRECATION_MESSAGE); - } - String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final GetIndexRequest getIndexRequest = new GetIndexRequest(); getIndexRequest.indices(indices); @@ -85,13 +74,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC protected Set responseParams() { return Settings.FORMAT_PARAMS; } - - @Override - protected Set responseParams(RestApiVersion restApiVersion) { - if (restApiVersion == RestApiVersion.V_7) { - return COMPATIBLE_RESPONSE_PARAMS; - } else { - return responseParams(); - } - } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 5f40bea92f818..242bcd399413b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -14,7 +14,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.rest.BaseRestHandler; @@ -28,7 +27,6 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.rest.RestRequest.Method.HEAD; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; @ServerlessScope(Scope.PUBLIC) @@ -58,26 +56,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7) { - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.param(INCLUDE_TYPE_NAME_PARAMETER); - deprecationLogger.compatibleCritical("get_mapping_with_types", INCLUDE_TYPE_DEPRECATION_MSG); - } - final String[] types = request.paramAsStringArrayOrEmptyIfAll("type"); - if (request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY) == false && types.length > 0) { - throw new IllegalArgumentException( - "Types cannot be provided in get mapping requests, unless include_type_name is set to true." - ); - } - if (request.method().equals(HEAD)) { - deprecationLogger.compatibleCritical( - "get_mapping_types_removal", - "Type exists requests are deprecated, as types have been deprecated." - ); - } - } final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); - final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); getMappingsRequest.indices(indices); getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java index dcd4cda21f969..b5050eb8007a8 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -76,11 +75,6 @@ public boolean allowSystemIndexAccessByDefault() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("types")) { - deprecationLogger.compatibleCritical("indices_stats_types", TYPES_DEPRECATION_MESSAGE); - request.param("types"); - } - IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); boolean forbidClosedIndices = request.paramAsBoolean("forbid_closed_indices", true); IndicesOptions defaultIndicesOption = forbidClosedIndices diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index f70d9351e69c9..defec2fefc615 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -21,12 +21,12 @@ import java.io.IOException; import java.util.List; -import java.util.Map; import static java.util.Arrays.asList; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; +import static org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction.prepareMappings; public class RestPutIndexTemplateAction extends BaseRestHandler { @@ -53,42 +53,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(request.param("name")); - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("template")) { - deprecationLogger.compatibleCritical( - "template_parameter_deprecation", - "Deprecated parameter [template] used, replaced by [index_patterns]" - ); - putRequest.patterns(List.of(request.param("template"))); - } else { - putRequest.patterns(asList(request.paramAsStringArray("index_patterns", Strings.EMPTY_ARRAY))); - } + putRequest.patterns(asList(request.paramAsStringArray("index_patterns", Strings.EMPTY_ARRAY))); putRequest.order(request.paramAsInt("order", putRequest.order())); putRequest.masterNodeTimeout(getMasterNodeTimeout(request)); putRequest.create(request.paramAsBoolean("create", false)); putRequest.cause(request.param("cause", "")); - - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - if (request.getRestApiVersion() == RestApiVersion.V_7) { - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - deprecationLogger.compatibleCritical("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE); - } - boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - if (includeTypeName) { - sourceAsMap = RestCreateIndexAction.prepareMappingsV7(sourceAsMap, request); - } else { - sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); - } - } else { - sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); - } - if (request.getRestApiVersion() == RestApiVersion.V_7 && sourceAsMap.containsKey("template")) { - deprecationLogger.compatibleCritical( - "template_field_deprecation", - "Deprecated field [template] used, replaced by [index_patterns]" - ); - putRequest.patterns(List.of((String) sourceAsMap.remove("template"))); - } - putRequest.source(sourceAsMap); + putRequest.source(prepareMappings(XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2())); return channel -> client.admin().indices().putTemplate(putRequest, new RestToXContentListener<>(channel)); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 014e761acc388..ee11f7f520463 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -15,8 +15,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -27,6 +25,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.index.mapper.MapperService.isMappingSourceTyped; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; @@ -60,27 +59,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC PutMappingRequest putMappingRequest = new PutMappingRequest(indices); Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - if (request.getRestApiVersion() == RestApiVersion.V_7) { - final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - deprecationLogger.compatibleCritical("put_mapping_with_types", TYPES_DEPRECATION_MESSAGE); - } - final String type = request.param("type"); - if (includeTypeName == false && (type != null || isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap))) { - throw new IllegalArgumentException( - "Types cannot be provided in put mapping requests, unless the include_type_name parameter is set to true." - ); - } - - Map mappingSource = prepareV7Mappings(includeTypeName, sourceAsMap); - putMappingRequest.source(mappingSource); - } else { - if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) { - throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); - } - putMappingRequest.source(sourceAsMap); + if (isMappingSourceTyped(SINGLE_MAPPING_NAME, sourceAsMap)) { + throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); } - + putMappingRequest.source(sourceAsMap); putMappingRequest.ackTimeout(getAckTimeout(request)); putMappingRequest.masterNodeTimeout(getMasterNodeTimeout(request)); putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java index c6afc8e041d01..ee1710f39ce41 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java @@ -14,8 +14,6 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -40,15 +38,6 @@ public abstract class RestResizeHandler extends BaseRestHandler { @Override public final RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("copy_settings")) { - deprecationLogger.compatibleCritical("copy_settings", "parameter [copy_settings] is deprecated and will be removed in 8.0.0"); - - final String rawCopySettings = request.param("copy_settings"); - final boolean copySettings = Booleans.parseBoolean(rawCopySettings); - if (copySettings == false) { - throw new IllegalArgumentException("parameter [copy_settings] can not be explicitly set to [false]"); - } - } final ResizeRequest resizeRequest = new ResizeRequest(request.param("target"), request.param("index")); resizeRequest.setResizeType(getResizeType()); request.applyContentParser(resizeRequest::fromXContent); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index c4df68098190f..942844dd1dd16 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -59,9 +58,8 @@ public Set supportedCapabilities() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final boolean includeTypeName = includeTypeName(request); RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index")); - request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser)); + request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(parser)); rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false)); rolloverIndexRequest.lazy(request.paramAsBoolean("lazy", false)); rolloverIndexRequest.ackTimeout(getAckTimeout(request)); @@ -71,7 +69,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (failureStore) { rolloverIndexRequest.setIndicesOptions( IndicesOptions.builder(rolloverIndexRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build() ); } @@ -83,14 +81,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC .rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel)); } - private static boolean includeTypeName(RestRequest request) { - boolean includeTypeName = false; - if (request.getRestApiVersion() == RestApiVersion.V_7) { - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - deprecationLogger.compatibleCritical("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); - } - includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - } - return includeTypeName; - } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java index 8784fad3405d0..8bc7e9aa76551 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; @@ -57,11 +56,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - deprecationLogger.compatibleCritical("validate_query_with_types", TYPES_DEPRECATION_MESSAGE); - request.param("type"); - } - ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest(Strings.splitStringByCommaToArray(request.param("index"))); validateQueryRequest.indicesOptions(IndicesOptions.fromRequest(request, validateQueryRequest.indicesOptions())); validateQueryRequest.explain(request.paramAsBoolean("explain", false)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index f97ea1db7a036..a968ea4520f40 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.rest.RestRequest; @@ -171,11 +170,7 @@ protected Table getTableWithHeader(final RestRequest request) { table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); - if (request.getRestApiVersion() == RestApiVersion.V_7) { - table.addCell("dataset.size", "default:false;text-align:right;desc:total size of dataset"); - } else { - table.addCell("dataset.size", "text-align:right;desc:total size of dataset"); - } + table.addCell("dataset.size", "text-align:right;desc:total size of dataset"); table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java index 72f786004e0bd..3bee9c5f3b04c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.Table; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.ListenableFuture; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.bulk.stats.BulkStats; import org.elasticsearch.index.cache.query.QueryCacheStats; @@ -119,12 +118,7 @@ protected Table getTableWithHeader(final RestRequest request) { .addCell("state", "default:true;alias:st;desc:shard state") .addCell("docs", "alias:d,dc;text-align:right;desc:number of docs in shard") .addCell("store", "alias:sto;text-align:right;desc:store size of shard (how much disk it uses)") - .addCell( - "dataset", - request.getRestApiVersion() == RestApiVersion.V_7 - ? "default:false;text-align:right;desc:total size of dataset" - : "text-align:right;desc:total size of dataset" - ) + .addCell("dataset", "text-align:right;desc:total size of dataset") .addCell("ip", "default:true;desc:ip of node where it lives") .addCell("id", "default:false;desc:unique id of node where it lives") .addCell("node", "default:true;alias:n;desc:name of node where it lives"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index 0b8e64f5eab4a..1e80e6de60d65 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; @@ -91,9 +90,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { if (request.isStreamedContent() == false) { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - } BulkRequest bulkRequest = new BulkRequest(); String defaultIndex = request.param("index"); String defaultRouting = request.param("routing"); @@ -108,26 +104,26 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC boolean defaultRequireDataStream = request.paramAsBoolean(DocWriteRequest.REQUIRE_DATA_STREAM, false); bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); - bulkRequest.add( - request.requiredContent(), - defaultIndex, - defaultRouting, - defaultFetchSourceContext, - defaultPipeline, - defaultRequireAlias, - defaultRequireDataStream, - defaultListExecutedPipelines, - allowExplicitIndex, - request.getXContentType(), - request.getRestApiVersion() - ); + try { + bulkRequest.add( + request.requiredContent(), + defaultIndex, + defaultRouting, + defaultFetchSourceContext, + defaultPipeline, + defaultRequireAlias, + defaultRequireDataStream, + defaultListExecutedPipelines, + allowExplicitIndex, + request.getXContentType(), + request.getRestApiVersion() + ); + } catch (Exception e) { + return channel -> new RestToXContentListener<>(channel).onFailure(parseFailureException(e)); + } return channel -> client.bulk(bulkRequest, new RestRefCountedChunkedToXContentListener<>(channel)); } else { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - } - String waitForActiveShards = request.param("wait_for_active_shards"); TimeValue timeout = request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT); String refresh = request.param("refresh"); @@ -135,6 +131,15 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } } + private static Exception parseFailureException(Exception e) { + if (e instanceof IllegalArgumentException) { + return e; + } else { + // TODO: Maybe improve in follow-up to be XContentParseException and include line number and column + return new ElasticsearchParseException("could not parse bulk request body", e); + } + } + static class ChunkHandler implements BaseRestHandler.RequestBodyChunkConsumer { private final boolean allowExplicitIndex; @@ -227,9 +232,7 @@ public void handleChunk(RestChannel channel, ReleasableBytesReference chunk, boo } catch (Exception e) { shortCircuit(); - new RestToXContentListener<>(channel).onFailure( - new ElasticsearchParseException("could not parse bulk request body", e) - ); + new RestToXContentListener<>(channel).onFailure(parseFailureException(e)); return; } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java index 3ee1810967153..3c795c635056d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -42,9 +41,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - } DeleteRequest deleteRequest = new DeleteRequest(request.param("index"), request.param("id")); deleteRequest.routing(request.param("routing")); deleteRequest.timeout(request.paramAsTime("timeout", DeleteRequest.DEFAULT_TIMEOUT)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java index cc2b820eb05f2..4e234c4cbcc3d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -44,12 +43,7 @@ public List routes() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7) { - request.param("type"); // consume and ignore the type - } - GetRequest getRequest = new GetRequest(request.param("index"), request.param("id")); - getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh())); getRequest.routing(request.param("routing")); getRequest.preference(request.param("preference")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java index e6567d7fdf592..a09fcbd0c5273 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; @@ -57,11 +56,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); // consume and ignore the type - deprecationLogger.compatibleCritical("get_source_with_types", TYPES_DEPRECATION_MESSAGE); - } - final GetRequest getRequest = new GetRequest(request.param("index"), request.param("id")); getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh())); getRequest.routing(request.param("routing")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 9931c10b38f3f..c2437dcb96fa6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -105,10 +104,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7) { - request.param("type"); // consume and ignore the type - } - IndexRequest indexRequest = new IndexRequest(request.param("index")); indexRequest.id(request.param("id")); indexRequest.routing(request.param("routing")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java index 0ceb9ebab7397..12769ef1c73e9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -55,9 +54,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.param("type") != null) { - request.param("type"); - } MultiGetRequest multiGetRequest = new MultiGetRequest(); multiGetRequest.refresh(request.paramAsBoolean("refresh", multiGetRequest.refresh())); multiGetRequest.preference(request.param("preference")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java index 0e23362e3f5df..65aa1869a41e4 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java @@ -14,9 +14,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.DeprecationRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; @@ -50,10 +48,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - deprecationLogger.compatibleCritical(DeprecationRestHandler.DEPRECATED_ROUTE_KEY, TYPES_DEPRECATION_MESSAGE); - } MultiTermVectorsRequest multiTermVectorsRequest = new MultiTermVectorsRequest(); TermVectorsRequest template = new TermVectorsRequest().index(request.param("index")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java index 1fbf35856589b..8e41e1cd09674 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -55,11 +54,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - } - TermVectorsRequest termVectorsRequest; - termVectorsRequest = new TermVectorsRequest(request.param("index"), request.param("id")); + TermVectorsRequest termVectorsRequest = new TermVectorsRequest(request.param("index"), request.param("id")); if (request.hasContentOrSourceParam()) { try (XContentParser parser = request.contentOrSourceParamParser()) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java index 682d5b5c55c3f..57b3a89b2303b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -45,9 +44,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - } UpdateRequest updateRequest = new UpdateRequest(request.param("index"), request.param("id")); updateRequest.routing(request.param("routing")); updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java index 0c3680e09e6bf..23da666a39a7e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -57,10 +56,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - deprecationLogger.compatibleCritical("count_with_types", TYPES_DEPRECATION_MESSAGE); - request.param("type"); - } SearchRequest countRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index"))); countRequest.indicesOptions(IndicesOptions.fromRequest(request, countRequest.indicesOptions())); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).trackTotalHits(true); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java index 4d88d115e1da7..c5dc047933ea6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.explain.ExplainResponse; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -47,11 +46,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - } ExplainRequest explainRequest = new ExplainRequest(request.param("index"), request.param("id")); - explainRequest.parent(request.param("parent")); explainRequest.routing(request.param("routing")); explainRequest.preference(request.param("preference")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index a58904c2649d9..aeb182978e1eb 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Tuple; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; @@ -110,10 +109,6 @@ public static MultiSearchRequest parseRequest( Predicate clusterSupportsFeature, TriFunction extraParamParser ) throws IOException { - if (restRequest.getRestApiVersion() == RestApiVersion.V_7 && restRequest.hasParam("type")) { - restRequest.param("type"); - } - MultiSearchRequest multiRequest = new MultiSearchRequest(); IndicesOptions indicesOptions = IndicesOptions.fromRequest(restRequest, multiRequest.indicesOptions()); multiRequest.indicesOptions(indicesOptions); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 4fff9229372ea..80a85d3b9b748 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -10,7 +10,6 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.TransportSearchAction; @@ -20,7 +19,6 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.rest.BaseRestHandler; @@ -101,12 +99,10 @@ public Set supportedCapabilities() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - SearchRequest searchRequest; - if (request.hasParam("min_compatible_shard_node")) { - searchRequest = new SearchRequest(Version.fromString(request.param("min_compatible_shard_node"))); - } else { - searchRequest = new SearchRequest(); - } + SearchRequest searchRequest = new SearchRequest(); + // access the BwC param, but just drop it + // this might be set by old clients + request.param("min_compatible_shard_node"); /* * We have to pull out the call to `source().size(size)` because * _update_by_query and _delete_by_query uses this same parsing @@ -169,11 +165,6 @@ public static void parseSearchRequest( IntConsumer setSize, @Nullable SearchUsageHolder searchUsageHolder ) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - request.param("type"); - deprecationLogger.compatibleCritical("search_with_types", TYPES_DEPRECATION_MESSAGE); - } - if (searchRequest.source() == null) { searchRequest.source(new SearchSourceBuilder()); } @@ -250,19 +241,8 @@ private static void parseSearchSource(final SearchSourceBuilder searchSourceBuil searchSourceBuilder.from(request.paramAsInt("from", 0)); } if (request.hasParam("size")) { - int size = request.paramAsInt("size", SearchService.DEFAULT_SIZE); - if (request.getRestApiVersion() == RestApiVersion.V_7 && size == -1) { - // we treat -1 as not-set, but deprecate it to be able to later remove this funny extra treatment - deprecationLogger.compatibleCritical( - "search-api-size-1", - "Using search size of -1 is deprecated and will be removed in future versions. " - + "Instead, don't use the `size` parameter if you don't want to set it explicitly." - ); - } else { - setSize.accept(size); - } + setSize.accept(request.paramAsInt("size", SearchService.DEFAULT_SIZE)); } - if (request.hasParam("explain")) { searchSourceBuilder.explain(request.paramAsBoolean("explain", null)); } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 1521b17a81766..8ac35f7c40caa 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -291,7 +291,7 @@ static int determineMaximumNumberOfSlices( ToLongFunction fieldCardinality ) { return executor instanceof ThreadPoolExecutor tpe - && tpe.getQueue().isEmpty() + && tpe.getQueue().size() <= tpe.getMaximumPoolSize() && isParallelCollectionSupportedForResults(resultsType, request.source(), fieldCardinality, enableQueryPhaseParallelCollection) ? tpe.getMaximumPoolSize() : 1; diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 381b4bc0e9008..1611c95d99df4 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.SimpleRefCounted; import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -816,9 +815,6 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t if (index != null) { builder.field(Fields._INDEX, RemoteClusterAware.buildRemoteIndexName(clusterAlias, index)); } - if (builder.getRestApiVersion() == RestApiVersion.V_7 && metaFields.containsKey(MapperService.TYPE_FIELD_NAME) == false) { - builder.field(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME); - } if (id != null) { builder.field(Fields._ID, id); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java index 043fab6f4f122..b2ca4a10dc4b3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.search.aggregations.Aggregator.BucketComparator; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; import org.elasticsearch.search.aggregations.support.AggregationPath; @@ -591,15 +590,6 @@ public static BucketOrder parseOrderParam(XContentParser parser) throws IOExcept if (orderKey == null) { throw new ParsingException(parser.getTokenLocation(), "Must specify at least one field for [order]"); } - // _term and _time order deprecated in 6.0; replaced by _key - if (parser.getRestApiVersion() == RestApiVersion.V_7 && ("_term".equals(orderKey) || "_time".equals(orderKey))) { - deprecationLogger.compatibleCritical( - "_term_and_time_key_removal", - "Deprecated aggregation order key [{}] used, replaced by [_key]", - orderKey - ); - return orderAsc ? KEY_ASC : KEY_DESC; - } return switch (orderKey) { case "_key" -> orderAsc ? KEY_ASC : KEY_DESC; case "_count" -> orderAsc ? COUNT_ASC : COUNT_DESC; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 9f94ec1452019..6ceb02f0e797f 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -24,8 +24,8 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -99,10 +99,11 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField TIMEOUT_FIELD = new ParseField("timeout"); public static final ParseField TERMINATE_AFTER_FIELD = new ParseField("terminate_after"); public static final ParseField QUERY_FIELD = new ParseField("query"); - public static final ParseField SUB_SEARCHES_FIELD = new ParseField("sub_searches"); + @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_RELEVANCE) // remove [sub_searches] and [rank] support in 10.0 + public static final ParseField SUB_SEARCHES_FIELD = new ParseField("sub_searches").withAllDeprecated("retriever"); + public static final ParseField RANK_FIELD = new ParseField("rank").withAllDeprecated("retriever"); public static final ParseField POST_FILTER_FIELD = new ParseField("post_filter"); public static final ParseField KNN_FIELD = new ParseField("knn"); - public static final ParseField RANK_FIELD = new ParseField("rank"); public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); public static final ParseField VERSION_FIELD = new ParseField("version"); public static final ParseField SEQ_NO_PRIMARY_TERM_FIELD = new ParseField("seq_no_primary_term"); @@ -1341,17 +1342,7 @@ private SearchSourceBuilder parseXContent( if (FROM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { from(parser.intValue()); } else if (SIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - int parsedSize = parser.intValue(); - if (parser.getRestApiVersion() == RestApiVersion.V_7 && parsedSize == -1) { - // we treat -1 as not-set, but deprecate it to be able to later remove this funny extra treatment - deprecationLogger.compatibleCritical( - "search-api-size-1", - "Using search size of -1 is deprecated and will be removed in future versions. " - + "Instead, don't use the `size` parameter if you don't want to set it explicitly." - ); - } else { - size(parsedSize); - } + size(parser.intValue()); } else if (TIMEOUT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { timeout = TimeValue.parseTimeValue(parser.text(), null, TIMEOUT_FIELD.getPreferredName()); } else if (TERMINATE_AFTER_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -1457,99 +1448,79 @@ private SearchSourceBuilder parseXContent( scriptFields.add(new ScriptField(parser)); } searchUsage.trackSectionUsage(SCRIPT_FIELDS_FIELD.getPreferredName()); - } else if (parser.getRestApiVersion() == RestApiVersion.V_7 - && INDICES_BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - deprecationLogger.compatibleCritical( - "indices_boost_object_format", - "Object format in indices_boost is deprecated, please use array format instead" - ); + } else if (AGGREGATIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler()) + || AGGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + aggregations = AggregatorFactories.parseAggregators(parser); + if (aggregations.count() > 0) { + searchUsage.trackSectionUsage(AGGS_FIELD.getPreferredName()); + } + } else if (HIGHLIGHT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + highlightBuilder = HighlightBuilder.fromXContent(parser); + if (highlightBuilder.fields().size() > 0) { + searchUsage.trackSectionUsage(HIGHLIGHT_FIELD.getPreferredName()); + } + } else if (SUGGEST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + suggestBuilder = SuggestBuilder.fromXContent(parser); + if (suggestBuilder.getSuggestions().size() > 0) { + searchUsage.trackSectionUsage(SUGGEST_FIELD.getPreferredName()); + } + } else if (SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + sorts = new ArrayList<>(SortBuilder.fromXContent(parser)); + } else if (RESCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + rescoreBuilders = new ArrayList<>(); + rescoreBuilders.add(RescorerBuilder.parseFromXContent(parser, searchUsage::trackRescorerUsage)); + searchUsage.trackSectionUsage(RESCORE_FIELD.getPreferredName()); + } else if (EXT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + extBuilders = new ArrayList<>(); + String extSectionName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue()) { - indexBoosts.add(new IndexBoost(currentFieldName, parser.floatValue())); + extSectionName = parser.currentName(); } else { - throw new ParsingException( - parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + currentFieldName + "].", - parser.getTokenLocation() - ); - } - } - searchUsage.trackSectionUsage(INDICES_BOOST_FIELD.getPreferredName()); - } else if (AGGREGATIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler()) - || AGGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - aggregations = AggregatorFactories.parseAggregators(parser); - if (aggregations.count() > 0) { - searchUsage.trackSectionUsage(AGGS_FIELD.getPreferredName()); - } - } else if (HIGHLIGHT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - highlightBuilder = HighlightBuilder.fromXContent(parser); - if (highlightBuilder.fields().size() > 0) { - searchUsage.trackSectionUsage(HIGHLIGHT_FIELD.getPreferredName()); - } - } else if (SUGGEST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - suggestBuilder = SuggestBuilder.fromXContent(parser); - if (suggestBuilder.getSuggestions().size() > 0) { - searchUsage.trackSectionUsage(SUGGEST_FIELD.getPreferredName()); - } - } else if (SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - sorts = new ArrayList<>(SortBuilder.fromXContent(parser)); - } else if (RESCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - rescoreBuilders = new ArrayList<>(); - rescoreBuilders.add(RescorerBuilder.parseFromXContent(parser, searchUsage::trackRescorerUsage)); - searchUsage.trackSectionUsage(RESCORE_FIELD.getPreferredName()); - } else if (EXT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - extBuilders = new ArrayList<>(); - String extSectionName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - extSectionName = parser.currentName(); - } else { - SearchExtBuilder searchExtBuilder = parser.namedObject(SearchExtBuilder.class, extSectionName, null); - if (searchExtBuilder.getWriteableName().equals(extSectionName) == false) { - throw new IllegalStateException( - "The parsed [" - + searchExtBuilder.getClass().getName() - + "] object has a different writeable name compared to the name of the section that " - + " it was parsed from: found [" - + searchExtBuilder.getWriteableName() - + "] expected [" - + extSectionName - + "]" - ); - } - extBuilders.add(searchExtBuilder); + SearchExtBuilder searchExtBuilder = parser.namedObject(SearchExtBuilder.class, extSectionName, null); + if (searchExtBuilder.getWriteableName().equals(extSectionName) == false) { + throw new IllegalStateException( + "The parsed [" + + searchExtBuilder.getClass().getName() + + "] object has a different writeable name compared to the name of the section that " + + " it was parsed from: found [" + + searchExtBuilder.getWriteableName() + + "] expected [" + + extSectionName + + "]" + ); } + extBuilders.add(searchExtBuilder); } - if (extBuilders.size() > 0) { - searchUsage.trackSectionUsage(EXT_FIELD.getPreferredName()); - } - } else if (SLICE.match(currentFieldName, parser.getDeprecationHandler())) { - sliceBuilder = SliceBuilder.fromXContent(parser); - if (sliceBuilder.getField() != null || sliceBuilder.getId() != -1 || sliceBuilder.getMax() != -1) { - searchUsage.trackSectionUsage(SLICE.getPreferredName()); - } - } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) { - collapse = CollapseBuilder.fromXContent(parser); - if (collapse.getField() != null) { - searchUsage.trackSectionUsage(COLLAPSE.getPreferredName()); - } - } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) { - pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser); - searchUsage.trackSectionUsage(POINT_IN_TIME.getPreferredName()); - } else if (RUNTIME_MAPPINGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - runtimeMappings = parser.map(); - if (runtimeMappings.size() > 0) { - searchUsage.trackSectionUsage(RUNTIME_MAPPINGS_FIELD.getPreferredName()); - } - } else { - throw new ParsingException( - parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + currentFieldName + "].", - parser.getTokenLocation() - ); } + if (extBuilders.size() > 0) { + searchUsage.trackSectionUsage(EXT_FIELD.getPreferredName()); + } + } else if (SLICE.match(currentFieldName, parser.getDeprecationHandler())) { + sliceBuilder = SliceBuilder.fromXContent(parser); + if (sliceBuilder.getField() != null || sliceBuilder.getId() != -1 || sliceBuilder.getMax() != -1) { + searchUsage.trackSectionUsage(SLICE.getPreferredName()); + } + } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) { + collapse = CollapseBuilder.fromXContent(parser); + if (collapse.getField() != null) { + searchUsage.trackSectionUsage(COLLAPSE.getPreferredName()); + } + } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) { + pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser); + searchUsage.trackSectionUsage(POINT_IN_TIME.getPreferredName()); + } else if (RUNTIME_MAPPINGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + runtimeMappings = parser.map(); + if (runtimeMappings.size() > 0) { + searchUsage.trackSectionUsage(RUNTIME_MAPPINGS_FIELD.getPreferredName()); + } + } else { + throw new ParsingException( + parser.getTokenLocation(), + "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation() + ); + } } else if (token == XContentParser.Token.START_ARRAY) { if (STORED_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { storedFieldsContext = StoredFieldsContext.fromXContent(STORED_FIELDS_FIELD.getPreferredName(), parser); diff --git a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java index 203f3facfd025..9c938fe6b1aab 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java @@ -199,6 +199,9 @@ public ActionRequestValidationException validate( validationException ); } + for (RetrieverSource innerRetriever : innerRetrievers) { + validationException = innerRetriever.retriever().validate(source, validationException, allowPartialSearchResults); + } return validationException; } diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 56871764ab0d4..6640f0f858404 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; @@ -427,18 +426,7 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String fieldName = currentName; } else if (token == XContentParser.Token.START_OBJECT) { - if (parser.getRestApiVersion() == RestApiVersion.V_7 - && NESTED_FILTER_FIELD.match(currentName, parser.getDeprecationHandler())) { - deprecationLogger.compatibleCritical( - "nested_filter", - "[nested_filter] has been removed in favour of the [nested] parameter" - ); - throw new ParsingException( - parser.getTokenLocation(), - "[nested_filter] has been removed in favour of the [nested] parameter", - currentName - ); - } else if (NESTED_FIELD.match(currentName, parser.getDeprecationHandler())) { + if (NESTED_FIELD.match(currentName, parser.getDeprecationHandler())) { nestedSort = NestedSortBuilder.fromXContent(parser); } else { // the json in the format of -> field : { lat : 30, lon : 12 } @@ -454,18 +442,7 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String geoPoints.add(GeoUtils.parseGeoPoint(parser)); } } else if (token.isValue()) { - if (parser.getRestApiVersion() == RestApiVersion.V_7 - && NESTED_PATH_FIELD.match(currentName, parser.getDeprecationHandler())) { - deprecationLogger.compatibleCritical( - "nested_path", - "[nested_path] has been removed in favour of the [nested] parameter" - ); - throw new ParsingException( - parser.getTokenLocation(), - "[nested_path] has been removed in favour of the [nested] parameter", - currentName - ); - } else if (ORDER_FIELD.match(currentName, parser.getDeprecationHandler())) { + if (ORDER_FIELD.match(currentName, parser.getDeprecationHandler())) { order = SortOrder.fromString(parser.text()); } else if (UNIT_FIELD.match(currentName, parser.getDeprecationHandler())) { unit = DistanceUnit.fromString(parser.text()); diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 31739850e2d35..2c6be01c851e4 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.action.search.VersionMismatchException; import org.elasticsearch.action.support.replication.ReplicationOperation; import org.elasticsearch.client.internal.AbstractClientHeadersTestCase; import org.elasticsearch.cluster.action.shard.ShardStateAction; @@ -816,7 +815,7 @@ public void testIds() { ids.put(158, PeerRecoveryNotFound.class); ids.put(159, NodeHealthCheckFailureException.class); ids.put(160, NoSeedNodeLeftException.class); - ids.put(161, VersionMismatchException.class); + ids.put(161, null); // was org.elasticsearch.action.search.VersionMismatchException.class ids.put(162, ElasticsearchAuthenticationProcessingError.class); ids.put(163, RepositoryConflictException.class); ids.put(164, VersionConflictException.class); diff --git a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java index a9cc6c07e68c8..c5d995710bcec 100644 --- a/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java @@ -114,13 +114,6 @@ public void testTypeWhenCompatible() throws IOException { ) { // DocWriteResponse is abstract so we have to sneak a subclass in here to test it. }; - try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { - response.toXContent(builder, ToXContent.EMPTY_PARAMS); - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - assertThat(parser.map(), hasEntry(MapperService.TYPE_FIELD_NAME, MapperService.SINGLE_MAPPING_NAME)); - } - } try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) { response.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java index 4cfd9b66306ad..a75b50e3a88f4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java @@ -82,9 +82,6 @@ public void testIndicesOptions() { ); assertThat(getIndexRequest.indicesOptions().wildcardOptions(), equalTo(IndicesOptions.strictExpandOpen().wildcardOptions())); assertThat(getIndexRequest.indicesOptions().gatekeeperOptions(), equalTo(IndicesOptions.strictExpandOpen().gatekeeperOptions())); - assertThat( - getIndexRequest.indicesOptions().failureStoreOptions(), - equalTo(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true).build()) - ); + assertThat(getIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA_AND_FAILURE)); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index 3f0a6b40ce60a..7640c1e7af308 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -17,8 +17,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -63,31 +61,8 @@ public void testToXContentIncludesType() throws Exception { FieldMappingMetadata fieldMappingMetadata = new FieldMappingMetadata("my field", new BytesArray("{}")); mappings.put("index", Collections.singletonMap("field", fieldMappingMetadata)); GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings); - ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "true")); + ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("include_type_name", "true")); - // v7 with include_type_name attaches _doc - try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { - response.toXContent(builder, params); - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - @SuppressWarnings("unchecked") - Map>> index = (Map>>) parser.map() - .get("index"); - assertThat(index.get("mappings"), hasKey(MapperService.SINGLE_MAPPING_NAME)); - assertThat(index.get("mappings").get(MapperService.SINGLE_MAPPING_NAME), hasKey("field")); - } - } - - // v7 with no include_type_name do not attach _doc - try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { - response.toXContent(builder, ToXContent.EMPTY_PARAMS); - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - @SuppressWarnings("unchecked") - Map> index = (Map>) parser.map().get("index"); - assertThat(index.get("mappings"), hasKey("field")); - } - } // v8 does not have _doc, even when include_type_name is present // (although this throws unconsumed parameter exception in RestGetFieldMappingsAction) try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 848e46f2b3366..b9fdb13958632 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.IndicesOptions.FailureStoreOptions; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasAction; @@ -747,14 +747,14 @@ public void testValidation() throws Exception { final String defaultRolloverIndexName; final boolean useDataStream = randomBoolean(); final Metadata.Builder builder = Metadata.builder(); - var failureStoreOptions = FailureStoreOptions.DEFAULT; + var defaultSelectorOptions = IndicesOptions.SelectorOptions.DEFAULT; if (useDataStream) { DataStream dataStream = DataStreamTestHelper.randomInstance() // ensure no replicate data stream .promoteDataStream(); rolloverTarget = dataStream.getName(); if (dataStream.isFailureStoreEnabled() && randomBoolean()) { - failureStoreOptions = new FailureStoreOptions(false, true); + defaultSelectorOptions = IndicesOptions.SelectorOptions.ONLY_FAILURES; sourceIndexName = dataStream.getFailureStoreWriteIndex().getName(); defaultRolloverIndexName = DataStream.getDefaultFailureStoreName( dataStream.getName(), @@ -815,7 +815,7 @@ public void testValidation() throws Exception { true, null, null, - failureStoreOptions.includeFailureIndices() + IndicesOptions.SelectorOptions.ONLY_FAILURES.equals(defaultSelectorOptions) ); newIndexName = newIndexName == null ? defaultRolloverIndexName : newIndexName; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index a7fa81eb24a57..08e92c833dc85 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -33,7 +34,9 @@ import org.junit.Before; import java.io.IOException; +import java.util.EnumSet; import java.util.Map; +import java.util.Set; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -66,7 +69,7 @@ public void testConditionsParsing() throws Exception { .endObject() .endObject(); try (var parser = createParser(builder)) { - request.fromXContent(false, parser); + request.fromXContent(parser); } Map> conditions = request.getConditions().getConditions(); assertThat(conditions.size(), equalTo(10)); @@ -120,7 +123,7 @@ public void testParsingWithIndexSettings() throws Exception { .endObject() .endObject(); try (var parser = createParser(builder)) { - request.fromXContent(false, parser); + request.fromXContent(parser); } Map> conditions = request.getConditions().getConditions(); assertThat(conditions.size(), equalTo(3)); @@ -143,7 +146,7 @@ public void testTypelessMappingParsing() throws Exception { .endObject(); try (var parser = createParser(builder)) { - request.fromXContent(false, parser); + request.fromXContent(parser); } CreateIndexRequest createIndexRequest = request.getCreateIndexRequest(); String mapping = createIndexRequest.mappings(); @@ -176,7 +179,12 @@ public void testSerialize() throws Exception { originalRequest.lazy(randomBoolean()); originalRequest.setIndicesOptions( IndicesOptions.builder(originalRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(randomBoolean(), randomBoolean())) + .selectorOptions( + IndicesOptions.SelectorOptions.builder() + .setDefaultSelectors( + EnumSet.copyOf(randomNonEmptySubsetOf(Set.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES))) + ) + ) .build() ); @@ -188,10 +196,7 @@ public void testSerialize() throws Exception { assertThat(cloneRequest.getNewIndexName(), equalTo(originalRequest.getNewIndexName())); assertThat(cloneRequest.getRolloverTarget(), equalTo(originalRequest.getRolloverTarget())); assertThat(cloneRequest.isLazy(), equalTo(originalRequest.isLazy())); - assertThat( - cloneRequest.indicesOptions().failureStoreOptions(), - equalTo(originalRequest.indicesOptions().failureStoreOptions()) - ); + assertThat(cloneRequest.indicesOptions().selectorOptions(), equalTo(originalRequest.indicesOptions().selectorOptions())); for (Map.Entry> entry : cloneRequest.getConditions().getConditions().entrySet()) { Condition condition = originalRequest.getConditions().getConditions().get(entry.getKey()); // here we compare the string representation as there is some information loss when serializing @@ -216,7 +221,7 @@ public void testUnknownFields() throws IOException { BytesReference mutated = XContentTestUtils.insertRandomFields(xContentType, BytesReference.bytes(builder), null, random()); expectThrows(XContentParseException.class, () -> { try (var parser = createParser(xContentType.xContent(), mutated)) { - request.fromXContent(false, parser); + request.fromXContent(parser); } }); } @@ -261,7 +266,7 @@ public void testValidation() { RolloverRequest rolloverRequest = new RolloverRequest("alias-index", "new-index-name"); rolloverRequest.setIndicesOptions( IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(new IndicesOptions.FailureStoreOptions(true, true)) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .build() ); ActionRequestValidationException validationException = rolloverRequest.validate(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java index 788265d353698..bc7fe7a62a1da 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; @@ -34,10 +33,8 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; -import java.util.UUID; import static org.elasticsearch.ElasticsearchExceptionTests.assertDeepEquals; import static org.elasticsearch.ElasticsearchExceptionTests.randomExceptions; @@ -95,28 +92,6 @@ public static void parseInnerToXContent(XContentParser parser, DocWriteResponse. } } - public void testBulkItemResponseShouldContainTypeInV7CompatibilityMode() throws IOException { - BulkItemResponse bulkItemResponse = BulkItemResponse.success( - randomInt(), - DocWriteRequest.OpType.INDEX, - new IndexResponse( - new ShardId(randomAlphaOfLength(8), UUID.randomUUID().toString(), randomInt()), - randomAlphaOfLength(4), - randomNonNegativeLong(), - randomNonNegativeLong(), - randomNonNegativeLong(), - true - ) - ); - XContentBuilder xContentBuilder = bulkItemResponse.toXContent( - XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7), - ToXContent.EMPTY_PARAMS - ); - - String json = BytesReference.bytes(xContentBuilder).utf8ToString(); - assertThat(json, containsString("\"_type\":\"_doc\"")); - } - public void testFailureToString() { Failure failure = new Failure("index", "id", new RuntimeException("test")); String toString = failure.toString(); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java index b63c88f623e21..d279fa5030a8c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -13,26 +13,18 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TotalHits; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.cluster.node.VersionInformation; import org.elasticsearch.cluster.routing.GroupShardsIterator; -import org.elasticsearch.cluster.routing.RecoverySource; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.lucene.grouping.TopFieldGroups; import org.elasticsearch.search.DocValueFormat; @@ -47,24 +39,17 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalAggregationTestCase; -import org.elasticsearch.test.VersionUtils; -import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.transport.Transport; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import static java.util.Collections.singletonList; -import static org.elasticsearch.test.VersionUtils.allVersions; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; public class SearchQueryThenFetchAsyncActionTests extends ESTestCase { @@ -257,484 +242,4 @@ public void run() { assertThat(((FieldDoc) phase.sortedTopDocs().scoreDocs()[0]).fields[0], equalTo(0)); } } - - public void testMinimumVersionSameAsNewVersion() throws Exception { - var newVersion = VersionInformation.CURRENT; - var oldVersion = new VersionInformation( - VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()), - IndexVersions.MINIMUM_COMPATIBLE, - IndexVersionUtils.randomCompatibleVersion(random()) - ); - testMixedVersionsShardsSearch(newVersion, oldVersion, newVersion.nodeVersion()); - } - - public void testMinimumVersionBetweenNewAndOldVersion() throws Exception { - var oldVersion = new VersionInformation( - VersionUtils.getFirstVersion(), - IndexVersions.MINIMUM_COMPATIBLE, - IndexVersionUtils.randomCompatibleVersion(random()) - ); - - var newVersion = new VersionInformation( - VersionUtils.maxCompatibleVersion(VersionUtils.getFirstVersion()), - IndexVersions.MINIMUM_COMPATIBLE, - IndexVersion.current() - ); - - var minVersion = VersionUtils.randomVersionBetween( - random(), - allVersions().get(allVersions().indexOf(oldVersion.nodeVersion()) + 1), - newVersion.nodeVersion() - ); - - testMixedVersionsShardsSearch(newVersion, oldVersion, minVersion); - } - - private void testMixedVersionsShardsSearch(VersionInformation oldVersion, VersionInformation newVersion, Version minVersion) - throws Exception { - final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider( - 0, - System.nanoTime(), - System::nanoTime - ); - int numConcurrent = randomIntBetween(1, 4); - - Map lookup = new ConcurrentHashMap<>(); - DiscoveryNode newVersionNode = DiscoveryNodeUtils.builder("node1").version(newVersion).build(); - DiscoveryNode oldVersionNode = DiscoveryNodeUtils.builder("node2").version(oldVersion).build(); - lookup.put("node1", new SearchAsyncActionTests.MockConnection(newVersionNode)); - lookup.put("node2", new SearchAsyncActionTests.MockConnection(oldVersionNode)); - - OriginalIndices idx = new OriginalIndices(new String[] { "idx" }, SearchRequest.DEFAULT_INDICES_OPTIONS); - ArrayList list = new ArrayList<>(); - ShardRouting routingNewVersionShard = ShardRouting.newUnassigned( - new ShardId(new Index("idx", "_na_"), 0), - true, - RecoverySource.EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), - ShardRouting.Role.DEFAULT - ); - routingNewVersionShard = routingNewVersionShard.initialize(newVersionNode.getId(), "p0", 0); - routingNewVersionShard.started(); - list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 0), singletonList(routingNewVersionShard), idx)); - - ShardRouting routingOldVersionShard = ShardRouting.newUnassigned( - new ShardId(new Index("idx", "_na_"), 1), - true, - RecoverySource.EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), - ShardRouting.Role.DEFAULT - ); - routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p1", 0); - routingOldVersionShard.started(); - list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 1), singletonList(routingOldVersionShard), idx)); - - GroupShardsIterator shardsIter = new GroupShardsIterator<>(list); - final SearchRequest searchRequest = new SearchRequest(minVersion); - searchRequest.setMaxConcurrentShardRequests(numConcurrent); - searchRequest.setBatchedReduceSize(2); - searchRequest.source(new SearchSourceBuilder().size(1)); - searchRequest.allowPartialSearchResults(false); - - SearchTransportService searchTransportService = new SearchTransportService(null, null, null); - SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder()); - SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); - try ( - QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( - searchRequest, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - new NoopCircuitBreaker(CircuitBreaker.REQUEST), - controller, - task::isCancelled, - task.getProgressListener(), - shardsIter.size(), - exc -> {} - ) - ) { - final List responses = new ArrayList<>(); - SearchQueryThenFetchAsyncAction newSearchAsyncAction = new SearchQueryThenFetchAsyncAction( - logger, - null, - searchTransportService, - (clusterAlias, node) -> lookup.get(node), - Collections.singletonMap("_na_", AliasFilter.EMPTY), - Collections.emptyMap(), - EsExecutors.DIRECT_EXECUTOR_SERVICE, - resultConsumer, - searchRequest, - new ActionListener<>() { - @Override - public void onFailure(Exception e) { - responses.add(e); - } - - public void onResponse(SearchResponse response) { - responses.add(response); - } - - ; - }, - shardsIter, - timeProvider, - new ClusterState.Builder(new ClusterName("test")).build(), - task, - SearchResponse.Clusters.EMPTY, - null - ); - - newSearchAsyncAction.start(); - assertThat(responses, hasSize(1)); - assertThat(responses.get(0), instanceOf(SearchPhaseExecutionException.class)); - SearchPhaseExecutionException e = (SearchPhaseExecutionException) responses.get(0); - assertThat(e.getCause(), instanceOf(VersionMismatchException.class)); - assertThat( - e.getCause().getMessage(), - equalTo("One of the shards is incompatible with the required minimum version [" + minVersion + "]") - ); - } - } - - public void testMinimumVersionSameAsOldVersion() throws Exception { - var newVersion = VersionInformation.CURRENT; - var oldVersion = new VersionInformation( - VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()), - IndexVersions.MINIMUM_COMPATIBLE, - IndexVersionUtils.randomCompatibleVersion(random()) - ); - Version minVersion = oldVersion.nodeVersion(); - - final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider( - 0, - System.nanoTime(), - System::nanoTime - ); - AtomicInteger successfulOps = new AtomicInteger(); - - Map lookup = new ConcurrentHashMap<>(); - DiscoveryNode newVersionNode = DiscoveryNodeUtils.builder("node1").version(newVersion).build(); - DiscoveryNode oldVersionNode = DiscoveryNodeUtils.builder("node2").version(oldVersion).build(); - lookup.put("node1", new SearchAsyncActionTests.MockConnection(newVersionNode)); - lookup.put("node2", new SearchAsyncActionTests.MockConnection(oldVersionNode)); - - OriginalIndices idx = new OriginalIndices(new String[] { "idx" }, SearchRequest.DEFAULT_INDICES_OPTIONS); - ArrayList list = new ArrayList<>(); - ShardRouting routingNewVersionShard = ShardRouting.newUnassigned( - new ShardId(new Index("idx", "_na_"), 0), - true, - RecoverySource.EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), - ShardRouting.Role.DEFAULT - ); - routingNewVersionShard = routingNewVersionShard.initialize(newVersionNode.getId(), "p0", 0); - routingNewVersionShard.started(); - list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 0), singletonList(routingNewVersionShard), idx)); - - ShardRouting routingOldVersionShard = ShardRouting.newUnassigned( - new ShardId(new Index("idx", "_na_"), 1), - true, - RecoverySource.EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), - ShardRouting.Role.DEFAULT - ); - routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p1", 0); - routingOldVersionShard.started(); - list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 1), singletonList(routingOldVersionShard), idx)); - - GroupShardsIterator shardsIter = new GroupShardsIterator<>(list); - final SearchRequest searchRequest = new SearchRequest(minVersion); - searchRequest.allowPartialSearchResults(false); - searchRequest.source(new SearchSourceBuilder().size(1).sort(SortBuilders.fieldSort("timestamp"))); - - SearchTransportService searchTransportService = new SearchTransportService(null, null, null) { - @Override - public void sendExecuteQuery( - Transport.Connection connection, - ShardSearchRequest request, - SearchTask task, - ActionListener listener - ) { - int shardId = request.shardId().id(); - QuerySearchResult queryResult = new QuerySearchResult( - new ShardSearchContextId("N/A", 123), - new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null), - null - ); - try { - SortField sortField = new SortField("timestamp", SortField.Type.LONG); - if (shardId == 0) { - queryResult.topDocs( - new TopDocsAndMaxScore( - new TopFieldDocs( - new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), - new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, - new SortField[] { sortField } - ), - Float.NaN - ), - new DocValueFormat[] { DocValueFormat.RAW } - ); - } else if (shardId == 1) { - queryResult.topDocs( - new TopDocsAndMaxScore( - new TopFieldDocs( - new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), - new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, - new SortField[] { sortField } - ), - Float.NaN - ), - new DocValueFormat[] { DocValueFormat.RAW } - ); - } - queryResult.from(0); - queryResult.size(1); - successfulOps.incrementAndGet(); - queryResult.incRef(); - new Thread(() -> ActionListener.respondAndRelease(listener, queryResult)).start(); - } finally { - queryResult.decRef(); - } - } - }; - SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder()); - SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); - try ( - QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( - searchRequest, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - new NoopCircuitBreaker(CircuitBreaker.REQUEST), - controller, - task::isCancelled, - task.getProgressListener(), - shardsIter.size(), - exc -> {} - ) - ) { - CountDownLatch latch = new CountDownLatch(1); - SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction( - logger, - null, - searchTransportService, - (clusterAlias, node) -> lookup.get(node), - Collections.singletonMap("_na_", AliasFilter.EMPTY), - Collections.emptyMap(), - EsExecutors.DIRECT_EXECUTOR_SERVICE, - resultConsumer, - searchRequest, - null, - shardsIter, - timeProvider, - new ClusterState.Builder(new ClusterName("test")).build(), - task, - SearchResponse.Clusters.EMPTY, - null - ) { - @Override - protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { - return new SearchPhase("test") { - @Override - public void run() { - latch.countDown(); - } - }; - } - }; - - action.start(); - latch.await(); - assertThat(successfulOps.get(), equalTo(2)); - SearchPhaseController.ReducedQueryPhase phase = action.results.reduce(); - assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1)); - assertThat(phase.totalHits().value, equalTo(2L)); - assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); - } - } - - public void testMinimumVersionShardDuringPhaseExecution() throws Exception { - var newVersion = VersionInformation.CURRENT; - var oldVersion = new VersionInformation( - VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()), - IndexVersions.MINIMUM_COMPATIBLE, - IndexVersionUtils.randomCompatibleVersion(random()) - ); - - Version minVersion = newVersion.nodeVersion(); - - final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider( - 0, - System.nanoTime(), - System::nanoTime - ); - AtomicInteger successfulOps = new AtomicInteger(); - - Map lookup = new ConcurrentHashMap<>(); - DiscoveryNode newVersionNode1 = DiscoveryNodeUtils.builder("node1").version(newVersion).build(); - DiscoveryNode newVersionNode2 = DiscoveryNodeUtils.builder("node2").version(newVersion).build(); - DiscoveryNode oldVersionNode = DiscoveryNodeUtils.builder("node3").version(oldVersion).build(); - lookup.put("node1", new SearchAsyncActionTests.MockConnection(newVersionNode1)); - lookup.put("node2", new SearchAsyncActionTests.MockConnection(newVersionNode2)); - lookup.put("node3", new SearchAsyncActionTests.MockConnection(oldVersionNode)); - - OriginalIndices idx = new OriginalIndices(new String[] { "idx" }, SearchRequest.DEFAULT_INDICES_OPTIONS); - ArrayList list = new ArrayList<>(); - ShardRouting routingNewVersionShard1 = ShardRouting.newUnassigned( - new ShardId(new Index("idx", "_na_"), 0), - true, - RecoverySource.EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), - ShardRouting.Role.DEFAULT - ); - routingNewVersionShard1 = routingNewVersionShard1.initialize(newVersionNode1.getId(), "p0", 0); - routingNewVersionShard1.started(); - list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 0), singletonList(routingNewVersionShard1), idx)); - - ShardRouting routingNewVersionShard2 = ShardRouting.newUnassigned( - new ShardId(new Index("idx", "_na_"), 1), - true, - RecoverySource.EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), - ShardRouting.Role.DEFAULT - ); - routingNewVersionShard2 = routingNewVersionShard2.initialize(newVersionNode2.getId(), "p1", 0); - routingNewVersionShard2.started(); - list.add(new SearchShardIterator(null, new ShardId(new Index("idx", "_na_"), 1), singletonList(routingNewVersionShard2), idx)); - - GroupShardsIterator shardsIter = new GroupShardsIterator<>(list); - final SearchRequest searchRequest = new SearchRequest(minVersion); - searchRequest.allowPartialSearchResults(false); - searchRequest.source(new SearchSourceBuilder().size(1).sort(SortBuilders.fieldSort("timestamp"))); - - SearchTransportService searchTransportService = new SearchTransportService(null, null, null) { - @Override - public void sendExecuteQuery( - Transport.Connection connection, - ShardSearchRequest request, - SearchTask task, - ActionListener listener - ) { - int shardId = request.shardId().id(); - QuerySearchResult queryResult = new QuerySearchResult( - new ShardSearchContextId("N/A", 123), - new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null), - null - ); - try { - SortField sortField = new SortField("timestamp", SortField.Type.LONG); - if (shardId == 0) { - queryResult.topDocs( - new TopDocsAndMaxScore( - new TopFieldDocs( - new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), - new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, - new SortField[] { sortField } - ), - Float.NaN - ), - new DocValueFormat[] { DocValueFormat.RAW } - ); - } else if (shardId == 1) { - queryResult.topDocs( - new TopDocsAndMaxScore( - new TopFieldDocs( - new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), - new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, - new SortField[] { sortField } - ), - Float.NaN - ), - new DocValueFormat[] { DocValueFormat.RAW } - ); - } - queryResult.from(0); - queryResult.size(1); - successfulOps.incrementAndGet(); - queryResult.incRef(); - new Thread(() -> ActionListener.respondAndRelease(listener, queryResult)).start(); - } finally { - queryResult.decRef(); - } - } - }; - SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder()); - SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); - - CountDownLatch latch = new CountDownLatch(1); - try ( - QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( - searchRequest, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - new NoopCircuitBreaker(CircuitBreaker.REQUEST), - controller, - task::isCancelled, - task.getProgressListener(), - shardsIter.size(), - exc -> {} - ) - ) { - SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction( - logger, - null, - searchTransportService, - (clusterAlias, node) -> lookup.get(node), - Collections.singletonMap("_na_", AliasFilter.EMPTY), - Collections.emptyMap(), - EsExecutors.DIRECT_EXECUTOR_SERVICE, - resultConsumer, - searchRequest, - null, - shardsIter, - timeProvider, - new ClusterState.Builder(new ClusterName("test")).build(), - task, - SearchResponse.Clusters.EMPTY, - null - ) { - @Override - protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { - return new SearchPhase("test") { - @Override - public void run() { - latch.countDown(); - } - }; - } - }; - ShardRouting routingOldVersionShard = ShardRouting.newUnassigned( - new ShardId(new Index("idx", "_na_"), 2), - true, - RecoverySource.EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), - ShardRouting.Role.DEFAULT - ); - SearchShardIterator shardIt = new SearchShardIterator( - null, - new ShardId(new Index("idx", "_na_"), 2), - singletonList(routingOldVersionShard), - idx - ); - routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p2", 0); - routingOldVersionShard.started(); - action.start(); - latch.await(); - assertThat(successfulOps.get(), equalTo(2)); - SearchPhaseController.ReducedQueryPhase phase = action.results.reduce(); - assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1)); - assertThat(phase.totalHits().value, equalTo(2L)); - assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); - - SearchShardTarget searchShardTarget = new SearchShardTarget("node3", shardIt.shardId(), null); - SearchActionListener listener = new SearchActionListener(searchShardTarget, 0) { - @Override - public void onFailure(Exception e) {} - - @Override - protected void innerOnResponse(SearchPhaseResult response) {} - }; - Exception e = expectThrows( - VersionMismatchException.class, - () -> action.executePhaseOnShard(shardIt, searchShardTarget, listener) - ); - assertThat(e.getMessage(), equalTo("One of the shards is incompatible with the required minimum version [" + minVersion + "]")); - } - } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index 23c956e6e52f2..3079b6d4b0371 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TransportVersionUtils; -import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -438,33 +437,6 @@ public QueryBuilder topDocsQuery() { assertEquals(1, validationErrors.validationErrors().size()); assertEquals("using [point in time] is not allowed in a scroll context", validationErrors.validationErrors().get(0)); } - { - // Minimum compatible shard node version with ccs_minimize_roundtrips - SearchRequest searchRequest; - boolean isMinCompatibleShardVersion = randomBoolean(); - if (isMinCompatibleShardVersion) { - searchRequest = new SearchRequest(VersionUtils.randomVersion(random())); - } else { - searchRequest = new SearchRequest(); - } - - boolean shouldSetCcsMinimizeRoundtrips = randomBoolean(); - if (shouldSetCcsMinimizeRoundtrips) { - searchRequest.setCcsMinimizeRoundtrips(true); - } - ActionRequestValidationException validationErrors = searchRequest.validate(); - - if (isMinCompatibleShardVersion && shouldSetCcsMinimizeRoundtrips) { - assertNotNull(validationErrors); - assertEquals(1, validationErrors.validationErrors().size()); - assertEquals( - "[ccs_minimize_roundtrips] cannot be [true] when setting a minimum compatible shard version", - validationErrors.validationErrors().get(0) - ); - } else { - assertNull(validationErrors); - } - } { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 06434a0c90518..70682cfd41d82 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -486,7 +486,7 @@ private MockTransportService[] startTransport( threadPool ); mockTransportServices[i] = remoteSeedTransport; - DiscoveryNode remoteSeedNode = remoteSeedTransport.getLocalDiscoNode(); + DiscoveryNode remoteSeedNode = remoteSeedTransport.getLocalNode(); knownNodes.add(remoteSeedNode); nodes[i] = remoteSeedNode; settingsBuilder.put("cluster.remote.remote" + i + ".seeds", remoteSeedNode.getAddress().toString()); diff --git a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java index 437899a1fae1c..1784ab863bf1c 100644 --- a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java @@ -10,7 +10,6 @@ package org.elasticsearch.action.support; import org.elasticsearch.action.support.IndicesOptions.ConcreteTargetOptions; -import org.elasticsearch.action.support.IndicesOptions.FailureStoreOptions; import org.elasticsearch.action.support.IndicesOptions.GatekeeperOptions; import org.elasticsearch.action.support.IndicesOptions.WildcardOptions; import org.elasticsearch.common.bytes.BytesReference; @@ -31,9 +30,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static org.hamcrest.CoreMatchers.equalTo; @@ -57,8 +58,12 @@ public void testSerialization() throws Exception { .allowAliasToMultipleIndices(randomBoolean()) .allowClosedIndices(randomBoolean()) ) - .failureStoreOptions( - FailureStoreOptions.builder().includeRegularIndices(randomBoolean()).includeFailureIndices(randomBoolean()) + .selectorOptions( + IndicesOptions.SelectorOptions.builder() + .setDefaultSelectors( + EnumSet.copyOf(randomNonEmptySubsetOf(Set.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES))) + ) + .build() ) .build(); @@ -345,9 +350,11 @@ public void testToXContent() throws IOException { randomBoolean() ); GatekeeperOptions gatekeeperOptions = new GatekeeperOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()); - FailureStoreOptions failureStoreOptions = new IndicesOptions.FailureStoreOptions(randomBoolean(), randomBoolean()); + IndicesOptions.SelectorOptions selectorOptions = new IndicesOptions.SelectorOptions( + EnumSet.copyOf(randomNonEmptySubsetOf(Set.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES))) + ); - IndicesOptions indicesOptions = new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, failureStoreOptions); + IndicesOptions indicesOptions = new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, selectorOptions); XContentType type = randomFrom(XContentType.values()); BytesReference xContentBytes = toXContentBytes(indicesOptions, type); @@ -362,7 +369,15 @@ public void testToXContent() throws IOException { assertThat(map.get("ignore_unavailable"), equalTo(concreteTargetOptions.allowUnavailableTargets())); assertThat(map.get("allow_no_indices"), equalTo(wildcardOptions.allowEmptyExpressions())); assertThat(map.get("ignore_throttled"), equalTo(gatekeeperOptions.ignoreThrottled())); - assertThat(map.get("failure_store"), equalTo(failureStoreOptions.displayValue())); + String displayValue; + if (IndicesOptions.SelectorOptions.DATA_AND_FAILURE.equals(selectorOptions)) { + displayValue = "include"; + } else if (IndicesOptions.SelectorOptions.ONLY_DATA.equals(selectorOptions)) { + displayValue = "exclude"; + } else { + displayValue = "only"; + } + assertThat(map.get("failure_store"), equalTo(displayValue)); } public void testFromXContent() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index bddbe259e0ef3..da19bd68e288a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -2319,7 +2319,8 @@ public void testIgnoreThrottled() { new IndicesOptions( IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS, IndicesOptions.WildcardOptions.DEFAULT, - IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).build() + IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).build(), + IndicesOptions.SelectorOptions.DEFAULT ), "ind*", "test-index" @@ -2757,7 +2758,7 @@ public void testDataStreamsWithFailureStore() { // Test include failure store with an exact data stream name { IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN) - .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .build(); Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream"); assertThat(result.length, equalTo(4)); @@ -2771,7 +2772,7 @@ public void testDataStreamsWithFailureStore() { // We expect that they will be skipped { IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN) - .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowFailureIndices(false).build()) .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) .build(); @@ -2785,7 +2786,7 @@ public void testDataStreamsWithFailureStore() { // We expect an error { IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN) - .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowFailureIndices(false).build()) .build(); FailureIndexNotSupportedException failureIndexNotSupportedException = expectThrows( @@ -2801,7 +2802,7 @@ public void testDataStreamsWithFailureStore() { // Test only failure store with an exact data stream name { IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN) - .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(false).includeFailureIndices(true)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build(); Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream"); assertThat(result.length, equalTo(2)); @@ -2828,7 +2829,7 @@ public void testDataStreamsWithFailureStore() { // Test include failure store without any expressions { IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN) - .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .build(); Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true); assertThat(result.length, equalTo(5)); @@ -2848,7 +2849,7 @@ public void testDataStreamsWithFailureStore() { // Test only failure store without any expressions { IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN) - .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(false).includeFailureIndices(true)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build(); Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true); assertThat(result.length, equalTo(2)); @@ -2881,7 +2882,7 @@ public void testDataStreamsWithFailureStore() { // Test include failure store with wildcard expression { IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN) - .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)) + .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE) .build(); Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*"); assertThat(result.length, equalTo(5)); @@ -2901,7 +2902,7 @@ public void testDataStreamsWithFailureStore() { // Test only failure store with wildcard expression { IndicesOptions indicesOptions = IndicesOptions.builder(IndicesOptions.STRICT_EXPAND_OPEN) - .failureStoreOptions(IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(false).includeFailureIndices(true)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build(); Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-*"); assertThat(result.length, equalTo(2)); diff --git a/server/src/test/java/org/elasticsearch/common/BackoffPolicyTests.java b/server/src/test/java/org/elasticsearch/common/BackoffPolicyTests.java index 0cbbcdc0f1674..9ffd05a5336d6 100644 --- a/server/src/test/java/org/elasticsearch/common/BackoffPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/common/BackoffPolicyTests.java @@ -77,6 +77,37 @@ public void testExponentialBackOff() { } } + public void testLinearBackoffWithLimit() { + long incrementMillis = randomIntBetween(10, 500); + long limitMillis = randomIntBetween(1000, 5000); + int maxNumberOfRetries = randomIntBetween(0, 30); + BackoffPolicy timeValues = BackoffPolicy.linearBackoff( + timeValueMillis(incrementMillis), + maxNumberOfRetries, + timeValueMillis(limitMillis) + ); + int counter = 0; + for (TimeValue timeValue : timeValues) { + counter++; + long unlimitedValue = counter * incrementMillis; + long expectedValue = Math.min(unlimitedValue, limitMillis); + assertEquals(timeValueMillis(expectedValue), timeValue); + } + assertEquals(counter, maxNumberOfRetries); + } + + public void testLinearBackoffWithoutLimit() { + long incrementMillis = randomIntBetween(10, 500); + int maxNumberOfRetries = randomIntBetween(0, 30); + BackoffPolicy timeValues = BackoffPolicy.linearBackoff(timeValueMillis(incrementMillis), maxNumberOfRetries, null); + int counter = 0; + for (TimeValue timeValue : timeValues) { + counter++; + assertEquals(timeValueMillis(counter * incrementMillis), timeValue); + } + assertEquals(counter, maxNumberOfRetries); + } + public void testNoBackoff() { BackoffPolicy noBackoff = BackoffPolicy.noBackoff(); int numberOfBackoffsToPerform = randomIntBetween(1, 3); diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java index 8a66bb1464a5b..23fc788a89bde 100644 --- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java @@ -13,14 +13,24 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class LogsIndexModeTests extends ESTestCase { public void testLogsIndexModeSetting() { assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGSDB)); } - public void testSortField() { + public void testDefaultHostNameSortField() { + final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings()); + assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); + final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); + assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true)); + assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), containsString("host.name")); + } + + public void testCustomSortField() { final Settings sortSettings = Settings.builder() .put(buildSettings()) .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id") @@ -29,7 +39,9 @@ public void testSortField() { assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB)); - assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()))); + assertThat(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()), equalTo("agent_id")); + assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(false)); + assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), not(containsString("host"))); } public void testSortMode() { diff --git a/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java b/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java index 38552d0a58683..e0d94632f06c1 100644 --- a/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java +++ b/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; @@ -26,10 +25,8 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.RandomObjects; import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; import java.util.ArrayList; @@ -114,56 +111,6 @@ public void testToXContent() throws IOException { } } - public void testToCompatibleXContent() throws IOException { - { - GetResult getResult = new GetResult( - "index", - "id", - 0, - 1, - 1, - true, - new BytesArray(""" - { "field1" : "value1", "field2":"value2"}"""), - singletonMap("field1", new DocumentField("field1", singletonList("value1"))), - singletonMap("field1", new DocumentField("metafield", singletonList("metavalue"))) - ); - - try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { - getResult.toXContent(builder, ToXContent.EMPTY_PARAMS); - String output = Strings.toString(builder); - assertEquals(XContentHelper.stripWhitespace(""" - { - "_index": "index", - "_type": "_doc", - "_id": "id", - "_version": 1, - "_seq_no": 0, - "_primary_term": 1, - "metafield": "metavalue", - "found": true, - "_source": { - "field1": "value1", - "field2": "value2" - }, - "fields": { - "field1": [ "value1" ] - } - }"""), XContentHelper.stripWhitespace(output)); - } - } - { - GetResult getResult = new GetResult("index", "id", UNASSIGNED_SEQ_NO, 0, 1, false, null, null, null); - - try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { - getResult.toXContent(builder, ToXContent.EMPTY_PARAMS); - String output = Strings.toString(builder); - assertEquals(""" - {"_index":"index","_type":"_doc","_id":"id","found":false}""", output); - } - } - } - public void testToAndFromXContentEmbedded() throws Exception { XContentType xContentType = randomFrom(XContentType.values()); Tuple tuple = randomGetResult(xContentType); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 5eac5acdca286..934744ef3ef96 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -932,6 +932,36 @@ public void testConflictingFieldNameAfterArray() throws IOException { {"path":{"id":0.1,"to":{"id":[1,20,3,10]}}}""", syntheticSource); } + public void testArrayWithNestedObjects() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").startObject("properties"); + { + b.startObject("to").field("type", "nested").startObject("properties"); + { + b.startObject("id").field("type", "integer").field("synthetic_source_keep", "arrays").endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + { + b.startObject().startArray("to"); + { + b.startObject().array("id", 1, 20, 3).endObject(); + b.startObject().field("id", 10).endObject(); + } + b.endArray().endObject(); + b.startObject().startObject("to").field("id", "0.1").endObject().endObject(); + } + b.endArray(); + }); + assertEquals(""" + {"path":{"to":[{"id":[1,20,3]},{"id":10},{"id":0}]}}""", syntheticSource); + } + public void testArrayWithinArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 3adaf398624de..d83fdbd5dd46b 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.support.ActionTestUtils; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.internal.Client; @@ -48,6 +49,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Strings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -424,7 +426,7 @@ public void testDelete() { public void testValidateNoIngestInfo() throws Exception { IngestService ingestService = createWithProcessors(); - PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """ + PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """ {"processors": [{"set" : {"field": "_field", "value": "_value"}}]}"""); var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2(); @@ -965,7 +967,7 @@ public void testGetPipelines() { public void testValidateProcessorTypeOnAllNodes() throws Exception { IngestService ingestService = createWithProcessors(); - PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """ + PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """ { "processors": [ { @@ -1009,7 +1011,7 @@ public void testValidateConfigurationExceptions() { // ordinary validation issues happen at processor construction time throw newConfigurationException("fail_validation", tag, "no_property_name", "validation failure reason"); })); - PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """ + PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """ { "processors": [ { @@ -1043,7 +1045,7 @@ public void extraValidation() throws Exception { } }; })); - PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """ + PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """ { "processors": [ { @@ -1067,6 +1069,32 @@ public void extraValidation() throws Exception { assertEquals("fail_extra_validation", e.getMetadata("es.processor_type").get(0)); } + public void testValidatePipelineName() throws Exception { + IngestService ingestService = createWithProcessors(); + for (Character badChar : List.of('\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',')) { + PutPipelineRequest putRequest = new PutPipelineRequest( + TimeValue.timeValueSeconds(10), + AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + "_id", + new BytesArray(""" + {"description":"test processor","processors":[{"set":{"field":"_field","value":"_value"}}]}"""), + XContentType.JSON + ); + var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2(); + DiscoveryNode node1 = DiscoveryNodeUtils.create("_node_id1", buildNewFakeTransportAddress(), Map.of(), Set.of()); + Map ingestInfos = new HashMap<>(); + ingestInfos.put(node1, new IngestInfo(List.of(new ProcessorInfo("set")))); + final String name = randomAlphaOfLength(5) + badChar + randomAlphaOfLength(5); + ingestService.validatePipeline(ingestInfos, name, pipelineConfig); + assertCriticalWarnings( + "Pipeline name [" + + name + + "] will be disallowed in a future version for the following reason: must not contain the following characters" + + " [' ','\"','*',',','/','<','>','?','\\','|']" + ); + } + } + public void testExecuteIndexPipelineExistsButFailedParsing() { IngestService ingestService = createWithProcessors( Map.of("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") { diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java index b716f11b5fffb..88c83df3e20fc 100644 --- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java @@ -89,7 +89,6 @@ protected SearchSourceBuilder createSearchSourceBuilder() { return RandomSearchRequestGenerator.randomSearchSourceBuilder( HighlightBuilderTests::randomHighlighterBuilder, SuggestBuilderTests::randomSuggestBuilder, - TestRankBuilder::randomRankBuilder, QueryRescorerBuilderTests::randomRescoreBuilder, randomExtBuilders, CollapseBuilderTests::randomCollapseBuilder, diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index 0e4945e8bb8d1..a474c1dc38c50 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -81,6 +81,7 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.function.Function; import java.util.function.Supplier; import java.util.function.ToLongFunction; @@ -507,10 +508,10 @@ public void testNewIdLoaderWithTsdbAndRoutingPathMatch() throws Exception { } } - public void testDetermineMaximumNumberOfSlices() { + private static ShardSearchRequest createParallelRequest() { IndexShard indexShard = mock(IndexShard.class); when(indexShard.shardId()).thenReturn(new ShardId("index", "uuid", 0)); - ShardSearchRequest parallelReq = new ShardSearchRequest( + return new ShardSearchRequest( OriginalIndices.NONE, new SearchRequest().allowPartialSearchResults(randomBoolean()), indexShard.shardId(), @@ -521,69 +522,74 @@ public void testDetermineMaximumNumberOfSlices() { System.currentTimeMillis(), null ); - ShardSearchRequest singleSliceReq = new ShardSearchRequest( - OriginalIndices.NONE, - new SearchRequest().allowPartialSearchResults(randomBoolean()) - .source(new SearchSourceBuilder().sort(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME))), - indexShard.shardId(), - 0, - 1, - AliasFilter.EMPTY, - 1f, - System.currentTimeMillis(), - null - ); - int executorPoolSize = randomIntBetween(1, 100); - ExecutorService threadPoolExecutor = EsExecutors.newFixed( - "test", - executorPoolSize, - 0, - Thread::new, - new ThreadContext(Settings.EMPTY), - EsExecutors.TaskTrackingConfig.DO_NOT_TRACK - ); - ExecutorService notThreadPoolExecutor = Executors.newWorkStealingPool(); - ToLongFunction fieldCardinality = name -> -1; + } + + public void testDetermineMaximumNumberOfSlicesNoExecutor() { + ToLongFunction fieldCardinality = name -> { throw new UnsupportedOperationException(); }; assertEquals( - executorPoolSize, + 1, DefaultSearchContext.determineMaximumNumberOfSlices( - threadPoolExecutor, - parallelReq, + null, + createParallelRequest(), SearchService.ResultsType.DFS, - true, + randomBoolean(), fieldCardinality ) ); assertEquals( - executorPoolSize, + 1, DefaultSearchContext.determineMaximumNumberOfSlices( - threadPoolExecutor, - singleSliceReq, - SearchService.ResultsType.DFS, - true, + null, + createParallelRequest(), + SearchService.ResultsType.QUERY, + randomBoolean(), fieldCardinality ) ); + } + + public void testDetermineMaximumNumberOfSlicesNotThreadPoolExecutor() { + ExecutorService notThreadPoolExecutor = Executors.newWorkStealingPool(); + ToLongFunction fieldCardinality = name -> { throw new UnsupportedOperationException(); }; assertEquals( 1, - DefaultSearchContext.determineMaximumNumberOfSlices(null, parallelReq, SearchService.ResultsType.DFS, true, fieldCardinality) + DefaultSearchContext.determineMaximumNumberOfSlices( + notThreadPoolExecutor, + createParallelRequest(), + SearchService.ResultsType.DFS, + randomBoolean(), + fieldCardinality + ) ); assertEquals( - executorPoolSize, + 1, DefaultSearchContext.determineMaximumNumberOfSlices( - threadPoolExecutor, - parallelReq, + notThreadPoolExecutor, + createParallelRequest(), SearchService.ResultsType.QUERY, - true, + randomBoolean(), fieldCardinality ) ); + } + + public void testDetermineMaximumNumberOfSlicesEnableQueryPhaseParallelCollection() { + int executorPoolSize = randomIntBetween(1, 100); + ThreadPoolExecutor threadPoolExecutor = EsExecutors.newFixed( + "test", + executorPoolSize, + 0, + Thread::new, + new ThreadContext(Settings.EMPTY), + EsExecutors.TaskTrackingConfig.DO_NOT_TRACK + ); + ToLongFunction fieldCardinality = name -> -1; assertEquals( - 1, + executorPoolSize, DefaultSearchContext.determineMaximumNumberOfSlices( threadPoolExecutor, - singleSliceReq, + createParallelRequest(), SearchService.ResultsType.QUERY, true, fieldCardinality @@ -592,54 +598,133 @@ public void testDetermineMaximumNumberOfSlices() { assertEquals( 1, DefaultSearchContext.determineMaximumNumberOfSlices( - notThreadPoolExecutor, - parallelReq, - SearchService.ResultsType.DFS, - true, + threadPoolExecutor, + createParallelRequest(), + SearchService.ResultsType.QUERY, + false, fieldCardinality ) ); - assertEquals( executorPoolSize, DefaultSearchContext.determineMaximumNumberOfSlices( threadPoolExecutor, - parallelReq, + createParallelRequest(), SearchService.ResultsType.DFS, - false, + randomBoolean(), fieldCardinality ) ); - assertEquals( + } + + public void testDetermineMaximumNumberOfSlicesSingleSortByField() { + IndexShard indexShard = mock(IndexShard.class); + when(indexShard.shardId()).thenReturn(new ShardId("index", "uuid", 0)); + ShardSearchRequest singleSliceReq = new ShardSearchRequest( + OriginalIndices.NONE, + new SearchRequest().allowPartialSearchResults(randomBoolean()) + .source(new SearchSourceBuilder().sort(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME))), + indexShard.shardId(), + 0, 1, - DefaultSearchContext.determineMaximumNumberOfSlices(null, parallelReq, SearchService.ResultsType.DFS, false, fieldCardinality) + AliasFilter.EMPTY, + 1f, + System.currentTimeMillis(), + null ); + ToLongFunction fieldCardinality = name -> { throw new UnsupportedOperationException(); }; + int executorPoolSize = randomIntBetween(1, 100); + ThreadPoolExecutor threadPoolExecutor = EsExecutors.newFixed( + "test", + executorPoolSize, + 0, + Thread::new, + new ThreadContext(Settings.EMPTY), + EsExecutors.TaskTrackingConfig.DO_NOT_TRACK + ); + // DFS concurrency does not rely on slices, hence it kicks in regardless of the request (supportsParallelCollection is not called) assertEquals( - 1, + executorPoolSize, DefaultSearchContext.determineMaximumNumberOfSlices( threadPoolExecutor, - parallelReq, - SearchService.ResultsType.QUERY, - false, + singleSliceReq, + SearchService.ResultsType.DFS, + true, fieldCardinality ) ); - assertEquals( - 1, - DefaultSearchContext.determineMaximumNumberOfSlices(null, parallelReq, SearchService.ResultsType.QUERY, false, fieldCardinality) - ); assertEquals( 1, DefaultSearchContext.determineMaximumNumberOfSlices( - notThreadPoolExecutor, - parallelReq, - SearchService.ResultsType.DFS, - false, + threadPoolExecutor, + singleSliceReq, + SearchService.ResultsType.QUERY, + true, fieldCardinality ) ); } + public void testDetermineMaximumNumberOfSlicesWithQueue() { + int executorPoolSize = randomIntBetween(1, 100); + ThreadPoolExecutor threadPoolExecutor = EsExecutors.newFixed( + "test", + executorPoolSize, + 1000, + Thread::new, + new ThreadContext(Settings.EMPTY), + EsExecutors.TaskTrackingConfig.DO_NOT_TRACK + ); + ToLongFunction fieldCardinality = name -> { throw new UnsupportedOperationException(); }; + + for (int i = 0; i < executorPoolSize; i++) { + assertTrue(threadPoolExecutor.getQueue().offer(() -> {})); + assertEquals( + executorPoolSize, + DefaultSearchContext.determineMaximumNumberOfSlices( + threadPoolExecutor, + createParallelRequest(), + SearchService.ResultsType.DFS, + true, + fieldCardinality + ) + ); + assertEquals( + executorPoolSize, + DefaultSearchContext.determineMaximumNumberOfSlices( + threadPoolExecutor, + createParallelRequest(), + SearchService.ResultsType.QUERY, + true, + fieldCardinality + ) + ); + } + for (int i = 0; i < 100; i++) { + assertTrue(threadPoolExecutor.getQueue().offer(() -> {})); + assertEquals( + 1, + DefaultSearchContext.determineMaximumNumberOfSlices( + threadPoolExecutor, + createParallelRequest(), + SearchService.ResultsType.DFS, + true, + fieldCardinality + ) + ); + assertEquals( + 1, + DefaultSearchContext.determineMaximumNumberOfSlices( + threadPoolExecutor, + createParallelRequest(), + SearchService.ResultsType.QUERY, + true, + fieldCardinality + ) + ); + } + } + public void testIsParallelCollectionSupportedForResults() { SearchSourceBuilder searchSourceBuilderOrNull = randomBoolean() ? null : new SearchSourceBuilder(); ToLongFunction fieldCardinality = name -> -1; diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 240a677f4cbfd..380b5189b3efc 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -464,18 +463,6 @@ public void testToXContentWithPointInTime() throws IOException { } public void testParseIndicesBoost() throws IOException { - { - String restContent = """ - { "indices_boost": {"foo": 1.0, "bar": 2.0}}"""; - try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, restContent, RestApiVersion.V_7)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); - assertEquals(2, searchSourceBuilder.indexBoosts().size()); - assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0)); - assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1)); - assertCriticalWarnings("Object format in indices_boost is deprecated, please use array format instead"); - } - } - { String restContent = """ { @@ -554,16 +541,6 @@ public void testNegativeSizeErrors() throws IOException { ); assertThat(ex.getMessage(), containsString(Integer.toString(boundedRandomSize))); } - - restContent = "{\"size\" : -1}"; - try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, restContent, RestApiVersion.V_7)) { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().parseXContent(parser, true, nf -> false); - assertEquals(-1, searchSourceBuilder.size()); - } - assertCriticalWarnings( - "Using search size of -1 is deprecated and will be removed in future versions. Instead, don't use the `size` " - + "parameter if you don't want to set it explicitly." - ); } public void testNegativeTerminateAfter() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java index d9fe421bafb46..4e4d2158a9574 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java @@ -74,7 +74,6 @@ public void testParseSearchRequest() throws IOException { () -> null, () -> null, () -> null, - () -> null, Collections::emptyList, () -> null, () -> null diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java index 78d76476d06fc..e5cdecd25ef34 100644 --- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java @@ -157,13 +157,13 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, parentTransportService.addSendBehavior(sendRequestBehavior); - AbstractSimpleTransportTestCase.connectToNode(parentTransportService, childTransportService.getLocalDiscoNode()); + AbstractSimpleTransportTestCase.connectToNode(parentTransportService, childTransportService.getLocalNode()); final CancellableTask parentTask = (CancellableTask) parentTransportService.getTaskManager() .register("transport", "internal:testAction", new ParentRequest()); parentTransportService.sendChildRequest( - childTransportService.getLocalDiscoNode(), + childTransportService.getLocalNode(), "internal:testAction[c]", new EmptyRequest(), parentTask, @@ -172,7 +172,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, ); try (MockLog mockLog = MockLog.capture(TaskCancellationService.class)) { - for (MockLog.LoggingExpectation expectation : expectations.apply(childTransportService.getLocalDiscoNode())) { + for (MockLog.LoggingExpectation expectation : expectations.apply(childTransportService.getLocalNode())) { mockLog.addExpectation(expectation); } diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java index 43dce7d406ba2..d7cf38828b7ba 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareClientTests.java @@ -89,7 +89,7 @@ public void testRemoteTaskCancellationOnFailedResponse() throws Exception { ) { remoteTransport.getTaskManager().setTaskCancellationService(new TaskCancellationService(remoteTransport)); Settings.Builder builder = Settings.builder(); - builder.putList("cluster.remote.cluster1.seeds", remoteTransport.getLocalDiscoNode().getAddress().toString()); + builder.putList("cluster.remote.cluster1.seeds", remoteTransport.getLocalNode().getAddress().toString()); try ( MockTransportService localService = MockTransportService.createNewService( builder.build(), @@ -163,11 +163,11 @@ public void testSearchShards() throws Exception { MockTransportService seedTransport = startTransport("seed_node", knownNodes); MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes) ) { - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); + knownNodes.add(seedTransport.getLocalNode()); + knownNodes.add(discoverableTransport.getLocalNode()); Collections.shuffle(knownNodes, random()); Settings.Builder builder = Settings.builder(); - builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString()); + builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalNode().getAddress().toString()); try ( MockTransportService service = MockTransportService.createNewService( builder.build(), @@ -216,11 +216,11 @@ public void testSearchShardsThreadContextHeader() { MockTransportService seedTransport = startTransport("seed_node", knownNodes); MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes) ) { - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); + knownNodes.add(seedTransport.getLocalNode()); + knownNodes.add(discoverableTransport.getLocalNode()); Collections.shuffle(knownNodes, random()); Settings.Builder builder = Settings.builder(); - builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString()); + builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalNode().getAddress().toString()); try ( MockTransportService service = MockTransportService.createNewService( builder.build(), diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index ff0742c89bba9..0efaef7015649 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -73,7 +73,7 @@ public void testConnectAndExecuteRequest() throws Exception { remoteSettings ) ) { - DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode(); + DiscoveryNode remoteNode = remoteTransport.getLocalNode(); Settings localSettings = Settings.builder() .put(onlyRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)) @@ -152,7 +152,7 @@ public void testEnsureWeReconnect() throws Exception { remoteSettings ) ) { - DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode(); + DiscoveryNode remoteNode = remoteTransport.getLocalNode(); Settings localSettings = Settings.builder() .put(onlyRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)) .put("cluster.remote.test.seeds", remoteNode.getAddress().getAddress() + ":" + remoteNode.getAddress().getPort()) @@ -251,7 +251,7 @@ public void testQuicklySkipUnavailableClusters() throws Exception { remoteSettings ) ) { - DiscoveryNode remoteNode = remoteTransport.getLocalDiscoNode(); + DiscoveryNode remoteNode = remoteTransport.getLocalNode(); Settings localSettings = Settings.builder() .put(onlyRole(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)) diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 7a259cf3100f0..21346bb93ef8e 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -311,11 +311,11 @@ public void testCloseWhileConcurrentlyConnecting() throws IOException, Interrupt TransportVersion.current() ) ) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - DiscoveryNode seedNode1 = seedTransport1.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); - knownNodes.add(discoverableTransport.getLocalDiscoNode()); - knownNodes.add(seedTransport1.getLocalDiscoNode()); + DiscoveryNode seedNode = seedTransport.getLocalNode(); + DiscoveryNode seedNode1 = seedTransport1.getLocalNode(); + knownNodes.add(seedTransport.getLocalNode()); + knownNodes.add(discoverableTransport.getLocalNode()); + knownNodes.add(seedTransport1.getLocalNode()); Collections.shuffle(knownNodes, random()); List seedNodes = addresses(seedNode1, seedNode); Collections.shuffle(seedNodes, random()); @@ -447,9 +447,9 @@ private void doTestGetConnectionInfo(boolean hasClusterCredentials) throws Excep seedTransportSettings ) ) { - DiscoveryNode node1 = transport1.getLocalDiscoNode(); - DiscoveryNode node2 = transport3.getLocalDiscoNode(); - DiscoveryNode node3 = transport2.getLocalDiscoNode(); + DiscoveryNode node1 = transport1.getLocalNode(); + DiscoveryNode node2 = transport3.getLocalNode(); + DiscoveryNode node3 = transport2.getLocalNode(); if (hasClusterCredentials) { node1 = node1.withTransportAddress(transport1.boundRemoteAccessAddress().publishAddress()); node2 = node2.withTransportAddress(transport3.boundRemoteAccessAddress().publishAddress()); @@ -645,7 +645,7 @@ private void doTestCollectNodes(boolean hasClusterCredentials) throws Exception seedTransportSettings ) ) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); + DiscoveryNode seedNode = seedTransport.getLocalNode(); if (hasClusterCredentials) { seedNode = seedNode.withTransportAddress(seedTransport.boundRemoteAccessAddress().publishAddress()); } @@ -725,8 +725,8 @@ public void testNoChannelsExceptREG() throws Exception { TransportVersion.current() ) ) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); + DiscoveryNode seedNode = seedTransport.getLocalNode(); + knownNodes.add(seedTransport.getLocalNode()); try ( MockTransportService service = MockTransportService.createNewService( Settings.EMPTY, diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java index 11243ba088f8f..3633128c45bfa 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java @@ -141,10 +141,10 @@ public void testGroupClusterIndices() throws IOException { TransportVersion.current() ) ) { - DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode(); - DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode(); - knownNodes.add(cluster1Transport.getLocalDiscoNode()); - knownNodes.add(cluster2Transport.getLocalDiscoNode()); + DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode(); + DiscoveryNode cluster2Seed = cluster2Transport.getLocalNode(); + knownNodes.add(cluster1Transport.getLocalNode()); + knownNodes.add(cluster2Transport.getLocalNode()); Collections.shuffle(knownNodes, random()); try ( @@ -343,10 +343,10 @@ public void testGroupIndices() throws IOException { TransportVersion.current() ) ) { - DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode(); - DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode(); - knownNodes.add(cluster1Transport.getLocalDiscoNode()); - knownNodes.add(cluster2Transport.getLocalDiscoNode()); + DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode(); + DiscoveryNode cluster2Seed = cluster2Transport.getLocalNode(); + knownNodes.add(cluster1Transport.getLocalNode()); + knownNodes.add(cluster2Transport.getLocalNode()); Collections.shuffle(knownNodes, random()); try ( @@ -453,10 +453,10 @@ public void testIncrementallyAddClusters() throws IOException { TransportVersion.current() ) ) { - DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode(); - DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode(); - knownNodes.add(cluster1Transport.getLocalDiscoNode()); - knownNodes.add(cluster2Transport.getLocalDiscoNode()); + DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode(); + DiscoveryNode cluster2Seed = cluster2Transport.getLocalNode(); + knownNodes.add(cluster1Transport.getLocalNode()); + knownNodes.add(cluster2Transport.getLocalNode()); Collections.shuffle(knownNodes, random()); try ( @@ -526,8 +526,8 @@ public void testDefaultPingSchedule() throws IOException { TransportVersion.current() ) ) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); - knownNodes.add(seedTransport.getLocalDiscoNode()); + DiscoveryNode seedNode = seedTransport.getLocalNode(); + knownNodes.add(seedTransport.getLocalNode()); TimeValue pingSchedule; Settings.Builder settingsBuilder = Settings.builder(); settingsBuilder.putList("cluster.remote.cluster_1.seeds", seedNode.getAddress().toString()); @@ -582,10 +582,10 @@ public void testCustomPingSchedule() throws IOException { TransportVersion.current() ) ) { - DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode(); - DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode(); - knownNodes.add(cluster1Transport.getLocalDiscoNode()); - knownNodes.add(cluster2Transport.getLocalDiscoNode()); + DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode(); + DiscoveryNode cluster2Seed = cluster2Transport.getLocalNode(); + knownNodes.add(cluster1Transport.getLocalNode()); + knownNodes.add(cluster2Transport.getLocalNode()); Collections.shuffle(knownNodes, random()); Settings.Builder settingsBuilder = Settings.builder(); if (randomBoolean()) { @@ -635,8 +635,8 @@ public void testChangeSettings() throws Exception { TransportVersion.current() ) ) { - DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode(); - knownNodes.add(cluster1Transport.getLocalDiscoNode()); + DiscoveryNode cluster1Seed = cluster1Transport.getLocalNode(); + knownNodes.add(cluster1Transport.getLocalNode()); Collections.shuffle(knownNodes, random()); try ( @@ -716,10 +716,10 @@ public void testRemoteNodeAttribute() throws IOException, InterruptedException { gateway ) ) { - final DiscoveryNode c1N1Node = c1N1.getLocalDiscoNode(); - final DiscoveryNode c1N2Node = c1N2.getLocalDiscoNode(); - final DiscoveryNode c2N1Node = c2N1.getLocalDiscoNode(); - final DiscoveryNode c2N2Node = c2N2.getLocalDiscoNode(); + final DiscoveryNode c1N1Node = c1N1.getLocalNode(); + final DiscoveryNode c1N2Node = c1N2.getLocalNode(); + final DiscoveryNode c2N1Node = c2N1.getLocalNode(); + final DiscoveryNode c2N2Node = c2N2.getLocalNode(); knownNodes.add(c1N1Node); knownNodes.add(c1N2Node); knownNodes.add(c2N1Node); @@ -809,10 +809,10 @@ public void testRemoteNodeRoles() throws IOException, InterruptedException { data ) ) { - final DiscoveryNode c1N1Node = c1N1.getLocalDiscoNode(); - final DiscoveryNode c1N2Node = c1N2.getLocalDiscoNode(); - final DiscoveryNode c2N1Node = c2N1.getLocalDiscoNode(); - final DiscoveryNode c2N2Node = c2N2.getLocalDiscoNode(); + final DiscoveryNode c1N1Node = c1N1.getLocalNode(); + final DiscoveryNode c1N2Node = c1N2.getLocalNode(); + final DiscoveryNode c2N1Node = c2N1.getLocalNode(); + final DiscoveryNode c2N2Node = c2N2.getLocalNode(); knownNodes.add(c1N1Node); knownNodes.add(c1N2Node); knownNodes.add(c2N1Node); @@ -906,10 +906,10 @@ public void testCollectNodes() throws InterruptedException, IOException { settings ) ) { - final DiscoveryNode c1N1Node = c1N1.getLocalDiscoNode(); - final DiscoveryNode c1N2Node = c1N2.getLocalDiscoNode(); - final DiscoveryNode c2N1Node = c2N1.getLocalDiscoNode(); - final DiscoveryNode c2N2Node = c2N2.getLocalDiscoNode(); + final DiscoveryNode c1N1Node = c1N1.getLocalNode(); + final DiscoveryNode c1N2Node = c1N2.getLocalNode(); + final DiscoveryNode c2N1Node = c2N1.getLocalNode(); + final DiscoveryNode c2N2Node = c2N2.getLocalNode(); knownNodes_c1.add(c1N1Node); knownNodes_c1.add(c1N2Node); knownNodes_c2.add(c2N1Node); @@ -1170,8 +1170,8 @@ public void testReconnectWhenStrategySettingsUpdated() throws Exception { ) ) { - final DiscoveryNode node0 = cluster_node_0.getLocalDiscoNode(); - final DiscoveryNode node1 = cluster_node_1.getLocalDiscoNode(); + final DiscoveryNode node0 = cluster_node_0.getLocalNode(); + final DiscoveryNode node1 = cluster_node_1.getLocalNode(); knownNodes.add(node0); knownNodes.add(node1); Collections.shuffle(knownNodes, random()); @@ -1267,10 +1267,10 @@ public void testSkipUnavailable() { TransportVersion.current() ) ) { - DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); + DiscoveryNode seedNode = seedTransport.getLocalNode(); knownNodes.add(seedNode); Settings.Builder builder = Settings.builder(); - builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalDiscoNode().getAddress().toString()); + builder.putList("cluster.remote.cluster1.seeds", seedTransport.getLocalNode().getAddress().toString()); try ( MockTransportService service = MockTransportService.createNewService( builder.build(), @@ -1353,8 +1353,8 @@ public void testUseDifferentTransportProfileForCredentialsProtectedRemoteCluster ); MockTransportService c2 = startTransport("cluster_2", knownNodes, VersionInformation.CURRENT, TransportVersion.current()); ) { - final DiscoveryNode c1Node = c1.getLocalDiscoNode().withTransportAddress(c1.boundRemoteAccessAddress().publishAddress()); - final DiscoveryNode c2Node = c2.getLocalDiscoNode(); + final DiscoveryNode c1Node = c1.getLocalNode().withTransportAddress(c1.boundRemoteAccessAddress().publishAddress()); + final DiscoveryNode c2Node = c2.getLocalNode(); final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("cluster.remote.cluster_1.credentials", randomAlphaOfLength(10)); @@ -1441,7 +1441,7 @@ public void testUpdateRemoteClusterCredentialsRebuildsConnectionWithCorrectProfi .build() ) ) { - final DiscoveryNode discoNode = c.getLocalDiscoNode().withTransportAddress(c.boundRemoteAccessAddress().publishAddress()); + final DiscoveryNode discoNode = c.getLocalNode().withTransportAddress(c.boundRemoteAccessAddress().publishAddress()); try ( MockTransportService transportService = MockTransportService.createNewService( Settings.EMPTY, @@ -1518,8 +1518,8 @@ public void testUpdateRemoteClusterCredentialsRebuildsMultipleConnectionsDespite .build() ) ) { - final DiscoveryNode c1DiscoNode = c1.getLocalDiscoNode().withTransportAddress(c1.boundRemoteAccessAddress().publishAddress()); - final DiscoveryNode c2DiscoNode = c2.getLocalDiscoNode().withTransportAddress(c2.boundRemoteAccessAddress().publishAddress()); + final DiscoveryNode c1DiscoNode = c1.getLocalNode().withTransportAddress(c1.boundRemoteAccessAddress().publishAddress()); + final DiscoveryNode c2DiscoNode = c2.getLocalNode().withTransportAddress(c2.boundRemoteAccessAddress().publishAddress()); try ( MockTransportService transportService = MockTransportService.createNewService( Settings.EMPTY, @@ -1636,7 +1636,7 @@ public void testLogsConnectionResult() throws IOException { assertThatLogger( () -> clusterSettings.applySettings( - Settings.builder().putList("cluster.remote.remote_1.seeds", remote.getLocalDiscoNode().getAddress().toString()).build() + Settings.builder().putList("cluster.remote.remote_1.seeds", remote.getLocalNode().getAddress().toString()).build() ), RemoteClusterService.class, new MockLog.SeenEventExpectation( diff --git a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java index 398bb5f2a9106..46585ac382583 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java @@ -85,15 +85,15 @@ public void setUp() throws Exception { threadPool = new TestThreadPool(getClass().getName()); serviceA = buildService(version0, transportVersion0); // this one supports dynamic tracer updates serviceA.taskManager.setTaskCancellationService(new TaskCancellationService(serviceA)); - nodeA = serviceA.getLocalDiscoNode(); + nodeA = serviceA.getLocalNode(); serviceB = buildService(version1, transportVersion1); // this one doesn't support dynamic tracer updates serviceB.taskManager.setTaskCancellationService(new TaskCancellationService(serviceB)); - nodeB = serviceB.getLocalDiscoNode(); + nodeB = serviceB.getLocalNode(); serviceC = buildService(version1, transportVersion1); // this one doesn't support dynamic tracer updates serviceC.taskManager.setTaskCancellationService(new TaskCancellationService(serviceC)); - nodeC = serviceC.getLocalDiscoNode(); + nodeC = serviceC.getLocalNode(); serviceD = buildService(version1, transportVersion1); - nodeD = serviceD.getLocalDiscoNode(); + nodeD = serviceD.getLocalNode(); } @Override diff --git a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java index 59205aa546cd2..3f3e0c0a25578 100644 --- a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java +++ b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java @@ -11,20 +11,18 @@ import com.sun.net.httpserver.HttpServer; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; import org.junit.rules.ExternalResource; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.security.MessageDigest; +import java.util.List; +import java.util.Objects; /** * This fixture is used to simulate a maxmind-provided server for downloading maxmind geoip database files from the @@ -32,21 +30,17 @@ */ public class EnterpriseGeoIpHttpFixture extends ExternalResource { - private final Path source; - private final String[] databaseTypes; + private final List maxmindDatabaseTypes; + private final List ipinfoDatabaseTypes; private HttpServer server; /* - * The values in databaseTypes must be in DatabaseConfiguration.MAXMIND_NAMES, and must be one of the databases copied in the - * copyFiles method of thisi class. + * The values in maxmindDatabaseTypes must be in DatabaseConfiguration.MAXMIND_NAMES, and the ipinfoDatabaseTypes + * must be in DatabaseConfiguration.IPINFO_NAMES. */ - public EnterpriseGeoIpHttpFixture(String... databaseTypes) { - this.databaseTypes = databaseTypes; - try { - this.source = Files.createTempDirectory("source"); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + public EnterpriseGeoIpHttpFixture(List maxmindDatabaseTypes, List ipinfoDatabaseTypes) { + this.maxmindDatabaseTypes = List.copyOf(maxmindDatabaseTypes); + this.ipinfoDatabaseTypes = List.copyOf(ipinfoDatabaseTypes); } public String getAddress() { @@ -55,7 +49,6 @@ public String getAddress() { @Override protected void before() throws Throwable { - copyFiles(); this.server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); // for expediency reasons, it is handy to have this test fixture be able to serve the dual purpose of actually stubbing @@ -64,26 +57,33 @@ protected void before() throws Throwable { this.server.createContext("/", exchange -> { String response = "[]"; // an empty json array exchange.sendResponseHeaders(200, response.length()); - try (OutputStream os = exchange.getResponseBody()) { - os.write(response.getBytes(StandardCharsets.UTF_8)); + try (OutputStream out = exchange.getResponseBody()) { + out.write(response.getBytes(StandardCharsets.UTF_8)); } }); // register the file types for the download fixture - for (String databaseType : databaseTypes) { - createContextForEnterpriseDatabase(databaseType); + for (String databaseType : maxmindDatabaseTypes) { + createContextForMaxmindDatabase(databaseType); + } + for (String databaseType : ipinfoDatabaseTypes) { + createContextForIpinfoDatabase(databaseType); } server.start(); } - private void createContextForEnterpriseDatabase(String databaseType) { + private static InputStream fixtureStream(String name) { + return Objects.requireNonNull(GeoIpHttpFixture.class.getResourceAsStream(name)); + } + + private void createContextForMaxmindDatabase(String databaseType) { this.server.createContext("/" + databaseType + "/download", exchange -> { exchange.sendResponseHeaders(200, 0); if (exchange.getRequestURI().toString().contains("sha256")) { MessageDigest sha256 = MessageDigests.sha256(); - try (InputStream inputStream = GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/" + databaseType + ".tgz")) { - sha256.update(inputStream.readAllBytes()); + try (InputStream in = fixtureStream("/geoip-fixture/" + databaseType + ".tgz")) { + sha256.update(in.readAllBytes()); } exchange.getResponseBody() .write( @@ -93,10 +93,33 @@ private void createContextForEnterpriseDatabase(String databaseType) { ); } else { try ( - OutputStream outputStream = exchange.getResponseBody(); - InputStream inputStream = GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/" + databaseType + ".tgz") + OutputStream out = exchange.getResponseBody(); + InputStream in = fixtureStream("/geoip-fixture/" + databaseType + ".tgz") + ) { + in.transferTo(out); + } + } + exchange.getResponseBody().close(); + }); + } + + private void createContextForIpinfoDatabase(String databaseType) { + this.server.createContext("/free/" + databaseType + ".mmdb", exchange -> { + exchange.sendResponseHeaders(200, 0); + if (exchange.getRequestURI().toString().contains("checksum")) { + MessageDigest md5 = MessageDigests.md5(); + try (InputStream in = fixtureStream("/ipinfo-fixture/ip_" + databaseType + "_sample.mmdb")) { + md5.update(in.readAllBytes()); + } + exchange.getResponseBody().write(Strings.format(""" + { "checksums": { "md5": "%s" } } + """, MessageDigests.toHexString(md5.digest())).getBytes(StandardCharsets.UTF_8)); + } else { + try ( + OutputStream out = exchange.getResponseBody(); + InputStream in = fixtureStream("/ipinfo-fixture/ip_" + databaseType + "_sample.mmdb") ) { - inputStream.transferTo(outputStream); + in.transferTo(out); } } exchange.getResponseBody().close(); @@ -107,14 +130,4 @@ private void createContextForEnterpriseDatabase(String databaseType) { protected void after() { server.stop(0); } - - private void copyFiles() throws Exception { - for (String databaseType : databaseTypes) { - Files.copy( - GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoIP2-City.tgz"), - source.resolve(databaseType + ".tgz"), - StandardCopyOption.REPLACE_EXISTING - ); - } - } } diff --git a/test/fixtures/geoip-fixture/src/main/resources/ipinfo-fixture/ip_asn_sample.mmdb b/test/fixtures/geoip-fixture/src/main/resources/ipinfo-fixture/ip_asn_sample.mmdb new file mode 100644 index 0000000000000..3e1fc49ba48a5 Binary files /dev/null and b/test/fixtures/geoip-fixture/src/main/resources/ipinfo-fixture/ip_asn_sample.mmdb differ diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index a9ee0317ce1ee..8bc2666bcfe3b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -302,8 +302,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) {} mapperMetrics ); - if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping() != null) { - mapperService.merge(null, indexSettings.getMode().getDefaultMapping(), MapperService.MergeReason.MAPPING_UPDATE); + if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping(indexSettings) != null) { + mapperService.merge( + null, + indexSettings.getMode().getDefaultMapping(indexSettings), + MapperService.MergeReason.MAPPING_UPDATE + ); } return mapperService; diff --git a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java index 568f386a81fd1..ec85feb200984 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java @@ -156,11 +156,11 @@ protected void checkTransientErrorsDuringRecoveryAreRetried(String recoveryActio Runnable connectionBreaker = () -> { // Always break connection from source to remote to ensure that actions are retried logger.info("--> closing connections from source node to target node"); - blueTransportService.disconnectFromNode(redTransportService.getLocalDiscoNode()); + blueTransportService.disconnectFromNode(redTransportService.getLocalNode()); if (randomBoolean()) { // Sometimes break connection from remote to source to ensure that recovery is re-established logger.info("--> closing connections from target node to source node"); - redTransportService.disconnectFromNode(blueTransportService.getLocalDiscoNode()); + redTransportService.disconnectFromNode(blueTransportService.getLocalNode()); } }; TransientReceiveRejected handlingBehavior = new TransientReceiveRejected(recoveryActionToBlock, recoveryStarted, connectionBreaker); @@ -258,13 +258,13 @@ public void checkDisconnectsWhileRecovering(String recoveryActionToBlock) throws blueMockTransportService.addRequestHandlingBehavior(recoveryActionToBlock, (handler, request, channel, task) -> { logger.info("--> preventing {} response by closing response channel", recoveryActionToBlock); requestFailed.countDown(); - redMockTransportService.disconnectFromNode(blueMockTransportService.getLocalDiscoNode()); + redMockTransportService.disconnectFromNode(blueMockTransportService.getLocalNode()); handler.messageReceived(request, channel, task); }); redMockTransportService.addRequestHandlingBehavior(recoveryActionToBlock, (handler, request, channel, task) -> { logger.info("--> preventing {} response by closing response channel", recoveryActionToBlock); requestFailed.countDown(); - blueMockTransportService.disconnectFromNode(redMockTransportService.getLocalDiscoNode()); + blueMockTransportService.disconnectFromNode(redMockTransportService.getLocalNode()); handler.messageReceived(request, channel, task); }); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java deleted file mode 100644 index cda571c0ef4b5..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.logsdb.datageneration; - -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.logsdb.datageneration.fields.TopLevelObjectFieldDataGenerator; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Map; - -/** - * Entry point of data generation logic. - * Every instance of generator generates a random mapping and a document generation routine - * that produces randomly generated documents valid for this mapping. - */ -public class DataGenerator { - private final TopLevelObjectFieldDataGenerator topLevelGenerator; - - public DataGenerator(DataGeneratorSpecification specification) { - this.topLevelGenerator = new TopLevelObjectFieldDataGenerator(specification); - } - - /** - * Writes a fully built mapping document (enclosed in a top-level object) to a provided builder. - * @param mapping destination - * @throws IOException - */ - public void writeMapping(XContentBuilder mapping) throws IOException { - mapping.startObject().field("_doc"); - topLevelGenerator.mappingWriter(Map.of()).accept(mapping); - mapping.endObject(); - } - - /** - * Writes a fully built mapping document (enclosed in a top-level object) to a provided builder. - * Allows customizing parameters of top level object mapper. - * @param mapping destination - * @param customMappingParameters writer of custom mapping parameters of top level object mapping - * @throws IOException - */ - public void writeMapping(XContentBuilder mapping, Map customMappingParameters) throws IOException { - mapping.startObject().field("_doc"); - topLevelGenerator.mappingWriter(customMappingParameters).accept(mapping); - mapping.endObject(); - } - - /** - * Generates a document and writes it to a provided builder. New document is generated every time. - * @param document - * @throws IOException - */ - public void generateDocument(XContentBuilder document) throws IOException { - topLevelGenerator.fieldValueGenerator(b -> {}).accept(document); - } - - /** - * Generates a document and writes it to a provided builder. New document is generated every time. - * Supports appending custom content to generated document (e.g. a custom generated field). - * @param document - * @param customDocumentModifications - * @throws IOException - */ - public void generateDocument(XContentBuilder document, CheckedConsumer customDocumentModifications) - throws IOException { - topLevelGenerator.fieldValueGenerator(customDocumentModifications).accept(document); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java index 97eb8b91a6e64..f6ff7c521afcc 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java @@ -18,7 +18,7 @@ import java.util.List; /** - * Allows configuring behavior of {@link DataGenerator}. + * Allows configuring behavior of data generation components. * @param dataSource source of generated data * @param maxFieldCountPerLevel maximum number of fields that an individual object in mapping has. * Applies to subobjects. diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java new file mode 100644 index 0000000000000..9b2878ff7bfc8 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; + +import java.util.ArrayList; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Generator that generates a valid random document that follows the structure of provided {@link Template}. + */ +public class DocumentGenerator { + private final DataGeneratorSpecification specification; + + private final DataSourceResponse.ObjectArrayGenerator objectArrayGenerator; + + public DocumentGenerator(DataGeneratorSpecification specification) { + this.specification = specification; + + this.objectArrayGenerator = specification.dataSource().get(new DataSourceRequest.ObjectArrayGenerator()); + } + + /** + * Generates a valid random document following the provided template. + * @param template template for the document + * @param mapping generated mapping that will be applied to the destination index of this document + * @return document as a map where subobjects are represented as nested maps + */ + public Map generate(Template template, Mapping mapping) { + var documentMap = new TreeMap(); + for (var predefinedField : specification.predefinedFields()) { + documentMap.put(predefinedField.name(), predefinedField.generator(specification.dataSource()).generateValue()); + } + + generateFields(documentMap, template.template(), new Context("", mapping.lookup())); + return documentMap; + } + + private void generateFields(Map document, Map template, Context context) { + for (var entry : template.entrySet()) { + String fieldName = entry.getKey(); + Template.Entry templateEntry = entry.getValue(); + + if (templateEntry instanceof Template.Leaf leaf) { + // Unsigned long does not play well when dynamically mapped because + // it gets mapped as just long and large values fail to index. + // Just skip it. + if (leaf.type() == FieldType.UNSIGNED_LONG && context.mappingLookup().get(context.pathTo(fieldName)) == null) { + continue; + } + + var generator = leaf.type().generator(fieldName, specification.dataSource()); + + document.put(fieldName, generator.generateValue()); + } else if (templateEntry instanceof Template.Object object) { + Optional arrayLength = objectArrayGenerator.lengthGenerator().get(); + + if (arrayLength.isPresent()) { + var children = new ArrayList<>(arrayLength.get()); + document.put(object.name(), children); + + for (int i = 0; i < arrayLength.get(); i++) { + children.add(generateObject(object, context)); + } + } else { + document.put(object.name(), generateObject(object, context)); + } + } + } + } + + private Map generateObject(Template.Object object, Context context) { + var children = new TreeMap(); + generateFields(children, object.children(), context.stepIntoObject(object.name())); + return children; + } + + record Context(String path, Map> mappingLookup) { + Context stepIntoObject(String name) { + return new Context(pathTo(name), mappingLookup); + } + + String pathTo(String leafFieldName) { + return path.isEmpty() ? leafFieldName : path + "." + leafFieldName; + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java index 3d7a96f55cbce..7e28a0a0fab25 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java @@ -9,21 +9,11 @@ package org.elasticsearch.logsdb.datageneration; -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; - /** - * Entity responsible for generating a valid randomized mapping for a field - * and a generator of field values valid for this mapping. + * Entity responsible for generating a valid value for a field. * - * Generator is expected to produce the same mapping per instance of generator. - * Function returned by {@link FieldDataGenerator#fieldValueGenerator() } is expected - * to produce a randomized value each time. + * Generator is expected to produce a different value on every call. */ public interface FieldDataGenerator { - CheckedConsumer mappingWriter(); - - CheckedConsumer fieldValueGenerator(); + Object generateValue(); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java index 4304a8c77e60b..96b75f29382e2 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java @@ -10,7 +10,6 @@ package org.elasticsearch.logsdb.datageneration; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.FloatFieldDataGenerator; @@ -26,33 +25,40 @@ * Lists all leaf field types that are supported for data generation. */ public enum FieldType { - KEYWORD, - LONG, - UNSIGNED_LONG, - INTEGER, - SHORT, - BYTE, - DOUBLE, - FLOAT, - HALF_FLOAT, - SCALED_FLOAT; + KEYWORD("keyword"), + LONG("long"), + UNSIGNED_LONG("unsigned_long"), + INTEGER("integer"), + SHORT("short"), + BYTE("byte"), + DOUBLE("double"), + FLOAT("float"), + HALF_FLOAT("half_float"), + SCALED_FLOAT("scaled_float"); - public FieldDataGenerator generator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + private final String name; + + FieldType(String name) { + this.name = name; + } + + public FieldDataGenerator generator(String fieldName, DataSource dataSource) { return switch (this) { - case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case LONG -> new LongFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case SHORT -> new ShortFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case BYTE -> new ByteFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case DOUBLE -> new DoubleFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); - case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource); + case LONG -> new LongFieldDataGenerator(fieldName, dataSource); + case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource); + case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource); + case SHORT -> new ShortFieldDataGenerator(fieldName, dataSource); + case BYTE -> new ByteFieldDataGenerator(fieldName, dataSource); + case DOUBLE -> new DoubleFieldDataGenerator(fieldName, dataSource); + case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource); + case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource); + case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource); }; } + + @Override + public String toString() { + return name; + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Mapping.java similarity index 51% rename from server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java rename to test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Mapping.java index 69ea4484ae691..c20c99d7cb0ba 100644 --- a/server/src/main/java/org/elasticsearch/action/search/VersionMismatchException.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Mapping.java @@ -7,21 +7,13 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.action.search; +package org.elasticsearch.logsdb.datageneration; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.io.stream.StreamInput; +import java.util.Map; -import java.io.IOException; - -public class VersionMismatchException extends ElasticsearchException { - - public VersionMismatchException(String msg, Object... args) { - super(msg, args); - } - - public VersionMismatchException(StreamInput in) throws IOException { - super(in); - } - -} +/** + * Contains generated mapping and supporting data. + * @param raw mapping represented as a possibly nested map (maps represent (sub-)objects) + * @param lookup supporting data structure that represent mapping in a flat form (full path to field -> mapping parameters) + */ +public record Mapping(Map raw, Map> lookup) {} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java new file mode 100644 index 0000000000000..bdb80ee09868f --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * Generator that generates a valid random mapping that follows the structure of provided {@link Template}. + * Mapping will contain all fields from the template with generated mapping parameters. + */ +public class MappingGenerator { + private final DataGeneratorSpecification specification; + + private final DataSourceResponse.DynamicMappingGenerator dynamicMappingGenerator; + + public MappingGenerator(DataGeneratorSpecification specification) { + this.specification = specification; + + this.dynamicMappingGenerator = specification.dataSource().get(new DataSourceRequest.DynamicMappingGenerator()); + } + + /** + * Generates a valid random mapping following the provided template. + * @param template template for the mapping + * @return {@link Mapping} + */ + public Mapping generate(Template template) { + var lookup = new TreeMap>(); + + // Top level mapping parameters + var mappingParametersGenerator = specification.dataSource() + .get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false, ObjectMapper.Subobjects.ENABLED)) + .mappingGenerator(); + + var topLevelMappingParameters = mappingParametersGenerator.get(); + // Top-level object can't be disabled because @timestamp is a required field in data streams. + topLevelMappingParameters.remove("enabled"); + + var rawMapping = new TreeMap(); + + var childrenMapping = new TreeMap(); + for (var predefinedField : specification.predefinedFields()) { + if (predefinedField.mapping() != null) { + childrenMapping.put(predefinedField.name(), predefinedField.mapping()); + lookup.put(predefinedField.name(), predefinedField.mapping()); + } + } + topLevelMappingParameters.put("properties", childrenMapping); + + rawMapping.put("_doc", topLevelMappingParameters); + + if (specification.fullyDynamicMapping()) { + // Has to be "true" for fully dynamic mapping + topLevelMappingParameters.remove("dynamic"); + + return new Mapping(rawMapping, lookup); + } + + var dynamicMapping = topLevelMappingParameters.getOrDefault("dynamic", "true").equals("strict") + ? DynamicMapping.FORBIDDEN + : DynamicMapping.SUPPORTED; + var subobjects = ObjectMapper.Subobjects.from(topLevelMappingParameters.getOrDefault("subobjects", "true")); + + generateMapping(childrenMapping, lookup, template.template(), new Context(new HashSet<>(), "", subobjects, dynamicMapping)); + + return new Mapping(rawMapping, lookup); + } + + private void generateMapping( + Map mapping, + Map> lookup, + Map template, + Context context + ) { + for (var entry : template.entrySet()) { + String fieldName = entry.getKey(); + Template.Entry templateEntry = entry.getValue(); + + var mappingParameters = new TreeMap(); + + boolean isDynamicallyMapped = isDynamicallyMapped(templateEntry, context); + // Simply skip this field if it is dynamic. + // Lookup will contain null signaling dynamic mapping as well. + if (isDynamicallyMapped) { + continue; + } + + if (templateEntry instanceof Template.Leaf leaf) { + // For simplicity we only copy to keyword fields, synthetic source logic to handle copy_to is generic. + if (leaf.type() == FieldType.KEYWORD) { + context.addCopyToCandidate(fieldName); + } + + var mappingParametersGenerator = specification.dataSource() + .get( + new DataSourceRequest.LeafMappingParametersGenerator( + fieldName, + leaf.type(), + context.eligibleCopyToDestinations(), + context.parentDynamicMapping() + ) + ) + .mappingGenerator(); + + mappingParameters.put("type", leaf.type().toString()); + mappingParameters.putAll(mappingParametersGenerator.get()); + + } else if (templateEntry instanceof Template.Object object) { + var mappingParametersGenerator = specification.dataSource() + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, object.nested(), context.parentSubobjects())) + .mappingGenerator(); + + mappingParameters.put("type", object.nested() ? "nested" : "object"); + mappingParameters.putAll(mappingParametersGenerator.get()); + + var childrenMapping = new TreeMap(); + mappingParameters.put("properties", childrenMapping); + generateMapping( + childrenMapping, + lookup, + object.children(), + context.stepIntoObject(object.name(), object.nested(), mappingParameters) + ); + } + + mapping.put(fieldName, mappingParameters); + lookup.put(context.pathTo(fieldName), Map.copyOf(mappingParameters)); + } + } + + private boolean isDynamicallyMapped(Template.Entry templateEntry, Context context) { + return context.parentDynamicMapping != DynamicMapping.FORBIDDEN + && dynamicMappingGenerator.generator().apply(templateEntry instanceof Template.Object); + } + + record Context( + Set eligibleCopyToDestinations, + String path, + ObjectMapper.Subobjects parentSubobjects, + DynamicMapping parentDynamicMapping + ) { + Context stepIntoObject(String name, boolean nested, Map mappingParameters) { + var subobjects = determineSubobjects(mappingParameters); + var dynamicMapping = determineDynamicMapping(mappingParameters); + + // copy_to can't be used across nested documents so all currently eligible fields are not eligible inside nested document. + return new Context(nested ? new HashSet<>() : eligibleCopyToDestinations, pathTo(name), subobjects, dynamicMapping); + } + + void addCopyToCandidate(String leafFieldName) { + eligibleCopyToDestinations.add(pathTo(leafFieldName)); + } + + String pathTo(String leafFieldName) { + return path.isEmpty() ? leafFieldName : path + "." + leafFieldName; + } + + private DynamicMapping determineDynamicMapping(Map mappingParameters) { + if (parentDynamicMapping == DynamicMapping.FORCED) { + return DynamicMapping.FORCED; + } + + var dynamicParameter = mappingParameters.get("dynamic"); + // Inherited from parent + if (dynamicParameter == null) { + return parentDynamicMapping; + } + + return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED; + } + + private ObjectMapper.Subobjects determineSubobjects(Map mappingParameters) { + if (parentSubobjects == ObjectMapper.Subobjects.DISABLED) { + return ObjectMapper.Subobjects.DISABLED; + } + + return ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true")); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java new file mode 100644 index 0000000000000..9bb2d34958a27 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration; + +import java.util.Map; + +/** + * A template used to generate mapping and documents for a test. + * Template encodes object structure, names of objects/fields and type of leaf fields. + * Having such a template allow to create interchangeable random mappings with different parameters + * but the same structure in order to f.e. test introduction of a new parameter. + * @param template actual template data + */ +public record Template(Map template) { + sealed interface Entry permits Leaf, Object {} + + record Leaf(String name, FieldType type) implements Entry {} + + record Object(String name, boolean nested, Map children) implements Entry {} +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java new file mode 100644 index 0000000000000..f0a3a866a2673 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +public class TemplateGenerator { + private final DataGeneratorSpecification specification; + private final DataSourceResponse.ChildFieldGenerator childFieldGenerator; + private final Supplier fieldTypeGenerator; + + public TemplateGenerator(DataGeneratorSpecification specification) { + this.specification = specification; + this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification)); + this.fieldTypeGenerator = specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator()).generator(); + } + + public Template generate() { + var map = new HashMap(); + + generateChildFields(map, 0, new AtomicInteger(0)); + return new Template(map); + } + + private void generateChildFields(Map mapping, int depth, AtomicInteger nestedFieldsCount) { + var existingFieldNames = new HashSet(); + // no child fields is legal + var childFieldsCount = childFieldGenerator.generateChildFieldCount(); + + for (int i = 0; i < childFieldsCount; i++) { + var fieldName = generateFieldName(existingFieldNames); + + if (depth < specification.maxObjectDepth() && childFieldGenerator.generateRegularSubObject()) { + var children = new HashMap(); + mapping.put(fieldName, new Template.Object(fieldName, false, children)); + generateChildFields(children, depth + 1, nestedFieldsCount); + } else if (depth <= specification.maxObjectDepth() + && nestedFieldsCount.get() < specification.nestedFieldsLimit() + && childFieldGenerator.generateNestedSubObject()) { + nestedFieldsCount.incrementAndGet(); + + var children = new HashMap(); + mapping.put(fieldName, new Template.Object(fieldName, true, children)); + generateChildFields(children, depth + 1, nestedFieldsCount); + } else { + var fieldTypeInfo = fieldTypeGenerator.get(); + mapping.put(fieldName, new Template.Leaf(fieldName, fieldTypeInfo.fieldType())); + } + } + } + + private String generateFieldName(Set existingFields) { + var fieldName = childFieldGenerator.generateFieldName(); + while (existingFields.contains(fieldName)) { + fieldName = childFieldGenerator.generateFieldName(); + } + existingFields.add(fieldName); + + return fieldName; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index 9a9c610e091e2..df28282fca407 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -73,4 +73,8 @@ default DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReque default DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { return null; } + + default DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) { + return null; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 8dee5876aa207..d77925f097b5a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -93,7 +93,7 @@ public DataSourceResponse.ChildFieldGenerator accept(DataSourceHandler handler) } } - record FieldTypeGenerator(DynamicMapping dynamicMapping) implements DataSourceRequest { + record FieldTypeGenerator() implements DataSourceRequest { public DataSourceResponse.FieldTypeGenerator accept(DataSourceHandler handler) { return handler.handle(this); } @@ -123,4 +123,10 @@ public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHand return handler.handle(this); } } + + record DynamicMappingGenerator() implements DataSourceRequest { + public DataSourceResponse.DynamicMappingGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index a7653c10bbf43..fa8f56b3e071b 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -52,7 +52,7 @@ interface ChildFieldGenerator extends DataSourceResponse { } record FieldTypeGenerator(Supplier generator) implements DataSourceResponse { - public record FieldTypeInfo(FieldType fieldType, boolean dynamic) {} + public record FieldTypeInfo(FieldType fieldType) {} } record ObjectArrayGenerator(Supplier> lengthGenerator) implements DataSourceResponse {} @@ -60,4 +60,6 @@ record ObjectArrayGenerator(Supplier> lengthGenerator) impleme record LeafMappingParametersGenerator(Supplier> mappingGenerator) implements DataSourceResponse {} record ObjectMappingParametersGenerator(Supplier> mappingGenerator) implements DataSourceResponse {} + + record DynamicMappingGenerator(Function generator) implements DataSourceResponse {} } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java index 62c4ef87e8a15..56ec676e53d55 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java @@ -14,7 +14,6 @@ import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; import static org.elasticsearch.test.ESTestCase.randomDouble; @@ -26,24 +25,25 @@ public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFiel return new DataSourceResponse.ChildFieldGenerator() { @Override public int generateChildFieldCount() { + // no child fields is legal return ESTestCase.randomIntBetween(0, request.specification().maxFieldCountPerLevel()); } @Override public boolean generateDynamicSubObject() { - // Using a static 5% change, this is just a chosen value that can be tweaked. + // Using a static 5% chance, this is just a chosen value that can be tweaked. return randomDouble() <= 0.05; } @Override public boolean generateNestedSubObject() { - // Using a static 5% change, this is just a chosen value that can be tweaked. + // Using a static 5% chance, this is just a chosen value that can be tweaked. return randomDouble() <= 0.05; } @Override public boolean generateRegularSubObject() { - // Using a static 5% change, this is just a chosen value that can be tweaked. + // Using a static 5% chance, this is just a chosen value that can be tweaked. return randomDouble() <= 0.05; } @@ -60,21 +60,10 @@ public String generateFieldName() { @Override public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { - Supplier generator = switch (request.dynamicMapping()) { - case FORBIDDEN -> () -> generateFieldTypeInfo(false); - case FORCED -> () -> generateFieldTypeInfo(true); - case SUPPORTED -> () -> generateFieldTypeInfo(ESTestCase.randomBoolean()); - }; - - return new DataSourceResponse.FieldTypeGenerator(generator); - } - - private static DataSourceResponse.FieldTypeGenerator.FieldTypeInfo generateFieldTypeInfo(boolean isDynamic) { - var excluded = isDynamic ? EXCLUDED_FROM_DYNAMIC_MAPPING : Set.of(); - var fieldType = ESTestCase.randomValueOtherThanMany(excluded::contains, () -> ESTestCase.randomFrom(FieldType.values())); - - return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(fieldType, isDynamic); + return new DataSourceResponse.FieldTypeGenerator( + () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(FieldType.values())) + ); } @Override @@ -87,4 +76,12 @@ public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectAr return Optional.empty(); }); } + + @Override + public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) { + // Using a static 5% chance for objects, this is just a chosen value that can be tweaked. + return new DataSourceResponse.DynamicMappingGenerator( + isObject -> isObject ? ESTestCase.randomDouble() <= 0.05 : ESTestCase.randomBoolean() + ); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java deleted file mode 100644 index c1ec15a3479b3..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.logsdb.datageneration.fields; - -import org.elasticsearch.index.mapper.ObjectMapper; -import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; - -import java.util.HashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -class Context { - private final DataGeneratorSpecification specification; - - private final DataSourceResponse.ChildFieldGenerator childFieldGenerator; - private final DataSourceResponse.ObjectArrayGenerator objectArrayGenerator; - - private final String path; - private final int objectDepth; - // We don't need atomicity, but we need to pass counter by reference to accumulate total value from sub-objects. - private final AtomicInteger nestedFieldsCount; - private final Set eligibleCopyToDestinations; - private final DynamicMapping parentDynamicMapping; - private final ObjectMapper.Subobjects currentSubobjectsConfig; - - Context( - DataGeneratorSpecification specification, - DynamicMapping parentDynamicMapping, - ObjectMapper.Subobjects currentSubobjectsConfig - ) { - this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping, currentSubobjectsConfig); - } - - private Context( - DataGeneratorSpecification specification, - String path, - int objectDepth, - AtomicInteger nestedFieldsCount, - Set eligibleCopyToDestinations, - DynamicMapping parentDynamicMapping, - ObjectMapper.Subobjects currentSubobjectsConfig - ) { - this.specification = specification; - this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification)); - this.objectArrayGenerator = specification.dataSource().get(new DataSourceRequest.ObjectArrayGenerator()); - this.path = path; - this.objectDepth = objectDepth; - this.nestedFieldsCount = nestedFieldsCount; - this.eligibleCopyToDestinations = eligibleCopyToDestinations; - this.parentDynamicMapping = parentDynamicMapping; - this.currentSubobjectsConfig = currentSubobjectsConfig; - } - - public DataGeneratorSpecification specification() { - return specification; - } - - public DataSourceResponse.ChildFieldGenerator childFieldGenerator() { - return childFieldGenerator; - } - - public DataSourceResponse.FieldTypeGenerator fieldTypeGenerator(DynamicMapping dynamicMapping) { - return specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator(dynamicMapping)); - } - - public Context subObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { - return new Context( - specification, - pathToField(name), - objectDepth + 1, - nestedFieldsCount, - eligibleCopyToDestinations, - dynamicMapping, - subobjects - ); - } - - public Context nestedObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { - nestedFieldsCount.incrementAndGet(); - // copy_to can't be used across nested documents so all currently eligible fields are not eligible inside nested document. - return new Context( - specification, - pathToField(name), - objectDepth + 1, - nestedFieldsCount, - new HashSet<>(), - dynamicMapping, - subobjects - ); - } - - public boolean shouldAddDynamicObjectField(DynamicMapping dynamicMapping) { - if (objectDepth >= specification.maxObjectDepth() || dynamicMapping == DynamicMapping.FORBIDDEN) { - return false; - } - - return childFieldGenerator.generateDynamicSubObject(); - } - - public boolean shouldAddObjectField() { - if (objectDepth >= specification.maxObjectDepth() || parentDynamicMapping == DynamicMapping.FORCED) { - return false; - } - - return childFieldGenerator.generateRegularSubObject(); - } - - public boolean shouldAddNestedField(ObjectMapper.Subobjects subobjects) { - if (objectDepth >= specification.maxObjectDepth() - || nestedFieldsCount.get() >= specification.nestedFieldsLimit() - || parentDynamicMapping == DynamicMapping.FORCED - || subobjects == ObjectMapper.Subobjects.DISABLED) { - return false; - } - - return childFieldGenerator.generateNestedSubObject(); - } - - public Optional generateObjectArray() { - if (objectDepth == 0) { - return Optional.empty(); - } - - return objectArrayGenerator.lengthGenerator().get(); - } - - public DynamicMapping determineDynamicMapping(Map mappingParameters) { - if (parentDynamicMapping == DynamicMapping.FORCED) { - return DynamicMapping.FORCED; - } - - var dynamicParameter = mappingParameters.get("dynamic"); - // Inherited from parent - if (dynamicParameter == null) { - return parentDynamicMapping; - } - - return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED; - } - - public ObjectMapper.Subobjects determineSubobjects(Map mappingParameters) { - if (currentSubobjectsConfig == ObjectMapper.Subobjects.DISABLED) { - return ObjectMapper.Subobjects.DISABLED; - } - - return ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true")); - } - - public Set getEligibleCopyToDestinations() { - return eligibleCopyToDestinations; - } - - public void markFieldAsEligibleForCopyTo(String field) { - eligibleCopyToDestinations.add(pathToField(field)); - } - - private String pathToField(String field) { - return path.isEmpty() ? field : path + "." + field; - } - - public ObjectMapper.Subobjects getCurrentSubobjectsConfig() { - return currentSubobjectsConfig; - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java deleted file mode 100644 index 83a68519d5de1..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.logsdb.datageneration.fields; - -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.index.mapper.ObjectMapper; -import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Generic generator for any type of object field (e.g. "object", "nested"). - */ -public class GenericSubObjectFieldDataGenerator { - private final Context context; - - GenericSubObjectFieldDataGenerator(Context context) { - this.context = context; - } - - List generateChildFields(DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { - var existingFieldNames = new HashSet(); - // no child fields is legal - var childFieldsCount = context.childFieldGenerator().generateChildFieldCount(); - var result = new ArrayList(childFieldsCount); - - for (int i = 0; i < childFieldsCount; i++) { - var fieldName = generateFieldName(existingFieldNames); - - if (context.shouldAddDynamicObjectField(dynamicMapping)) { - result.add( - new ChildField( - fieldName, - new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED, subobjects)), - true - ) - ); - } else if (context.shouldAddObjectField()) { - result.add( - new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping, subobjects)), false) - ); - } else if (context.shouldAddNestedField(subobjects)) { - result.add( - new ChildField( - fieldName, - new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping, subobjects)), - false - ) - ); - } else { - var fieldTypeInfo = context.fieldTypeGenerator(dynamicMapping).generator().get(); - - // For simplicity we only copy to keyword fields, synthetic source logic to handle copy_to is generic. - if (fieldTypeInfo.fieldType() == FieldType.KEYWORD) { - context.markFieldAsEligibleForCopyTo(fieldName); - } - - var mappingParametersGenerator = context.specification() - .dataSource() - .get( - new DataSourceRequest.LeafMappingParametersGenerator( - fieldName, - fieldTypeInfo.fieldType(), - context.getEligibleCopyToDestinations(), - dynamicMapping - ) - ); - var generator = fieldTypeInfo.fieldType() - .generator(fieldName, context.specification().dataSource(), mappingParametersGenerator); - result.add(new ChildField(fieldName, generator, fieldTypeInfo.dynamic())); - } - } - - return result; - } - - List generateChildFields(List predefinedFields) { - return predefinedFields.stream() - .map(pf -> new ChildField(pf.name(), pf.generator(context.specification().dataSource()), false)) - .toList(); - } - - static void writeChildFieldsMapping(XContentBuilder mapping, List childFields) throws IOException { - for (var childField : childFields) { - if (childField.dynamic() == false) { - mapping.field(childField.fieldName); - childField.generator.mappingWriter().accept(mapping); - } - } - } - - static void writeObjectsData(XContentBuilder document, Context context, CheckedConsumer objectWriter) - throws IOException { - var optionalLength = context.generateObjectArray(); - if (optionalLength.isPresent()) { - int size = optionalLength.get(); - - document.startArray(); - for (int i = 0; i < size; i++) { - objectWriter.accept(document); - } - document.endArray(); - } else { - objectWriter.accept(document); - } - } - - static void writeSingleObject(XContentBuilder document, Iterable childFields) throws IOException { - document.startObject(); - writeChildFieldsData(document, childFields); - document.endObject(); - } - - static void writeChildFieldsData(XContentBuilder document, Iterable childFields) throws IOException { - for (var childField : childFields) { - document.field(childField.fieldName); - childField.generator.fieldValueGenerator().accept(document); - } - } - - private String generateFieldName(Set existingFields) { - var fieldName = context.childFieldGenerator().generateFieldName(); - while (existingFields.contains(fieldName)) { - fieldName = context.childFieldGenerator().generateFieldName(); - } - existingFields.add(fieldName); - - return fieldName; - } - - record ChildField(String fieldName, FieldDataGenerator generator, boolean dynamic) {} -} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java deleted file mode 100644 index 69853debf9b77..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.logsdb.datageneration.fields; - -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -public class NestedFieldDataGenerator implements FieldDataGenerator { - private final Context context; - private final Map mappingParameters; - private final List childFields; - - NestedFieldDataGenerator(Context context) { - this.context = context; - - this.mappingParameters = context.specification() - .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true, context.getCurrentSubobjectsConfig())) - .mappingGenerator() - .get(); - var dynamicMapping = context.determineDynamicMapping(mappingParameters); - var subobjects = context.determineSubobjects(mappingParameters); - - var genericGenerator = new GenericSubObjectFieldDataGenerator(context); - this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); - } - - @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject(); - - b.field("type", "nested"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.startObject("properties"); - GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, childFields); - b.endObject(); - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - CheckedConsumer objectWriter = object -> GenericSubObjectFieldDataGenerator.writeSingleObject( - object, - childFields - ); - return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java deleted file mode 100644 index 701642c57619b..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.logsdb.datageneration.fields; - -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -public class ObjectFieldDataGenerator implements FieldDataGenerator { - private final Context context; - private final Map mappingParameters; - private final List childFields; - - ObjectFieldDataGenerator(Context context) { - this.context = context; - - this.mappingParameters = context.specification() - .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false, context.getCurrentSubobjectsConfig())) - .mappingGenerator() - .get(); - var dynamicMapping = context.determineDynamicMapping(mappingParameters); - var subobjects = context.determineSubobjects(mappingParameters); - - var genericGenerator = new GenericSubObjectFieldDataGenerator(context); - this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); - } - - @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject(); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.startObject("properties"); - GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, childFields); - b.endObject(); - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - CheckedConsumer objectWriter = object -> GenericSubObjectFieldDataGenerator.writeSingleObject( - object, - childFields - ); - return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java index 57e3ce3ce2a86..948f1cf80b35c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java @@ -12,37 +12,21 @@ import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import java.util.Set; +import java.util.Map; public interface PredefinedField { String name(); - FieldDataGenerator generator(DataSource dataSource); + FieldType fieldType(); - record WithType(String fieldName, FieldType fieldType, DynamicMapping dynamicMapping) implements PredefinedField { - @Override - public String name() { - return fieldName; - } + Map mapping(); - @Override - public FieldDataGenerator generator(DataSource dataSource) { - // copy_to currently not supported for predefined fields, use WithGenerator if needed - var mappingParametersGenerator = dataSource.get( - new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of(), dynamicMapping) - ); - return fieldType().generator(fieldName, dataSource, mappingParametersGenerator); - } - } - - record WithGenerator(String fieldName, FieldDataGenerator generator) implements PredefinedField { - @Override - public String name() { - return fieldName; - } + FieldDataGenerator generator(DataSource dataSource); + record WithGenerator(String name, FieldType fieldType, Map mapping, FieldDataGenerator generator) + implements + PredefinedField { @Override public FieldDataGenerator generator(DataSource dataSource) { return generator; diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java deleted file mode 100644 index 1374362df7f4a..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.logsdb.datageneration.fields; - -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.index.mapper.ObjectMapper; -import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class TopLevelObjectFieldDataGenerator { - private final Context context; - private final Map mappingParameters; - - // Child fields of top level object that were explicitly requested, they have predefined name and type. - private final List predefinedFields; - // Child fields of top level object that are generated and merged with predefined fields. - private final List generatedChildFields; - - public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification) { - DynamicMapping dynamicMapping; - if (specification.fullyDynamicMapping()) { - dynamicMapping = DynamicMapping.FORCED; - this.mappingParameters = Map.of(); - } else { - this.mappingParameters = new HashMap<>( - // Value of subobjects here is for a parent of this object. - // Since there is no parent we pass ENABLED to allow to set subobjects to any value at top level. - specification.dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false, ObjectMapper.Subobjects.ENABLED)) - .mappingGenerator() - .get() - ); - // Top-level object can't be disabled because @timestamp is a required field in data streams. - this.mappingParameters.remove("enabled"); - - dynamicMapping = mappingParameters.getOrDefault("dynamic", "true").equals("strict") - ? DynamicMapping.FORBIDDEN - : DynamicMapping.SUPPORTED; - } - var subobjects = ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true")); - - // Value of subobjects here is for a parent of this object. - // Since there is no parent we pass ENABLED to allow to set subobjects to any value at top level. - this.context = new Context(specification, dynamicMapping, ObjectMapper.Subobjects.ENABLED); - var genericGenerator = new GenericSubObjectFieldDataGenerator(context); - - this.predefinedFields = genericGenerator.generateChildFields(specification.predefinedFields()); - this.generatedChildFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); - } - - public CheckedConsumer mappingWriter(Map customMappingParameters) { - return b -> { - b.startObject(); - - var mergedParameters = Stream.of(this.mappingParameters, customMappingParameters) - .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (l, r) -> r)); - - for (var entry : mergedParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.startObject("properties"); - GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, predefinedFields); - GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, generatedChildFields); - b.endObject(); - - b.endObject(); - }; - } - - public CheckedConsumer fieldValueGenerator( - CheckedConsumer customDocumentModification - ) { - CheckedConsumer objectWriter = b -> { - b.startObject(); - - customDocumentModification.accept(b); - GenericSubObjectFieldDataGenerator.writeChildFieldsData(b, predefinedFields); - GenericSubObjectFieldDataGenerator.writeChildFieldsData(b, generatedChildFields); - - b.endObject(); - }; - return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java index 08cbfa32bcdb2..4ead8ffd0b718 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class ByteFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public ByteFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public ByteFieldDataGenerator(String fieldName, DataSource dataSource) { var bytes = dataSource.get(new DataSourceRequest.ByteGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> bytes.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "byte"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java index 0c486360d0003..cf2c4f6abdbf4 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class DoubleFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public DoubleFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public DoubleFieldDataGenerator(String fieldName, DataSource dataSource) { var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "double"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java index 97397013dd303..b59d5ceabb188 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class FloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public FloatFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public FloatFieldDataGenerator(String fieldName, DataSource dataSource) { var floats = dataSource.get(new DataSourceRequest.FloatGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> floats.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "float"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java index 7a8da5fd93117..e2ed299f1a4dc 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class HalfFloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public HalfFloatFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public HalfFloatFieldDataGenerator(String fieldName, DataSource dataSource) { var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> halfFloats.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "half_float"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java index 9a942fca3ba14..f2fe8ed8362e5 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class IntegerFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public IntegerFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public IntegerFieldDataGenerator(String fieldName, DataSource dataSource) { var ints = dataSource.get(new DataSourceRequest.IntegerGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> ints.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "integer"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java index 8a37c73d00397..8dc4d8b8767c4 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class KeywordFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public KeywordFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) { var strings = dataSource.get(new DataSourceRequest.StringGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> strings.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "keyword"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java index 1c240fd10a953..f17610e501ed7 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class LongFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public LongFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public LongFieldDataGenerator(String fieldName, DataSource dataSource) { var longs = dataSource.get(new DataSourceRequest.LongGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> longs.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "long"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java index 359a725aed941..008dd04179dcd 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class ScaledFloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public ScaledFloatFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public ScaledFloatFieldDataGenerator(String fieldName, DataSource dataSource) { var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "scaled_float"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java index 8a056f00701d6..85bff2c85e538 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class ShortFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public ShortFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public ShortFieldDataGenerator(String fieldName, DataSource dataSource) { var shorts = dataSource.get(new DataSourceRequest.ShortGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> shorts.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "short"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java index d836d5625a89c..329f684bef70d 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java @@ -9,49 +9,25 @@ package org.elasticsearch.logsdb.datageneration.fields.leaf; -import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Map; import java.util.function.Supplier; public class UnsignedLongFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; - private final Map mappingParameters; - public UnsignedLongFieldDataGenerator( - String fieldName, - DataSource dataSource, - DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator - ) { + public UnsignedLongFieldDataGenerator(String fieldName, DataSource dataSource) { var unsignedLongs = dataSource.get(new DataSourceRequest.UnsignedLongGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> unsignedLongs.generator().get()); - this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override - public CheckedConsumer mappingWriter() { - return b -> { - b.startObject().field("type", "unsigned_long"); - - for (var entry : mappingParameters.entrySet()) { - b.field(entry.getKey(), entry.getValue()); - } - - b.endObject(); - }; - } - - @Override - public CheckedConsumer fieldValueGenerator() { - return b -> b.value(valueGenerator.get()); + public Object generateValue() { + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index b59f1a5e5f029..363d34ca3ff86 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -23,13 +23,11 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.builder.SubSearchSourceBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.rank.RankBuilder; import org.elasticsearch.search.rescore.RescorerBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; import org.elasticsearch.search.slice.SliceBuilder; @@ -122,7 +120,6 @@ public static SearchRequest randomSearchRequest(Supplier ra public static SearchSourceBuilder randomSearchSourceBuilder( Supplier randomHighlightBuilder, Supplier randomSuggestBuilder, - Supplier rankContextBuilderSupplier, Supplier> randomRescoreBuilder, Supplier> randomExtBuilders, Supplier randomCollapseBuilder, @@ -250,17 +247,6 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } if (randomBoolean()) { builder.query(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20))); - } else if (randomBoolean()) { - builder.subSearches( - List.of( - new SubSearchSourceBuilder( - QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)) - ), - new SubSearchSourceBuilder( - QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)) - ) - ) - ); } if (randomBoolean()) { builder.postFilter(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20))); @@ -354,9 +340,6 @@ public static SearchSourceBuilder randomSearchSourceBuilder( if (randomBoolean()) { builder.suggest(randomSuggestBuilder.get()); } - if (randomBoolean()) { - builder.rankBuilder(rankContextBuilderSupplier.get()); - } if (randomBoolean()) { int numRescores = randomIntBetween(1, 5); for (int i = 0; i < numRescores; i++) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index c4e1c6c7a0681..fd376fcd07688 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -830,9 +830,4 @@ protected void doClose() throws IOException { assertTrue(ThreadPool.terminate(testExecutor, 10, TimeUnit.SECONDS)); } } - - public DiscoveryNode getLocalDiscoNode() { - return this.getLocalNode(); - } - } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 34f67ac78a41c..4595fbf286077 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -668,7 +668,7 @@ public void testVoidMessageCompressed() throws Exception { ) .build(); ConnectionProfile connectionProfile = ConnectionProfile.buildDefaultConnectionProfile(settingsWithCompress); - connectToNode(serviceC, serviceA.getLocalDiscoNode(), connectionProfile); + connectToNode(serviceC, serviceA.getLocalNode(), connectionProfile); Future res = submitRequest( serviceC, @@ -725,7 +725,7 @@ public void testHelloWorldCompressed() throws Exception { ) .build(); ConnectionProfile connectionProfile = ConnectionProfile.buildDefaultConnectionProfile(settingsWithCompress); - connectToNode(serviceC, serviceA.getLocalDiscoNode(), connectionProfile); + connectToNode(serviceC, serviceA.getLocalNode(), connectionProfile); Future res = submitRequest( serviceC, @@ -795,8 +795,8 @@ public void testIndexingDataCompression() throws Exception { ) .build(); ConnectionProfile connectionProfile = ConnectionProfile.buildDefaultConnectionProfile(settingsWithCompress); - connectToNode(serviceC, serviceA.getLocalDiscoNode(), connectionProfile); - connectToNode(serviceA, serviceC.getLocalDiscoNode(), connectionProfile); + connectToNode(serviceC, serviceA.getLocalNode(), connectionProfile); + connectToNode(serviceA, serviceC.getLocalNode(), connectionProfile); TransportResponseHandler responseHandler = new TransportResponseHandler<>() { @Override @@ -821,14 +821,14 @@ public void handleException(TransportException exp) { Future compressed = submitRequest( serviceC, - serviceA.getLocalDiscoNode(), + serviceA.getLocalNode(), "internal:sayHello", new StringMessageRequest(text, -1, true), responseHandler ); Future uncompressed = submitRequest( serviceA, - serviceC.getLocalDiscoNode(), + serviceC.getLocalNode(), "internal:sayHello", new StringMessageRequest(text, -1, false), responseHandler @@ -1049,7 +1049,7 @@ public void onAfter() { ignoringRequestHandler ); serviceB = newService; - nodeB = newService.getLocalDiscoNode(); + nodeB = newService.getLocalNode(); connectToNode(serviceB, nodeA); connectToNode(serviceA, nodeB); } else if (serviceA.nodeConnected(nodeB)) { @@ -3419,7 +3419,7 @@ public void sendRequest( ) { final CountDownLatch latch = new CountDownLatch(1); serviceC.connectToNode( - serviceA.getLocalDiscoNode(), + serviceA.getLocalNode(), ConnectionProfile.buildDefaultConnectionProfile(Settings.EMPTY), new ActionListener<>() { @Override diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java similarity index 65% rename from test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java rename to test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java index 1178a32e46d31..bac61daf554d5 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java @@ -17,25 +17,27 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -public class DataGeneratorSnapshotTests extends ESTestCase { +public class DataGenerationSnapshotTests extends ESTestCase { public void testSnapshot() throws Exception { - var dataGenerator = new DataGenerator( - DataGeneratorSpecification.builder() - .withDataSourceHandlers(List.of(new DataSourceOverrides())) - .withMaxFieldCountPerLevel(5) - .withMaxObjectDepth(2) - .build() - ); + var specification = DataGeneratorSpecification.builder() + .withDataSourceHandlers(List.of(new DataSourceOverrides())) + .withMaxFieldCountPerLevel(5) + .withMaxObjectDepth(2) + .build(); - var mapping = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint(); - dataGenerator.writeMapping(mapping); + var template = new TemplateGenerator(specification).generate(); + var mapping = new MappingGenerator(specification).generate(template); - var document = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint(); - dataGenerator.generateDocument(document); + var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint(); + mappingXContent.map(mapping.raw()); + + var documentXContent = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint(); + documentXContent.map(new DocumentGenerator(specification).generate(template, mapping)); var expectedMapping = """ { @@ -49,53 +51,66 @@ public void testSnapshot() throws Exception { "dynamic" : "false", "properties" : { "f3" : { - "type" : "keyword", - "store" : "true" + "store" : "true", + "type" : "keyword" }, "f4" : { - "type" : "long", - "index" : "false" + "index" : "false", + "type" : "long" } - } + }, + "type" : "object" }, "f5" : { "dynamic" : "false", "properties" : { "f6" : { - "type" : "keyword", - "store" : "true" + "store" : "true", + "type" : "keyword" }, "f7" : { - "type" : "long", - "index" : "false" + "dynamic" : "false", + "properties" : { + "f8" : { + "index" : "false", + "type" : "long" + }, + "f9" : { + "store" : "true", + "type" : "keyword" + } + }, + "type" : "nested" } - } + }, + "type" : "object" } - } + }, + "type" : "object" }, - "f8" : { - "type" : "nested", + "f10" : { "dynamic" : "false", "properties" : { - "f9" : { - "type" : "nested", + "f11" : { "dynamic" : "false", "properties" : { - "f10" : { - "type" : "keyword", - "store" : "true" + "f12" : { + "index" : "false", + "type" : "long" }, - "f11" : { - "type" : "long", - "index" : "false" + "f13" : { + "store" : "true", + "type" : "keyword" } - } + }, + "type" : "nested" }, - "f12" : { - "type" : "keyword", - "store" : "true" + "f14" : { + "index" : "false", + "type" : "long" } - } + }, + "type" : "nested" } } } @@ -103,48 +118,48 @@ public void testSnapshot() throws Exception { var expectedDocument = """ { - "f1" : [ + "f1" : { + "f2" : { + "f3" : null, + "f4" : 3 + }, + "f5" : { + "f6" : [ + "string4", + "string5" + ], + "f7" : { + "f8" : null, + "f9" : "string6" + } + } + }, + "f10" : [ { - "f2" : { - "f3" : [ + "f11" : { + "f12" : [ null, - "string1" + 0 ], - "f4" : 0 + "f13" : "string1" }, - "f5" : { - "f6" : "string2", - "f7" : null - } + "f14" : 1 }, { - "f2" : { - "f3" : [ - "string3", - "string4" - ], - "f4" : 1 + "f11" : { + "f12" : null, + "f13" : [ + "string2", + "string3" + ] }, - "f5" : { - "f6" : null, - "f7" : 2 - } + "f14" : 2 } - ], - "f8" : { - "f9" : { - "f10" : [ - "string5", - "string6" - ], - "f11" : null - }, - "f12" : "string7" - } + ] }"""; - assertEquals(expectedMapping, Strings.toString(mapping)); - assertEquals(expectedDocument, Strings.toString(document)); + assertEquals(expectedMapping, Strings.toString(mappingXContent)); + assertEquals(expectedDocument, Strings.toString(documentXContent)); } private static class DataSourceOverrides implements DataSourceHandler { @@ -173,7 +188,6 @@ public DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper reque @Override public DataSourceResponse.ArrayWrapper handle(DataSourceRequest.ArrayWrapper request) { - return new DataSourceResponse.ArrayWrapper((values) -> () -> { if (generateArrayChecks++ % 4 == 0) { // we have nulls so can't use List.of @@ -206,14 +220,19 @@ public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeG return new DataSourceResponse.FieldTypeGenerator(() -> { if (fieldType == FieldType.KEYWORD) { fieldType = FieldType.LONG; - return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD, false); + return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD); } fieldType = FieldType.KEYWORD; - return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG, false); + return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG); }); } + @Override + public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.DynamicMappingGenerator request) { + return new DataSourceResponse.DynamicMappingGenerator((ignored) -> false); + } + @Override public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { if (request.fieldType() == FieldType.KEYWORD) { @@ -229,7 +248,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques @Override public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { - return new DataSourceResponse.ObjectMappingParametersGenerator(() -> Map.of("dynamic", "false")); + return new DataSourceResponse.ObjectMappingParametersGenerator(() -> new HashMap<>(Map.of("dynamic", "false"))); } } diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java similarity index 69% rename from test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java rename to test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java index b5fa66bf3006b..d9750328ff3fa 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; -import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; @@ -28,20 +27,8 @@ import java.util.List; import java.util.Optional; -public class DataGeneratorTests extends ESTestCase { - public void testDataGeneratorSanity() throws IOException { - var dataGenerator = new DataGenerator(DataGeneratorSpecification.buildDefault()); - - var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); - dataGenerator.writeMapping(mapping); - - for (int i = 0; i < 1000; i++) { - var document = XContentBuilder.builder(XContentType.JSON.xContent()); - dataGenerator.generateDocument(document); - } - } - - public void testDataGeneratorProducesValidMappingAndDocument() throws IOException { +public class DataGenerationTests extends ESTestCase { + public void testDataGenerationProducesValidMappingAndDocument() throws IOException { // Make sure objects, nested objects and all field types are covered. var testChildFieldGenerator = new DataSourceResponse.ChildFieldGenerator() { private boolean dynamicSubObjectCovered = false; @@ -101,43 +88,34 @@ public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFiel @Override public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { - if (request.dynamicMapping() == DynamicMapping.FORBIDDEN || request.dynamicMapping() == DynamicMapping.SUPPORTED) { - return new DataSourceResponse.FieldTypeGenerator( - () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo( - FieldType.values()[generatedFields++ % FieldType.values().length], - false - ) - ); - } - - return new DataSourceResponse.FieldTypeGenerator(() -> { - var fieldType = FieldType.values()[generatedFields++ % FieldType.values().length]; - // Does not really work with dynamic mapping. - if (fieldType == FieldType.UNSIGNED_LONG) { - fieldType = FieldType.values()[generatedFields++ % FieldType.values().length]; - } + return new DataSourceResponse.FieldTypeGenerator( + () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo( + FieldType.values()[generatedFields++ % FieldType.values().length] + ) + ); - return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(fieldType, true); - }); } }; - var dataGenerator = new DataGenerator( - DataGeneratorSpecification.builder().withDataSourceHandlers(List.of(dataSourceOverride)).build() - ); + var specification = DataGeneratorSpecification.builder().withDataSourceHandlers(List.of(dataSourceOverride)).build(); + + var documentGenerator = new DocumentGenerator(specification); - var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); - dataGenerator.writeMapping(mapping); + var template = new TemplateGenerator(specification).generate(); + var mapping = new MappingGenerator(specification).generate(template); + + var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()); + mappingXContent.map(mapping.raw()); var mappingService = new MapperServiceTestCase() { @Override protected Collection getPlugins() { return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin()); } - }.createMapperService(mapping); + }.createMapperService(mappingXContent); var document = XContentBuilder.builder(XContentType.JSON.xContent()); - dataGenerator.generateDocument(document); + document.map(documentGenerator.generate(template, mapping)); mappingService.documentMapper().parse(new SourceToParse("1", BytesReference.bytes(document), XContentType.JSON)); } @@ -187,19 +165,17 @@ public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectAr @Override public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { return new DataSourceResponse.FieldTypeGenerator( - () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG, false) + () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG) ); } }; - - var dataGenerator = new DataGenerator( - DataGeneratorSpecification.builder().withDataSourceHandlers(List.of(dataSourceOverride)).withMaxObjectDepth(2).build() - ); - - var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); - dataGenerator.writeMapping(mapping); - - var document = XContentBuilder.builder(XContentType.JSON.xContent()); - dataGenerator.generateDocument(document); + var specification = DataGeneratorSpecification.builder() + .withDataSourceHandlers(List.of(dataSourceOverride)) + .withMaxObjectDepth(2) + .build(); + + var template = new TemplateGenerator(specification).generate(); + var mapping = new MappingGenerator(specification).generate(template); + var ignored = new DocumentGenerator(specification).generate(template, mapping); } } diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/TemplateGeneratorTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/TemplateGeneratorTests.java new file mode 100644 index 0000000000000..3f57addcac51e --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/TemplateGeneratorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +public class TemplateGeneratorTests extends ESTestCase { + public void testSanity() { + var specification = DataGeneratorSpecification.buildDefault(); + var generator = new TemplateGenerator(specification); + + var template = generator.generate(); + validateMappingTemplate(template.template()); + } + + private void validateMappingTemplate(Map template) { + // Just a high level sanity check, we test that mapping and documents make sense in DataGenerationTests. + for (var entry : template.entrySet()) { + assertNotNull(entry.getKey()); + assertFalse(entry.getKey().isEmpty()); + switch (entry.getValue()) { + case Template.Leaf leaf -> { + assertEquals(entry.getKey(), leaf.name()); + assertNotNull(leaf.type()); + } + case Template.Object object -> { + assertEquals(entry.getKey(), object.name()); + assertNotNull(object.children()); + validateMappingTemplate(object.children()); + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java index 5bf35dea51af6..baec748141903 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.protocol.xpack.license.GetLicenseRequest; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -26,7 +27,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.core.RestApiVersion.V_7; import static org.elasticsearch.core.RestApiVersion.V_8; import static org.elasticsearch.core.RestApiVersion.onOrAfter; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -57,18 +57,12 @@ public String getName() { * The licenses are sorted by latest issue_date */ @Override + @UpdateForV9(owner = UpdateForV9.Owner.SECURITY) // remove support for accept_enterprise param public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final Map overrideParams = Maps.newMapWithExpectedSize(2); overrideParams.put(License.REST_VIEW_MODE, "true"); overrideParams.put(License.LICENSE_VERSION_MODE, String.valueOf(License.VERSION_CURRENT)); - if (request.getRestApiVersion() == V_7) { - // Hide enterprise licenses by default, there is an opt-in flag to show them - final boolean hideEnterprise = request.paramAsBoolean("accept_enterprise", false) == false; - final int licenseVersion = hideEnterprise ? License.VERSION_CRYPTO_ALGORITHMS : License.VERSION_CURRENT; - overrideParams.put(License.LICENSE_VERSION_MODE, String.valueOf(licenseVersion)); - overrideParams.put(License.XCONTENT_HIDE_ENTERPRISE, String.valueOf(hideEnterprise)); - } // In 7.x, there was an opt-in flag to show "enterprise" licenses. In 8.0 the flag is deprecated and can only be true // TODO Remove this from 9.0 if (request.hasParam("accept_enterprise")) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java index 2f9b125352e9c..b20f1a9d9ce23 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java @@ -14,8 +14,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.license.License; import org.elasticsearch.protocol.xpack.license.LicenseStatus; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -206,24 +204,8 @@ public int hashCode() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("uid", uid); - - if (builder.getRestApiVersion() == RestApiVersion.V_7 && params.paramAsBoolean("accept_enterprise", false) == false) { - if (License.LicenseType.ENTERPRISE.getTypeName().equals(type)) { - builder.field("type", License.LicenseType.PLATINUM.getTypeName()); - } else { - builder.field("type", type); - } - - if (License.OperationMode.ENTERPRISE.description().equals(mode)) { - builder.field("mode", License.OperationMode.PLATINUM.description()); - } else { - builder.field("mode", mode); - } - } else { - builder.field("type", type); - builder.field("mode", mode); - } - + builder.field("type", type); + builder.field("mode", mode); builder.field("status", status.label()); if (expiryDate != BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS) { builder.timestampFieldsFromUnixEpochMillis("expiry_date_in_millis", "expiry_date", expiryDate); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java index 3e6c00eeadba4..d648dd1c7edf8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java @@ -127,7 +127,7 @@ public void performAction( if (targetFailureStore) { rolloverRequest.setIndicesOptions( IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build() ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java index 7b751994222b1..67f65481ef63e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java @@ -248,7 +248,7 @@ RolloverRequest createRolloverRequest( if (targetFailureStore) { rolloverRequest.setIndicesOptions( IndicesOptions.builder(rolloverRequest.indicesOptions()) - .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false)) + .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES) .build() ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java index 381fb30b65e68..ca81509920ca5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java @@ -179,11 +179,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Job.ID.getPreferredName(), jobId); builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep()); builder.field(FORCE.getPreferredName(), force); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(DEPRECATED_ALLOW_NO_JOBS_PARAM, allowNoMatch); - } else { - builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); - } + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java index 12565ea49e058..12c8696a50626 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java @@ -243,11 +243,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (end != null) { builder.field(END.getPreferredName(), String.valueOf(end)); } - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(DEPRECATED_ALLOW_NO_JOBS_PARAM, allowNoMatch); - } else { - builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); - } + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java index 2fd00a5ea3983..cb89cfa8cd0e7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java @@ -169,11 +169,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); builder.field(TIMEOUT.getPreferredName(), stopTimeout.getStringRep()); builder.field(FORCE.getPreferredName(), force); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.field(DEPRECATED_ALLOW_NO_DATAFEEDS_PARAM, allowNoMatch); - } else { - builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); - } + builder.field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java index 15958e9396d81..afb17644303bb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java @@ -308,8 +308,8 @@ public void onFailure(Exception e) { verify(indicesClient, Mockito.only()).rolloverIndex(requestCaptor.capture(), Mockito.any()); RolloverRequest request = requestCaptor.getValue(); - assertThat(request.indicesOptions().failureStoreOptions().includeFailureIndices(), equalTo(failureStoreIndex)); - assertThat(request.indicesOptions().failureStoreOptions().includeRegularIndices(), not(equalTo(failureStoreIndex))); + assertThat(request.indicesOptions().includeFailureIndices(), equalTo(failureStoreIndex)); + assertThat(request.indicesOptions().includeRegularIndices(), not(equalTo(failureStoreIndex))); } public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index 51f92bcda7da4..c0141da2432ce 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -59,7 +59,6 @@ import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; -import static org.elasticsearch.xpack.ql.plugin.TransportActionUtils.executeRequestWithRetryAttempt; public final class TransportEqlSearchAction extends HandledTransportAction implements @@ -236,22 +235,11 @@ public static void operation( new TaskId(nodeId, task.getId()), task ); - executeRequestWithRetryAttempt( - clusterService, - listener::onFailure, - onFailure -> planExecutor.eql( - cfg, - request.query(), - params, - wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())), onFailure) - ), - node -> transportService.sendRequest( - node, - EqlSearchAction.NAME, - request, - new ActionListenerResponseHandler<>(listener, EqlSearchResponse::new, EsExecutors.DIRECT_EXECUTOR_SERVICE) - ), - log + planExecutor.eql( + cfg, + request.query(), + params, + wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())), listener::onFailure) ); } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index b23703c6d8b66..cb1a7b2eb6fe0 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -266,6 +266,8 @@ public enum DataType { .sorted(Comparator.comparing(DataType::typeName)) .toList(); + private static final Collection STRING_TYPES = DataType.types().stream().filter(DataType::isString).toList(); + private static final Map NAME_TO_TYPE = TYPES.stream().collect(toUnmodifiableMap(DataType::typeName, t -> t)); private static final Map ES_TO_TYPE; @@ -292,6 +294,10 @@ public static Collection types() { return TYPES; } + public static Collection stringTypes() { + return STRING_TYPES; + } + /** * Resolve a type from a name. This name is sometimes user supplied, * like in the case of {@code ::} and is sometimes the name diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java index 78b395503e700..7c91a506697c1 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java @@ -30,11 +30,9 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; -import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT; -import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime; @@ -62,7 +60,7 @@ public static Converter converterFor(DataType from, DataType to) { return DefaultConverter.TO_NULL; } // proper converters - if (to == KEYWORD || to == TEXT) { + if (isString(to)) { return conversionToString(from); } if (to == LONG) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java index 1fc2430393c98..c66c960dd8a99 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/spatial/CentroidPointAggregator.java @@ -58,7 +58,7 @@ public static void evaluateIntermediate(CentroidState state, DriverContext drive } public static Block evaluateFinal(CentroidState state, DriverContext driverContext) { - return driverContext.blockFactory().newConstantBytesRefBlockWith(state.encodeCentroidResult(), 1); + return state.toBlock(driverContext.blockFactory()); } public static void combineStates(GroupingCentroidState current, int groupId, GroupingCentroidState state, int statePosition) { @@ -181,10 +181,14 @@ public void add(double x, double dx, double y, double dy, long count) { this.count += count; } - protected BytesRef encodeCentroidResult() { - double x = xSum.value() / count; - double y = ySum.value() / count; - return encode(x, y); + protected Block toBlock(BlockFactory blockFactory) { + if (count > 0) { + double x = xSum.value() / count; + double y = ySum.value() / count; + return blockFactory.newConstantBytesRefBlockWith(encode(x, y), 1); + } else { + return blockFactory.newConstantNullBlock(1); + } } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java index ab785e739d080..9e07f9c8f5faf 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java @@ -441,7 +441,7 @@ public void sendResponse(TransportResponse transportResponse) { PlainActionFuture sourceCompletionFuture = new PlainActionFuture<>(); sourceHandler.addCompletionListener(sourceCompletionFuture); ExchangeSinkHandler sinkHandler = exchange1.createSinkHandler(exchangeId, randomIntBetween(1, 128)); - Transport.Connection connection = node0.getConnection(node1.getLocalDiscoNode()); + Transport.Connection connection = node0.getConnection(node1.getLocalNode()); sourceHandler.addRemoteSink(exchange0.newRemoteSink(task, exchangeId, node0, connection), randomIntBetween(1, 5)); Exception err = expectThrows( Exception.class, diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 4fa6ac3009654..e3199649a91be 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -32,6 +32,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -670,6 +671,10 @@ public void testErrorMessageForArrayValuesInParams() throws IOException { } public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOException { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); bulkLoadTestData(10); // positive var query = requestObjectBuilder().query( diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json index 98a3794d977e2..cd572012c6b3a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json @@ -12,11 +12,17 @@ "centroid": { "type": "geo_point" }, + "lk": { + "type": "keyword" + }, "location": { "type": "geo_point" }, "subset": { "type": "geo_point" + }, + "disjoint": { + "type": "geo_point" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv index c5d7c7f4ee305..efb0aaaa29a4b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv @@ -1,14 +1,14 @@ -id:l, intersects:boolean, within:boolean, centroid:geo_point, location:geo_point, subset:geo_point -0, true, true, "POINT(0.0 5.0)", ["POINT(5 5)", "POINT(-5 5)"], "POINT(5 5)" -1, true, true, "POINT(0.5 0.5)", ["POINT(0 1)","POINT(1 0)"], "POINT(0 1)" -2, true, true, "POINT(9.0 9.0)", "POINT(9 9)", "POINT(9 9)" -3, true, true, "POINT(0.0 0.0)", ["POINT(-9 -9)","POINT(9 9)"], "POINT(-9 -9)" -4, true, false, "POINT(10.0 10.0)", ["POINT(5 5)", "POINT(15 15)"], "POINT(5 5)" -5, true, false, "POINT(5.5 5.5)", ["POINT(0 0)","POINT(11 11)"], "POINT(0 0)" -6, true, false, "POINT(0.0 -5.0)", ["POINT(-9 -19)","POINT(9 9)"], "POINT(-9 -19)" -7, false, false, "POINT(10.0 10.0)", ["POINT(5 15)", "POINT(15 5)"], "POINT(5 15)" -8, false, false, "POINT(5.5 5.5)", ["POINT(0 11)","POINT(11 0)"], "POINT(0 11)" -9, false, false, "POINT(19.0 9.0)", "POINT(19 9)", "POINT(19 9)" -10, false, false, "POINT(5.0 -5.0)", ["POINT(-9 -19)","POINT(19 9)"], "POINT(-9 -19)" -11, true, false, "POINT(0.0 0.0)", ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"], ["POINT(0 0)", "POINT(55 55)"] -12, true, false, "POINT(0.0 0.0)", ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"], ["POINT(0 0)", "POINT(55 54)"] +id:l, intersects:boolean, within:boolean, centroid:geo_point, lk:keyword, location:geo_point, subset:geo_point, disjoint:geo_point +0, true, true, "POINT(0.0 5.0)", ["POINT(5 5)", "POINT(-5 5)"], ["POINT(5 5)", "POINT(-5 5)"], "POINT(5 5)", ["POINT(55 55)", "POINT(65 65)"] +1, true, true, "POINT(0.5 0.5)", ["POINT(0 1)","POINT(1 0)"], ["POINT(0 1)","POINT(1 0)"], "POINT(0 1)", ["POINT(55 55)", "POINT(65 65)"] +2, true, true, "POINT(9.0 9.0)", "POINT(9 9)", "POINT(9 9)", "POINT(9 9)", ["POINT(55 55)", "POINT(65 65)"] +3, true, true, "POINT(0.0 0.0)", ["POINT(-9 -9)","POINT(9 9)"], ["POINT(-9 -9)","POINT(9 9)"], "POINT(-9 -9)", ["POINT(55 55)", "POINT(65 65)"] +4, true, false, "POINT(10.0 10.0)", ["POINT(5 5)", "POINT(15 15)"], ["POINT(5 5)", "POINT(15 15)"], "POINT(5 5)", ["POINT(55 55)", "POINT(65 65)"] +5, true, false, "POINT(5.5 5.5)", ["POINT(0 0)","POINT(11 11)"], ["POINT(0 0)","POINT(11 11)"], "POINT(0 0)", ["POINT(55 55)", "POINT(65 65)"] +6, true, false, "POINT(0.0 -5.0)", ["POINT(-9 -19)","POINT(9 9)"], ["POINT(-9 -19)","POINT(9 9)"], "POINT(-9 -19)", ["POINT(55 55)", "POINT(65 65)"] +7, false, false, "POINT(10.0 10.0)", ["POINT(5 15)", "POINT(15 5)"], ["POINT(5 15)", "POINT(15 5)"], "POINT(5 15)", ["POINT(55 55)", "POINT(65 65)"] +8, false, false, "POINT(5.5 5.5)", ["POINT(0 11)","POINT(11 0)"], ["POINT(0 11)","POINT(11 0)"], "POINT(0 11)", ["POINT(55 55)", "POINT(65 65)"] +9, false, false, "POINT(19.0 9.0)", "POINT(19 9)", "POINT(19 9)", "POINT(19 9)", ["POINT(55 55)", "POINT(65 65)"] +10, false, false, "POINT(5.0 -5.0)", ["POINT(-9 -19)","POINT(19 9)"], ["POINT(-9 -19)","POINT(19 9)"], "POINT(-9 -19)", ["POINT(55 55)", "POINT(65 65)"] +11, true, false, "POINT(0.0 0.0)", ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"], ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"], ["POINT(0 0)", "POINT(55 55)"], ["POINT(55 55)", "POINT(65 65)"] +12, true, false, "POINT(0.0 0.0)", ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"], ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"], ["POINT(0 0)", "POINT(55 54)"], ["POINT(55 55)", "POINT(65 65)"] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv-spec index 6a3521c558fa9..ed9434e9241fc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_points.csv-spec @@ -60,6 +60,92 @@ id:l | intersects:boolean | within:boolean | st_intersects:boolean | st_within:b 12 | true | false | true | false | false | "POINT(0.0 0.0)" | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] ; +spatialDistance +required_capability: st_distance +required_capability: spatial_distance_supports_multivalues + +FROM multivalue_points +| EVAL distance_origin=ST_DISTANCE(location, TO_GEOPOINT("POINT(0 0)")) +| EVAL distance_subset=ST_DISTANCE(location, subset) +| EVAL distance_disjoint=ST_DISTANCE(location, disjoint) +| EVAL distance_far=ST_DISTANCE(location, TO_GEOPOINT("POINT(55 55)")) +| KEEP id, distance_origin, distance_subset, distance_disjoint, distance_far, location, subset, disjoint +| SORT id +; + +id:l | distance_origin:d | distance_subset:d | distance_disjoint:d | distance_far:d | location:geo_point | subset:geo_point | disjoint:geo_point +0 | 785768.2986429982 | 0.0 | 7114305.127148048 | 7114305.127148048 | ["POINT(5 5)", "POINT(-5 5)"] | "POINT(5 5)" | ["POINT(55 55)", "POINT(65 65)"] +1 | 111195.07310665186 | 0.0 | 7775410.295475619 | 7775410.295475619 | ["POINT(0 1)","POINT(1 0)"] | "POINT(0 1)" | ["POINT(55 55)", "POINT(65 65)"] +2 | 1412359.654429245 | 0.0 | 6511042.96320646 | 6511042.96320646 | "POINT(9 9)" | "POINT(9 9)" | ["POINT(55 55)", "POINT(65 65)"] +3 | 1412359.654429245 | 0.0 | 6511042.96320646 | 6511042.96320646 | ["POINT(-9 -9)","POINT(9 9)"] | "POINT(-9 -9)" | ["POINT(55 55)", "POINT(65 65)"] +4 | 785768.2986429982 | 0.0 | 5612483.563947934 | 5612483.563947934 | ["POINT(5 5)", "POINT(15 15)"] | "POINT(5 5)" | ["POINT(55 55)", "POINT(65 65)"] +5 | 0.0 | 0.0 | 6210584.082742179 | 6210584.082742179 | ["POINT(0 0)","POINT(11 11)"] | "POINT(0 0)" | ["POINT(55 55)", "POINT(65 65)"] +6 | 1412359.654429245 | 0.0 | 6511042.96320646 | 6511042.96320646 | ["POINT(-9 -19)","POINT(9 9)"] | "POINT(-9 -19)" | ["POINT(55 55)", "POINT(65 65)"] +7 | 1756128.8127267 | 0.0 | 6157935.487780502 | 6157935.487780502 | ["POINT(5 15)", "POINT(15 5)"] | "POINT(5 15)" | ["POINT(55 55)", "POINT(65 65)"] +8 | 1223145.8694147274 | 0.0 | 6823348.604540896 | 6823348.604540896 | ["POINT(0 11)","POINT(11 0)"] | "POINT(0 11)" | ["POINT(55 55)", "POINT(65 65)"] +9 | 2329806.5462053656 | 0.0 | 6014935.534393433 | 6014935.534393433 | "POINT(19 9)" | "POINT(19 9)" | ["POINT(55 55)", "POINT(65 65)"] +10 | 2329806.5462053656 | 0.0 | 6014935.534393433 | 6014935.534393433 | ["POINT(-9 -19)","POINT(19 9)"] | "POINT(-9 -19)" | ["POINT(55 55)", "POINT(65 65)"] +11 | 0.0 | 0.0 | 0.0 | 0.0 | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] | ["POINT(0 0)", "POINT(55 55)"] | ["POINT(55 55)", "POINT(65 65)"] +12 | 0.0 | 0.0 | 0.0 | 0.0 | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] | ["POINT(0 0)", "POINT(55 54)"] | ["POINT(55 55)", "POINT(65 65)"] +; + +whereSpatialDistanceTwoFields +required_capability: st_distance +required_capability: spatial_distance_supports_multivalues + +FROM multivalue_points +| WHERE ST_DISTANCE(location, disjoint) < 6200000 +| KEEP id, location +| SORT id +; + +id:l | location:geo_point +4 | ["POINT(5 5)", "POINT(15 15)"] +7 | ["POINT(5 15)", "POINT(15 5)"] +9 | "POINT(19 9)" +10 | ["POINT(-9 -19)","POINT(19 9)"] +11 | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] +12 | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] +; + +whereSpatialDistanceKeyword +required_capability: st_distance +required_capability: spatial_distance_supports_multivalues + +FROM multivalue_points +| EVAL lk=lk::geo_point +| WHERE ST_DISTANCE(lk, disjoint) < 6200000 +| KEEP id, location +| SORT id +; + +id:l | location:geo_point +4 | ["POINT(5 5)", "POINT(15 15)"] +7 | ["POINT(5 15)", "POINT(15 5)"] +9 | "POINT(19 9)" +10 | ["POINT(-9 -19)","POINT(19 9)"] +11 | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] +12 | ["POINT(0 0)", "POINT(55 55)", "POINT(-55 -55)"] +; + +spatialDistanceFilterTwoFieldStats +required_capability: st_distance +required_capability: spatial_distance_supports_multivalues + +FROM multivalue_points +| EVAL distance_disjoint=ST_DISTANCE(location, disjoint) +| WHERE distance_disjoint > 0 +| EVAL kkm = TO_INTEGER(distance_disjoint / 1000000) +| STATS c=COUNT(*) BY kkm +| SORT kkm ASC +; + +c:long | kkm:integer +5 | 6 +5 | 7 +1 | 8 +; + whereIntersectsPolygon required_capability: st_intersects diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index b72c8bcb05ae9..c1c4538c7393d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -616,6 +616,42 @@ location:geo_point | city_location:geo_point | count:long POINT (0 0) | POINT (0 0) | 1 ; +airportCityLocationPointIntersectionCentroidGroups +required_capability: st_intersects + +FROM airports_mp +| WHERE ST_INTERSECTS(location, city_location) +| STATS location=ST_CENTROID_AGG(location), city_location=ST_CENTROID_AGG(city_location), count=COUNT() BY country +; + +location:geo_point | city_location:geo_point | count:long | country:k +POINT (0 0) | POINT (0 0) | 1 | Atlantis +; + +airportCityLocationPointIntersectionNullCentroid +required_capability: st_intersects +required_capability: spatial_centroid_no_records + +FROM airports +| WHERE ST_INTERSECTS(location, city_location) +| STATS location=ST_CENTROID_AGG(location), city_location=ST_CENTROID_AGG(city_location), count=COUNT() +; + +location:geo_point | city_location:geo_point | count:long +null | null | 0 +; + +airportCityLocationPointIntersectionNullCentroidGroups +required_capability: st_intersects + +FROM airports +| WHERE ST_INTERSECTS(location, city_location) +| STATS location=ST_CENTROID_AGG(location), city_location=ST_CENTROID_AGG(city_location), count=COUNT() BY country +; + +location:geo_point | city_location:geo_point | count:long | country:k +; + ############################################### # Tests for ST_DISJOINT on GEO_POINT type @@ -1948,14 +1984,15 @@ wkt:keyword | pt:cartesian_point cartesianCentroidFromAirportsAfterPointContainsPolygonPredicate required_capability: st_contains_within +required_capability: spatial_centroid_no_records FROM airports_web | WHERE ST_CONTAINS(location, TO_CARTESIANSHAPE("POLYGON((4700000 1600000, 4800000 1600000, 4800000 1700000, 4700000 1700000, 4700000 1600000))")) | STATS centroid=ST_CENTROID_AGG(location), count=COUNT() ; -centroid:cartesian_point | count:long -POINT (NaN NaN) | 0 +centroid:cartesian_point | count:long +null | 0 ; cartesianPointContainsPolygonPredicate @@ -2222,17 +2259,19 @@ wkt:keyword | pt:cartesian_point | distance:double twoCitiesPointDistanceCartesian required_capability: st_distance +required_capability: spatial_distance_supports_multivalues ROW p1 = TO_CARTESIANPOINT("POINT(-90.82814 29.79511)"), p2 = TO_CARTESIANPOINT("POINT(-90.79731509999999 29.8835389)") | EVAL d = ST_DISTANCE(p1, p2) ; p1:cartesian_point | p2:cartesian_point | d:double -POINT (-90.82814 29.79511) | POINT (-90.79731509999999 29.8835389) | 0.09364744959271905 +POINT (-90.82814 29.79511) | POINT (-90.79731509999999 29.8835389) | 0.09364636296011444 ; airportCartesianCityLocationPointDistance required_capability: st_distance +required_capability: spatial_distance_supports_multivalues FROM airports_web | EVAL distance = ST_DISTANCE(location, TO_CARTESIANPOINT("POINT(1402900 7490000)")) @@ -2240,12 +2279,13 @@ FROM airports_web | STATS distance=AVG(distance), min=min(distance), max=max(distance), count=COUNT() ; -distance:double | min:double | max:double | count:long -676858.3463435044 | 7358.02077507206 | 971112.9731194031 | 12 +distance:double | min:double | max:double | count:long +676858.3629952326 | 7358.012830411482 | 971113.1663946278 | 12 ; airportCartesianDistanceToCityCopenhagen required_capability: st_distance +required_capability: spatial_distance_supports_multivalues // tag::st_distance-airports_web[] FROM airports_web @@ -2257,7 +2297,7 @@ FROM airports_web // tag::st_distance-airports_web-result[] abbrev:k | name:text | location:cartesian_point | distance:d -CPH | Copenhagen | POINT(1408119.2975413958 7484813.53657096) | 7358.02077507206 +CPH | Copenhagen | POINT(1408119.2975413958 7484813.53657096) | 7358.012830411482 // end::st_distance-airports_web-result[] ; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 051e83129d12d..ffab261d3c174 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -305,8 +305,8 @@ mode PROJECT_MODE; PROJECT_PIPE : PIPE -> type(PIPE), popMode; PROJECT_DOT: DOT -> type(DOT); PROJECT_COMMA : COMMA -> type(COMMA); -PROJECT_PARAM : PARAM -> type(PARAM); -PROJECT_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); +PROJECT_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM); +PROJECT_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); fragment UNQUOTED_ID_BODY_WITH_PATTERN : (LETTER | DIGIT | UNDERSCORE | ASTERISK) @@ -340,8 +340,8 @@ RENAME_PIPE : PIPE -> type(PIPE), popMode; RENAME_ASSIGN : ASSIGN -> type(ASSIGN); RENAME_COMMA : COMMA -> type(COMMA); RENAME_DOT: DOT -> type(DOT); -RENAME_PARAM : PARAM -> type(PARAM); -RENAME_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); +RENAME_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM); +RENAME_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); AS : 'as'; @@ -413,8 +413,8 @@ ENRICH_FIELD_QUOTED_IDENTIFIER : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) ; -ENRICH_FIELD_PARAM : PARAM -> type(PARAM); -ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); +ENRICH_FIELD_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM); +ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); ENRICH_FIELD_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN) @@ -431,8 +431,8 @@ ENRICH_FIELD_WS mode MVEXPAND_MODE; MVEXPAND_PIPE : PIPE -> type(PIPE), popMode; MVEXPAND_DOT: DOT -> type(DOT); -MVEXPAND_PARAM : PARAM -> type(PARAM); -MVEXPAND_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); +MVEXPAND_PARAM : {this.isDevVersion()}? PARAM -> type(PARAM); +MVEXPAND_NAMED_OR_POSITIONAL_PARAM : {this.isDevVersion()}? NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM); MVEXPAND_QUOTED_IDENTIFIER : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER) diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index 7d489417ab4ca..f9f994f4ab329 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -195,7 +195,7 @@ identifier identifierPattern : ID_PATTERN - | parameter + | {this.isDevVersion()}? parameter ; constant @@ -218,7 +218,7 @@ parameter identifierOrParameter : identifier - | parameter + | {this.isDevVersion()}? parameter ; limitCommand diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java index a3fc2fadbf227..4917b71464dce 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java @@ -10,7 +10,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; @@ -26,63 +25,42 @@ public final class StDistanceCartesianPointDocValuesAndConstantEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; - private final EvalOperator.ExpressionEvaluator leftValue; + private final EvalOperator.ExpressionEvaluator left; - private final Point rightValue; + private final Point right; private final DriverContext driverContext; private Warnings warnings; public StDistanceCartesianPointDocValuesAndConstantEvaluator(Source source, - EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator left, Point right, DriverContext driverContext) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; this.driverContext = driverContext; } @Override public Block eval(Page page) { - try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) { - LongVector leftValueVector = leftValueBlock.asVector(); - if (leftValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock); - } - return eval(page.getPositionCount(), leftValueVector); + try (LongBlock leftBlock = (LongBlock) left.eval(page)) { + return eval(page.getPositionCount(), leftBlock); } } - public DoubleBlock eval(int positionCount, LongBlock leftValueBlock) { + public DoubleBlock eval(int positionCount, LongBlock leftBlock) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { position: for (int p = 0; p < positionCount; p++) { - if (leftValueBlock.isNull(p)) { - result.appendNull(); - continue position; + boolean allBlocksAreNulls = true; + if (!leftBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (leftValueBlock.getValueCount(p) != 1) { - if (leftValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } + if (allBlocksAreNulls) { result.appendNull(); continue position; } try { - result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue)); - } catch (IllegalArgumentException e) { - warnings().registerException(e); - result.appendNull(); - } - } - return result.build(); - } - } - - public DoubleBlock eval(int positionCount, LongVector leftValueVector) { - try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - position: for (int p = 0; p < positionCount; p++) { - try { - result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue)); + StDistance.processCartesianPointDocValuesAndConstant(result, p, leftBlock, this.right); } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); @@ -94,12 +72,12 @@ public DoubleBlock eval(int positionCount, LongVector leftValueVector) { @Override public String toString() { - return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]"; } @Override public void close() { - Releasables.closeExpectNoException(leftValue); + Releasables.closeExpectNoException(left); } private Warnings warnings() { @@ -117,25 +95,24 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory leftValue; + private final EvalOperator.ExpressionEvaluator.Factory left; - private final Point rightValue; + private final Point right; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, - Point rightValue) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, Point right) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; } @Override public StDistanceCartesianPointDocValuesAndConstantEvaluator get(DriverContext context) { - return new StDistanceCartesianPointDocValuesAndConstantEvaluator(source, leftValue.get(context), rightValue, context); + return new StDistanceCartesianPointDocValuesAndConstantEvaluator(source, left.get(context), right, context); } @Override public String toString() { - return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceCartesianPointDocValuesAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java index 1da164436a2cb..03c8ba1a04ab6 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndSourceEvaluator.java @@ -4,17 +4,12 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; -import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; -import org.elasticsearch.compute.data.DoubleVector; import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; @@ -29,79 +24,47 @@ public final class StDistanceCartesianPointDocValuesAndSourceEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; - private final EvalOperator.ExpressionEvaluator leftValue; + private final EvalOperator.ExpressionEvaluator left; - private final EvalOperator.ExpressionEvaluator rightValue; + private final EvalOperator.ExpressionEvaluator right; private final DriverContext driverContext; private Warnings warnings; public StDistanceCartesianPointDocValuesAndSourceEvaluator(Source source, - EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue, + EvalOperator.ExpressionEvaluator left, EvalOperator.ExpressionEvaluator right, DriverContext driverContext) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; this.driverContext = driverContext; } @Override public Block eval(Page page) { - try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) { - try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) { - LongVector leftValueVector = leftValueBlock.asVector(); - if (leftValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); - } - BytesRefVector rightValueVector = rightValueBlock.asVector(); - if (rightValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); - } - return eval(page.getPositionCount(), leftValueVector, rightValueVector).asBlock(); + try (LongBlock leftBlock = (LongBlock) left.eval(page)) { + try (BytesRefBlock rightBlock = (BytesRefBlock) right.eval(page)) { + return eval(page.getPositionCount(), leftBlock, rightBlock); } } } - public DoubleBlock eval(int positionCount, LongBlock leftValueBlock, - BytesRefBlock rightValueBlock) { + public DoubleBlock eval(int positionCount, LongBlock leftBlock, BytesRefBlock rightBlock) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef rightValueScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - if (leftValueBlock.isNull(p)) { - result.appendNull(); - continue position; + boolean allBlocksAreNulls = true; + if (!leftBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (leftValueBlock.getValueCount(p) != 1) { - if (leftValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } - result.appendNull(); - continue position; + if (!rightBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (rightValueBlock.isNull(p)) { + if (allBlocksAreNulls) { result.appendNull(); continue position; } - if (rightValueBlock.getValueCount(p) != 1) { - if (rightValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } - result.appendNull(); - continue position; - } - result.appendDouble(StDistance.processCartesianPointDocValuesAndSource(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch))); - } - return result.build(); - } - } - - public DoubleVector eval(int positionCount, LongVector leftValueVector, - BytesRefVector rightValueVector) { - try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - BytesRef rightValueScratch = new BytesRef(); - position: for (int p = 0; p < positionCount; p++) { - result.appendDouble(p, StDistance.processCartesianPointDocValuesAndSource(leftValueVector.getLong(p), rightValueVector.getBytesRef(p, rightValueScratch))); + StDistance.processCartesianPointDocValuesAndSource(result, p, leftBlock, rightBlock); } return result.build(); } @@ -109,12 +72,12 @@ public DoubleVector eval(int positionCount, LongVector leftValueVector, @Override public String toString() { - return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]"; } @Override public void close() { - Releasables.closeExpectNoException(leftValue, rightValue); + Releasables.closeExpectNoException(left, right); } private Warnings warnings() { @@ -132,25 +95,25 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory leftValue; + private final EvalOperator.ExpressionEvaluator.Factory left; - private final EvalOperator.ExpressionEvaluator.Factory rightValue; + private final EvalOperator.ExpressionEvaluator.Factory right; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, - EvalOperator.ExpressionEvaluator.Factory rightValue) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, + EvalOperator.ExpressionEvaluator.Factory right) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; } @Override public StDistanceCartesianPointDocValuesAndSourceEvaluator get(DriverContext context) { - return new StDistanceCartesianPointDocValuesAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context); + return new StDistanceCartesianPointDocValuesAndSourceEvaluator(source, left.get(context), right.get(context), context); } @Override public String toString() { - return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceCartesianPointDocValuesAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java index 61329ad7606d0..1085f71e95b73 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java @@ -4,14 +4,11 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; -import java.io.IOException; import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; @@ -28,66 +25,43 @@ public final class StDistanceCartesianSourceAndConstantEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; - private final EvalOperator.ExpressionEvaluator leftValue; + private final EvalOperator.ExpressionEvaluator left; - private final Point rightValue; + private final Point right; private final DriverContext driverContext; private Warnings warnings; public StDistanceCartesianSourceAndConstantEvaluator(Source source, - EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator left, Point right, DriverContext driverContext) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; this.driverContext = driverContext; } @Override public Block eval(Page page) { - try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) { - BytesRefVector leftValueVector = leftValueBlock.asVector(); - if (leftValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock); - } - return eval(page.getPositionCount(), leftValueVector); + try (BytesRefBlock leftBlock = (BytesRefBlock) left.eval(page)) { + return eval(page.getPositionCount(), leftBlock); } } - public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock) { + public DoubleBlock eval(int positionCount, BytesRefBlock leftBlock) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef leftValueScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - if (leftValueBlock.isNull(p)) { - result.appendNull(); - continue position; + boolean allBlocksAreNulls = true; + if (!leftBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (leftValueBlock.getValueCount(p) != 1) { - if (leftValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } + if (allBlocksAreNulls) { result.appendNull(); continue position; } try { - result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue)); - } catch (IllegalArgumentException | IOException e) { - warnings().registerException(e); - result.appendNull(); - } - } - return result.build(); - } - } - - public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) { - try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef leftValueScratch = new BytesRef(); - position: for (int p = 0; p < positionCount; p++) { - try { - result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue)); - } catch (IllegalArgumentException | IOException e) { + StDistance.processCartesianSourceAndConstant(result, p, leftBlock, this.right); + } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); } @@ -98,12 +72,12 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) { @Override public String toString() { - return "StDistanceCartesianSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceCartesianSourceAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]"; } @Override public void close() { - Releasables.closeExpectNoException(leftValue); + Releasables.closeExpectNoException(left); } private Warnings warnings() { @@ -121,25 +95,24 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory leftValue; + private final EvalOperator.ExpressionEvaluator.Factory left; - private final Point rightValue; + private final Point right; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, - Point rightValue) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, Point right) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; } @Override public StDistanceCartesianSourceAndConstantEvaluator get(DriverContext context) { - return new StDistanceCartesianSourceAndConstantEvaluator(source, leftValue.get(context), rightValue, context); + return new StDistanceCartesianSourceAndConstantEvaluator(source, left.get(context), right, context); } @Override public String toString() { - return "StDistanceCartesianSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceCartesianSourceAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java index c18c9a56fa77a..c8554a3041c89 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndSourceEvaluator.java @@ -4,14 +4,11 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; -import java.io.IOException; import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; @@ -27,88 +24,49 @@ public final class StDistanceCartesianSourceAndSourceEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; - private final EvalOperator.ExpressionEvaluator leftValue; + private final EvalOperator.ExpressionEvaluator left; - private final EvalOperator.ExpressionEvaluator rightValue; + private final EvalOperator.ExpressionEvaluator right; private final DriverContext driverContext; private Warnings warnings; public StDistanceCartesianSourceAndSourceEvaluator(Source source, - EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue, + EvalOperator.ExpressionEvaluator left, EvalOperator.ExpressionEvaluator right, DriverContext driverContext) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; this.driverContext = driverContext; } @Override public Block eval(Page page) { - try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) { - try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) { - BytesRefVector leftValueVector = leftValueBlock.asVector(); - if (leftValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); - } - BytesRefVector rightValueVector = rightValueBlock.asVector(); - if (rightValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); - } - return eval(page.getPositionCount(), leftValueVector, rightValueVector); + try (BytesRefBlock leftBlock = (BytesRefBlock) left.eval(page)) { + try (BytesRefBlock rightBlock = (BytesRefBlock) right.eval(page)) { + return eval(page.getPositionCount(), leftBlock, rightBlock); } } } - public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock, - BytesRefBlock rightValueBlock) { + public DoubleBlock eval(int positionCount, BytesRefBlock leftBlock, BytesRefBlock rightBlock) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef leftValueScratch = new BytesRef(); - BytesRef rightValueScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - if (leftValueBlock.isNull(p)) { - result.appendNull(); - continue position; - } - if (leftValueBlock.getValueCount(p) != 1) { - if (leftValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } - result.appendNull(); - continue position; + boolean allBlocksAreNulls = true; + if (!leftBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (rightValueBlock.isNull(p)) { - result.appendNull(); - continue position; + if (!rightBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (rightValueBlock.getValueCount(p) != 1) { - if (rightValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } + if (allBlocksAreNulls) { result.appendNull(); continue position; } try { - result.appendDouble(StDistance.processCartesianSourceAndSource(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch))); - } catch (IllegalArgumentException | IOException e) { - warnings().registerException(e); - result.appendNull(); - } - } - return result.build(); - } - } - - public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector, - BytesRefVector rightValueVector) { - try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef leftValueScratch = new BytesRef(); - BytesRef rightValueScratch = new BytesRef(); - position: for (int p = 0; p < positionCount; p++) { - try { - result.appendDouble(StDistance.processCartesianSourceAndSource(leftValueVector.getBytesRef(p, leftValueScratch), rightValueVector.getBytesRef(p, rightValueScratch))); - } catch (IllegalArgumentException | IOException e) { + StDistance.processCartesianSourceAndSource(result, p, leftBlock, rightBlock); + } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); } @@ -119,12 +77,12 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector, @Override public String toString() { - return "StDistanceCartesianSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceCartesianSourceAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]"; } @Override public void close() { - Releasables.closeExpectNoException(leftValue, rightValue); + Releasables.closeExpectNoException(left, right); } private Warnings warnings() { @@ -142,25 +100,25 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory leftValue; + private final EvalOperator.ExpressionEvaluator.Factory left; - private final EvalOperator.ExpressionEvaluator.Factory rightValue; + private final EvalOperator.ExpressionEvaluator.Factory right; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, - EvalOperator.ExpressionEvaluator.Factory rightValue) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, + EvalOperator.ExpressionEvaluator.Factory right) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; } @Override public StDistanceCartesianSourceAndSourceEvaluator get(DriverContext context) { - return new StDistanceCartesianSourceAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context); + return new StDistanceCartesianSourceAndSourceEvaluator(source, left.get(context), right.get(context), context); } @Override public String toString() { - return "StDistanceCartesianSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceCartesianSourceAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java index 2ac1ff6aeb0d8..8f37d3157fac6 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java @@ -10,7 +10,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; @@ -26,63 +25,42 @@ public final class StDistanceGeoPointDocValuesAndConstantEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; - private final EvalOperator.ExpressionEvaluator leftValue; + private final EvalOperator.ExpressionEvaluator left; - private final Point rightValue; + private final Point right; private final DriverContext driverContext; private Warnings warnings; public StDistanceGeoPointDocValuesAndConstantEvaluator(Source source, - EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator left, Point right, DriverContext driverContext) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; this.driverContext = driverContext; } @Override public Block eval(Page page) { - try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) { - LongVector leftValueVector = leftValueBlock.asVector(); - if (leftValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock); - } - return eval(page.getPositionCount(), leftValueVector); + try (LongBlock leftBlock = (LongBlock) left.eval(page)) { + return eval(page.getPositionCount(), leftBlock); } } - public DoubleBlock eval(int positionCount, LongBlock leftValueBlock) { + public DoubleBlock eval(int positionCount, LongBlock leftBlock) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { position: for (int p = 0; p < positionCount; p++) { - if (leftValueBlock.isNull(p)) { - result.appendNull(); - continue position; + boolean allBlocksAreNulls = true; + if (!leftBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (leftValueBlock.getValueCount(p) != 1) { - if (leftValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } + if (allBlocksAreNulls) { result.appendNull(); continue position; } try { - result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue)); - } catch (IllegalArgumentException e) { - warnings().registerException(e); - result.appendNull(); - } - } - return result.build(); - } - } - - public DoubleBlock eval(int positionCount, LongVector leftValueVector) { - try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - position: for (int p = 0; p < positionCount; p++) { - try { - result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue)); + StDistance.processGeoPointDocValuesAndConstant(result, p, leftBlock, this.right); } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); @@ -94,12 +72,12 @@ public DoubleBlock eval(int positionCount, LongVector leftValueVector) { @Override public String toString() { - return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]"; } @Override public void close() { - Releasables.closeExpectNoException(leftValue); + Releasables.closeExpectNoException(left); } private Warnings warnings() { @@ -117,25 +95,24 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory leftValue; + private final EvalOperator.ExpressionEvaluator.Factory left; - private final Point rightValue; + private final Point right; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, - Point rightValue) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, Point right) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; } @Override public StDistanceGeoPointDocValuesAndConstantEvaluator get(DriverContext context) { - return new StDistanceGeoPointDocValuesAndConstantEvaluator(source, leftValue.get(context), rightValue, context); + return new StDistanceGeoPointDocValuesAndConstantEvaluator(source, left.get(context), right, context); } @Override public String toString() { - return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceGeoPointDocValuesAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java index 6758d888cc7d1..a7664987739e2 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndSourceEvaluator.java @@ -7,13 +7,10 @@ import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; @@ -28,85 +25,48 @@ public final class StDistanceGeoPointDocValuesAndSourceEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; - private final EvalOperator.ExpressionEvaluator leftValue; + private final EvalOperator.ExpressionEvaluator left; - private final EvalOperator.ExpressionEvaluator rightValue; + private final EvalOperator.ExpressionEvaluator right; private final DriverContext driverContext; private Warnings warnings; public StDistanceGeoPointDocValuesAndSourceEvaluator(Source source, - EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue, + EvalOperator.ExpressionEvaluator left, EvalOperator.ExpressionEvaluator right, DriverContext driverContext) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; this.driverContext = driverContext; } @Override public Block eval(Page page) { - try (LongBlock leftValueBlock = (LongBlock) leftValue.eval(page)) { - try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) { - LongVector leftValueVector = leftValueBlock.asVector(); - if (leftValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); - } - BytesRefVector rightValueVector = rightValueBlock.asVector(); - if (rightValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); - } - return eval(page.getPositionCount(), leftValueVector, rightValueVector); + try (LongBlock leftBlock = (LongBlock) left.eval(page)) { + try (BytesRefBlock rightBlock = (BytesRefBlock) right.eval(page)) { + return eval(page.getPositionCount(), leftBlock, rightBlock); } } } - public DoubleBlock eval(int positionCount, LongBlock leftValueBlock, - BytesRefBlock rightValueBlock) { + public DoubleBlock eval(int positionCount, LongBlock leftBlock, BytesRefBlock rightBlock) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef rightValueScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - if (leftValueBlock.isNull(p)) { - result.appendNull(); - continue position; + boolean allBlocksAreNulls = true; + if (!leftBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (leftValueBlock.getValueCount(p) != 1) { - if (leftValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } - result.appendNull(); - continue position; + if (!rightBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (rightValueBlock.isNull(p)) { + if (allBlocksAreNulls) { result.appendNull(); continue position; } - if (rightValueBlock.getValueCount(p) != 1) { - if (rightValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } - result.appendNull(); - continue position; - } - try { - result.appendDouble(StDistance.processGeoPointDocValuesAndSource(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch))); - } catch (IllegalArgumentException e) { - warnings().registerException(e); - result.appendNull(); - } - } - return result.build(); - } - } - - public DoubleBlock eval(int positionCount, LongVector leftValueVector, - BytesRefVector rightValueVector) { - try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef rightValueScratch = new BytesRef(); - position: for (int p = 0; p < positionCount; p++) { try { - result.appendDouble(StDistance.processGeoPointDocValuesAndSource(leftValueVector.getLong(p), rightValueVector.getBytesRef(p, rightValueScratch))); + StDistance.processGeoPointDocValuesAndSource(result, p, leftBlock, rightBlock); } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); @@ -118,12 +78,12 @@ public DoubleBlock eval(int positionCount, LongVector leftValueVector, @Override public String toString() { - return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]"; } @Override public void close() { - Releasables.closeExpectNoException(leftValue, rightValue); + Releasables.closeExpectNoException(left, right); } private Warnings warnings() { @@ -141,25 +101,25 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory leftValue; + private final EvalOperator.ExpressionEvaluator.Factory left; - private final EvalOperator.ExpressionEvaluator.Factory rightValue; + private final EvalOperator.ExpressionEvaluator.Factory right; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, - EvalOperator.ExpressionEvaluator.Factory rightValue) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, + EvalOperator.ExpressionEvaluator.Factory right) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; } @Override public StDistanceGeoPointDocValuesAndSourceEvaluator get(DriverContext context) { - return new StDistanceGeoPointDocValuesAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context); + return new StDistanceGeoPointDocValuesAndSourceEvaluator(source, left.get(context), right.get(context), context); } @Override public String toString() { - return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceGeoPointDocValuesAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java index 201c29cb04275..06e44f996daf5 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java @@ -4,14 +4,11 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; -import java.io.IOException; import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; @@ -28,66 +25,43 @@ public final class StDistanceGeoSourceAndConstantEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; - private final EvalOperator.ExpressionEvaluator leftValue; + private final EvalOperator.ExpressionEvaluator left; - private final Point rightValue; + private final Point right; private final DriverContext driverContext; private Warnings warnings; public StDistanceGeoSourceAndConstantEvaluator(Source source, - EvalOperator.ExpressionEvaluator leftValue, Point rightValue, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator left, Point right, DriverContext driverContext) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; this.driverContext = driverContext; } @Override public Block eval(Page page) { - try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) { - BytesRefVector leftValueVector = leftValueBlock.asVector(); - if (leftValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock); - } - return eval(page.getPositionCount(), leftValueVector); + try (BytesRefBlock leftBlock = (BytesRefBlock) left.eval(page)) { + return eval(page.getPositionCount(), leftBlock); } } - public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock) { + public DoubleBlock eval(int positionCount, BytesRefBlock leftBlock) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef leftValueScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - if (leftValueBlock.isNull(p)) { - result.appendNull(); - continue position; + boolean allBlocksAreNulls = true; + if (!leftBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (leftValueBlock.getValueCount(p) != 1) { - if (leftValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } + if (allBlocksAreNulls) { result.appendNull(); continue position; } try { - result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue)); - } catch (IllegalArgumentException | IOException e) { - warnings().registerException(e); - result.appendNull(); - } - } - return result.build(); - } - } - - public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) { - try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef leftValueScratch = new BytesRef(); - position: for (int p = 0; p < positionCount; p++) { - try { - result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue)); - } catch (IllegalArgumentException | IOException e) { + StDistance.processGeoSourceAndConstant(result, p, leftBlock, this.right); + } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); } @@ -98,12 +72,12 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) { @Override public String toString() { - return "StDistanceGeoSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceGeoSourceAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]"; } @Override public void close() { - Releasables.closeExpectNoException(leftValue); + Releasables.closeExpectNoException(left); } private Warnings warnings() { @@ -121,25 +95,24 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory leftValue; + private final EvalOperator.ExpressionEvaluator.Factory left; - private final Point rightValue; + private final Point right; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, - Point rightValue) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, Point right) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; } @Override public StDistanceGeoSourceAndConstantEvaluator get(DriverContext context) { - return new StDistanceGeoSourceAndConstantEvaluator(source, leftValue.get(context), rightValue, context); + return new StDistanceGeoSourceAndConstantEvaluator(source, left.get(context), right, context); } @Override public String toString() { - return "StDistanceGeoSourceAndConstantEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceGeoSourceAndConstantEvaluator[" + "left=" + left + ", right=" + right + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java index 9cbe30e22c3ed..31e20d9f42197 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndSourceEvaluator.java @@ -4,14 +4,11 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.spatial; -import java.io.IOException; import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; @@ -27,88 +24,48 @@ public final class StDistanceGeoSourceAndSourceEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; - private final EvalOperator.ExpressionEvaluator leftValue; + private final EvalOperator.ExpressionEvaluator left; - private final EvalOperator.ExpressionEvaluator rightValue; + private final EvalOperator.ExpressionEvaluator right; private final DriverContext driverContext; private Warnings warnings; - public StDistanceGeoSourceAndSourceEvaluator(Source source, - EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue, - DriverContext driverContext) { + public StDistanceGeoSourceAndSourceEvaluator(Source source, EvalOperator.ExpressionEvaluator left, + EvalOperator.ExpressionEvaluator right, DriverContext driverContext) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; this.driverContext = driverContext; } @Override public Block eval(Page page) { - try (BytesRefBlock leftValueBlock = (BytesRefBlock) leftValue.eval(page)) { - try (BytesRefBlock rightValueBlock = (BytesRefBlock) rightValue.eval(page)) { - BytesRefVector leftValueVector = leftValueBlock.asVector(); - if (leftValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); - } - BytesRefVector rightValueVector = rightValueBlock.asVector(); - if (rightValueVector == null) { - return eval(page.getPositionCount(), leftValueBlock, rightValueBlock); - } - return eval(page.getPositionCount(), leftValueVector, rightValueVector); + try (BytesRefBlock leftBlock = (BytesRefBlock) left.eval(page)) { + try (BytesRefBlock rightBlock = (BytesRefBlock) right.eval(page)) { + return eval(page.getPositionCount(), leftBlock, rightBlock); } } } - public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock, - BytesRefBlock rightValueBlock) { + public DoubleBlock eval(int positionCount, BytesRefBlock leftBlock, BytesRefBlock rightBlock) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef leftValueScratch = new BytesRef(); - BytesRef rightValueScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - if (leftValueBlock.isNull(p)) { - result.appendNull(); - continue position; - } - if (leftValueBlock.getValueCount(p) != 1) { - if (leftValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } - result.appendNull(); - continue position; + boolean allBlocksAreNulls = true; + if (!leftBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (rightValueBlock.isNull(p)) { - result.appendNull(); - continue position; + if (!rightBlock.isNull(p)) { + allBlocksAreNulls = false; } - if (rightValueBlock.getValueCount(p) != 1) { - if (rightValueBlock.getValueCount(p) > 1) { - warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); - } + if (allBlocksAreNulls) { result.appendNull(); continue position; } try { - result.appendDouble(StDistance.processGeoSourceAndSource(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValueBlock.getBytesRef(rightValueBlock.getFirstValueIndex(p), rightValueScratch))); - } catch (IllegalArgumentException | IOException e) { - warnings().registerException(e); - result.appendNull(); - } - } - return result.build(); - } - } - - public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector, - BytesRefVector rightValueVector) { - try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - BytesRef leftValueScratch = new BytesRef(); - BytesRef rightValueScratch = new BytesRef(); - position: for (int p = 0; p < positionCount; p++) { - try { - result.appendDouble(StDistance.processGeoSourceAndSource(leftValueVector.getBytesRef(p, leftValueScratch), rightValueVector.getBytesRef(p, rightValueScratch))); - } catch (IllegalArgumentException | IOException e) { + StDistance.processGeoSourceAndSource(result, p, leftBlock, rightBlock); + } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); } @@ -119,12 +76,12 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector, @Override public String toString() { - return "StDistanceGeoSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceGeoSourceAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]"; } @Override public void close() { - Releasables.closeExpectNoException(leftValue, rightValue); + Releasables.closeExpectNoException(left, right); } private Warnings warnings() { @@ -142,25 +99,25 @@ private Warnings warnings() { static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final Source source; - private final EvalOperator.ExpressionEvaluator.Factory leftValue; + private final EvalOperator.ExpressionEvaluator.Factory left; - private final EvalOperator.ExpressionEvaluator.Factory rightValue; + private final EvalOperator.ExpressionEvaluator.Factory right; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory leftValue, - EvalOperator.ExpressionEvaluator.Factory rightValue) { + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory left, + EvalOperator.ExpressionEvaluator.Factory right) { this.source = source; - this.leftValue = leftValue; - this.rightValue = rightValue; + this.left = left; + this.right = right; } @Override public StDistanceGeoSourceAndSourceEvaluator get(DriverContext context) { - return new StDistanceGeoSourceAndSourceEvaluator(source, leftValue.get(context), rightValue.get(context), context); + return new StDistanceGeoSourceAndSourceEvaluator(source, left.get(context), right.get(context), context); } @Override public String toString() { - return "StDistanceGeoSourceAndSourceEvaluator[" + "leftValue=" + leftValue + ", rightValue=" + rightValue + "]"; + return "StDistanceGeoSourceAndSourceEvaluator[" + "left=" + left + ", right=" + right + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f5baaef4f579d..c94791964fb90 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -170,11 +170,21 @@ public enum Cap { */ SPATIAL_PREDICATES_SUPPORT_MULTIVALUES, + /** + * Enable spatial distance function to support multi-values. Done in #114836. + */ + SPATIAL_DISTANCE_SUPPORTS_MULTIVALUES, + /** * Support a number of fixes and enhancements to spatial distance pushdown. Done in #112938. */ SPATIAL_DISTANCE_PUSHDOWN_ENHANCEMENTS, + /** + * Fix for spatial centroid when no records are found. + */ + SPATIAL_CENTROID_NO_RECORDS, + /** * Fix to GROK and DISSECT that allows extracting attributes with the same name as the input * https://github.com/elastic/elasticsearch/issues/110184 @@ -365,7 +375,7 @@ public enum Cap { /** * Support named parameters for field names. */ - NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES, + NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES(true), /** * Fix sorting not allowed on _source and counters. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java index 7224aa049093d..71aface993ab9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java @@ -184,7 +184,8 @@ private static QueryParams parseParams(XContentParser p) throws IOException { String paramName = entry.getKey(); checkParamNameValidity(paramName, errors, loc); - if (entry.getValue() instanceof Map values) {// parameter specified as key:value pairs + if (EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + && entry.getValue() instanceof Map values) {// parameter specified as key:value pairs Map paramElements = Maps.newMapWithExpectedSize(2); for (Object keyName : values.keySet()) { ParamParsingKey paramType = checkParamValueKeysValidity(keyName.toString(), errors, loc); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index 447df09942ca8..e67c406e26929 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -25,6 +25,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequest; @@ -72,12 +73,14 @@ public class EnrichPolicyResolver { private final IndexResolver indexResolver; private final TransportService transportService; private final ThreadPool threadPool; + private final RemoteClusterService remoteClusterService; public EnrichPolicyResolver(ClusterService clusterService, TransportService transportService, IndexResolver indexResolver) { this.clusterService = clusterService; this.transportService = transportService; this.indexResolver = indexResolver; this.threadPool = transportService.getThreadPool(); + this.remoteClusterService = transportService.getRemoteClusterService(); transportService.registerRequestHandler( RESOLVE_ACTION_NAME, threadPool.executor(ThreadPool.Names.SEARCH), @@ -257,22 +260,21 @@ private void lookupPolicies( // remote clusters if (remotePolicies.isEmpty() == false) { for (String cluster : remoteClusters) { - final Transport.Connection connection; - try { - connection = getRemoteConnection(cluster); - } catch (Exception e) { - refs.acquire().onFailure(e); - return; - } - transportService.sendRequest( - connection, - RESOLVE_ACTION_NAME, - new LookupRequest(cluster, remotePolicies), - TransportRequestOptions.EMPTY, - new ActionListenerResponseHandler<>( - refs.acquire(resp -> lookupResponses.put(cluster, resp)), - LookupResponse::new, - threadPool.executor(ThreadPool.Names.SEARCH) + ActionListener lookupListener = refs.acquire(resp -> lookupResponses.put(cluster, resp)); + getRemoteConnection( + cluster, + lookupListener.delegateFailureAndWrap( + (delegate, connection) -> transportService.sendRequest( + connection, + RESOLVE_ACTION_NAME, + new LookupRequest(cluster, remotePolicies), + TransportRequestOptions.EMPTY, + new ActionListenerResponseHandler<>( + delegate, + LookupResponse::new, + threadPool.executor(ThreadPool.Names.SEARCH) + ) + ) ) ); } @@ -389,13 +391,16 @@ protected Map availablePolicies() { return metadata == null ? Map.of() : metadata.getPolicies(); } - protected Transport.Connection getRemoteConnection(String cluster) { - return transportService.getRemoteClusterService().getConnection(cluster); + protected void getRemoteConnection(String cluster, ActionListener listener) { + remoteClusterService.maybeEnsureConnectedAndGetConnection( + cluster, + remoteClusterService.isSkipUnavailable(cluster) == false, + listener + ); } public Map> groupIndicesPerCluster(String[] indices) { - return transportService.getRemoteClusterService() - .groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, indices) + return remoteClusterService.groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, indices) .entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> Arrays.asList(e.getValue().indices()))); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 2550e5bdcf515..756000dfbb187 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -209,7 +209,7 @@ public AggregatorFunctionSupplier supplier(List inputChannels) { if (type == DataType.DOUBLE) { return new CountDistinctDoubleAggregatorFunctionSupplier(inputChannels, precision); } - if (type == DataType.KEYWORD || type == DataType.IP || type == DataType.VERSION || type == DataType.TEXT) { + if (DataType.isString(type) || type == DataType.IP || type == DataType.VERSION) { return new CountDistinctBytesRefAggregatorFunctionSupplier(inputChannels, precision); } throw EsqlIllegalArgumentException.illegalDataType(type); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index 47d74c71d9cc5..6119b2ce58465 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -128,7 +128,7 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { if (type == DataType.IP) { return new MaxIpAggregatorFunctionSupplier(inputChannels); } - if (type == DataType.VERSION || type == DataType.KEYWORD || type == DataType.TEXT) { + if (type == DataType.VERSION || DataType.isString(type)) { return new MaxBytesRefAggregatorFunctionSupplier(inputChannels); } throw EsqlIllegalArgumentException.illegalDataType(type); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index ce69decca8e81..a1492f79da393 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -128,7 +128,7 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { if (type == DataType.IP) { return new MinIpAggregatorFunctionSupplier(inputChannels); } - if (type == DataType.VERSION || type == DataType.KEYWORD || type == DataType.TEXT) { + if (type == DataType.VERSION || DataType.isString(type)) { return new MinBytesRefAggregatorFunctionSupplier(inputChannels); } throw EsqlIllegalArgumentException.illegalDataType(type); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java index 9d815d15accdc..d47ebeab4ca6c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java @@ -155,11 +155,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType == DataType.LONG || dataType == DataType.DATETIME) { return new GreatestLongEvaluator.Factory(source(), factories); } - if (dataType == DataType.KEYWORD - || dataType == DataType.TEXT - || dataType == DataType.IP - || dataType == DataType.VERSION - || dataType == DataType.UNSUPPORTED) { + if (DataType.isString(dataType) || dataType == DataType.IP || dataType == DataType.VERSION || dataType == DataType.UNSUPPORTED) { return new GreatestBytesRefEvaluator.Factory(source(), factories); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java index 435a14d0fef33..81c1419dcf788 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java @@ -154,11 +154,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType == DataType.LONG || dataType == DataType.DATETIME) { return new LeastLongEvaluator.Factory(source(), factories); } - if (dataType == DataType.KEYWORD - || dataType == DataType.TEXT - || dataType == DataType.IP - || dataType == DataType.VERSION - || dataType == DataType.UNSUPPORTED) { + if (DataType.isString(dataType) || dataType == DataType.IP || dataType == DataType.VERSION || dataType == DataType.UNSUPPORTED) { return new LeastBytesRefEvaluator.Factory(source(), factories); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java index 5401fcf188d4a..06815d738e82c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java @@ -48,7 +48,6 @@ public abstract class AbstractConvertFunction extends UnaryScalarFunction { // the numeric types convert functions need to handle; the other numeric types are converted upstream to one of these private static final List NUMERIC_TYPES = List.of(DataType.INTEGER, DataType.LONG, DataType.UNSIGNED_LONG, DataType.DOUBLE); - public static final List STRING_TYPES = DataType.types().stream().filter(DataType::isString).toList(); protected AbstractConvertFunction(Source source, Expression field) { super(source, field); @@ -90,9 +89,9 @@ private static String supportedTypesNames(Set types) { NUMERIC_TYPES.forEach(supportTypes::remove); } - if (types.containsAll(STRING_TYPES)) { + if (types.containsAll(DataType.stringTypes())) { supportedTypesNames.add("string"); - STRING_TYPES.forEach(supportTypes::remove); + DataType.stringTypes().forEach(supportTypes::remove); } supportTypes.forEach(t -> supportedTypesNames.add(t.nameUpper().toLowerCase(Locale.ROOT))); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java index ae9d3383bad39..3cf042a2db828 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java @@ -14,6 +14,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; @@ -77,8 +80,9 @@ protected CartesianDistanceCalculator() { @Override protected double distance(Point left, Point right) { - final double diffX = left.getX() - right.getX(); - final double diffY = left.getY() - right.getY(); + // Cast coordinates to float to mimic Lucene behaviour, so we get identical results + final double diffX = (double) ((float) left.getX()) - (double) ((float) right.getX()); + final double diffY = (double) ((float) left.getY()) - (double) ((float) right.getY()); return Math.sqrt(diffX * diffX + diffY * diffY); } } @@ -100,11 +104,6 @@ protected Double compare(BytesRef left, BytesRef right) throws IOException { protected abstract double distance(Point left, Point right); - protected double distance(long encoded, Geometry right) { - Point point = spatialCoordinateType.longAsPoint(encoded); - return distance(point, (Point) right); - } - protected double distance(Geometry left, Geometry right) { return distance((Point) left, (Point) right); } @@ -113,8 +112,112 @@ public double distance(BytesRef left, BytesRef right) { return distance(this.fromBytesRef(left), this.fromBytesRef(right)); } - public double distance(BytesRef left, Point right) { - return distance(this.fromBytesRef(left), right); + public void distanceSourceAndConstant(DoubleBlock.Builder results, int position, BytesRefBlock left, Point right) { + int valueCount = left.getValueCount(position); + if (valueCount < 1) { + results.appendNull(); + } else { + final BytesRef scratch = new BytesRef(); + final int firstValueIndex = left.getFirstValueIndex(position); + double distance = Double.MAX_VALUE; + if (valueCount == 1) { + distance = distance(fromBytesRef(left.getBytesRef(firstValueIndex, scratch)), right); + } else { + for (int i = 0; i < valueCount; i++) { + double value = distance(fromBytesRef(left.getBytesRef(firstValueIndex + i, scratch)), right); + if (value < distance) { + distance = value; + } + } + } + results.appendDouble(distance); + } + } + + public void distanceSourceAndSource(DoubleBlock.Builder results, int position, BytesRefBlock left, BytesRefBlock right) { + int leftCount = left.getValueCount(position); + int rightCount = right.getValueCount(position); + if (leftCount < 1 || rightCount < 1) { + results.appendNull(); + } else { + final BytesRef scratchLeft = new BytesRef(); + final BytesRef scratchRight = new BytesRef(); + final int leftFirstValueIndex = left.getFirstValueIndex(position); + final int rightFirstValueIndex = right.getFirstValueIndex(position); + double distance = Double.MAX_VALUE; + if (leftCount == 1 && rightCount == 1) { + distance = distance( + fromBytesRef(left.getBytesRef(leftFirstValueIndex, scratchLeft)), + fromBytesRef(right.getBytesRef(rightFirstValueIndex, scratchRight)) + ); + } else { + for (int i = 0; i < leftCount; i++) { + for (int j = 0; j < rightCount; j++) { + double value = distance( + fromBytesRef(left.getBytesRef(leftFirstValueIndex + i, scratchLeft)), + fromBytesRef(right.getBytesRef(rightFirstValueIndex + j, scratchRight)) + ); + if (value < distance) { + distance = value; + } + } + } + } + results.appendDouble(distance); + } + } + + public void distancePointDocValuesAndConstant(DoubleBlock.Builder results, int position, LongBlock left, Point right) { + int valueCount = left.getValueCount(position); + if (valueCount < 1) { + results.appendNull(); + } else { + final int firstValueIndex = left.getFirstValueIndex(position); + double distance = Double.MAX_VALUE; + if (valueCount == 1) { + distance = distance(spatialCoordinateType.longAsPoint(left.getLong(firstValueIndex)), right); + } else { + for (int i = 0; i < valueCount; i++) { + double value = distance(spatialCoordinateType.longAsPoint(left.getLong(firstValueIndex + i)), right); + if (value < distance) { + distance = value; + } + } + } + results.appendDouble(distance); + } + } + + public void distancePointDocValuesAndSource(DoubleBlock.Builder results, int position, LongBlock left, BytesRefBlock right) { + int leftCount = left.getValueCount(position); + int rightCount = right.getValueCount(position); + if (leftCount < 1 || rightCount < 1) { + results.appendNull(); + } else { + final BytesRef scratchRight = new BytesRef(); + final int leftFirstValueIndex = left.getFirstValueIndex(position); + final int rightFirstValueIndex = right.getFirstValueIndex(position); + double distance = Double.MAX_VALUE; + if (leftCount == 1 && rightCount == 1) { + distance = distance( + spatialCoordinateType.longAsPoint(left.getLong(leftFirstValueIndex)), + fromBytesRef(right.getBytesRef(rightFirstValueIndex, scratchRight)) + ); + } + for (int i = 0; i < leftCount; i++) { + for (int j = 0; j < rightCount; j++) { + double value = distance( + spatialCoordinateType.longAsPoint(left.getLong(leftFirstValueIndex + i)), + fromBytesRef(right.getBytesRef(rightFirstValueIndex + j, scratchRight)) + ); + if (value < distance) { + distance = value; + } + } + } + results.appendDouble(distance); + } + } } @@ -249,45 +352,43 @@ private EvalOperator.ExpressionEvaluator.Factory toEvaluator( throw EsqlIllegalArgumentException.illegalDataType(crsType().name()); } - @Evaluator(extraName = "GeoSourceAndConstant", warnExceptions = { IllegalArgumentException.class, IOException.class }) - static double processGeoSourceAndConstant(BytesRef leftValue, @Fixed Point rightValue) throws IOException { - return GEO.distance(leftValue, rightValue); + @Evaluator(extraName = "GeoSourceAndConstant", warnExceptions = { IllegalArgumentException.class }) + static void processGeoSourceAndConstant(DoubleBlock.Builder results, int p, BytesRefBlock left, @Fixed Point right) { + GEO.distanceSourceAndConstant(results, p, left, right); } - @Evaluator(extraName = "GeoSourceAndSource", warnExceptions = { IllegalArgumentException.class, IOException.class }) - static double processGeoSourceAndSource(BytesRef leftValue, BytesRef rightValue) throws IOException { - return GEO.distance(leftValue, rightValue); + @Evaluator(extraName = "GeoSourceAndSource", warnExceptions = { IllegalArgumentException.class }) + static void processGeoSourceAndSource(DoubleBlock.Builder results, int p, BytesRefBlock left, BytesRefBlock right) { + GEO.distanceSourceAndSource(results, p, left, right); } @Evaluator(extraName = "GeoPointDocValuesAndConstant", warnExceptions = { IllegalArgumentException.class }) - static double processGeoPointDocValuesAndConstant(long leftValue, @Fixed Point rightValue) { - return GEO.distance(leftValue, rightValue); + static void processGeoPointDocValuesAndConstant(DoubleBlock.Builder results, int p, LongBlock left, @Fixed Point right) { + GEO.distancePointDocValuesAndConstant(results, p, left, right); } @Evaluator(extraName = "GeoPointDocValuesAndSource", warnExceptions = { IllegalArgumentException.class }) - static double processGeoPointDocValuesAndSource(long leftValue, BytesRef rightValue) { - Geometry geometry = SpatialCoordinateTypes.UNSPECIFIED.wkbToGeometry(rightValue); - return GEO.distance(leftValue, geometry); + static void processGeoPointDocValuesAndSource(DoubleBlock.Builder results, int p, LongBlock left, BytesRefBlock right) { + GEO.distancePointDocValuesAndSource(results, p, left, right); } - @Evaluator(extraName = "CartesianSourceAndConstant", warnExceptions = { IllegalArgumentException.class, IOException.class }) - static double processCartesianSourceAndConstant(BytesRef leftValue, @Fixed Point rightValue) throws IOException { - return CARTESIAN.distance(leftValue, rightValue); + @Evaluator(extraName = "CartesianSourceAndConstant", warnExceptions = { IllegalArgumentException.class }) + static void processCartesianSourceAndConstant(DoubleBlock.Builder results, int p, BytesRefBlock left, @Fixed Point right) { + CARTESIAN.distanceSourceAndConstant(results, p, left, right); } - @Evaluator(extraName = "CartesianSourceAndSource", warnExceptions = { IllegalArgumentException.class, IOException.class }) - static double processCartesianSourceAndSource(BytesRef leftValue, BytesRef rightValue) throws IOException { - return CARTESIAN.distance(leftValue, rightValue); + @Evaluator(extraName = "CartesianSourceAndSource", warnExceptions = { IllegalArgumentException.class }) + static void processCartesianSourceAndSource(DoubleBlock.Builder results, int p, BytesRefBlock left, BytesRefBlock right) { + CARTESIAN.distanceSourceAndSource(results, p, left, right); } @Evaluator(extraName = "CartesianPointDocValuesAndConstant", warnExceptions = { IllegalArgumentException.class }) - static double processCartesianPointDocValuesAndConstant(long leftValue, @Fixed Point rightValue) { - return CARTESIAN.distance(leftValue, rightValue); + static void processCartesianPointDocValuesAndConstant(DoubleBlock.Builder results, int p, LongBlock left, @Fixed Point right) { + CARTESIAN.distancePointDocValuesAndConstant(results, p, left, right); } @Evaluator(extraName = "CartesianPointDocValuesAndSource") - static double processCartesianPointDocValuesAndSource(long leftValue, BytesRef rightValue) { - Geometry geometry = SpatialCoordinateTypes.UNSPECIFIED.wkbToGeometry(rightValue); - return CARTESIAN.distance(leftValue, geometry); + static void processCartesianPointDocValuesAndSource(DoubleBlock.Builder results, int p, LongBlock left, BytesRefBlock right) { + CARTESIAN.distancePointDocValuesAndSource(results, p, left, right); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java index d11f5c9b68532..f5704239993f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java @@ -34,7 +34,7 @@ public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout var leftEval = toEvaluator(bc.left(), layout); var rightEval = toEvaluator(bc.right(), layout); - if (leftType == DataType.KEYWORD || leftType == DataType.TEXT) { + if (DataType.isString(leftType)) { if (bc.right().foldable() && DataType.isString(rightType)) { BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold()); Automaton automaton = InsensitiveEquals.automaton(rightVal); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java index ec25c69deba5c..cde305e52a705 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java @@ -95,7 +95,7 @@ private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec) { } return comparison; }); - if (rewritten.equals(filterExec.condition()) == false) { + if (rewritten.equals(filterExec.condition()) == false && canPushToSource(rewritten, x -> false)) { return new FilterExec(filterExec.source(), esQueryExec, rewritten); } return filterExec; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index 1a0105b9951d2..2566da379af73 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -466,4 +466,4 @@ METRICS_MODE CLOSING_METRICS_MODE atn: -[4, 0, 120, 1471, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 4, 19, 578, 8, 19, 11, 19, 12, 19, 579, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 588, 8, 20, 10, 20, 12, 20, 591, 9, 20, 1, 20, 3, 20, 594, 8, 20, 1, 20, 3, 20, 597, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 606, 8, 21, 10, 21, 12, 21, 609, 9, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 4, 22, 617, 8, 22, 11, 22, 12, 22, 618, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 3, 28, 638, 8, 28, 1, 28, 4, 28, 641, 8, 28, 11, 28, 12, 28, 642, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 652, 8, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 659, 8, 33, 1, 34, 1, 34, 1, 34, 5, 34, 664, 8, 34, 10, 34, 12, 34, 667, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 675, 8, 34, 10, 34, 12, 34, 678, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 685, 8, 34, 1, 34, 3, 34, 688, 8, 34, 3, 34, 690, 8, 34, 1, 35, 4, 35, 693, 8, 35, 11, 35, 12, 35, 694, 1, 36, 4, 36, 698, 8, 36, 11, 36, 12, 36, 699, 1, 36, 1, 36, 5, 36, 704, 8, 36, 10, 36, 12, 36, 707, 9, 36, 1, 36, 1, 36, 4, 36, 711, 8, 36, 11, 36, 12, 36, 712, 1, 36, 4, 36, 716, 8, 36, 11, 36, 12, 36, 717, 1, 36, 1, 36, 5, 36, 722, 8, 36, 10, 36, 12, 36, 725, 9, 36, 3, 36, 727, 8, 36, 1, 36, 1, 36, 1, 36, 1, 36, 4, 36, 733, 8, 36, 11, 36, 12, 36, 734, 1, 36, 1, 36, 3, 36, 739, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 3, 74, 871, 8, 74, 1, 74, 5, 74, 874, 8, 74, 10, 74, 12, 74, 877, 9, 74, 1, 74, 1, 74, 4, 74, 881, 8, 74, 11, 74, 12, 74, 882, 3, 74, 885, 8, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 5, 77, 899, 8, 77, 10, 77, 12, 77, 902, 9, 77, 1, 77, 1, 77, 3, 77, 906, 8, 77, 1, 77, 4, 77, 909, 8, 77, 11, 77, 12, 77, 910, 3, 77, 913, 8, 77, 1, 78, 1, 78, 4, 78, 917, 8, 78, 11, 78, 12, 78, 918, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 3, 95, 996, 8, 95, 1, 96, 4, 96, 999, 8, 96, 11, 96, 12, 96, 1000, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 3, 107, 1048, 8, 107, 1, 108, 1, 108, 3, 108, 1052, 8, 108, 1, 108, 5, 108, 1055, 8, 108, 10, 108, 12, 108, 1058, 9, 108, 1, 108, 1, 108, 3, 108, 1062, 8, 108, 1, 108, 4, 108, 1065, 8, 108, 11, 108, 12, 108, 1066, 3, 108, 1069, 8, 108, 1, 109, 1, 109, 4, 109, 1073, 8, 109, 11, 109, 12, 109, 1074, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 129, 4, 129, 1158, 8, 129, 11, 129, 12, 129, 1159, 1, 129, 1, 129, 3, 129, 1164, 8, 129, 1, 129, 4, 129, 1167, 8, 129, 11, 129, 12, 129, 1168, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 4, 162, 1308, 8, 162, 11, 162, 12, 162, 1309, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 2, 607, 676, 0, 198, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 0, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 25, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 0, 163, 64, 165, 65, 167, 66, 169, 67, 171, 0, 173, 68, 175, 69, 177, 70, 179, 71, 181, 0, 183, 0, 185, 72, 187, 73, 189, 74, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 0, 203, 75, 205, 0, 207, 76, 209, 0, 211, 0, 213, 77, 215, 78, 217, 79, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 0, 231, 0, 233, 80, 235, 81, 237, 82, 239, 83, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 0, 253, 84, 255, 0, 257, 85, 259, 86, 261, 87, 263, 0, 265, 0, 267, 88, 269, 89, 271, 0, 273, 90, 275, 0, 277, 91, 279, 92, 281, 93, 283, 0, 285, 0, 287, 0, 289, 0, 291, 0, 293, 0, 295, 0, 297, 0, 299, 0, 301, 94, 303, 95, 305, 96, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 97, 321, 98, 323, 99, 325, 0, 327, 100, 329, 101, 331, 102, 333, 103, 335, 0, 337, 104, 339, 105, 341, 106, 343, 107, 345, 108, 347, 0, 349, 0, 351, 0, 353, 0, 355, 0, 357, 0, 359, 0, 361, 109, 363, 110, 365, 111, 367, 0, 369, 0, 371, 0, 373, 0, 375, 112, 377, 113, 379, 114, 381, 0, 383, 0, 385, 0, 387, 115, 389, 116, 391, 117, 393, 0, 395, 0, 397, 118, 399, 119, 401, 120, 403, 0, 405, 0, 407, 0, 409, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1499, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 1, 83, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 2, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 203, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 225, 1, 0, 0, 0, 4, 227, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 4, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 5, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 5, 261, 1, 0, 0, 0, 6, 263, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 269, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 6, 275, 1, 0, 0, 0, 6, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 6, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 7, 295, 1, 0, 0, 0, 7, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 8, 309, 1, 0, 0, 0, 8, 311, 1, 0, 0, 0, 8, 313, 1, 0, 0, 0, 8, 315, 1, 0, 0, 0, 8, 317, 1, 0, 0, 0, 8, 319, 1, 0, 0, 0, 8, 321, 1, 0, 0, 0, 8, 323, 1, 0, 0, 0, 9, 325, 1, 0, 0, 0, 9, 327, 1, 0, 0, 0, 9, 329, 1, 0, 0, 0, 9, 331, 1, 0, 0, 0, 9, 333, 1, 0, 0, 0, 10, 335, 1, 0, 0, 0, 10, 337, 1, 0, 0, 0, 10, 339, 1, 0, 0, 0, 10, 341, 1, 0, 0, 0, 10, 343, 1, 0, 0, 0, 10, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 11, 351, 1, 0, 0, 0, 11, 353, 1, 0, 0, 0, 11, 355, 1, 0, 0, 0, 11, 357, 1, 0, 0, 0, 11, 359, 1, 0, 0, 0, 11, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 11, 365, 1, 0, 0, 0, 12, 367, 1, 0, 0, 0, 12, 369, 1, 0, 0, 0, 12, 371, 1, 0, 0, 0, 12, 373, 1, 0, 0, 0, 12, 375, 1, 0, 0, 0, 12, 377, 1, 0, 0, 0, 12, 379, 1, 0, 0, 0, 13, 381, 1, 0, 0, 0, 13, 383, 1, 0, 0, 0, 13, 385, 1, 0, 0, 0, 13, 387, 1, 0, 0, 0, 13, 389, 1, 0, 0, 0, 13, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 14, 395, 1, 0, 0, 0, 14, 397, 1, 0, 0, 0, 14, 399, 1, 0, 0, 0, 14, 401, 1, 0, 0, 0, 14, 403, 1, 0, 0, 0, 14, 405, 1, 0, 0, 0, 14, 407, 1, 0, 0, 0, 14, 409, 1, 0, 0, 0, 15, 411, 1, 0, 0, 0, 17, 421, 1, 0, 0, 0, 19, 428, 1, 0, 0, 0, 21, 437, 1, 0, 0, 0, 23, 444, 1, 0, 0, 0, 25, 454, 1, 0, 0, 0, 27, 461, 1, 0, 0, 0, 29, 468, 1, 0, 0, 0, 31, 475, 1, 0, 0, 0, 33, 483, 1, 0, 0, 0, 35, 495, 1, 0, 0, 0, 37, 504, 1, 0, 0, 0, 39, 510, 1, 0, 0, 0, 41, 517, 1, 0, 0, 0, 43, 524, 1, 0, 0, 0, 45, 532, 1, 0, 0, 0, 47, 540, 1, 0, 0, 0, 49, 555, 1, 0, 0, 0, 51, 565, 1, 0, 0, 0, 53, 577, 1, 0, 0, 0, 55, 583, 1, 0, 0, 0, 57, 600, 1, 0, 0, 0, 59, 616, 1, 0, 0, 0, 61, 622, 1, 0, 0, 0, 63, 626, 1, 0, 0, 0, 65, 628, 1, 0, 0, 0, 67, 630, 1, 0, 0, 0, 69, 633, 1, 0, 0, 0, 71, 635, 1, 0, 0, 0, 73, 644, 1, 0, 0, 0, 75, 646, 1, 0, 0, 0, 77, 651, 1, 0, 0, 0, 79, 653, 1, 0, 0, 0, 81, 658, 1, 0, 0, 0, 83, 689, 1, 0, 0, 0, 85, 692, 1, 0, 0, 0, 87, 738, 1, 0, 0, 0, 89, 740, 1, 0, 0, 0, 91, 743, 1, 0, 0, 0, 93, 747, 1, 0, 0, 0, 95, 751, 1, 0, 0, 0, 97, 753, 1, 0, 0, 0, 99, 756, 1, 0, 0, 0, 101, 758, 1, 0, 0, 0, 103, 763, 1, 0, 0, 0, 105, 765, 1, 0, 0, 0, 107, 771, 1, 0, 0, 0, 109, 777, 1, 0, 0, 0, 111, 780, 1, 0, 0, 0, 113, 783, 1, 0, 0, 0, 115, 788, 1, 0, 0, 0, 117, 793, 1, 0, 0, 0, 119, 795, 1, 0, 0, 0, 121, 799, 1, 0, 0, 0, 123, 804, 1, 0, 0, 0, 125, 810, 1, 0, 0, 0, 127, 813, 1, 0, 0, 0, 129, 815, 1, 0, 0, 0, 131, 821, 1, 0, 0, 0, 133, 823, 1, 0, 0, 0, 135, 828, 1, 0, 0, 0, 137, 831, 1, 0, 0, 0, 139, 834, 1, 0, 0, 0, 141, 837, 1, 0, 0, 0, 143, 839, 1, 0, 0, 0, 145, 842, 1, 0, 0, 0, 147, 844, 1, 0, 0, 0, 149, 847, 1, 0, 0, 0, 151, 849, 1, 0, 0, 0, 153, 851, 1, 0, 0, 0, 155, 853, 1, 0, 0, 0, 157, 855, 1, 0, 0, 0, 159, 857, 1, 0, 0, 0, 161, 863, 1, 0, 0, 0, 163, 884, 1, 0, 0, 0, 165, 886, 1, 0, 0, 0, 167, 891, 1, 0, 0, 0, 169, 912, 1, 0, 0, 0, 171, 914, 1, 0, 0, 0, 173, 922, 1, 0, 0, 0, 175, 924, 1, 0, 0, 0, 177, 928, 1, 0, 0, 0, 179, 932, 1, 0, 0, 0, 181, 936, 1, 0, 0, 0, 183, 941, 1, 0, 0, 0, 185, 946, 1, 0, 0, 0, 187, 950, 1, 0, 0, 0, 189, 954, 1, 0, 0, 0, 191, 958, 1, 0, 0, 0, 193, 963, 1, 0, 0, 0, 195, 967, 1, 0, 0, 0, 197, 971, 1, 0, 0, 0, 199, 975, 1, 0, 0, 0, 201, 979, 1, 0, 0, 0, 203, 983, 1, 0, 0, 0, 205, 995, 1, 0, 0, 0, 207, 998, 1, 0, 0, 0, 209, 1002, 1, 0, 0, 0, 211, 1006, 1, 0, 0, 0, 213, 1010, 1, 0, 0, 0, 215, 1014, 1, 0, 0, 0, 217, 1018, 1, 0, 0, 0, 219, 1022, 1, 0, 0, 0, 221, 1027, 1, 0, 0, 0, 223, 1031, 1, 0, 0, 0, 225, 1035, 1, 0, 0, 0, 227, 1039, 1, 0, 0, 0, 229, 1047, 1, 0, 0, 0, 231, 1068, 1, 0, 0, 0, 233, 1072, 1, 0, 0, 0, 235, 1076, 1, 0, 0, 0, 237, 1080, 1, 0, 0, 0, 239, 1084, 1, 0, 0, 0, 241, 1088, 1, 0, 0, 0, 243, 1093, 1, 0, 0, 0, 245, 1097, 1, 0, 0, 0, 247, 1101, 1, 0, 0, 0, 249, 1105, 1, 0, 0, 0, 251, 1109, 1, 0, 0, 0, 253, 1113, 1, 0, 0, 0, 255, 1116, 1, 0, 0, 0, 257, 1120, 1, 0, 0, 0, 259, 1124, 1, 0, 0, 0, 261, 1128, 1, 0, 0, 0, 263, 1132, 1, 0, 0, 0, 265, 1137, 1, 0, 0, 0, 267, 1142, 1, 0, 0, 0, 269, 1147, 1, 0, 0, 0, 271, 1154, 1, 0, 0, 0, 273, 1163, 1, 0, 0, 0, 275, 1170, 1, 0, 0, 0, 277, 1174, 1, 0, 0, 0, 279, 1178, 1, 0, 0, 0, 281, 1182, 1, 0, 0, 0, 283, 1186, 1, 0, 0, 0, 285, 1192, 1, 0, 0, 0, 287, 1196, 1, 0, 0, 0, 289, 1200, 1, 0, 0, 0, 291, 1204, 1, 0, 0, 0, 293, 1208, 1, 0, 0, 0, 295, 1212, 1, 0, 0, 0, 297, 1216, 1, 0, 0, 0, 299, 1220, 1, 0, 0, 0, 301, 1224, 1, 0, 0, 0, 303, 1228, 1, 0, 0, 0, 305, 1232, 1, 0, 0, 0, 307, 1236, 1, 0, 0, 0, 309, 1241, 1, 0, 0, 0, 311, 1245, 1, 0, 0, 0, 313, 1249, 1, 0, 0, 0, 315, 1253, 1, 0, 0, 0, 317, 1257, 1, 0, 0, 0, 319, 1261, 1, 0, 0, 0, 321, 1265, 1, 0, 0, 0, 323, 1269, 1, 0, 0, 0, 325, 1273, 1, 0, 0, 0, 327, 1278, 1, 0, 0, 0, 329, 1283, 1, 0, 0, 0, 331, 1287, 1, 0, 0, 0, 333, 1291, 1, 0, 0, 0, 335, 1295, 1, 0, 0, 0, 337, 1300, 1, 0, 0, 0, 339, 1307, 1, 0, 0, 0, 341, 1311, 1, 0, 0, 0, 343, 1315, 1, 0, 0, 0, 345, 1319, 1, 0, 0, 0, 347, 1323, 1, 0, 0, 0, 349, 1328, 1, 0, 0, 0, 351, 1332, 1, 0, 0, 0, 353, 1336, 1, 0, 0, 0, 355, 1340, 1, 0, 0, 0, 357, 1345, 1, 0, 0, 0, 359, 1349, 1, 0, 0, 0, 361, 1353, 1, 0, 0, 0, 363, 1357, 1, 0, 0, 0, 365, 1361, 1, 0, 0, 0, 367, 1365, 1, 0, 0, 0, 369, 1371, 1, 0, 0, 0, 371, 1375, 1, 0, 0, 0, 373, 1379, 1, 0, 0, 0, 375, 1383, 1, 0, 0, 0, 377, 1387, 1, 0, 0, 0, 379, 1391, 1, 0, 0, 0, 381, 1395, 1, 0, 0, 0, 383, 1400, 1, 0, 0, 0, 385, 1406, 1, 0, 0, 0, 387, 1412, 1, 0, 0, 0, 389, 1416, 1, 0, 0, 0, 391, 1420, 1, 0, 0, 0, 393, 1424, 1, 0, 0, 0, 395, 1430, 1, 0, 0, 0, 397, 1436, 1, 0, 0, 0, 399, 1440, 1, 0, 0, 0, 401, 1444, 1, 0, 0, 0, 403, 1448, 1, 0, 0, 0, 405, 1454, 1, 0, 0, 0, 407, 1460, 1, 0, 0, 0, 409, 1466, 1, 0, 0, 0, 411, 412, 7, 0, 0, 0, 412, 413, 7, 1, 0, 0, 413, 414, 7, 2, 0, 0, 414, 415, 7, 2, 0, 0, 415, 416, 7, 3, 0, 0, 416, 417, 7, 4, 0, 0, 417, 418, 7, 5, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 0, 0, 0, 420, 16, 1, 0, 0, 0, 421, 422, 7, 0, 0, 0, 422, 423, 7, 6, 0, 0, 423, 424, 7, 7, 0, 0, 424, 425, 7, 8, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 1, 1, 0, 427, 18, 1, 0, 0, 0, 428, 429, 7, 3, 0, 0, 429, 430, 7, 9, 0, 0, 430, 431, 7, 6, 0, 0, 431, 432, 7, 1, 0, 0, 432, 433, 7, 4, 0, 0, 433, 434, 7, 10, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 2, 2, 0, 436, 20, 1, 0, 0, 0, 437, 438, 7, 3, 0, 0, 438, 439, 7, 11, 0, 0, 439, 440, 7, 12, 0, 0, 440, 441, 7, 13, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 3, 0, 0, 443, 22, 1, 0, 0, 0, 444, 445, 7, 3, 0, 0, 445, 446, 7, 14, 0, 0, 446, 447, 7, 8, 0, 0, 447, 448, 7, 13, 0, 0, 448, 449, 7, 12, 0, 0, 449, 450, 7, 1, 0, 0, 450, 451, 7, 9, 0, 0, 451, 452, 1, 0, 0, 0, 452, 453, 6, 4, 3, 0, 453, 24, 1, 0, 0, 0, 454, 455, 7, 15, 0, 0, 455, 456, 7, 6, 0, 0, 456, 457, 7, 7, 0, 0, 457, 458, 7, 16, 0, 0, 458, 459, 1, 0, 0, 0, 459, 460, 6, 5, 4, 0, 460, 26, 1, 0, 0, 0, 461, 462, 7, 17, 0, 0, 462, 463, 7, 6, 0, 0, 463, 464, 7, 7, 0, 0, 464, 465, 7, 18, 0, 0, 465, 466, 1, 0, 0, 0, 466, 467, 6, 6, 0, 0, 467, 28, 1, 0, 0, 0, 468, 469, 7, 18, 0, 0, 469, 470, 7, 3, 0, 0, 470, 471, 7, 3, 0, 0, 471, 472, 7, 8, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 7, 1, 0, 474, 30, 1, 0, 0, 0, 475, 476, 7, 13, 0, 0, 476, 477, 7, 1, 0, 0, 477, 478, 7, 16, 0, 0, 478, 479, 7, 1, 0, 0, 479, 480, 7, 5, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 8, 0, 0, 482, 32, 1, 0, 0, 0, 483, 484, 7, 16, 0, 0, 484, 485, 7, 11, 0, 0, 485, 486, 5, 95, 0, 0, 486, 487, 7, 3, 0, 0, 487, 488, 7, 14, 0, 0, 488, 489, 7, 8, 0, 0, 489, 490, 7, 12, 0, 0, 490, 491, 7, 9, 0, 0, 491, 492, 7, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 9, 5, 0, 494, 34, 1, 0, 0, 0, 495, 496, 7, 6, 0, 0, 496, 497, 7, 3, 0, 0, 497, 498, 7, 9, 0, 0, 498, 499, 7, 12, 0, 0, 499, 500, 7, 16, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 10, 6, 0, 503, 36, 1, 0, 0, 0, 504, 505, 7, 6, 0, 0, 505, 506, 7, 7, 0, 0, 506, 507, 7, 19, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 6, 11, 0, 0, 509, 38, 1, 0, 0, 0, 510, 511, 7, 2, 0, 0, 511, 512, 7, 10, 0, 0, 512, 513, 7, 7, 0, 0, 513, 514, 7, 19, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 12, 7, 0, 516, 40, 1, 0, 0, 0, 517, 518, 7, 2, 0, 0, 518, 519, 7, 7, 0, 0, 519, 520, 7, 6, 0, 0, 520, 521, 7, 5, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 13, 0, 0, 523, 42, 1, 0, 0, 0, 524, 525, 7, 2, 0, 0, 525, 526, 7, 5, 0, 0, 526, 527, 7, 12, 0, 0, 527, 528, 7, 5, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 14, 0, 0, 531, 44, 1, 0, 0, 0, 532, 533, 7, 19, 0, 0, 533, 534, 7, 10, 0, 0, 534, 535, 7, 3, 0, 0, 535, 536, 7, 6, 0, 0, 536, 537, 7, 3, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 6, 15, 0, 0, 539, 46, 1, 0, 0, 0, 540, 541, 4, 16, 0, 0, 541, 542, 7, 1, 0, 0, 542, 543, 7, 9, 0, 0, 543, 544, 7, 13, 0, 0, 544, 545, 7, 1, 0, 0, 545, 546, 7, 9, 0, 0, 546, 547, 7, 3, 0, 0, 547, 548, 7, 2, 0, 0, 548, 549, 7, 5, 0, 0, 549, 550, 7, 12, 0, 0, 550, 551, 7, 5, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 16, 0, 0, 554, 48, 1, 0, 0, 0, 555, 556, 4, 17, 1, 0, 556, 557, 7, 13, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 7, 0, 0, 559, 560, 7, 18, 0, 0, 560, 561, 7, 20, 0, 0, 561, 562, 7, 8, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 17, 8, 0, 564, 50, 1, 0, 0, 0, 565, 566, 4, 18, 2, 0, 566, 567, 7, 16, 0, 0, 567, 568, 7, 3, 0, 0, 568, 569, 7, 5, 0, 0, 569, 570, 7, 6, 0, 0, 570, 571, 7, 1, 0, 0, 571, 572, 7, 4, 0, 0, 572, 573, 7, 2, 0, 0, 573, 574, 1, 0, 0, 0, 574, 575, 6, 18, 9, 0, 575, 52, 1, 0, 0, 0, 576, 578, 8, 21, 0, 0, 577, 576, 1, 0, 0, 0, 578, 579, 1, 0, 0, 0, 579, 577, 1, 0, 0, 0, 579, 580, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 582, 6, 19, 0, 0, 582, 54, 1, 0, 0, 0, 583, 584, 5, 47, 0, 0, 584, 585, 5, 47, 0, 0, 585, 589, 1, 0, 0, 0, 586, 588, 8, 22, 0, 0, 587, 586, 1, 0, 0, 0, 588, 591, 1, 0, 0, 0, 589, 587, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 593, 1, 0, 0, 0, 591, 589, 1, 0, 0, 0, 592, 594, 5, 13, 0, 0, 593, 592, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 596, 1, 0, 0, 0, 595, 597, 5, 10, 0, 0, 596, 595, 1, 0, 0, 0, 596, 597, 1, 0, 0, 0, 597, 598, 1, 0, 0, 0, 598, 599, 6, 20, 10, 0, 599, 56, 1, 0, 0, 0, 600, 601, 5, 47, 0, 0, 601, 602, 5, 42, 0, 0, 602, 607, 1, 0, 0, 0, 603, 606, 3, 57, 21, 0, 604, 606, 9, 0, 0, 0, 605, 603, 1, 0, 0, 0, 605, 604, 1, 0, 0, 0, 606, 609, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 607, 605, 1, 0, 0, 0, 608, 610, 1, 0, 0, 0, 609, 607, 1, 0, 0, 0, 610, 611, 5, 42, 0, 0, 611, 612, 5, 47, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 6, 21, 10, 0, 614, 58, 1, 0, 0, 0, 615, 617, 7, 23, 0, 0, 616, 615, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 620, 1, 0, 0, 0, 620, 621, 6, 22, 10, 0, 621, 60, 1, 0, 0, 0, 622, 623, 5, 124, 0, 0, 623, 624, 1, 0, 0, 0, 624, 625, 6, 23, 11, 0, 625, 62, 1, 0, 0, 0, 626, 627, 7, 24, 0, 0, 627, 64, 1, 0, 0, 0, 628, 629, 7, 25, 0, 0, 629, 66, 1, 0, 0, 0, 630, 631, 5, 92, 0, 0, 631, 632, 7, 26, 0, 0, 632, 68, 1, 0, 0, 0, 633, 634, 8, 27, 0, 0, 634, 70, 1, 0, 0, 0, 635, 637, 7, 3, 0, 0, 636, 638, 7, 28, 0, 0, 637, 636, 1, 0, 0, 0, 637, 638, 1, 0, 0, 0, 638, 640, 1, 0, 0, 0, 639, 641, 3, 63, 24, 0, 640, 639, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 640, 1, 0, 0, 0, 642, 643, 1, 0, 0, 0, 643, 72, 1, 0, 0, 0, 644, 645, 5, 64, 0, 0, 645, 74, 1, 0, 0, 0, 646, 647, 5, 96, 0, 0, 647, 76, 1, 0, 0, 0, 648, 652, 8, 29, 0, 0, 649, 650, 5, 96, 0, 0, 650, 652, 5, 96, 0, 0, 651, 648, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 652, 78, 1, 0, 0, 0, 653, 654, 5, 95, 0, 0, 654, 80, 1, 0, 0, 0, 655, 659, 3, 65, 25, 0, 656, 659, 3, 63, 24, 0, 657, 659, 3, 79, 32, 0, 658, 655, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 658, 657, 1, 0, 0, 0, 659, 82, 1, 0, 0, 0, 660, 665, 5, 34, 0, 0, 661, 664, 3, 67, 26, 0, 662, 664, 3, 69, 27, 0, 663, 661, 1, 0, 0, 0, 663, 662, 1, 0, 0, 0, 664, 667, 1, 0, 0, 0, 665, 663, 1, 0, 0, 0, 665, 666, 1, 0, 0, 0, 666, 668, 1, 0, 0, 0, 667, 665, 1, 0, 0, 0, 668, 690, 5, 34, 0, 0, 669, 670, 5, 34, 0, 0, 670, 671, 5, 34, 0, 0, 671, 672, 5, 34, 0, 0, 672, 676, 1, 0, 0, 0, 673, 675, 8, 22, 0, 0, 674, 673, 1, 0, 0, 0, 675, 678, 1, 0, 0, 0, 676, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 679, 1, 0, 0, 0, 678, 676, 1, 0, 0, 0, 679, 680, 5, 34, 0, 0, 680, 681, 5, 34, 0, 0, 681, 682, 5, 34, 0, 0, 682, 684, 1, 0, 0, 0, 683, 685, 5, 34, 0, 0, 684, 683, 1, 0, 0, 0, 684, 685, 1, 0, 0, 0, 685, 687, 1, 0, 0, 0, 686, 688, 5, 34, 0, 0, 687, 686, 1, 0, 0, 0, 687, 688, 1, 0, 0, 0, 688, 690, 1, 0, 0, 0, 689, 660, 1, 0, 0, 0, 689, 669, 1, 0, 0, 0, 690, 84, 1, 0, 0, 0, 691, 693, 3, 63, 24, 0, 692, 691, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 692, 1, 0, 0, 0, 694, 695, 1, 0, 0, 0, 695, 86, 1, 0, 0, 0, 696, 698, 3, 63, 24, 0, 697, 696, 1, 0, 0, 0, 698, 699, 1, 0, 0, 0, 699, 697, 1, 0, 0, 0, 699, 700, 1, 0, 0, 0, 700, 701, 1, 0, 0, 0, 701, 705, 3, 103, 44, 0, 702, 704, 3, 63, 24, 0, 703, 702, 1, 0, 0, 0, 704, 707, 1, 0, 0, 0, 705, 703, 1, 0, 0, 0, 705, 706, 1, 0, 0, 0, 706, 739, 1, 0, 0, 0, 707, 705, 1, 0, 0, 0, 708, 710, 3, 103, 44, 0, 709, 711, 3, 63, 24, 0, 710, 709, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0, 712, 710, 1, 0, 0, 0, 712, 713, 1, 0, 0, 0, 713, 739, 1, 0, 0, 0, 714, 716, 3, 63, 24, 0, 715, 714, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717, 715, 1, 0, 0, 0, 717, 718, 1, 0, 0, 0, 718, 726, 1, 0, 0, 0, 719, 723, 3, 103, 44, 0, 720, 722, 3, 63, 24, 0, 721, 720, 1, 0, 0, 0, 722, 725, 1, 0, 0, 0, 723, 721, 1, 0, 0, 0, 723, 724, 1, 0, 0, 0, 724, 727, 1, 0, 0, 0, 725, 723, 1, 0, 0, 0, 726, 719, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 728, 1, 0, 0, 0, 728, 729, 3, 71, 28, 0, 729, 739, 1, 0, 0, 0, 730, 732, 3, 103, 44, 0, 731, 733, 3, 63, 24, 0, 732, 731, 1, 0, 0, 0, 733, 734, 1, 0, 0, 0, 734, 732, 1, 0, 0, 0, 734, 735, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 737, 3, 71, 28, 0, 737, 739, 1, 0, 0, 0, 738, 697, 1, 0, 0, 0, 738, 708, 1, 0, 0, 0, 738, 715, 1, 0, 0, 0, 738, 730, 1, 0, 0, 0, 739, 88, 1, 0, 0, 0, 740, 741, 7, 30, 0, 0, 741, 742, 7, 31, 0, 0, 742, 90, 1, 0, 0, 0, 743, 744, 7, 12, 0, 0, 744, 745, 7, 9, 0, 0, 745, 746, 7, 0, 0, 0, 746, 92, 1, 0, 0, 0, 747, 748, 7, 12, 0, 0, 748, 749, 7, 2, 0, 0, 749, 750, 7, 4, 0, 0, 750, 94, 1, 0, 0, 0, 751, 752, 5, 61, 0, 0, 752, 96, 1, 0, 0, 0, 753, 754, 5, 58, 0, 0, 754, 755, 5, 58, 0, 0, 755, 98, 1, 0, 0, 0, 756, 757, 5, 44, 0, 0, 757, 100, 1, 0, 0, 0, 758, 759, 7, 0, 0, 0, 759, 760, 7, 3, 0, 0, 760, 761, 7, 2, 0, 0, 761, 762, 7, 4, 0, 0, 762, 102, 1, 0, 0, 0, 763, 764, 5, 46, 0, 0, 764, 104, 1, 0, 0, 0, 765, 766, 7, 15, 0, 0, 766, 767, 7, 12, 0, 0, 767, 768, 7, 13, 0, 0, 768, 769, 7, 2, 0, 0, 769, 770, 7, 3, 0, 0, 770, 106, 1, 0, 0, 0, 771, 772, 7, 15, 0, 0, 772, 773, 7, 1, 0, 0, 773, 774, 7, 6, 0, 0, 774, 775, 7, 2, 0, 0, 775, 776, 7, 5, 0, 0, 776, 108, 1, 0, 0, 0, 777, 778, 7, 1, 0, 0, 778, 779, 7, 9, 0, 0, 779, 110, 1, 0, 0, 0, 780, 781, 7, 1, 0, 0, 781, 782, 7, 2, 0, 0, 782, 112, 1, 0, 0, 0, 783, 784, 7, 13, 0, 0, 784, 785, 7, 12, 0, 0, 785, 786, 7, 2, 0, 0, 786, 787, 7, 5, 0, 0, 787, 114, 1, 0, 0, 0, 788, 789, 7, 13, 0, 0, 789, 790, 7, 1, 0, 0, 790, 791, 7, 18, 0, 0, 791, 792, 7, 3, 0, 0, 792, 116, 1, 0, 0, 0, 793, 794, 5, 40, 0, 0, 794, 118, 1, 0, 0, 0, 795, 796, 7, 9, 0, 0, 796, 797, 7, 7, 0, 0, 797, 798, 7, 5, 0, 0, 798, 120, 1, 0, 0, 0, 799, 800, 7, 9, 0, 0, 800, 801, 7, 20, 0, 0, 801, 802, 7, 13, 0, 0, 802, 803, 7, 13, 0, 0, 803, 122, 1, 0, 0, 0, 804, 805, 7, 9, 0, 0, 805, 806, 7, 20, 0, 0, 806, 807, 7, 13, 0, 0, 807, 808, 7, 13, 0, 0, 808, 809, 7, 2, 0, 0, 809, 124, 1, 0, 0, 0, 810, 811, 7, 7, 0, 0, 811, 812, 7, 6, 0, 0, 812, 126, 1, 0, 0, 0, 813, 814, 5, 63, 0, 0, 814, 128, 1, 0, 0, 0, 815, 816, 7, 6, 0, 0, 816, 817, 7, 13, 0, 0, 817, 818, 7, 1, 0, 0, 818, 819, 7, 18, 0, 0, 819, 820, 7, 3, 0, 0, 820, 130, 1, 0, 0, 0, 821, 822, 5, 41, 0, 0, 822, 132, 1, 0, 0, 0, 823, 824, 7, 5, 0, 0, 824, 825, 7, 6, 0, 0, 825, 826, 7, 20, 0, 0, 826, 827, 7, 3, 0, 0, 827, 134, 1, 0, 0, 0, 828, 829, 5, 61, 0, 0, 829, 830, 5, 61, 0, 0, 830, 136, 1, 0, 0, 0, 831, 832, 5, 61, 0, 0, 832, 833, 5, 126, 0, 0, 833, 138, 1, 0, 0, 0, 834, 835, 5, 33, 0, 0, 835, 836, 5, 61, 0, 0, 836, 140, 1, 0, 0, 0, 837, 838, 5, 60, 0, 0, 838, 142, 1, 0, 0, 0, 839, 840, 5, 60, 0, 0, 840, 841, 5, 61, 0, 0, 841, 144, 1, 0, 0, 0, 842, 843, 5, 62, 0, 0, 843, 146, 1, 0, 0, 0, 844, 845, 5, 62, 0, 0, 845, 846, 5, 61, 0, 0, 846, 148, 1, 0, 0, 0, 847, 848, 5, 43, 0, 0, 848, 150, 1, 0, 0, 0, 849, 850, 5, 45, 0, 0, 850, 152, 1, 0, 0, 0, 851, 852, 5, 42, 0, 0, 852, 154, 1, 0, 0, 0, 853, 854, 5, 47, 0, 0, 854, 156, 1, 0, 0, 0, 855, 856, 5, 37, 0, 0, 856, 158, 1, 0, 0, 0, 857, 858, 7, 16, 0, 0, 858, 859, 7, 12, 0, 0, 859, 860, 7, 5, 0, 0, 860, 861, 7, 4, 0, 0, 861, 862, 7, 10, 0, 0, 862, 160, 1, 0, 0, 0, 863, 864, 3, 45, 15, 0, 864, 865, 1, 0, 0, 0, 865, 866, 6, 73, 12, 0, 866, 162, 1, 0, 0, 0, 867, 870, 3, 127, 56, 0, 868, 871, 3, 65, 25, 0, 869, 871, 3, 79, 32, 0, 870, 868, 1, 0, 0, 0, 870, 869, 1, 0, 0, 0, 871, 875, 1, 0, 0, 0, 872, 874, 3, 81, 33, 0, 873, 872, 1, 0, 0, 0, 874, 877, 1, 0, 0, 0, 875, 873, 1, 0, 0, 0, 875, 876, 1, 0, 0, 0, 876, 885, 1, 0, 0, 0, 877, 875, 1, 0, 0, 0, 878, 880, 3, 127, 56, 0, 879, 881, 3, 63, 24, 0, 880, 879, 1, 0, 0, 0, 881, 882, 1, 0, 0, 0, 882, 880, 1, 0, 0, 0, 882, 883, 1, 0, 0, 0, 883, 885, 1, 0, 0, 0, 884, 867, 1, 0, 0, 0, 884, 878, 1, 0, 0, 0, 885, 164, 1, 0, 0, 0, 886, 887, 5, 91, 0, 0, 887, 888, 1, 0, 0, 0, 888, 889, 6, 75, 0, 0, 889, 890, 6, 75, 0, 0, 890, 166, 1, 0, 0, 0, 891, 892, 5, 93, 0, 0, 892, 893, 1, 0, 0, 0, 893, 894, 6, 76, 11, 0, 894, 895, 6, 76, 11, 0, 895, 168, 1, 0, 0, 0, 896, 900, 3, 65, 25, 0, 897, 899, 3, 81, 33, 0, 898, 897, 1, 0, 0, 0, 899, 902, 1, 0, 0, 0, 900, 898, 1, 0, 0, 0, 900, 901, 1, 0, 0, 0, 901, 913, 1, 0, 0, 0, 902, 900, 1, 0, 0, 0, 903, 906, 3, 79, 32, 0, 904, 906, 3, 73, 29, 0, 905, 903, 1, 0, 0, 0, 905, 904, 1, 0, 0, 0, 906, 908, 1, 0, 0, 0, 907, 909, 3, 81, 33, 0, 908, 907, 1, 0, 0, 0, 909, 910, 1, 0, 0, 0, 910, 908, 1, 0, 0, 0, 910, 911, 1, 0, 0, 0, 911, 913, 1, 0, 0, 0, 912, 896, 1, 0, 0, 0, 912, 905, 1, 0, 0, 0, 913, 170, 1, 0, 0, 0, 914, 916, 3, 75, 30, 0, 915, 917, 3, 77, 31, 0, 916, 915, 1, 0, 0, 0, 917, 918, 1, 0, 0, 0, 918, 916, 1, 0, 0, 0, 918, 919, 1, 0, 0, 0, 919, 920, 1, 0, 0, 0, 920, 921, 3, 75, 30, 0, 921, 172, 1, 0, 0, 0, 922, 923, 3, 171, 78, 0, 923, 174, 1, 0, 0, 0, 924, 925, 3, 55, 20, 0, 925, 926, 1, 0, 0, 0, 926, 927, 6, 80, 10, 0, 927, 176, 1, 0, 0, 0, 928, 929, 3, 57, 21, 0, 929, 930, 1, 0, 0, 0, 930, 931, 6, 81, 10, 0, 931, 178, 1, 0, 0, 0, 932, 933, 3, 59, 22, 0, 933, 934, 1, 0, 0, 0, 934, 935, 6, 82, 10, 0, 935, 180, 1, 0, 0, 0, 936, 937, 3, 165, 75, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 83, 13, 0, 939, 940, 6, 83, 14, 0, 940, 182, 1, 0, 0, 0, 941, 942, 3, 61, 23, 0, 942, 943, 1, 0, 0, 0, 943, 944, 6, 84, 15, 0, 944, 945, 6, 84, 11, 0, 945, 184, 1, 0, 0, 0, 946, 947, 3, 59, 22, 0, 947, 948, 1, 0, 0, 0, 948, 949, 6, 85, 10, 0, 949, 186, 1, 0, 0, 0, 950, 951, 3, 55, 20, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 86, 10, 0, 953, 188, 1, 0, 0, 0, 954, 955, 3, 57, 21, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 87, 10, 0, 957, 190, 1, 0, 0, 0, 958, 959, 3, 61, 23, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 88, 15, 0, 961, 962, 6, 88, 11, 0, 962, 192, 1, 0, 0, 0, 963, 964, 3, 165, 75, 0, 964, 965, 1, 0, 0, 0, 965, 966, 6, 89, 13, 0, 966, 194, 1, 0, 0, 0, 967, 968, 3, 167, 76, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 90, 16, 0, 970, 196, 1, 0, 0, 0, 971, 972, 3, 337, 161, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 91, 17, 0, 974, 198, 1, 0, 0, 0, 975, 976, 3, 99, 42, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 92, 18, 0, 978, 200, 1, 0, 0, 0, 979, 980, 3, 95, 40, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 93, 19, 0, 982, 202, 1, 0, 0, 0, 983, 984, 7, 16, 0, 0, 984, 985, 7, 3, 0, 0, 985, 986, 7, 5, 0, 0, 986, 987, 7, 12, 0, 0, 987, 988, 7, 0, 0, 0, 988, 989, 7, 12, 0, 0, 989, 990, 7, 5, 0, 0, 990, 991, 7, 12, 0, 0, 991, 204, 1, 0, 0, 0, 992, 996, 8, 32, 0, 0, 993, 994, 5, 47, 0, 0, 994, 996, 8, 33, 0, 0, 995, 992, 1, 0, 0, 0, 995, 993, 1, 0, 0, 0, 996, 206, 1, 0, 0, 0, 997, 999, 3, 205, 95, 0, 998, 997, 1, 0, 0, 0, 999, 1000, 1, 0, 0, 0, 1000, 998, 1, 0, 0, 0, 1000, 1001, 1, 0, 0, 0, 1001, 208, 1, 0, 0, 0, 1002, 1003, 3, 207, 96, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1005, 6, 97, 20, 0, 1005, 210, 1, 0, 0, 0, 1006, 1007, 3, 83, 34, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 98, 21, 0, 1009, 212, 1, 0, 0, 0, 1010, 1011, 3, 55, 20, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 99, 10, 0, 1013, 214, 1, 0, 0, 0, 1014, 1015, 3, 57, 21, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 100, 10, 0, 1017, 216, 1, 0, 0, 0, 1018, 1019, 3, 59, 22, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 101, 10, 0, 1021, 218, 1, 0, 0, 0, 1022, 1023, 3, 61, 23, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 102, 15, 0, 1025, 1026, 6, 102, 11, 0, 1026, 220, 1, 0, 0, 0, 1027, 1028, 3, 103, 44, 0, 1028, 1029, 1, 0, 0, 0, 1029, 1030, 6, 103, 22, 0, 1030, 222, 1, 0, 0, 0, 1031, 1032, 3, 99, 42, 0, 1032, 1033, 1, 0, 0, 0, 1033, 1034, 6, 104, 18, 0, 1034, 224, 1, 0, 0, 0, 1035, 1036, 3, 127, 56, 0, 1036, 1037, 1, 0, 0, 0, 1037, 1038, 6, 105, 23, 0, 1038, 226, 1, 0, 0, 0, 1039, 1040, 3, 163, 74, 0, 1040, 1041, 1, 0, 0, 0, 1041, 1042, 6, 106, 24, 0, 1042, 228, 1, 0, 0, 0, 1043, 1048, 3, 65, 25, 0, 1044, 1048, 3, 63, 24, 0, 1045, 1048, 3, 79, 32, 0, 1046, 1048, 3, 153, 69, 0, 1047, 1043, 1, 0, 0, 0, 1047, 1044, 1, 0, 0, 0, 1047, 1045, 1, 0, 0, 0, 1047, 1046, 1, 0, 0, 0, 1048, 230, 1, 0, 0, 0, 1049, 1052, 3, 65, 25, 0, 1050, 1052, 3, 153, 69, 0, 1051, 1049, 1, 0, 0, 0, 1051, 1050, 1, 0, 0, 0, 1052, 1056, 1, 0, 0, 0, 1053, 1055, 3, 229, 107, 0, 1054, 1053, 1, 0, 0, 0, 1055, 1058, 1, 0, 0, 0, 1056, 1054, 1, 0, 0, 0, 1056, 1057, 1, 0, 0, 0, 1057, 1069, 1, 0, 0, 0, 1058, 1056, 1, 0, 0, 0, 1059, 1062, 3, 79, 32, 0, 1060, 1062, 3, 73, 29, 0, 1061, 1059, 1, 0, 0, 0, 1061, 1060, 1, 0, 0, 0, 1062, 1064, 1, 0, 0, 0, 1063, 1065, 3, 229, 107, 0, 1064, 1063, 1, 0, 0, 0, 1065, 1066, 1, 0, 0, 0, 1066, 1064, 1, 0, 0, 0, 1066, 1067, 1, 0, 0, 0, 1067, 1069, 1, 0, 0, 0, 1068, 1051, 1, 0, 0, 0, 1068, 1061, 1, 0, 0, 0, 1069, 232, 1, 0, 0, 0, 1070, 1073, 3, 231, 108, 0, 1071, 1073, 3, 171, 78, 0, 1072, 1070, 1, 0, 0, 0, 1072, 1071, 1, 0, 0, 0, 1073, 1074, 1, 0, 0, 0, 1074, 1072, 1, 0, 0, 0, 1074, 1075, 1, 0, 0, 0, 1075, 234, 1, 0, 0, 0, 1076, 1077, 3, 55, 20, 0, 1077, 1078, 1, 0, 0, 0, 1078, 1079, 6, 110, 10, 0, 1079, 236, 1, 0, 0, 0, 1080, 1081, 3, 57, 21, 0, 1081, 1082, 1, 0, 0, 0, 1082, 1083, 6, 111, 10, 0, 1083, 238, 1, 0, 0, 0, 1084, 1085, 3, 59, 22, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1087, 6, 112, 10, 0, 1087, 240, 1, 0, 0, 0, 1088, 1089, 3, 61, 23, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 113, 15, 0, 1091, 1092, 6, 113, 11, 0, 1092, 242, 1, 0, 0, 0, 1093, 1094, 3, 95, 40, 0, 1094, 1095, 1, 0, 0, 0, 1095, 1096, 6, 114, 19, 0, 1096, 244, 1, 0, 0, 0, 1097, 1098, 3, 99, 42, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 6, 115, 18, 0, 1100, 246, 1, 0, 0, 0, 1101, 1102, 3, 103, 44, 0, 1102, 1103, 1, 0, 0, 0, 1103, 1104, 6, 116, 22, 0, 1104, 248, 1, 0, 0, 0, 1105, 1106, 3, 127, 56, 0, 1106, 1107, 1, 0, 0, 0, 1107, 1108, 6, 117, 23, 0, 1108, 250, 1, 0, 0, 0, 1109, 1110, 3, 163, 74, 0, 1110, 1111, 1, 0, 0, 0, 1111, 1112, 6, 118, 24, 0, 1112, 252, 1, 0, 0, 0, 1113, 1114, 7, 12, 0, 0, 1114, 1115, 7, 2, 0, 0, 1115, 254, 1, 0, 0, 0, 1116, 1117, 3, 233, 109, 0, 1117, 1118, 1, 0, 0, 0, 1118, 1119, 6, 120, 25, 0, 1119, 256, 1, 0, 0, 0, 1120, 1121, 3, 55, 20, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 121, 10, 0, 1123, 258, 1, 0, 0, 0, 1124, 1125, 3, 57, 21, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 122, 10, 0, 1127, 260, 1, 0, 0, 0, 1128, 1129, 3, 59, 22, 0, 1129, 1130, 1, 0, 0, 0, 1130, 1131, 6, 123, 10, 0, 1131, 262, 1, 0, 0, 0, 1132, 1133, 3, 61, 23, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 124, 15, 0, 1135, 1136, 6, 124, 11, 0, 1136, 264, 1, 0, 0, 0, 1137, 1138, 3, 165, 75, 0, 1138, 1139, 1, 0, 0, 0, 1139, 1140, 6, 125, 13, 0, 1140, 1141, 6, 125, 26, 0, 1141, 266, 1, 0, 0, 0, 1142, 1143, 7, 7, 0, 0, 1143, 1144, 7, 9, 0, 0, 1144, 1145, 1, 0, 0, 0, 1145, 1146, 6, 126, 27, 0, 1146, 268, 1, 0, 0, 0, 1147, 1148, 7, 19, 0, 0, 1148, 1149, 7, 1, 0, 0, 1149, 1150, 7, 5, 0, 0, 1150, 1151, 7, 10, 0, 0, 1151, 1152, 1, 0, 0, 0, 1152, 1153, 6, 127, 27, 0, 1153, 270, 1, 0, 0, 0, 1154, 1155, 8, 34, 0, 0, 1155, 272, 1, 0, 0, 0, 1156, 1158, 3, 271, 128, 0, 1157, 1156, 1, 0, 0, 0, 1158, 1159, 1, 0, 0, 0, 1159, 1157, 1, 0, 0, 0, 1159, 1160, 1, 0, 0, 0, 1160, 1161, 1, 0, 0, 0, 1161, 1162, 3, 337, 161, 0, 1162, 1164, 1, 0, 0, 0, 1163, 1157, 1, 0, 0, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1166, 1, 0, 0, 0, 1165, 1167, 3, 271, 128, 0, 1166, 1165, 1, 0, 0, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1166, 1, 0, 0, 0, 1168, 1169, 1, 0, 0, 0, 1169, 274, 1, 0, 0, 0, 1170, 1171, 3, 273, 129, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1173, 6, 130, 28, 0, 1173, 276, 1, 0, 0, 0, 1174, 1175, 3, 55, 20, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 131, 10, 0, 1177, 278, 1, 0, 0, 0, 1178, 1179, 3, 57, 21, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 132, 10, 0, 1181, 280, 1, 0, 0, 0, 1182, 1183, 3, 59, 22, 0, 1183, 1184, 1, 0, 0, 0, 1184, 1185, 6, 133, 10, 0, 1185, 282, 1, 0, 0, 0, 1186, 1187, 3, 61, 23, 0, 1187, 1188, 1, 0, 0, 0, 1188, 1189, 6, 134, 15, 0, 1189, 1190, 6, 134, 11, 0, 1190, 1191, 6, 134, 11, 0, 1191, 284, 1, 0, 0, 0, 1192, 1193, 3, 95, 40, 0, 1193, 1194, 1, 0, 0, 0, 1194, 1195, 6, 135, 19, 0, 1195, 286, 1, 0, 0, 0, 1196, 1197, 3, 99, 42, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 136, 18, 0, 1199, 288, 1, 0, 0, 0, 1200, 1201, 3, 103, 44, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 137, 22, 0, 1203, 290, 1, 0, 0, 0, 1204, 1205, 3, 269, 127, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 138, 29, 0, 1207, 292, 1, 0, 0, 0, 1208, 1209, 3, 233, 109, 0, 1209, 1210, 1, 0, 0, 0, 1210, 1211, 6, 139, 25, 0, 1211, 294, 1, 0, 0, 0, 1212, 1213, 3, 173, 79, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 140, 30, 0, 1215, 296, 1, 0, 0, 0, 1216, 1217, 3, 127, 56, 0, 1217, 1218, 1, 0, 0, 0, 1218, 1219, 6, 141, 23, 0, 1219, 298, 1, 0, 0, 0, 1220, 1221, 3, 163, 74, 0, 1221, 1222, 1, 0, 0, 0, 1222, 1223, 6, 142, 24, 0, 1223, 300, 1, 0, 0, 0, 1224, 1225, 3, 55, 20, 0, 1225, 1226, 1, 0, 0, 0, 1226, 1227, 6, 143, 10, 0, 1227, 302, 1, 0, 0, 0, 1228, 1229, 3, 57, 21, 0, 1229, 1230, 1, 0, 0, 0, 1230, 1231, 6, 144, 10, 0, 1231, 304, 1, 0, 0, 0, 1232, 1233, 3, 59, 22, 0, 1233, 1234, 1, 0, 0, 0, 1234, 1235, 6, 145, 10, 0, 1235, 306, 1, 0, 0, 0, 1236, 1237, 3, 61, 23, 0, 1237, 1238, 1, 0, 0, 0, 1238, 1239, 6, 146, 15, 0, 1239, 1240, 6, 146, 11, 0, 1240, 308, 1, 0, 0, 0, 1241, 1242, 3, 103, 44, 0, 1242, 1243, 1, 0, 0, 0, 1243, 1244, 6, 147, 22, 0, 1244, 310, 1, 0, 0, 0, 1245, 1246, 3, 127, 56, 0, 1246, 1247, 1, 0, 0, 0, 1247, 1248, 6, 148, 23, 0, 1248, 312, 1, 0, 0, 0, 1249, 1250, 3, 163, 74, 0, 1250, 1251, 1, 0, 0, 0, 1251, 1252, 6, 149, 24, 0, 1252, 314, 1, 0, 0, 0, 1253, 1254, 3, 173, 79, 0, 1254, 1255, 1, 0, 0, 0, 1255, 1256, 6, 150, 30, 0, 1256, 316, 1, 0, 0, 0, 1257, 1258, 3, 169, 77, 0, 1258, 1259, 1, 0, 0, 0, 1259, 1260, 6, 151, 31, 0, 1260, 318, 1, 0, 0, 0, 1261, 1262, 3, 55, 20, 0, 1262, 1263, 1, 0, 0, 0, 1263, 1264, 6, 152, 10, 0, 1264, 320, 1, 0, 0, 0, 1265, 1266, 3, 57, 21, 0, 1266, 1267, 1, 0, 0, 0, 1267, 1268, 6, 153, 10, 0, 1268, 322, 1, 0, 0, 0, 1269, 1270, 3, 59, 22, 0, 1270, 1271, 1, 0, 0, 0, 1271, 1272, 6, 154, 10, 0, 1272, 324, 1, 0, 0, 0, 1273, 1274, 3, 61, 23, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 155, 15, 0, 1276, 1277, 6, 155, 11, 0, 1277, 326, 1, 0, 0, 0, 1278, 1279, 7, 1, 0, 0, 1279, 1280, 7, 9, 0, 0, 1280, 1281, 7, 15, 0, 0, 1281, 1282, 7, 7, 0, 0, 1282, 328, 1, 0, 0, 0, 1283, 1284, 3, 55, 20, 0, 1284, 1285, 1, 0, 0, 0, 1285, 1286, 6, 157, 10, 0, 1286, 330, 1, 0, 0, 0, 1287, 1288, 3, 57, 21, 0, 1288, 1289, 1, 0, 0, 0, 1289, 1290, 6, 158, 10, 0, 1290, 332, 1, 0, 0, 0, 1291, 1292, 3, 59, 22, 0, 1292, 1293, 1, 0, 0, 0, 1293, 1294, 6, 159, 10, 0, 1294, 334, 1, 0, 0, 0, 1295, 1296, 3, 167, 76, 0, 1296, 1297, 1, 0, 0, 0, 1297, 1298, 6, 160, 16, 0, 1298, 1299, 6, 160, 11, 0, 1299, 336, 1, 0, 0, 0, 1300, 1301, 5, 58, 0, 0, 1301, 338, 1, 0, 0, 0, 1302, 1308, 3, 73, 29, 0, 1303, 1308, 3, 63, 24, 0, 1304, 1308, 3, 103, 44, 0, 1305, 1308, 3, 65, 25, 0, 1306, 1308, 3, 79, 32, 0, 1307, 1302, 1, 0, 0, 0, 1307, 1303, 1, 0, 0, 0, 1307, 1304, 1, 0, 0, 0, 1307, 1305, 1, 0, 0, 0, 1307, 1306, 1, 0, 0, 0, 1308, 1309, 1, 0, 0, 0, 1309, 1307, 1, 0, 0, 0, 1309, 1310, 1, 0, 0, 0, 1310, 340, 1, 0, 0, 0, 1311, 1312, 3, 55, 20, 0, 1312, 1313, 1, 0, 0, 0, 1313, 1314, 6, 163, 10, 0, 1314, 342, 1, 0, 0, 0, 1315, 1316, 3, 57, 21, 0, 1316, 1317, 1, 0, 0, 0, 1317, 1318, 6, 164, 10, 0, 1318, 344, 1, 0, 0, 0, 1319, 1320, 3, 59, 22, 0, 1320, 1321, 1, 0, 0, 0, 1321, 1322, 6, 165, 10, 0, 1322, 346, 1, 0, 0, 0, 1323, 1324, 3, 61, 23, 0, 1324, 1325, 1, 0, 0, 0, 1325, 1326, 6, 166, 15, 0, 1326, 1327, 6, 166, 11, 0, 1327, 348, 1, 0, 0, 0, 1328, 1329, 3, 337, 161, 0, 1329, 1330, 1, 0, 0, 0, 1330, 1331, 6, 167, 17, 0, 1331, 350, 1, 0, 0, 0, 1332, 1333, 3, 99, 42, 0, 1333, 1334, 1, 0, 0, 0, 1334, 1335, 6, 168, 18, 0, 1335, 352, 1, 0, 0, 0, 1336, 1337, 3, 103, 44, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 169, 22, 0, 1339, 354, 1, 0, 0, 0, 1340, 1341, 3, 267, 126, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 170, 32, 0, 1343, 1344, 6, 170, 33, 0, 1344, 356, 1, 0, 0, 0, 1345, 1346, 3, 207, 96, 0, 1346, 1347, 1, 0, 0, 0, 1347, 1348, 6, 171, 20, 0, 1348, 358, 1, 0, 0, 0, 1349, 1350, 3, 83, 34, 0, 1350, 1351, 1, 0, 0, 0, 1351, 1352, 6, 172, 21, 0, 1352, 360, 1, 0, 0, 0, 1353, 1354, 3, 55, 20, 0, 1354, 1355, 1, 0, 0, 0, 1355, 1356, 6, 173, 10, 0, 1356, 362, 1, 0, 0, 0, 1357, 1358, 3, 57, 21, 0, 1358, 1359, 1, 0, 0, 0, 1359, 1360, 6, 174, 10, 0, 1360, 364, 1, 0, 0, 0, 1361, 1362, 3, 59, 22, 0, 1362, 1363, 1, 0, 0, 0, 1363, 1364, 6, 175, 10, 0, 1364, 366, 1, 0, 0, 0, 1365, 1366, 3, 61, 23, 0, 1366, 1367, 1, 0, 0, 0, 1367, 1368, 6, 176, 15, 0, 1368, 1369, 6, 176, 11, 0, 1369, 1370, 6, 176, 11, 0, 1370, 368, 1, 0, 0, 0, 1371, 1372, 3, 99, 42, 0, 1372, 1373, 1, 0, 0, 0, 1373, 1374, 6, 177, 18, 0, 1374, 370, 1, 0, 0, 0, 1375, 1376, 3, 103, 44, 0, 1376, 1377, 1, 0, 0, 0, 1377, 1378, 6, 178, 22, 0, 1378, 372, 1, 0, 0, 0, 1379, 1380, 3, 233, 109, 0, 1380, 1381, 1, 0, 0, 0, 1381, 1382, 6, 179, 25, 0, 1382, 374, 1, 0, 0, 0, 1383, 1384, 3, 55, 20, 0, 1384, 1385, 1, 0, 0, 0, 1385, 1386, 6, 180, 10, 0, 1386, 376, 1, 0, 0, 0, 1387, 1388, 3, 57, 21, 0, 1388, 1389, 1, 0, 0, 0, 1389, 1390, 6, 181, 10, 0, 1390, 378, 1, 0, 0, 0, 1391, 1392, 3, 59, 22, 0, 1392, 1393, 1, 0, 0, 0, 1393, 1394, 6, 182, 10, 0, 1394, 380, 1, 0, 0, 0, 1395, 1396, 3, 61, 23, 0, 1396, 1397, 1, 0, 0, 0, 1397, 1398, 6, 183, 15, 0, 1398, 1399, 6, 183, 11, 0, 1399, 382, 1, 0, 0, 0, 1400, 1401, 3, 207, 96, 0, 1401, 1402, 1, 0, 0, 0, 1402, 1403, 6, 184, 20, 0, 1403, 1404, 6, 184, 11, 0, 1404, 1405, 6, 184, 34, 0, 1405, 384, 1, 0, 0, 0, 1406, 1407, 3, 83, 34, 0, 1407, 1408, 1, 0, 0, 0, 1408, 1409, 6, 185, 21, 0, 1409, 1410, 6, 185, 11, 0, 1410, 1411, 6, 185, 34, 0, 1411, 386, 1, 0, 0, 0, 1412, 1413, 3, 55, 20, 0, 1413, 1414, 1, 0, 0, 0, 1414, 1415, 6, 186, 10, 0, 1415, 388, 1, 0, 0, 0, 1416, 1417, 3, 57, 21, 0, 1417, 1418, 1, 0, 0, 0, 1418, 1419, 6, 187, 10, 0, 1419, 390, 1, 0, 0, 0, 1420, 1421, 3, 59, 22, 0, 1421, 1422, 1, 0, 0, 0, 1422, 1423, 6, 188, 10, 0, 1423, 392, 1, 0, 0, 0, 1424, 1425, 3, 337, 161, 0, 1425, 1426, 1, 0, 0, 0, 1426, 1427, 6, 189, 17, 0, 1427, 1428, 6, 189, 11, 0, 1428, 1429, 6, 189, 9, 0, 1429, 394, 1, 0, 0, 0, 1430, 1431, 3, 99, 42, 0, 1431, 1432, 1, 0, 0, 0, 1432, 1433, 6, 190, 18, 0, 1433, 1434, 6, 190, 11, 0, 1434, 1435, 6, 190, 9, 0, 1435, 396, 1, 0, 0, 0, 1436, 1437, 3, 55, 20, 0, 1437, 1438, 1, 0, 0, 0, 1438, 1439, 6, 191, 10, 0, 1439, 398, 1, 0, 0, 0, 1440, 1441, 3, 57, 21, 0, 1441, 1442, 1, 0, 0, 0, 1442, 1443, 6, 192, 10, 0, 1443, 400, 1, 0, 0, 0, 1444, 1445, 3, 59, 22, 0, 1445, 1446, 1, 0, 0, 0, 1446, 1447, 6, 193, 10, 0, 1447, 402, 1, 0, 0, 0, 1448, 1449, 3, 173, 79, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1451, 6, 194, 11, 0, 1451, 1452, 6, 194, 0, 0, 1452, 1453, 6, 194, 30, 0, 1453, 404, 1, 0, 0, 0, 1454, 1455, 3, 169, 77, 0, 1455, 1456, 1, 0, 0, 0, 1456, 1457, 6, 195, 11, 0, 1457, 1458, 6, 195, 0, 0, 1458, 1459, 6, 195, 31, 0, 1459, 406, 1, 0, 0, 0, 1460, 1461, 3, 89, 37, 0, 1461, 1462, 1, 0, 0, 0, 1462, 1463, 6, 196, 11, 0, 1463, 1464, 6, 196, 0, 0, 1464, 1465, 6, 196, 35, 0, 1465, 408, 1, 0, 0, 0, 1466, 1467, 3, 61, 23, 0, 1467, 1468, 1, 0, 0, 0, 1468, 1469, 6, 197, 15, 0, 1469, 1470, 6, 197, 11, 0, 1470, 410, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 579, 589, 593, 596, 605, 607, 618, 637, 642, 651, 658, 663, 665, 676, 684, 687, 689, 694, 699, 705, 712, 717, 723, 726, 734, 738, 870, 875, 882, 884, 900, 905, 910, 912, 918, 995, 1000, 1047, 1051, 1056, 1061, 1066, 1068, 1072, 1074, 1159, 1163, 1168, 1307, 1309, 36, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 65, 0, 5, 0, 0, 7, 24, 0, 7, 66, 0, 7, 104, 0, 7, 33, 0, 7, 31, 0, 7, 76, 0, 7, 25, 0, 7, 35, 0, 7, 47, 0, 7, 64, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 28, 0] \ No newline at end of file +[4, 0, 120, 1479, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 4, 19, 578, 8, 19, 11, 19, 12, 19, 579, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 588, 8, 20, 10, 20, 12, 20, 591, 9, 20, 1, 20, 3, 20, 594, 8, 20, 1, 20, 3, 20, 597, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 606, 8, 21, 10, 21, 12, 21, 609, 9, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 4, 22, 617, 8, 22, 11, 22, 12, 22, 618, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 3, 28, 638, 8, 28, 1, 28, 4, 28, 641, 8, 28, 11, 28, 12, 28, 642, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 652, 8, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 659, 8, 33, 1, 34, 1, 34, 1, 34, 5, 34, 664, 8, 34, 10, 34, 12, 34, 667, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 675, 8, 34, 10, 34, 12, 34, 678, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 685, 8, 34, 1, 34, 3, 34, 688, 8, 34, 3, 34, 690, 8, 34, 1, 35, 4, 35, 693, 8, 35, 11, 35, 12, 35, 694, 1, 36, 4, 36, 698, 8, 36, 11, 36, 12, 36, 699, 1, 36, 1, 36, 5, 36, 704, 8, 36, 10, 36, 12, 36, 707, 9, 36, 1, 36, 1, 36, 4, 36, 711, 8, 36, 11, 36, 12, 36, 712, 1, 36, 4, 36, 716, 8, 36, 11, 36, 12, 36, 717, 1, 36, 1, 36, 5, 36, 722, 8, 36, 10, 36, 12, 36, 725, 9, 36, 3, 36, 727, 8, 36, 1, 36, 1, 36, 1, 36, 1, 36, 4, 36, 733, 8, 36, 11, 36, 12, 36, 734, 1, 36, 1, 36, 3, 36, 739, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 3, 74, 871, 8, 74, 1, 74, 5, 74, 874, 8, 74, 10, 74, 12, 74, 877, 9, 74, 1, 74, 1, 74, 4, 74, 881, 8, 74, 11, 74, 12, 74, 882, 3, 74, 885, 8, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 5, 77, 899, 8, 77, 10, 77, 12, 77, 902, 9, 77, 1, 77, 1, 77, 3, 77, 906, 8, 77, 1, 77, 4, 77, 909, 8, 77, 11, 77, 12, 77, 910, 3, 77, 913, 8, 77, 1, 78, 1, 78, 4, 78, 917, 8, 78, 11, 78, 12, 78, 918, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 3, 95, 996, 8, 95, 1, 96, 4, 96, 999, 8, 96, 11, 96, 12, 96, 1000, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 3, 107, 1050, 8, 107, 1, 108, 1, 108, 3, 108, 1054, 8, 108, 1, 108, 5, 108, 1057, 8, 108, 10, 108, 12, 108, 1060, 9, 108, 1, 108, 1, 108, 3, 108, 1064, 8, 108, 1, 108, 4, 108, 1067, 8, 108, 11, 108, 12, 108, 1068, 3, 108, 1071, 8, 108, 1, 109, 1, 109, 4, 109, 1075, 8, 109, 11, 109, 12, 109, 1076, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 129, 4, 129, 1162, 8, 129, 11, 129, 12, 129, 1163, 1, 129, 1, 129, 3, 129, 1168, 8, 129, 1, 129, 4, 129, 1171, 8, 129, 11, 129, 12, 129, 1172, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 4, 162, 1316, 8, 162, 11, 162, 12, 162, 1317, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 2, 607, 676, 0, 198, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 0, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 25, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 0, 163, 64, 165, 65, 167, 66, 169, 67, 171, 0, 173, 68, 175, 69, 177, 70, 179, 71, 181, 0, 183, 0, 185, 72, 187, 73, 189, 74, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 0, 203, 75, 205, 0, 207, 76, 209, 0, 211, 0, 213, 77, 215, 78, 217, 79, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 0, 231, 0, 233, 80, 235, 81, 237, 82, 239, 83, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 0, 253, 84, 255, 0, 257, 85, 259, 86, 261, 87, 263, 0, 265, 0, 267, 88, 269, 89, 271, 0, 273, 90, 275, 0, 277, 91, 279, 92, 281, 93, 283, 0, 285, 0, 287, 0, 289, 0, 291, 0, 293, 0, 295, 0, 297, 0, 299, 0, 301, 94, 303, 95, 305, 96, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 97, 321, 98, 323, 99, 325, 0, 327, 100, 329, 101, 331, 102, 333, 103, 335, 0, 337, 104, 339, 105, 341, 106, 343, 107, 345, 108, 347, 0, 349, 0, 351, 0, 353, 0, 355, 0, 357, 0, 359, 0, 361, 109, 363, 110, 365, 111, 367, 0, 369, 0, 371, 0, 373, 0, 375, 112, 377, 113, 379, 114, 381, 0, 383, 0, 385, 0, 387, 115, 389, 116, 391, 117, 393, 0, 395, 0, 397, 118, 399, 119, 401, 120, 403, 0, 405, 0, 407, 0, 409, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1507, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 1, 83, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 2, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 203, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 225, 1, 0, 0, 0, 4, 227, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 4, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 5, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 5, 261, 1, 0, 0, 0, 6, 263, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 269, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 6, 275, 1, 0, 0, 0, 6, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 6, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 7, 295, 1, 0, 0, 0, 7, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 8, 309, 1, 0, 0, 0, 8, 311, 1, 0, 0, 0, 8, 313, 1, 0, 0, 0, 8, 315, 1, 0, 0, 0, 8, 317, 1, 0, 0, 0, 8, 319, 1, 0, 0, 0, 8, 321, 1, 0, 0, 0, 8, 323, 1, 0, 0, 0, 9, 325, 1, 0, 0, 0, 9, 327, 1, 0, 0, 0, 9, 329, 1, 0, 0, 0, 9, 331, 1, 0, 0, 0, 9, 333, 1, 0, 0, 0, 10, 335, 1, 0, 0, 0, 10, 337, 1, 0, 0, 0, 10, 339, 1, 0, 0, 0, 10, 341, 1, 0, 0, 0, 10, 343, 1, 0, 0, 0, 10, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 11, 351, 1, 0, 0, 0, 11, 353, 1, 0, 0, 0, 11, 355, 1, 0, 0, 0, 11, 357, 1, 0, 0, 0, 11, 359, 1, 0, 0, 0, 11, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 11, 365, 1, 0, 0, 0, 12, 367, 1, 0, 0, 0, 12, 369, 1, 0, 0, 0, 12, 371, 1, 0, 0, 0, 12, 373, 1, 0, 0, 0, 12, 375, 1, 0, 0, 0, 12, 377, 1, 0, 0, 0, 12, 379, 1, 0, 0, 0, 13, 381, 1, 0, 0, 0, 13, 383, 1, 0, 0, 0, 13, 385, 1, 0, 0, 0, 13, 387, 1, 0, 0, 0, 13, 389, 1, 0, 0, 0, 13, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 14, 395, 1, 0, 0, 0, 14, 397, 1, 0, 0, 0, 14, 399, 1, 0, 0, 0, 14, 401, 1, 0, 0, 0, 14, 403, 1, 0, 0, 0, 14, 405, 1, 0, 0, 0, 14, 407, 1, 0, 0, 0, 14, 409, 1, 0, 0, 0, 15, 411, 1, 0, 0, 0, 17, 421, 1, 0, 0, 0, 19, 428, 1, 0, 0, 0, 21, 437, 1, 0, 0, 0, 23, 444, 1, 0, 0, 0, 25, 454, 1, 0, 0, 0, 27, 461, 1, 0, 0, 0, 29, 468, 1, 0, 0, 0, 31, 475, 1, 0, 0, 0, 33, 483, 1, 0, 0, 0, 35, 495, 1, 0, 0, 0, 37, 504, 1, 0, 0, 0, 39, 510, 1, 0, 0, 0, 41, 517, 1, 0, 0, 0, 43, 524, 1, 0, 0, 0, 45, 532, 1, 0, 0, 0, 47, 540, 1, 0, 0, 0, 49, 555, 1, 0, 0, 0, 51, 565, 1, 0, 0, 0, 53, 577, 1, 0, 0, 0, 55, 583, 1, 0, 0, 0, 57, 600, 1, 0, 0, 0, 59, 616, 1, 0, 0, 0, 61, 622, 1, 0, 0, 0, 63, 626, 1, 0, 0, 0, 65, 628, 1, 0, 0, 0, 67, 630, 1, 0, 0, 0, 69, 633, 1, 0, 0, 0, 71, 635, 1, 0, 0, 0, 73, 644, 1, 0, 0, 0, 75, 646, 1, 0, 0, 0, 77, 651, 1, 0, 0, 0, 79, 653, 1, 0, 0, 0, 81, 658, 1, 0, 0, 0, 83, 689, 1, 0, 0, 0, 85, 692, 1, 0, 0, 0, 87, 738, 1, 0, 0, 0, 89, 740, 1, 0, 0, 0, 91, 743, 1, 0, 0, 0, 93, 747, 1, 0, 0, 0, 95, 751, 1, 0, 0, 0, 97, 753, 1, 0, 0, 0, 99, 756, 1, 0, 0, 0, 101, 758, 1, 0, 0, 0, 103, 763, 1, 0, 0, 0, 105, 765, 1, 0, 0, 0, 107, 771, 1, 0, 0, 0, 109, 777, 1, 0, 0, 0, 111, 780, 1, 0, 0, 0, 113, 783, 1, 0, 0, 0, 115, 788, 1, 0, 0, 0, 117, 793, 1, 0, 0, 0, 119, 795, 1, 0, 0, 0, 121, 799, 1, 0, 0, 0, 123, 804, 1, 0, 0, 0, 125, 810, 1, 0, 0, 0, 127, 813, 1, 0, 0, 0, 129, 815, 1, 0, 0, 0, 131, 821, 1, 0, 0, 0, 133, 823, 1, 0, 0, 0, 135, 828, 1, 0, 0, 0, 137, 831, 1, 0, 0, 0, 139, 834, 1, 0, 0, 0, 141, 837, 1, 0, 0, 0, 143, 839, 1, 0, 0, 0, 145, 842, 1, 0, 0, 0, 147, 844, 1, 0, 0, 0, 149, 847, 1, 0, 0, 0, 151, 849, 1, 0, 0, 0, 153, 851, 1, 0, 0, 0, 155, 853, 1, 0, 0, 0, 157, 855, 1, 0, 0, 0, 159, 857, 1, 0, 0, 0, 161, 863, 1, 0, 0, 0, 163, 884, 1, 0, 0, 0, 165, 886, 1, 0, 0, 0, 167, 891, 1, 0, 0, 0, 169, 912, 1, 0, 0, 0, 171, 914, 1, 0, 0, 0, 173, 922, 1, 0, 0, 0, 175, 924, 1, 0, 0, 0, 177, 928, 1, 0, 0, 0, 179, 932, 1, 0, 0, 0, 181, 936, 1, 0, 0, 0, 183, 941, 1, 0, 0, 0, 185, 946, 1, 0, 0, 0, 187, 950, 1, 0, 0, 0, 189, 954, 1, 0, 0, 0, 191, 958, 1, 0, 0, 0, 193, 963, 1, 0, 0, 0, 195, 967, 1, 0, 0, 0, 197, 971, 1, 0, 0, 0, 199, 975, 1, 0, 0, 0, 201, 979, 1, 0, 0, 0, 203, 983, 1, 0, 0, 0, 205, 995, 1, 0, 0, 0, 207, 998, 1, 0, 0, 0, 209, 1002, 1, 0, 0, 0, 211, 1006, 1, 0, 0, 0, 213, 1010, 1, 0, 0, 0, 215, 1014, 1, 0, 0, 0, 217, 1018, 1, 0, 0, 0, 219, 1022, 1, 0, 0, 0, 221, 1027, 1, 0, 0, 0, 223, 1031, 1, 0, 0, 0, 225, 1035, 1, 0, 0, 0, 227, 1040, 1, 0, 0, 0, 229, 1049, 1, 0, 0, 0, 231, 1070, 1, 0, 0, 0, 233, 1074, 1, 0, 0, 0, 235, 1078, 1, 0, 0, 0, 237, 1082, 1, 0, 0, 0, 239, 1086, 1, 0, 0, 0, 241, 1090, 1, 0, 0, 0, 243, 1095, 1, 0, 0, 0, 245, 1099, 1, 0, 0, 0, 247, 1103, 1, 0, 0, 0, 249, 1107, 1, 0, 0, 0, 251, 1112, 1, 0, 0, 0, 253, 1117, 1, 0, 0, 0, 255, 1120, 1, 0, 0, 0, 257, 1124, 1, 0, 0, 0, 259, 1128, 1, 0, 0, 0, 261, 1132, 1, 0, 0, 0, 263, 1136, 1, 0, 0, 0, 265, 1141, 1, 0, 0, 0, 267, 1146, 1, 0, 0, 0, 269, 1151, 1, 0, 0, 0, 271, 1158, 1, 0, 0, 0, 273, 1167, 1, 0, 0, 0, 275, 1174, 1, 0, 0, 0, 277, 1178, 1, 0, 0, 0, 279, 1182, 1, 0, 0, 0, 281, 1186, 1, 0, 0, 0, 283, 1190, 1, 0, 0, 0, 285, 1196, 1, 0, 0, 0, 287, 1200, 1, 0, 0, 0, 289, 1204, 1, 0, 0, 0, 291, 1208, 1, 0, 0, 0, 293, 1212, 1, 0, 0, 0, 295, 1216, 1, 0, 0, 0, 297, 1220, 1, 0, 0, 0, 299, 1225, 1, 0, 0, 0, 301, 1230, 1, 0, 0, 0, 303, 1234, 1, 0, 0, 0, 305, 1238, 1, 0, 0, 0, 307, 1242, 1, 0, 0, 0, 309, 1247, 1, 0, 0, 0, 311, 1251, 1, 0, 0, 0, 313, 1256, 1, 0, 0, 0, 315, 1261, 1, 0, 0, 0, 317, 1265, 1, 0, 0, 0, 319, 1269, 1, 0, 0, 0, 321, 1273, 1, 0, 0, 0, 323, 1277, 1, 0, 0, 0, 325, 1281, 1, 0, 0, 0, 327, 1286, 1, 0, 0, 0, 329, 1291, 1, 0, 0, 0, 331, 1295, 1, 0, 0, 0, 333, 1299, 1, 0, 0, 0, 335, 1303, 1, 0, 0, 0, 337, 1308, 1, 0, 0, 0, 339, 1315, 1, 0, 0, 0, 341, 1319, 1, 0, 0, 0, 343, 1323, 1, 0, 0, 0, 345, 1327, 1, 0, 0, 0, 347, 1331, 1, 0, 0, 0, 349, 1336, 1, 0, 0, 0, 351, 1340, 1, 0, 0, 0, 353, 1344, 1, 0, 0, 0, 355, 1348, 1, 0, 0, 0, 357, 1353, 1, 0, 0, 0, 359, 1357, 1, 0, 0, 0, 361, 1361, 1, 0, 0, 0, 363, 1365, 1, 0, 0, 0, 365, 1369, 1, 0, 0, 0, 367, 1373, 1, 0, 0, 0, 369, 1379, 1, 0, 0, 0, 371, 1383, 1, 0, 0, 0, 373, 1387, 1, 0, 0, 0, 375, 1391, 1, 0, 0, 0, 377, 1395, 1, 0, 0, 0, 379, 1399, 1, 0, 0, 0, 381, 1403, 1, 0, 0, 0, 383, 1408, 1, 0, 0, 0, 385, 1414, 1, 0, 0, 0, 387, 1420, 1, 0, 0, 0, 389, 1424, 1, 0, 0, 0, 391, 1428, 1, 0, 0, 0, 393, 1432, 1, 0, 0, 0, 395, 1438, 1, 0, 0, 0, 397, 1444, 1, 0, 0, 0, 399, 1448, 1, 0, 0, 0, 401, 1452, 1, 0, 0, 0, 403, 1456, 1, 0, 0, 0, 405, 1462, 1, 0, 0, 0, 407, 1468, 1, 0, 0, 0, 409, 1474, 1, 0, 0, 0, 411, 412, 7, 0, 0, 0, 412, 413, 7, 1, 0, 0, 413, 414, 7, 2, 0, 0, 414, 415, 7, 2, 0, 0, 415, 416, 7, 3, 0, 0, 416, 417, 7, 4, 0, 0, 417, 418, 7, 5, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 0, 0, 0, 420, 16, 1, 0, 0, 0, 421, 422, 7, 0, 0, 0, 422, 423, 7, 6, 0, 0, 423, 424, 7, 7, 0, 0, 424, 425, 7, 8, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 1, 1, 0, 427, 18, 1, 0, 0, 0, 428, 429, 7, 3, 0, 0, 429, 430, 7, 9, 0, 0, 430, 431, 7, 6, 0, 0, 431, 432, 7, 1, 0, 0, 432, 433, 7, 4, 0, 0, 433, 434, 7, 10, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 2, 2, 0, 436, 20, 1, 0, 0, 0, 437, 438, 7, 3, 0, 0, 438, 439, 7, 11, 0, 0, 439, 440, 7, 12, 0, 0, 440, 441, 7, 13, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 3, 0, 0, 443, 22, 1, 0, 0, 0, 444, 445, 7, 3, 0, 0, 445, 446, 7, 14, 0, 0, 446, 447, 7, 8, 0, 0, 447, 448, 7, 13, 0, 0, 448, 449, 7, 12, 0, 0, 449, 450, 7, 1, 0, 0, 450, 451, 7, 9, 0, 0, 451, 452, 1, 0, 0, 0, 452, 453, 6, 4, 3, 0, 453, 24, 1, 0, 0, 0, 454, 455, 7, 15, 0, 0, 455, 456, 7, 6, 0, 0, 456, 457, 7, 7, 0, 0, 457, 458, 7, 16, 0, 0, 458, 459, 1, 0, 0, 0, 459, 460, 6, 5, 4, 0, 460, 26, 1, 0, 0, 0, 461, 462, 7, 17, 0, 0, 462, 463, 7, 6, 0, 0, 463, 464, 7, 7, 0, 0, 464, 465, 7, 18, 0, 0, 465, 466, 1, 0, 0, 0, 466, 467, 6, 6, 0, 0, 467, 28, 1, 0, 0, 0, 468, 469, 7, 18, 0, 0, 469, 470, 7, 3, 0, 0, 470, 471, 7, 3, 0, 0, 471, 472, 7, 8, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 7, 1, 0, 474, 30, 1, 0, 0, 0, 475, 476, 7, 13, 0, 0, 476, 477, 7, 1, 0, 0, 477, 478, 7, 16, 0, 0, 478, 479, 7, 1, 0, 0, 479, 480, 7, 5, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 8, 0, 0, 482, 32, 1, 0, 0, 0, 483, 484, 7, 16, 0, 0, 484, 485, 7, 11, 0, 0, 485, 486, 5, 95, 0, 0, 486, 487, 7, 3, 0, 0, 487, 488, 7, 14, 0, 0, 488, 489, 7, 8, 0, 0, 489, 490, 7, 12, 0, 0, 490, 491, 7, 9, 0, 0, 491, 492, 7, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 9, 5, 0, 494, 34, 1, 0, 0, 0, 495, 496, 7, 6, 0, 0, 496, 497, 7, 3, 0, 0, 497, 498, 7, 9, 0, 0, 498, 499, 7, 12, 0, 0, 499, 500, 7, 16, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 10, 6, 0, 503, 36, 1, 0, 0, 0, 504, 505, 7, 6, 0, 0, 505, 506, 7, 7, 0, 0, 506, 507, 7, 19, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 6, 11, 0, 0, 509, 38, 1, 0, 0, 0, 510, 511, 7, 2, 0, 0, 511, 512, 7, 10, 0, 0, 512, 513, 7, 7, 0, 0, 513, 514, 7, 19, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 12, 7, 0, 516, 40, 1, 0, 0, 0, 517, 518, 7, 2, 0, 0, 518, 519, 7, 7, 0, 0, 519, 520, 7, 6, 0, 0, 520, 521, 7, 5, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 13, 0, 0, 523, 42, 1, 0, 0, 0, 524, 525, 7, 2, 0, 0, 525, 526, 7, 5, 0, 0, 526, 527, 7, 12, 0, 0, 527, 528, 7, 5, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 14, 0, 0, 531, 44, 1, 0, 0, 0, 532, 533, 7, 19, 0, 0, 533, 534, 7, 10, 0, 0, 534, 535, 7, 3, 0, 0, 535, 536, 7, 6, 0, 0, 536, 537, 7, 3, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 6, 15, 0, 0, 539, 46, 1, 0, 0, 0, 540, 541, 4, 16, 0, 0, 541, 542, 7, 1, 0, 0, 542, 543, 7, 9, 0, 0, 543, 544, 7, 13, 0, 0, 544, 545, 7, 1, 0, 0, 545, 546, 7, 9, 0, 0, 546, 547, 7, 3, 0, 0, 547, 548, 7, 2, 0, 0, 548, 549, 7, 5, 0, 0, 549, 550, 7, 12, 0, 0, 550, 551, 7, 5, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 16, 0, 0, 554, 48, 1, 0, 0, 0, 555, 556, 4, 17, 1, 0, 556, 557, 7, 13, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 7, 0, 0, 559, 560, 7, 18, 0, 0, 560, 561, 7, 20, 0, 0, 561, 562, 7, 8, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 17, 8, 0, 564, 50, 1, 0, 0, 0, 565, 566, 4, 18, 2, 0, 566, 567, 7, 16, 0, 0, 567, 568, 7, 3, 0, 0, 568, 569, 7, 5, 0, 0, 569, 570, 7, 6, 0, 0, 570, 571, 7, 1, 0, 0, 571, 572, 7, 4, 0, 0, 572, 573, 7, 2, 0, 0, 573, 574, 1, 0, 0, 0, 574, 575, 6, 18, 9, 0, 575, 52, 1, 0, 0, 0, 576, 578, 8, 21, 0, 0, 577, 576, 1, 0, 0, 0, 578, 579, 1, 0, 0, 0, 579, 577, 1, 0, 0, 0, 579, 580, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 582, 6, 19, 0, 0, 582, 54, 1, 0, 0, 0, 583, 584, 5, 47, 0, 0, 584, 585, 5, 47, 0, 0, 585, 589, 1, 0, 0, 0, 586, 588, 8, 22, 0, 0, 587, 586, 1, 0, 0, 0, 588, 591, 1, 0, 0, 0, 589, 587, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 593, 1, 0, 0, 0, 591, 589, 1, 0, 0, 0, 592, 594, 5, 13, 0, 0, 593, 592, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 596, 1, 0, 0, 0, 595, 597, 5, 10, 0, 0, 596, 595, 1, 0, 0, 0, 596, 597, 1, 0, 0, 0, 597, 598, 1, 0, 0, 0, 598, 599, 6, 20, 10, 0, 599, 56, 1, 0, 0, 0, 600, 601, 5, 47, 0, 0, 601, 602, 5, 42, 0, 0, 602, 607, 1, 0, 0, 0, 603, 606, 3, 57, 21, 0, 604, 606, 9, 0, 0, 0, 605, 603, 1, 0, 0, 0, 605, 604, 1, 0, 0, 0, 606, 609, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 607, 605, 1, 0, 0, 0, 608, 610, 1, 0, 0, 0, 609, 607, 1, 0, 0, 0, 610, 611, 5, 42, 0, 0, 611, 612, 5, 47, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 6, 21, 10, 0, 614, 58, 1, 0, 0, 0, 615, 617, 7, 23, 0, 0, 616, 615, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 620, 1, 0, 0, 0, 620, 621, 6, 22, 10, 0, 621, 60, 1, 0, 0, 0, 622, 623, 5, 124, 0, 0, 623, 624, 1, 0, 0, 0, 624, 625, 6, 23, 11, 0, 625, 62, 1, 0, 0, 0, 626, 627, 7, 24, 0, 0, 627, 64, 1, 0, 0, 0, 628, 629, 7, 25, 0, 0, 629, 66, 1, 0, 0, 0, 630, 631, 5, 92, 0, 0, 631, 632, 7, 26, 0, 0, 632, 68, 1, 0, 0, 0, 633, 634, 8, 27, 0, 0, 634, 70, 1, 0, 0, 0, 635, 637, 7, 3, 0, 0, 636, 638, 7, 28, 0, 0, 637, 636, 1, 0, 0, 0, 637, 638, 1, 0, 0, 0, 638, 640, 1, 0, 0, 0, 639, 641, 3, 63, 24, 0, 640, 639, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 640, 1, 0, 0, 0, 642, 643, 1, 0, 0, 0, 643, 72, 1, 0, 0, 0, 644, 645, 5, 64, 0, 0, 645, 74, 1, 0, 0, 0, 646, 647, 5, 96, 0, 0, 647, 76, 1, 0, 0, 0, 648, 652, 8, 29, 0, 0, 649, 650, 5, 96, 0, 0, 650, 652, 5, 96, 0, 0, 651, 648, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 652, 78, 1, 0, 0, 0, 653, 654, 5, 95, 0, 0, 654, 80, 1, 0, 0, 0, 655, 659, 3, 65, 25, 0, 656, 659, 3, 63, 24, 0, 657, 659, 3, 79, 32, 0, 658, 655, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 658, 657, 1, 0, 0, 0, 659, 82, 1, 0, 0, 0, 660, 665, 5, 34, 0, 0, 661, 664, 3, 67, 26, 0, 662, 664, 3, 69, 27, 0, 663, 661, 1, 0, 0, 0, 663, 662, 1, 0, 0, 0, 664, 667, 1, 0, 0, 0, 665, 663, 1, 0, 0, 0, 665, 666, 1, 0, 0, 0, 666, 668, 1, 0, 0, 0, 667, 665, 1, 0, 0, 0, 668, 690, 5, 34, 0, 0, 669, 670, 5, 34, 0, 0, 670, 671, 5, 34, 0, 0, 671, 672, 5, 34, 0, 0, 672, 676, 1, 0, 0, 0, 673, 675, 8, 22, 0, 0, 674, 673, 1, 0, 0, 0, 675, 678, 1, 0, 0, 0, 676, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 679, 1, 0, 0, 0, 678, 676, 1, 0, 0, 0, 679, 680, 5, 34, 0, 0, 680, 681, 5, 34, 0, 0, 681, 682, 5, 34, 0, 0, 682, 684, 1, 0, 0, 0, 683, 685, 5, 34, 0, 0, 684, 683, 1, 0, 0, 0, 684, 685, 1, 0, 0, 0, 685, 687, 1, 0, 0, 0, 686, 688, 5, 34, 0, 0, 687, 686, 1, 0, 0, 0, 687, 688, 1, 0, 0, 0, 688, 690, 1, 0, 0, 0, 689, 660, 1, 0, 0, 0, 689, 669, 1, 0, 0, 0, 690, 84, 1, 0, 0, 0, 691, 693, 3, 63, 24, 0, 692, 691, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 692, 1, 0, 0, 0, 694, 695, 1, 0, 0, 0, 695, 86, 1, 0, 0, 0, 696, 698, 3, 63, 24, 0, 697, 696, 1, 0, 0, 0, 698, 699, 1, 0, 0, 0, 699, 697, 1, 0, 0, 0, 699, 700, 1, 0, 0, 0, 700, 701, 1, 0, 0, 0, 701, 705, 3, 103, 44, 0, 702, 704, 3, 63, 24, 0, 703, 702, 1, 0, 0, 0, 704, 707, 1, 0, 0, 0, 705, 703, 1, 0, 0, 0, 705, 706, 1, 0, 0, 0, 706, 739, 1, 0, 0, 0, 707, 705, 1, 0, 0, 0, 708, 710, 3, 103, 44, 0, 709, 711, 3, 63, 24, 0, 710, 709, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0, 712, 710, 1, 0, 0, 0, 712, 713, 1, 0, 0, 0, 713, 739, 1, 0, 0, 0, 714, 716, 3, 63, 24, 0, 715, 714, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717, 715, 1, 0, 0, 0, 717, 718, 1, 0, 0, 0, 718, 726, 1, 0, 0, 0, 719, 723, 3, 103, 44, 0, 720, 722, 3, 63, 24, 0, 721, 720, 1, 0, 0, 0, 722, 725, 1, 0, 0, 0, 723, 721, 1, 0, 0, 0, 723, 724, 1, 0, 0, 0, 724, 727, 1, 0, 0, 0, 725, 723, 1, 0, 0, 0, 726, 719, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 728, 1, 0, 0, 0, 728, 729, 3, 71, 28, 0, 729, 739, 1, 0, 0, 0, 730, 732, 3, 103, 44, 0, 731, 733, 3, 63, 24, 0, 732, 731, 1, 0, 0, 0, 733, 734, 1, 0, 0, 0, 734, 732, 1, 0, 0, 0, 734, 735, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 737, 3, 71, 28, 0, 737, 739, 1, 0, 0, 0, 738, 697, 1, 0, 0, 0, 738, 708, 1, 0, 0, 0, 738, 715, 1, 0, 0, 0, 738, 730, 1, 0, 0, 0, 739, 88, 1, 0, 0, 0, 740, 741, 7, 30, 0, 0, 741, 742, 7, 31, 0, 0, 742, 90, 1, 0, 0, 0, 743, 744, 7, 12, 0, 0, 744, 745, 7, 9, 0, 0, 745, 746, 7, 0, 0, 0, 746, 92, 1, 0, 0, 0, 747, 748, 7, 12, 0, 0, 748, 749, 7, 2, 0, 0, 749, 750, 7, 4, 0, 0, 750, 94, 1, 0, 0, 0, 751, 752, 5, 61, 0, 0, 752, 96, 1, 0, 0, 0, 753, 754, 5, 58, 0, 0, 754, 755, 5, 58, 0, 0, 755, 98, 1, 0, 0, 0, 756, 757, 5, 44, 0, 0, 757, 100, 1, 0, 0, 0, 758, 759, 7, 0, 0, 0, 759, 760, 7, 3, 0, 0, 760, 761, 7, 2, 0, 0, 761, 762, 7, 4, 0, 0, 762, 102, 1, 0, 0, 0, 763, 764, 5, 46, 0, 0, 764, 104, 1, 0, 0, 0, 765, 766, 7, 15, 0, 0, 766, 767, 7, 12, 0, 0, 767, 768, 7, 13, 0, 0, 768, 769, 7, 2, 0, 0, 769, 770, 7, 3, 0, 0, 770, 106, 1, 0, 0, 0, 771, 772, 7, 15, 0, 0, 772, 773, 7, 1, 0, 0, 773, 774, 7, 6, 0, 0, 774, 775, 7, 2, 0, 0, 775, 776, 7, 5, 0, 0, 776, 108, 1, 0, 0, 0, 777, 778, 7, 1, 0, 0, 778, 779, 7, 9, 0, 0, 779, 110, 1, 0, 0, 0, 780, 781, 7, 1, 0, 0, 781, 782, 7, 2, 0, 0, 782, 112, 1, 0, 0, 0, 783, 784, 7, 13, 0, 0, 784, 785, 7, 12, 0, 0, 785, 786, 7, 2, 0, 0, 786, 787, 7, 5, 0, 0, 787, 114, 1, 0, 0, 0, 788, 789, 7, 13, 0, 0, 789, 790, 7, 1, 0, 0, 790, 791, 7, 18, 0, 0, 791, 792, 7, 3, 0, 0, 792, 116, 1, 0, 0, 0, 793, 794, 5, 40, 0, 0, 794, 118, 1, 0, 0, 0, 795, 796, 7, 9, 0, 0, 796, 797, 7, 7, 0, 0, 797, 798, 7, 5, 0, 0, 798, 120, 1, 0, 0, 0, 799, 800, 7, 9, 0, 0, 800, 801, 7, 20, 0, 0, 801, 802, 7, 13, 0, 0, 802, 803, 7, 13, 0, 0, 803, 122, 1, 0, 0, 0, 804, 805, 7, 9, 0, 0, 805, 806, 7, 20, 0, 0, 806, 807, 7, 13, 0, 0, 807, 808, 7, 13, 0, 0, 808, 809, 7, 2, 0, 0, 809, 124, 1, 0, 0, 0, 810, 811, 7, 7, 0, 0, 811, 812, 7, 6, 0, 0, 812, 126, 1, 0, 0, 0, 813, 814, 5, 63, 0, 0, 814, 128, 1, 0, 0, 0, 815, 816, 7, 6, 0, 0, 816, 817, 7, 13, 0, 0, 817, 818, 7, 1, 0, 0, 818, 819, 7, 18, 0, 0, 819, 820, 7, 3, 0, 0, 820, 130, 1, 0, 0, 0, 821, 822, 5, 41, 0, 0, 822, 132, 1, 0, 0, 0, 823, 824, 7, 5, 0, 0, 824, 825, 7, 6, 0, 0, 825, 826, 7, 20, 0, 0, 826, 827, 7, 3, 0, 0, 827, 134, 1, 0, 0, 0, 828, 829, 5, 61, 0, 0, 829, 830, 5, 61, 0, 0, 830, 136, 1, 0, 0, 0, 831, 832, 5, 61, 0, 0, 832, 833, 5, 126, 0, 0, 833, 138, 1, 0, 0, 0, 834, 835, 5, 33, 0, 0, 835, 836, 5, 61, 0, 0, 836, 140, 1, 0, 0, 0, 837, 838, 5, 60, 0, 0, 838, 142, 1, 0, 0, 0, 839, 840, 5, 60, 0, 0, 840, 841, 5, 61, 0, 0, 841, 144, 1, 0, 0, 0, 842, 843, 5, 62, 0, 0, 843, 146, 1, 0, 0, 0, 844, 845, 5, 62, 0, 0, 845, 846, 5, 61, 0, 0, 846, 148, 1, 0, 0, 0, 847, 848, 5, 43, 0, 0, 848, 150, 1, 0, 0, 0, 849, 850, 5, 45, 0, 0, 850, 152, 1, 0, 0, 0, 851, 852, 5, 42, 0, 0, 852, 154, 1, 0, 0, 0, 853, 854, 5, 47, 0, 0, 854, 156, 1, 0, 0, 0, 855, 856, 5, 37, 0, 0, 856, 158, 1, 0, 0, 0, 857, 858, 7, 16, 0, 0, 858, 859, 7, 12, 0, 0, 859, 860, 7, 5, 0, 0, 860, 861, 7, 4, 0, 0, 861, 862, 7, 10, 0, 0, 862, 160, 1, 0, 0, 0, 863, 864, 3, 45, 15, 0, 864, 865, 1, 0, 0, 0, 865, 866, 6, 73, 12, 0, 866, 162, 1, 0, 0, 0, 867, 870, 3, 127, 56, 0, 868, 871, 3, 65, 25, 0, 869, 871, 3, 79, 32, 0, 870, 868, 1, 0, 0, 0, 870, 869, 1, 0, 0, 0, 871, 875, 1, 0, 0, 0, 872, 874, 3, 81, 33, 0, 873, 872, 1, 0, 0, 0, 874, 877, 1, 0, 0, 0, 875, 873, 1, 0, 0, 0, 875, 876, 1, 0, 0, 0, 876, 885, 1, 0, 0, 0, 877, 875, 1, 0, 0, 0, 878, 880, 3, 127, 56, 0, 879, 881, 3, 63, 24, 0, 880, 879, 1, 0, 0, 0, 881, 882, 1, 0, 0, 0, 882, 880, 1, 0, 0, 0, 882, 883, 1, 0, 0, 0, 883, 885, 1, 0, 0, 0, 884, 867, 1, 0, 0, 0, 884, 878, 1, 0, 0, 0, 885, 164, 1, 0, 0, 0, 886, 887, 5, 91, 0, 0, 887, 888, 1, 0, 0, 0, 888, 889, 6, 75, 0, 0, 889, 890, 6, 75, 0, 0, 890, 166, 1, 0, 0, 0, 891, 892, 5, 93, 0, 0, 892, 893, 1, 0, 0, 0, 893, 894, 6, 76, 11, 0, 894, 895, 6, 76, 11, 0, 895, 168, 1, 0, 0, 0, 896, 900, 3, 65, 25, 0, 897, 899, 3, 81, 33, 0, 898, 897, 1, 0, 0, 0, 899, 902, 1, 0, 0, 0, 900, 898, 1, 0, 0, 0, 900, 901, 1, 0, 0, 0, 901, 913, 1, 0, 0, 0, 902, 900, 1, 0, 0, 0, 903, 906, 3, 79, 32, 0, 904, 906, 3, 73, 29, 0, 905, 903, 1, 0, 0, 0, 905, 904, 1, 0, 0, 0, 906, 908, 1, 0, 0, 0, 907, 909, 3, 81, 33, 0, 908, 907, 1, 0, 0, 0, 909, 910, 1, 0, 0, 0, 910, 908, 1, 0, 0, 0, 910, 911, 1, 0, 0, 0, 911, 913, 1, 0, 0, 0, 912, 896, 1, 0, 0, 0, 912, 905, 1, 0, 0, 0, 913, 170, 1, 0, 0, 0, 914, 916, 3, 75, 30, 0, 915, 917, 3, 77, 31, 0, 916, 915, 1, 0, 0, 0, 917, 918, 1, 0, 0, 0, 918, 916, 1, 0, 0, 0, 918, 919, 1, 0, 0, 0, 919, 920, 1, 0, 0, 0, 920, 921, 3, 75, 30, 0, 921, 172, 1, 0, 0, 0, 922, 923, 3, 171, 78, 0, 923, 174, 1, 0, 0, 0, 924, 925, 3, 55, 20, 0, 925, 926, 1, 0, 0, 0, 926, 927, 6, 80, 10, 0, 927, 176, 1, 0, 0, 0, 928, 929, 3, 57, 21, 0, 929, 930, 1, 0, 0, 0, 930, 931, 6, 81, 10, 0, 931, 178, 1, 0, 0, 0, 932, 933, 3, 59, 22, 0, 933, 934, 1, 0, 0, 0, 934, 935, 6, 82, 10, 0, 935, 180, 1, 0, 0, 0, 936, 937, 3, 165, 75, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 83, 13, 0, 939, 940, 6, 83, 14, 0, 940, 182, 1, 0, 0, 0, 941, 942, 3, 61, 23, 0, 942, 943, 1, 0, 0, 0, 943, 944, 6, 84, 15, 0, 944, 945, 6, 84, 11, 0, 945, 184, 1, 0, 0, 0, 946, 947, 3, 59, 22, 0, 947, 948, 1, 0, 0, 0, 948, 949, 6, 85, 10, 0, 949, 186, 1, 0, 0, 0, 950, 951, 3, 55, 20, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 86, 10, 0, 953, 188, 1, 0, 0, 0, 954, 955, 3, 57, 21, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 87, 10, 0, 957, 190, 1, 0, 0, 0, 958, 959, 3, 61, 23, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 88, 15, 0, 961, 962, 6, 88, 11, 0, 962, 192, 1, 0, 0, 0, 963, 964, 3, 165, 75, 0, 964, 965, 1, 0, 0, 0, 965, 966, 6, 89, 13, 0, 966, 194, 1, 0, 0, 0, 967, 968, 3, 167, 76, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 90, 16, 0, 970, 196, 1, 0, 0, 0, 971, 972, 3, 337, 161, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 91, 17, 0, 974, 198, 1, 0, 0, 0, 975, 976, 3, 99, 42, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 92, 18, 0, 978, 200, 1, 0, 0, 0, 979, 980, 3, 95, 40, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 93, 19, 0, 982, 202, 1, 0, 0, 0, 983, 984, 7, 16, 0, 0, 984, 985, 7, 3, 0, 0, 985, 986, 7, 5, 0, 0, 986, 987, 7, 12, 0, 0, 987, 988, 7, 0, 0, 0, 988, 989, 7, 12, 0, 0, 989, 990, 7, 5, 0, 0, 990, 991, 7, 12, 0, 0, 991, 204, 1, 0, 0, 0, 992, 996, 8, 32, 0, 0, 993, 994, 5, 47, 0, 0, 994, 996, 8, 33, 0, 0, 995, 992, 1, 0, 0, 0, 995, 993, 1, 0, 0, 0, 996, 206, 1, 0, 0, 0, 997, 999, 3, 205, 95, 0, 998, 997, 1, 0, 0, 0, 999, 1000, 1, 0, 0, 0, 1000, 998, 1, 0, 0, 0, 1000, 1001, 1, 0, 0, 0, 1001, 208, 1, 0, 0, 0, 1002, 1003, 3, 207, 96, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1005, 6, 97, 20, 0, 1005, 210, 1, 0, 0, 0, 1006, 1007, 3, 83, 34, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 98, 21, 0, 1009, 212, 1, 0, 0, 0, 1010, 1011, 3, 55, 20, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 99, 10, 0, 1013, 214, 1, 0, 0, 0, 1014, 1015, 3, 57, 21, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 100, 10, 0, 1017, 216, 1, 0, 0, 0, 1018, 1019, 3, 59, 22, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 101, 10, 0, 1021, 218, 1, 0, 0, 0, 1022, 1023, 3, 61, 23, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 102, 15, 0, 1025, 1026, 6, 102, 11, 0, 1026, 220, 1, 0, 0, 0, 1027, 1028, 3, 103, 44, 0, 1028, 1029, 1, 0, 0, 0, 1029, 1030, 6, 103, 22, 0, 1030, 222, 1, 0, 0, 0, 1031, 1032, 3, 99, 42, 0, 1032, 1033, 1, 0, 0, 0, 1033, 1034, 6, 104, 18, 0, 1034, 224, 1, 0, 0, 0, 1035, 1036, 4, 105, 3, 0, 1036, 1037, 3, 127, 56, 0, 1037, 1038, 1, 0, 0, 0, 1038, 1039, 6, 105, 23, 0, 1039, 226, 1, 0, 0, 0, 1040, 1041, 4, 106, 4, 0, 1041, 1042, 3, 163, 74, 0, 1042, 1043, 1, 0, 0, 0, 1043, 1044, 6, 106, 24, 0, 1044, 228, 1, 0, 0, 0, 1045, 1050, 3, 65, 25, 0, 1046, 1050, 3, 63, 24, 0, 1047, 1050, 3, 79, 32, 0, 1048, 1050, 3, 153, 69, 0, 1049, 1045, 1, 0, 0, 0, 1049, 1046, 1, 0, 0, 0, 1049, 1047, 1, 0, 0, 0, 1049, 1048, 1, 0, 0, 0, 1050, 230, 1, 0, 0, 0, 1051, 1054, 3, 65, 25, 0, 1052, 1054, 3, 153, 69, 0, 1053, 1051, 1, 0, 0, 0, 1053, 1052, 1, 0, 0, 0, 1054, 1058, 1, 0, 0, 0, 1055, 1057, 3, 229, 107, 0, 1056, 1055, 1, 0, 0, 0, 1057, 1060, 1, 0, 0, 0, 1058, 1056, 1, 0, 0, 0, 1058, 1059, 1, 0, 0, 0, 1059, 1071, 1, 0, 0, 0, 1060, 1058, 1, 0, 0, 0, 1061, 1064, 3, 79, 32, 0, 1062, 1064, 3, 73, 29, 0, 1063, 1061, 1, 0, 0, 0, 1063, 1062, 1, 0, 0, 0, 1064, 1066, 1, 0, 0, 0, 1065, 1067, 3, 229, 107, 0, 1066, 1065, 1, 0, 0, 0, 1067, 1068, 1, 0, 0, 0, 1068, 1066, 1, 0, 0, 0, 1068, 1069, 1, 0, 0, 0, 1069, 1071, 1, 0, 0, 0, 1070, 1053, 1, 0, 0, 0, 1070, 1063, 1, 0, 0, 0, 1071, 232, 1, 0, 0, 0, 1072, 1075, 3, 231, 108, 0, 1073, 1075, 3, 171, 78, 0, 1074, 1072, 1, 0, 0, 0, 1074, 1073, 1, 0, 0, 0, 1075, 1076, 1, 0, 0, 0, 1076, 1074, 1, 0, 0, 0, 1076, 1077, 1, 0, 0, 0, 1077, 234, 1, 0, 0, 0, 1078, 1079, 3, 55, 20, 0, 1079, 1080, 1, 0, 0, 0, 1080, 1081, 6, 110, 10, 0, 1081, 236, 1, 0, 0, 0, 1082, 1083, 3, 57, 21, 0, 1083, 1084, 1, 0, 0, 0, 1084, 1085, 6, 111, 10, 0, 1085, 238, 1, 0, 0, 0, 1086, 1087, 3, 59, 22, 0, 1087, 1088, 1, 0, 0, 0, 1088, 1089, 6, 112, 10, 0, 1089, 240, 1, 0, 0, 0, 1090, 1091, 3, 61, 23, 0, 1091, 1092, 1, 0, 0, 0, 1092, 1093, 6, 113, 15, 0, 1093, 1094, 6, 113, 11, 0, 1094, 242, 1, 0, 0, 0, 1095, 1096, 3, 95, 40, 0, 1096, 1097, 1, 0, 0, 0, 1097, 1098, 6, 114, 19, 0, 1098, 244, 1, 0, 0, 0, 1099, 1100, 3, 99, 42, 0, 1100, 1101, 1, 0, 0, 0, 1101, 1102, 6, 115, 18, 0, 1102, 246, 1, 0, 0, 0, 1103, 1104, 3, 103, 44, 0, 1104, 1105, 1, 0, 0, 0, 1105, 1106, 6, 116, 22, 0, 1106, 248, 1, 0, 0, 0, 1107, 1108, 4, 117, 5, 0, 1108, 1109, 3, 127, 56, 0, 1109, 1110, 1, 0, 0, 0, 1110, 1111, 6, 117, 23, 0, 1111, 250, 1, 0, 0, 0, 1112, 1113, 4, 118, 6, 0, 1113, 1114, 3, 163, 74, 0, 1114, 1115, 1, 0, 0, 0, 1115, 1116, 6, 118, 24, 0, 1116, 252, 1, 0, 0, 0, 1117, 1118, 7, 12, 0, 0, 1118, 1119, 7, 2, 0, 0, 1119, 254, 1, 0, 0, 0, 1120, 1121, 3, 233, 109, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 120, 25, 0, 1123, 256, 1, 0, 0, 0, 1124, 1125, 3, 55, 20, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 121, 10, 0, 1127, 258, 1, 0, 0, 0, 1128, 1129, 3, 57, 21, 0, 1129, 1130, 1, 0, 0, 0, 1130, 1131, 6, 122, 10, 0, 1131, 260, 1, 0, 0, 0, 1132, 1133, 3, 59, 22, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 123, 10, 0, 1135, 262, 1, 0, 0, 0, 1136, 1137, 3, 61, 23, 0, 1137, 1138, 1, 0, 0, 0, 1138, 1139, 6, 124, 15, 0, 1139, 1140, 6, 124, 11, 0, 1140, 264, 1, 0, 0, 0, 1141, 1142, 3, 165, 75, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 125, 13, 0, 1144, 1145, 6, 125, 26, 0, 1145, 266, 1, 0, 0, 0, 1146, 1147, 7, 7, 0, 0, 1147, 1148, 7, 9, 0, 0, 1148, 1149, 1, 0, 0, 0, 1149, 1150, 6, 126, 27, 0, 1150, 268, 1, 0, 0, 0, 1151, 1152, 7, 19, 0, 0, 1152, 1153, 7, 1, 0, 0, 1153, 1154, 7, 5, 0, 0, 1154, 1155, 7, 10, 0, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 6, 127, 27, 0, 1157, 270, 1, 0, 0, 0, 1158, 1159, 8, 34, 0, 0, 1159, 272, 1, 0, 0, 0, 1160, 1162, 3, 271, 128, 0, 1161, 1160, 1, 0, 0, 0, 1162, 1163, 1, 0, 0, 0, 1163, 1161, 1, 0, 0, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1165, 1, 0, 0, 0, 1165, 1166, 3, 337, 161, 0, 1166, 1168, 1, 0, 0, 0, 1167, 1161, 1, 0, 0, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1170, 1, 0, 0, 0, 1169, 1171, 3, 271, 128, 0, 1170, 1169, 1, 0, 0, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1170, 1, 0, 0, 0, 1172, 1173, 1, 0, 0, 0, 1173, 274, 1, 0, 0, 0, 1174, 1175, 3, 273, 129, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 130, 28, 0, 1177, 276, 1, 0, 0, 0, 1178, 1179, 3, 55, 20, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 131, 10, 0, 1181, 278, 1, 0, 0, 0, 1182, 1183, 3, 57, 21, 0, 1183, 1184, 1, 0, 0, 0, 1184, 1185, 6, 132, 10, 0, 1185, 280, 1, 0, 0, 0, 1186, 1187, 3, 59, 22, 0, 1187, 1188, 1, 0, 0, 0, 1188, 1189, 6, 133, 10, 0, 1189, 282, 1, 0, 0, 0, 1190, 1191, 3, 61, 23, 0, 1191, 1192, 1, 0, 0, 0, 1192, 1193, 6, 134, 15, 0, 1193, 1194, 6, 134, 11, 0, 1194, 1195, 6, 134, 11, 0, 1195, 284, 1, 0, 0, 0, 1196, 1197, 3, 95, 40, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 135, 19, 0, 1199, 286, 1, 0, 0, 0, 1200, 1201, 3, 99, 42, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 136, 18, 0, 1203, 288, 1, 0, 0, 0, 1204, 1205, 3, 103, 44, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 137, 22, 0, 1207, 290, 1, 0, 0, 0, 1208, 1209, 3, 269, 127, 0, 1209, 1210, 1, 0, 0, 0, 1210, 1211, 6, 138, 29, 0, 1211, 292, 1, 0, 0, 0, 1212, 1213, 3, 233, 109, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 139, 25, 0, 1215, 294, 1, 0, 0, 0, 1216, 1217, 3, 173, 79, 0, 1217, 1218, 1, 0, 0, 0, 1218, 1219, 6, 140, 30, 0, 1219, 296, 1, 0, 0, 0, 1220, 1221, 4, 141, 7, 0, 1221, 1222, 3, 127, 56, 0, 1222, 1223, 1, 0, 0, 0, 1223, 1224, 6, 141, 23, 0, 1224, 298, 1, 0, 0, 0, 1225, 1226, 4, 142, 8, 0, 1226, 1227, 3, 163, 74, 0, 1227, 1228, 1, 0, 0, 0, 1228, 1229, 6, 142, 24, 0, 1229, 300, 1, 0, 0, 0, 1230, 1231, 3, 55, 20, 0, 1231, 1232, 1, 0, 0, 0, 1232, 1233, 6, 143, 10, 0, 1233, 302, 1, 0, 0, 0, 1234, 1235, 3, 57, 21, 0, 1235, 1236, 1, 0, 0, 0, 1236, 1237, 6, 144, 10, 0, 1237, 304, 1, 0, 0, 0, 1238, 1239, 3, 59, 22, 0, 1239, 1240, 1, 0, 0, 0, 1240, 1241, 6, 145, 10, 0, 1241, 306, 1, 0, 0, 0, 1242, 1243, 3, 61, 23, 0, 1243, 1244, 1, 0, 0, 0, 1244, 1245, 6, 146, 15, 0, 1245, 1246, 6, 146, 11, 0, 1246, 308, 1, 0, 0, 0, 1247, 1248, 3, 103, 44, 0, 1248, 1249, 1, 0, 0, 0, 1249, 1250, 6, 147, 22, 0, 1250, 310, 1, 0, 0, 0, 1251, 1252, 4, 148, 9, 0, 1252, 1253, 3, 127, 56, 0, 1253, 1254, 1, 0, 0, 0, 1254, 1255, 6, 148, 23, 0, 1255, 312, 1, 0, 0, 0, 1256, 1257, 4, 149, 10, 0, 1257, 1258, 3, 163, 74, 0, 1258, 1259, 1, 0, 0, 0, 1259, 1260, 6, 149, 24, 0, 1260, 314, 1, 0, 0, 0, 1261, 1262, 3, 173, 79, 0, 1262, 1263, 1, 0, 0, 0, 1263, 1264, 6, 150, 30, 0, 1264, 316, 1, 0, 0, 0, 1265, 1266, 3, 169, 77, 0, 1266, 1267, 1, 0, 0, 0, 1267, 1268, 6, 151, 31, 0, 1268, 318, 1, 0, 0, 0, 1269, 1270, 3, 55, 20, 0, 1270, 1271, 1, 0, 0, 0, 1271, 1272, 6, 152, 10, 0, 1272, 320, 1, 0, 0, 0, 1273, 1274, 3, 57, 21, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 153, 10, 0, 1276, 322, 1, 0, 0, 0, 1277, 1278, 3, 59, 22, 0, 1278, 1279, 1, 0, 0, 0, 1279, 1280, 6, 154, 10, 0, 1280, 324, 1, 0, 0, 0, 1281, 1282, 3, 61, 23, 0, 1282, 1283, 1, 0, 0, 0, 1283, 1284, 6, 155, 15, 0, 1284, 1285, 6, 155, 11, 0, 1285, 326, 1, 0, 0, 0, 1286, 1287, 7, 1, 0, 0, 1287, 1288, 7, 9, 0, 0, 1288, 1289, 7, 15, 0, 0, 1289, 1290, 7, 7, 0, 0, 1290, 328, 1, 0, 0, 0, 1291, 1292, 3, 55, 20, 0, 1292, 1293, 1, 0, 0, 0, 1293, 1294, 6, 157, 10, 0, 1294, 330, 1, 0, 0, 0, 1295, 1296, 3, 57, 21, 0, 1296, 1297, 1, 0, 0, 0, 1297, 1298, 6, 158, 10, 0, 1298, 332, 1, 0, 0, 0, 1299, 1300, 3, 59, 22, 0, 1300, 1301, 1, 0, 0, 0, 1301, 1302, 6, 159, 10, 0, 1302, 334, 1, 0, 0, 0, 1303, 1304, 3, 167, 76, 0, 1304, 1305, 1, 0, 0, 0, 1305, 1306, 6, 160, 16, 0, 1306, 1307, 6, 160, 11, 0, 1307, 336, 1, 0, 0, 0, 1308, 1309, 5, 58, 0, 0, 1309, 338, 1, 0, 0, 0, 1310, 1316, 3, 73, 29, 0, 1311, 1316, 3, 63, 24, 0, 1312, 1316, 3, 103, 44, 0, 1313, 1316, 3, 65, 25, 0, 1314, 1316, 3, 79, 32, 0, 1315, 1310, 1, 0, 0, 0, 1315, 1311, 1, 0, 0, 0, 1315, 1312, 1, 0, 0, 0, 1315, 1313, 1, 0, 0, 0, 1315, 1314, 1, 0, 0, 0, 1316, 1317, 1, 0, 0, 0, 1317, 1315, 1, 0, 0, 0, 1317, 1318, 1, 0, 0, 0, 1318, 340, 1, 0, 0, 0, 1319, 1320, 3, 55, 20, 0, 1320, 1321, 1, 0, 0, 0, 1321, 1322, 6, 163, 10, 0, 1322, 342, 1, 0, 0, 0, 1323, 1324, 3, 57, 21, 0, 1324, 1325, 1, 0, 0, 0, 1325, 1326, 6, 164, 10, 0, 1326, 344, 1, 0, 0, 0, 1327, 1328, 3, 59, 22, 0, 1328, 1329, 1, 0, 0, 0, 1329, 1330, 6, 165, 10, 0, 1330, 346, 1, 0, 0, 0, 1331, 1332, 3, 61, 23, 0, 1332, 1333, 1, 0, 0, 0, 1333, 1334, 6, 166, 15, 0, 1334, 1335, 6, 166, 11, 0, 1335, 348, 1, 0, 0, 0, 1336, 1337, 3, 337, 161, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 167, 17, 0, 1339, 350, 1, 0, 0, 0, 1340, 1341, 3, 99, 42, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 168, 18, 0, 1343, 352, 1, 0, 0, 0, 1344, 1345, 3, 103, 44, 0, 1345, 1346, 1, 0, 0, 0, 1346, 1347, 6, 169, 22, 0, 1347, 354, 1, 0, 0, 0, 1348, 1349, 3, 267, 126, 0, 1349, 1350, 1, 0, 0, 0, 1350, 1351, 6, 170, 32, 0, 1351, 1352, 6, 170, 33, 0, 1352, 356, 1, 0, 0, 0, 1353, 1354, 3, 207, 96, 0, 1354, 1355, 1, 0, 0, 0, 1355, 1356, 6, 171, 20, 0, 1356, 358, 1, 0, 0, 0, 1357, 1358, 3, 83, 34, 0, 1358, 1359, 1, 0, 0, 0, 1359, 1360, 6, 172, 21, 0, 1360, 360, 1, 0, 0, 0, 1361, 1362, 3, 55, 20, 0, 1362, 1363, 1, 0, 0, 0, 1363, 1364, 6, 173, 10, 0, 1364, 362, 1, 0, 0, 0, 1365, 1366, 3, 57, 21, 0, 1366, 1367, 1, 0, 0, 0, 1367, 1368, 6, 174, 10, 0, 1368, 364, 1, 0, 0, 0, 1369, 1370, 3, 59, 22, 0, 1370, 1371, 1, 0, 0, 0, 1371, 1372, 6, 175, 10, 0, 1372, 366, 1, 0, 0, 0, 1373, 1374, 3, 61, 23, 0, 1374, 1375, 1, 0, 0, 0, 1375, 1376, 6, 176, 15, 0, 1376, 1377, 6, 176, 11, 0, 1377, 1378, 6, 176, 11, 0, 1378, 368, 1, 0, 0, 0, 1379, 1380, 3, 99, 42, 0, 1380, 1381, 1, 0, 0, 0, 1381, 1382, 6, 177, 18, 0, 1382, 370, 1, 0, 0, 0, 1383, 1384, 3, 103, 44, 0, 1384, 1385, 1, 0, 0, 0, 1385, 1386, 6, 178, 22, 0, 1386, 372, 1, 0, 0, 0, 1387, 1388, 3, 233, 109, 0, 1388, 1389, 1, 0, 0, 0, 1389, 1390, 6, 179, 25, 0, 1390, 374, 1, 0, 0, 0, 1391, 1392, 3, 55, 20, 0, 1392, 1393, 1, 0, 0, 0, 1393, 1394, 6, 180, 10, 0, 1394, 376, 1, 0, 0, 0, 1395, 1396, 3, 57, 21, 0, 1396, 1397, 1, 0, 0, 0, 1397, 1398, 6, 181, 10, 0, 1398, 378, 1, 0, 0, 0, 1399, 1400, 3, 59, 22, 0, 1400, 1401, 1, 0, 0, 0, 1401, 1402, 6, 182, 10, 0, 1402, 380, 1, 0, 0, 0, 1403, 1404, 3, 61, 23, 0, 1404, 1405, 1, 0, 0, 0, 1405, 1406, 6, 183, 15, 0, 1406, 1407, 6, 183, 11, 0, 1407, 382, 1, 0, 0, 0, 1408, 1409, 3, 207, 96, 0, 1409, 1410, 1, 0, 0, 0, 1410, 1411, 6, 184, 20, 0, 1411, 1412, 6, 184, 11, 0, 1412, 1413, 6, 184, 34, 0, 1413, 384, 1, 0, 0, 0, 1414, 1415, 3, 83, 34, 0, 1415, 1416, 1, 0, 0, 0, 1416, 1417, 6, 185, 21, 0, 1417, 1418, 6, 185, 11, 0, 1418, 1419, 6, 185, 34, 0, 1419, 386, 1, 0, 0, 0, 1420, 1421, 3, 55, 20, 0, 1421, 1422, 1, 0, 0, 0, 1422, 1423, 6, 186, 10, 0, 1423, 388, 1, 0, 0, 0, 1424, 1425, 3, 57, 21, 0, 1425, 1426, 1, 0, 0, 0, 1426, 1427, 6, 187, 10, 0, 1427, 390, 1, 0, 0, 0, 1428, 1429, 3, 59, 22, 0, 1429, 1430, 1, 0, 0, 0, 1430, 1431, 6, 188, 10, 0, 1431, 392, 1, 0, 0, 0, 1432, 1433, 3, 337, 161, 0, 1433, 1434, 1, 0, 0, 0, 1434, 1435, 6, 189, 17, 0, 1435, 1436, 6, 189, 11, 0, 1436, 1437, 6, 189, 9, 0, 1437, 394, 1, 0, 0, 0, 1438, 1439, 3, 99, 42, 0, 1439, 1440, 1, 0, 0, 0, 1440, 1441, 6, 190, 18, 0, 1441, 1442, 6, 190, 11, 0, 1442, 1443, 6, 190, 9, 0, 1443, 396, 1, 0, 0, 0, 1444, 1445, 3, 55, 20, 0, 1445, 1446, 1, 0, 0, 0, 1446, 1447, 6, 191, 10, 0, 1447, 398, 1, 0, 0, 0, 1448, 1449, 3, 57, 21, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1451, 6, 192, 10, 0, 1451, 400, 1, 0, 0, 0, 1452, 1453, 3, 59, 22, 0, 1453, 1454, 1, 0, 0, 0, 1454, 1455, 6, 193, 10, 0, 1455, 402, 1, 0, 0, 0, 1456, 1457, 3, 173, 79, 0, 1457, 1458, 1, 0, 0, 0, 1458, 1459, 6, 194, 11, 0, 1459, 1460, 6, 194, 0, 0, 1460, 1461, 6, 194, 30, 0, 1461, 404, 1, 0, 0, 0, 1462, 1463, 3, 169, 77, 0, 1463, 1464, 1, 0, 0, 0, 1464, 1465, 6, 195, 11, 0, 1465, 1466, 6, 195, 0, 0, 1466, 1467, 6, 195, 31, 0, 1467, 406, 1, 0, 0, 0, 1468, 1469, 3, 89, 37, 0, 1469, 1470, 1, 0, 0, 0, 1470, 1471, 6, 196, 11, 0, 1471, 1472, 6, 196, 0, 0, 1472, 1473, 6, 196, 35, 0, 1473, 408, 1, 0, 0, 0, 1474, 1475, 3, 61, 23, 0, 1475, 1476, 1, 0, 0, 0, 1476, 1477, 6, 197, 15, 0, 1477, 1478, 6, 197, 11, 0, 1478, 410, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 579, 589, 593, 596, 605, 607, 618, 637, 642, 651, 658, 663, 665, 676, 684, 687, 689, 694, 699, 705, 712, 717, 723, 726, 734, 738, 870, 875, 882, 884, 900, 905, 910, 912, 918, 995, 1000, 1049, 1053, 1058, 1063, 1068, 1070, 1074, 1076, 1163, 1167, 1172, 1315, 1317, 36, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 65, 0, 5, 0, 0, 7, 24, 0, 7, 66, 0, 7, 104, 0, 7, 33, 0, 7, 31, 0, 7, 76, 0, 7, 25, 0, 7, 35, 0, 7, 47, 0, 7, 64, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 28, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index 305126ddfae2d..cef4bc5378aaa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -227,6 +227,22 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex); case 18: return DEV_METRICS_sempred((RuleContext)_localctx, predIndex); + case 105: + return PROJECT_PARAM_sempred((RuleContext)_localctx, predIndex); + case 106: + return PROJECT_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex); + case 117: + return RENAME_PARAM_sempred((RuleContext)_localctx, predIndex); + case 118: + return RENAME_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex); + case 141: + return ENRICH_FIELD_PARAM_sempred((RuleContext)_localctx, predIndex); + case 142: + return ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex); + case 148: + return MVEXPAND_PARAM_sempred((RuleContext)_localctx, predIndex); + case 149: + return MVEXPAND_NAMED_OR_POSITIONAL_PARAM_sempred((RuleContext)_localctx, predIndex); } return true; } @@ -251,9 +267,65 @@ private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { } return true; } + private boolean PROJECT_PARAM_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 3: + return this.isDevVersion(); + } + return true; + } + private boolean PROJECT_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 4: + return this.isDevVersion(); + } + return true; + } + private boolean RENAME_PARAM_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 5: + return this.isDevVersion(); + } + return true; + } + private boolean RENAME_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 6: + return this.isDevVersion(); + } + return true; + } + private boolean ENRICH_FIELD_PARAM_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 7: + return this.isDevVersion(); + } + return true; + } + private boolean ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 8: + return this.isDevVersion(); + } + return true; + } + private boolean MVEXPAND_PARAM_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 9: + return this.isDevVersion(); + } + return true; + } + private boolean MVEXPAND_NAMED_OR_POSITIONAL_PARAM_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 10: + return this.isDevVersion(); + } + return true; + } public static final String _serializedATN = - "\u0004\u0000x\u05bf\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0004\u0000x\u05c7\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ @@ -387,35 +459,36 @@ private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { "b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001"+ "d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001f\u0001"+ "g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001"+ - "i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001k\u0001k\u0003"+ - "k\u0418\bk\u0001l\u0001l\u0003l\u041c\bl\u0001l\u0005l\u041f\bl\nl\fl"+ - "\u0422\tl\u0001l\u0001l\u0003l\u0426\bl\u0001l\u0004l\u0429\bl\u000bl"+ - "\fl\u042a\u0003l\u042d\bl\u0001m\u0001m\u0004m\u0431\bm\u000bm\fm\u0432"+ - "\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001"+ - "p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001"+ - "r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001t\u0001"+ - "u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001w\u0001"+ - "w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001z\u0001"+ - "z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001|\u0001"+ - "|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001~\u0001"+ - "~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001"+ - "\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001\u0081\u0004\u0081\u0486"+ - "\b\u0081\u000b\u0081\f\u0081\u0487\u0001\u0081\u0001\u0081\u0003\u0081"+ - "\u048c\b\u0081\u0001\u0081\u0004\u0081\u048f\b\u0081\u000b\u0081\f\u0081"+ - "\u0490\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083\u0001"+ - "\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001\u0084\u0001"+ - "\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0086\u0001"+ - "\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087\u0001"+ - "\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001\u0088\u0001"+ - "\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a\u0001"+ - "\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001\u008b\u0001"+ - "\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008d\u0001"+ - "\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e\u0001\u008e\u0001"+ - "\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u0090\u0001"+ - "\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001\u0091\u0001\u0091\u0001"+ - "\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001"+ - "\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001"+ - "\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001"+ + "i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001"+ + "k\u0001k\u0003k\u041a\bk\u0001l\u0001l\u0003l\u041e\bl\u0001l\u0005l\u0421"+ + "\bl\nl\fl\u0424\tl\u0001l\u0001l\u0003l\u0428\bl\u0001l\u0004l\u042b\b"+ + "l\u000bl\fl\u042c\u0003l\u042f\bl\u0001m\u0001m\u0004m\u0433\bm\u000b"+ + "m\fm\u0434\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001"+ + "p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001q\u0001r\u0001"+ + "r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001"+ + "t\u0001u\u0001u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001"+ + "v\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001"+ + "y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001"+ + "|\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001}\u0001"+ + "~\u0001~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001"+ + "\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001"+ + "\u0081\u0004\u0081\u048a\b\u0081\u000b\u0081\f\u0081\u048b\u0001\u0081"+ + "\u0001\u0081\u0003\u0081\u0490\b\u0081\u0001\u0081\u0004\u0081\u0493\b"+ + "\u0081\u000b\u0081\f\u0081\u0494\u0001\u0082\u0001\u0082\u0001\u0082\u0001"+ + "\u0082\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001"+ + "\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001"+ + "\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001"+ + "\u0086\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001"+ + "\u0088\u0001\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001"+ + "\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001"+ + "\u008b\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+ + "\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001"+ + "\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001"+ + "\u008f\u0001\u008f\u0001\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001"+ + "\u0090\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001"+ + "\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001"+ + "\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001"+ + "\u0094\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001"+ "\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097\u0001\u0097\u0001"+ "\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0098\u0001"+ "\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u009a\u0001\u009a\u0001"+ @@ -425,7 +498,7 @@ private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { "\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f\u0001"+ "\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a1\u0001"+ "\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0004"+ - "\u00a2\u051c\b\u00a2\u000b\u00a2\f\u00a2\u051d\u0001\u00a3\u0001\u00a3"+ + "\u00a2\u0524\b\u00a2\u000b\u00a2\f\u00a2\u0525\u0001\u00a3\u0001\u00a3"+ "\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4"+ "\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a6\u0001\u00a6"+ "\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001\u00a7"+ @@ -486,7 +559,7 @@ private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { "\r \u0001\u000009\u0002\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000"+ "\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000"+ "YYyy\u000b\u0000\t\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000"+ - "\t\n\r\r \"#,,//::<<>?\\\\||\u05db\u0000\u000f\u0001\u0000\u0000\u0000"+ + "\t\n\r\r \"#,,//::<<>?\\\\||\u05e3\u0000\u000f\u0001\u0000\u0000\u0000"+ "\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000"+ "\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000"+ "\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000"+ @@ -623,53 +696,53 @@ private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { "\u0000\u00d5\u03f2\u0001\u0000\u0000\u0000\u00d7\u03f6\u0001\u0000\u0000"+ "\u0000\u00d9\u03fa\u0001\u0000\u0000\u0000\u00db\u03fe\u0001\u0000\u0000"+ "\u0000\u00dd\u0403\u0001\u0000\u0000\u0000\u00df\u0407\u0001\u0000\u0000"+ - "\u0000\u00e1\u040b\u0001\u0000\u0000\u0000\u00e3\u040f\u0001\u0000\u0000"+ - "\u0000\u00e5\u0417\u0001\u0000\u0000\u0000\u00e7\u042c\u0001\u0000\u0000"+ - "\u0000\u00e9\u0430\u0001\u0000\u0000\u0000\u00eb\u0434\u0001\u0000\u0000"+ - "\u0000\u00ed\u0438\u0001\u0000\u0000\u0000\u00ef\u043c\u0001\u0000\u0000"+ - "\u0000\u00f1\u0440\u0001\u0000\u0000\u0000\u00f3\u0445\u0001\u0000\u0000"+ - "\u0000\u00f5\u0449\u0001\u0000\u0000\u0000\u00f7\u044d\u0001\u0000\u0000"+ - "\u0000\u00f9\u0451\u0001\u0000\u0000\u0000\u00fb\u0455\u0001\u0000\u0000"+ - "\u0000\u00fd\u0459\u0001\u0000\u0000\u0000\u00ff\u045c\u0001\u0000\u0000"+ - "\u0000\u0101\u0460\u0001\u0000\u0000\u0000\u0103\u0464\u0001\u0000\u0000"+ - "\u0000\u0105\u0468\u0001\u0000\u0000\u0000\u0107\u046c\u0001\u0000\u0000"+ - "\u0000\u0109\u0471\u0001\u0000\u0000\u0000\u010b\u0476\u0001\u0000\u0000"+ - "\u0000\u010d\u047b\u0001\u0000\u0000\u0000\u010f\u0482\u0001\u0000\u0000"+ - "\u0000\u0111\u048b\u0001\u0000\u0000\u0000\u0113\u0492\u0001\u0000\u0000"+ - "\u0000\u0115\u0496\u0001\u0000\u0000\u0000\u0117\u049a\u0001\u0000\u0000"+ - "\u0000\u0119\u049e\u0001\u0000\u0000\u0000\u011b\u04a2\u0001\u0000\u0000"+ - "\u0000\u011d\u04a8\u0001\u0000\u0000\u0000\u011f\u04ac\u0001\u0000\u0000"+ - "\u0000\u0121\u04b0\u0001\u0000\u0000\u0000\u0123\u04b4\u0001\u0000\u0000"+ - "\u0000\u0125\u04b8\u0001\u0000\u0000\u0000\u0127\u04bc\u0001\u0000\u0000"+ - "\u0000\u0129\u04c0\u0001\u0000\u0000\u0000\u012b\u04c4\u0001\u0000\u0000"+ - "\u0000\u012d\u04c8\u0001\u0000\u0000\u0000\u012f\u04cc\u0001\u0000\u0000"+ - "\u0000\u0131\u04d0\u0001\u0000\u0000\u0000\u0133\u04d4\u0001\u0000\u0000"+ - "\u0000\u0135\u04d9\u0001\u0000\u0000\u0000\u0137\u04dd\u0001\u0000\u0000"+ - "\u0000\u0139\u04e1\u0001\u0000\u0000\u0000\u013b\u04e5\u0001\u0000\u0000"+ - "\u0000\u013d\u04e9\u0001\u0000\u0000\u0000\u013f\u04ed\u0001\u0000\u0000"+ - "\u0000\u0141\u04f1\u0001\u0000\u0000\u0000\u0143\u04f5\u0001\u0000\u0000"+ - "\u0000\u0145\u04f9\u0001\u0000\u0000\u0000\u0147\u04fe\u0001\u0000\u0000"+ - "\u0000\u0149\u0503\u0001\u0000\u0000\u0000\u014b\u0507\u0001\u0000\u0000"+ - "\u0000\u014d\u050b\u0001\u0000\u0000\u0000\u014f\u050f\u0001\u0000\u0000"+ - "\u0000\u0151\u0514\u0001\u0000\u0000\u0000\u0153\u051b\u0001\u0000\u0000"+ - "\u0000\u0155\u051f\u0001\u0000\u0000\u0000\u0157\u0523\u0001\u0000\u0000"+ - "\u0000\u0159\u0527\u0001\u0000\u0000\u0000\u015b\u052b\u0001\u0000\u0000"+ - "\u0000\u015d\u0530\u0001\u0000\u0000\u0000\u015f\u0534\u0001\u0000\u0000"+ - "\u0000\u0161\u0538\u0001\u0000\u0000\u0000\u0163\u053c\u0001\u0000\u0000"+ - "\u0000\u0165\u0541\u0001\u0000\u0000\u0000\u0167\u0545\u0001\u0000\u0000"+ - "\u0000\u0169\u0549\u0001\u0000\u0000\u0000\u016b\u054d\u0001\u0000\u0000"+ - "\u0000\u016d\u0551\u0001\u0000\u0000\u0000\u016f\u0555\u0001\u0000\u0000"+ - "\u0000\u0171\u055b\u0001\u0000\u0000\u0000\u0173\u055f\u0001\u0000\u0000"+ - "\u0000\u0175\u0563\u0001\u0000\u0000\u0000\u0177\u0567\u0001\u0000\u0000"+ - "\u0000\u0179\u056b\u0001\u0000\u0000\u0000\u017b\u056f\u0001\u0000\u0000"+ - "\u0000\u017d\u0573\u0001\u0000\u0000\u0000\u017f\u0578\u0001\u0000\u0000"+ - "\u0000\u0181\u057e\u0001\u0000\u0000\u0000\u0183\u0584\u0001\u0000\u0000"+ - "\u0000\u0185\u0588\u0001\u0000\u0000\u0000\u0187\u058c\u0001\u0000\u0000"+ - "\u0000\u0189\u0590\u0001\u0000\u0000\u0000\u018b\u0596\u0001\u0000\u0000"+ - "\u0000\u018d\u059c\u0001\u0000\u0000\u0000\u018f\u05a0\u0001\u0000\u0000"+ - "\u0000\u0191\u05a4\u0001\u0000\u0000\u0000\u0193\u05a8\u0001\u0000\u0000"+ - "\u0000\u0195\u05ae\u0001\u0000\u0000\u0000\u0197\u05b4\u0001\u0000\u0000"+ - "\u0000\u0199\u05ba\u0001\u0000\u0000\u0000\u019b\u019c\u0007\u0000\u0000"+ + "\u0000\u00e1\u040b\u0001\u0000\u0000\u0000\u00e3\u0410\u0001\u0000\u0000"+ + "\u0000\u00e5\u0419\u0001\u0000\u0000\u0000\u00e7\u042e\u0001\u0000\u0000"+ + "\u0000\u00e9\u0432\u0001\u0000\u0000\u0000\u00eb\u0436\u0001\u0000\u0000"+ + "\u0000\u00ed\u043a\u0001\u0000\u0000\u0000\u00ef\u043e\u0001\u0000\u0000"+ + "\u0000\u00f1\u0442\u0001\u0000\u0000\u0000\u00f3\u0447\u0001\u0000\u0000"+ + "\u0000\u00f5\u044b\u0001\u0000\u0000\u0000\u00f7\u044f\u0001\u0000\u0000"+ + "\u0000\u00f9\u0453\u0001\u0000\u0000\u0000\u00fb\u0458\u0001\u0000\u0000"+ + "\u0000\u00fd\u045d\u0001\u0000\u0000\u0000\u00ff\u0460\u0001\u0000\u0000"+ + "\u0000\u0101\u0464\u0001\u0000\u0000\u0000\u0103\u0468\u0001\u0000\u0000"+ + "\u0000\u0105\u046c\u0001\u0000\u0000\u0000\u0107\u0470\u0001\u0000\u0000"+ + "\u0000\u0109\u0475\u0001\u0000\u0000\u0000\u010b\u047a\u0001\u0000\u0000"+ + "\u0000\u010d\u047f\u0001\u0000\u0000\u0000\u010f\u0486\u0001\u0000\u0000"+ + "\u0000\u0111\u048f\u0001\u0000\u0000\u0000\u0113\u0496\u0001\u0000\u0000"+ + "\u0000\u0115\u049a\u0001\u0000\u0000\u0000\u0117\u049e\u0001\u0000\u0000"+ + "\u0000\u0119\u04a2\u0001\u0000\u0000\u0000\u011b\u04a6\u0001\u0000\u0000"+ + "\u0000\u011d\u04ac\u0001\u0000\u0000\u0000\u011f\u04b0\u0001\u0000\u0000"+ + "\u0000\u0121\u04b4\u0001\u0000\u0000\u0000\u0123\u04b8\u0001\u0000\u0000"+ + "\u0000\u0125\u04bc\u0001\u0000\u0000\u0000\u0127\u04c0\u0001\u0000\u0000"+ + "\u0000\u0129\u04c4\u0001\u0000\u0000\u0000\u012b\u04c9\u0001\u0000\u0000"+ + "\u0000\u012d\u04ce\u0001\u0000\u0000\u0000\u012f\u04d2\u0001\u0000\u0000"+ + "\u0000\u0131\u04d6\u0001\u0000\u0000\u0000\u0133\u04da\u0001\u0000\u0000"+ + "\u0000\u0135\u04df\u0001\u0000\u0000\u0000\u0137\u04e3\u0001\u0000\u0000"+ + "\u0000\u0139\u04e8\u0001\u0000\u0000\u0000\u013b\u04ed\u0001\u0000\u0000"+ + "\u0000\u013d\u04f1\u0001\u0000\u0000\u0000\u013f\u04f5\u0001\u0000\u0000"+ + "\u0000\u0141\u04f9\u0001\u0000\u0000\u0000\u0143\u04fd\u0001\u0000\u0000"+ + "\u0000\u0145\u0501\u0001\u0000\u0000\u0000\u0147\u0506\u0001\u0000\u0000"+ + "\u0000\u0149\u050b\u0001\u0000\u0000\u0000\u014b\u050f\u0001\u0000\u0000"+ + "\u0000\u014d\u0513\u0001\u0000\u0000\u0000\u014f\u0517\u0001\u0000\u0000"+ + "\u0000\u0151\u051c\u0001\u0000\u0000\u0000\u0153\u0523\u0001\u0000\u0000"+ + "\u0000\u0155\u0527\u0001\u0000\u0000\u0000\u0157\u052b\u0001\u0000\u0000"+ + "\u0000\u0159\u052f\u0001\u0000\u0000\u0000\u015b\u0533\u0001\u0000\u0000"+ + "\u0000\u015d\u0538\u0001\u0000\u0000\u0000\u015f\u053c\u0001\u0000\u0000"+ + "\u0000\u0161\u0540\u0001\u0000\u0000\u0000\u0163\u0544\u0001\u0000\u0000"+ + "\u0000\u0165\u0549\u0001\u0000\u0000\u0000\u0167\u054d\u0001\u0000\u0000"+ + "\u0000\u0169\u0551\u0001\u0000\u0000\u0000\u016b\u0555\u0001\u0000\u0000"+ + "\u0000\u016d\u0559\u0001\u0000\u0000\u0000\u016f\u055d\u0001\u0000\u0000"+ + "\u0000\u0171\u0563\u0001\u0000\u0000\u0000\u0173\u0567\u0001\u0000\u0000"+ + "\u0000\u0175\u056b\u0001\u0000\u0000\u0000\u0177\u056f\u0001\u0000\u0000"+ + "\u0000\u0179\u0573\u0001\u0000\u0000\u0000\u017b\u0577\u0001\u0000\u0000"+ + "\u0000\u017d\u057b\u0001\u0000\u0000\u0000\u017f\u0580\u0001\u0000\u0000"+ + "\u0000\u0181\u0586\u0001\u0000\u0000\u0000\u0183\u058c\u0001\u0000\u0000"+ + "\u0000\u0185\u0590\u0001\u0000\u0000\u0000\u0187\u0594\u0001\u0000\u0000"+ + "\u0000\u0189\u0598\u0001\u0000\u0000\u0000\u018b\u059e\u0001\u0000\u0000"+ + "\u0000\u018d\u05a4\u0001\u0000\u0000\u0000\u018f\u05a8\u0001\u0000\u0000"+ + "\u0000\u0191\u05ac\u0001\u0000\u0000\u0000\u0193\u05b0\u0001\u0000\u0000"+ + "\u0000\u0195\u05b6\u0001\u0000\u0000\u0000\u0197\u05bc\u0001\u0000\u0000"+ + "\u0000\u0199\u05c2\u0001\u0000\u0000\u0000\u019b\u019c\u0007\u0000\u0000"+ "\u0000\u019c\u019d\u0007\u0001\u0000\u0000\u019d\u019e\u0007\u0002\u0000"+ "\u0000\u019e\u019f\u0007\u0002\u0000\u0000\u019f\u01a0\u0007\u0003\u0000"+ "\u0000\u01a0\u01a1\u0007\u0004\u0000\u0000\u01a1\u01a2\u0007\u0005\u0000"+ @@ -981,233 +1054,237 @@ private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { "\u0000\u0000\u0403\u0404\u0003g,\u0000\u0404\u0405\u0001\u0000\u0000\u0000"+ "\u0405\u0406\u0006g\u0016\u0000\u0406\u00de\u0001\u0000\u0000\u0000\u0407"+ "\u0408\u0003c*\u0000\u0408\u0409\u0001\u0000\u0000\u0000\u0409\u040a\u0006"+ - "h\u0012\u0000\u040a\u00e0\u0001\u0000\u0000\u0000\u040b\u040c\u0003\u007f"+ - "8\u0000\u040c\u040d\u0001\u0000\u0000\u0000\u040d\u040e\u0006i\u0017\u0000"+ - "\u040e\u00e2\u0001\u0000\u0000\u0000\u040f\u0410\u0003\u00a3J\u0000\u0410"+ - "\u0411\u0001\u0000\u0000\u0000\u0411\u0412\u0006j\u0018\u0000\u0412\u00e4"+ - "\u0001\u0000\u0000\u0000\u0413\u0418\u0003A\u0019\u0000\u0414\u0418\u0003"+ - "?\u0018\u0000\u0415\u0418\u0003O \u0000\u0416\u0418\u0003\u0099E\u0000"+ - "\u0417\u0413\u0001\u0000\u0000\u0000\u0417\u0414\u0001\u0000\u0000\u0000"+ - "\u0417\u0415\u0001\u0000\u0000\u0000\u0417\u0416\u0001\u0000\u0000\u0000"+ - "\u0418\u00e6\u0001\u0000\u0000\u0000\u0419\u041c\u0003A\u0019\u0000\u041a"+ - "\u041c\u0003\u0099E\u0000\u041b\u0419\u0001\u0000\u0000\u0000\u041b\u041a"+ - "\u0001\u0000\u0000\u0000\u041c\u0420\u0001\u0000\u0000\u0000\u041d\u041f"+ - "\u0003\u00e5k\u0000\u041e\u041d\u0001\u0000\u0000\u0000\u041f\u0422\u0001"+ - "\u0000\u0000\u0000\u0420\u041e\u0001\u0000\u0000\u0000\u0420\u0421\u0001"+ - "\u0000\u0000\u0000\u0421\u042d\u0001\u0000\u0000\u0000\u0422\u0420\u0001"+ - "\u0000\u0000\u0000\u0423\u0426\u0003O \u0000\u0424\u0426\u0003I\u001d"+ - "\u0000\u0425\u0423\u0001\u0000\u0000\u0000\u0425\u0424\u0001\u0000\u0000"+ - "\u0000\u0426\u0428\u0001\u0000\u0000\u0000\u0427\u0429\u0003\u00e5k\u0000"+ - "\u0428\u0427\u0001\u0000\u0000\u0000\u0429\u042a\u0001\u0000\u0000\u0000"+ - "\u042a\u0428\u0001\u0000\u0000\u0000\u042a\u042b\u0001\u0000\u0000\u0000"+ - "\u042b\u042d\u0001\u0000\u0000\u0000\u042c\u041b\u0001\u0000\u0000\u0000"+ - "\u042c\u0425\u0001\u0000\u0000\u0000\u042d\u00e8\u0001\u0000\u0000\u0000"+ - "\u042e\u0431\u0003\u00e7l\u0000\u042f\u0431\u0003\u00abN\u0000\u0430\u042e"+ - "\u0001\u0000\u0000\u0000\u0430\u042f\u0001\u0000\u0000\u0000\u0431\u0432"+ - "\u0001\u0000\u0000\u0000\u0432\u0430\u0001\u0000\u0000\u0000\u0432\u0433"+ - "\u0001\u0000\u0000\u0000\u0433\u00ea\u0001\u0000\u0000\u0000\u0434\u0435"+ - "\u00037\u0014\u0000\u0435\u0436\u0001\u0000\u0000\u0000\u0436\u0437\u0006"+ - "n\n\u0000\u0437\u00ec\u0001\u0000\u0000\u0000\u0438\u0439\u00039\u0015"+ - "\u0000\u0439\u043a\u0001\u0000\u0000\u0000\u043a\u043b\u0006o\n\u0000"+ - "\u043b\u00ee\u0001\u0000\u0000\u0000\u043c\u043d\u0003;\u0016\u0000\u043d"+ - "\u043e\u0001\u0000\u0000\u0000\u043e\u043f\u0006p\n\u0000\u043f\u00f0"+ - "\u0001\u0000\u0000\u0000\u0440\u0441\u0003=\u0017\u0000\u0441\u0442\u0001"+ - "\u0000\u0000\u0000\u0442\u0443\u0006q\u000f\u0000\u0443\u0444\u0006q\u000b"+ - "\u0000\u0444\u00f2\u0001\u0000\u0000\u0000\u0445\u0446\u0003_(\u0000\u0446"+ - "\u0447\u0001\u0000\u0000\u0000\u0447\u0448\u0006r\u0013\u0000\u0448\u00f4"+ - "\u0001\u0000\u0000\u0000\u0449\u044a\u0003c*\u0000\u044a\u044b\u0001\u0000"+ - "\u0000\u0000\u044b\u044c\u0006s\u0012\u0000\u044c\u00f6\u0001\u0000\u0000"+ - "\u0000\u044d\u044e\u0003g,\u0000\u044e\u044f\u0001\u0000\u0000\u0000\u044f"+ - "\u0450\u0006t\u0016\u0000\u0450\u00f8\u0001\u0000\u0000\u0000\u0451\u0452"+ - "\u0003\u007f8\u0000\u0452\u0453\u0001\u0000\u0000\u0000\u0453\u0454\u0006"+ - "u\u0017\u0000\u0454\u00fa\u0001\u0000\u0000\u0000\u0455\u0456\u0003\u00a3"+ - "J\u0000\u0456\u0457\u0001\u0000\u0000\u0000\u0457\u0458\u0006v\u0018\u0000"+ - "\u0458\u00fc\u0001\u0000\u0000\u0000\u0459\u045a\u0007\f\u0000\u0000\u045a"+ - "\u045b\u0007\u0002\u0000\u0000\u045b\u00fe\u0001\u0000\u0000\u0000\u045c"+ - "\u045d\u0003\u00e9m\u0000\u045d\u045e\u0001\u0000\u0000\u0000\u045e\u045f"+ - "\u0006x\u0019\u0000\u045f\u0100\u0001\u0000\u0000\u0000\u0460\u0461\u0003"+ - "7\u0014\u0000\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463\u0006y\n"+ - "\u0000\u0463\u0102\u0001\u0000\u0000\u0000\u0464\u0465\u00039\u0015\u0000"+ - "\u0465\u0466\u0001\u0000\u0000\u0000\u0466\u0467\u0006z\n\u0000\u0467"+ - "\u0104\u0001\u0000\u0000\u0000\u0468\u0469\u0003;\u0016\u0000\u0469\u046a"+ - "\u0001\u0000\u0000\u0000\u046a\u046b\u0006{\n\u0000\u046b\u0106\u0001"+ - "\u0000\u0000\u0000\u046c\u046d\u0003=\u0017\u0000\u046d\u046e\u0001\u0000"+ - "\u0000\u0000\u046e\u046f\u0006|\u000f\u0000\u046f\u0470\u0006|\u000b\u0000"+ - "\u0470\u0108\u0001\u0000\u0000\u0000\u0471\u0472\u0003\u00a5K\u0000\u0472"+ - "\u0473\u0001\u0000\u0000\u0000\u0473\u0474\u0006}\r\u0000\u0474\u0475"+ - "\u0006}\u001a\u0000\u0475\u010a\u0001\u0000\u0000\u0000\u0476\u0477\u0007"+ - "\u0007\u0000\u0000\u0477\u0478\u0007\t\u0000\u0000\u0478\u0479\u0001\u0000"+ - "\u0000\u0000\u0479\u047a\u0006~\u001b\u0000\u047a\u010c\u0001\u0000\u0000"+ - "\u0000\u047b\u047c\u0007\u0013\u0000\u0000\u047c\u047d\u0007\u0001\u0000"+ - "\u0000\u047d\u047e\u0007\u0005\u0000\u0000\u047e\u047f\u0007\n\u0000\u0000"+ - "\u047f\u0480\u0001\u0000\u0000\u0000\u0480\u0481\u0006\u007f\u001b\u0000"+ - "\u0481\u010e\u0001\u0000\u0000\u0000\u0482\u0483\b\"\u0000\u0000\u0483"+ - "\u0110\u0001\u0000\u0000\u0000\u0484\u0486\u0003\u010f\u0080\u0000\u0485"+ - "\u0484\u0001\u0000\u0000\u0000\u0486\u0487\u0001\u0000\u0000\u0000\u0487"+ - "\u0485\u0001\u0000\u0000\u0000\u0487\u0488\u0001\u0000\u0000\u0000\u0488"+ - "\u0489\u0001\u0000\u0000\u0000\u0489\u048a\u0003\u0151\u00a1\u0000\u048a"+ - "\u048c\u0001\u0000\u0000\u0000\u048b\u0485\u0001\u0000\u0000\u0000\u048b"+ - "\u048c\u0001\u0000\u0000\u0000\u048c\u048e\u0001\u0000\u0000\u0000\u048d"+ - "\u048f\u0003\u010f\u0080\u0000\u048e\u048d\u0001\u0000\u0000\u0000\u048f"+ - "\u0490\u0001\u0000\u0000\u0000\u0490\u048e\u0001\u0000\u0000\u0000\u0490"+ - "\u0491\u0001\u0000\u0000\u0000\u0491\u0112\u0001\u0000\u0000\u0000\u0492"+ - "\u0493\u0003\u0111\u0081\u0000\u0493\u0494\u0001\u0000\u0000\u0000\u0494"+ - "\u0495\u0006\u0082\u001c\u0000\u0495\u0114\u0001\u0000\u0000\u0000\u0496"+ - "\u0497\u00037\u0014\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498\u0499"+ - "\u0006\u0083\n\u0000\u0499\u0116\u0001\u0000\u0000\u0000\u049a\u049b\u0003"+ - "9\u0015\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d\u0006\u0084"+ - "\n\u0000\u049d\u0118\u0001\u0000\u0000\u0000\u049e\u049f\u0003;\u0016"+ - "\u0000\u049f\u04a0\u0001\u0000\u0000\u0000\u04a0\u04a1\u0006\u0085\n\u0000"+ - "\u04a1\u011a\u0001\u0000\u0000\u0000\u04a2\u04a3\u0003=\u0017\u0000\u04a3"+ - "\u04a4\u0001\u0000\u0000\u0000\u04a4\u04a5\u0006\u0086\u000f\u0000\u04a5"+ - "\u04a6\u0006\u0086\u000b\u0000\u04a6\u04a7\u0006\u0086\u000b\u0000\u04a7"+ - "\u011c\u0001\u0000\u0000\u0000\u04a8\u04a9\u0003_(\u0000\u04a9\u04aa\u0001"+ - "\u0000\u0000\u0000\u04aa\u04ab\u0006\u0087\u0013\u0000\u04ab\u011e\u0001"+ - "\u0000\u0000\u0000\u04ac\u04ad\u0003c*\u0000\u04ad\u04ae\u0001\u0000\u0000"+ - "\u0000\u04ae\u04af\u0006\u0088\u0012\u0000\u04af\u0120\u0001\u0000\u0000"+ - "\u0000\u04b0\u04b1\u0003g,\u0000\u04b1\u04b2\u0001\u0000\u0000\u0000\u04b2"+ - "\u04b3\u0006\u0089\u0016\u0000\u04b3\u0122\u0001\u0000\u0000\u0000\u04b4"+ - "\u04b5\u0003\u010d\u007f\u0000\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6"+ - "\u04b7\u0006\u008a\u001d\u0000\u04b7\u0124\u0001\u0000\u0000\u0000\u04b8"+ - "\u04b9\u0003\u00e9m\u0000\u04b9\u04ba\u0001\u0000\u0000\u0000\u04ba\u04bb"+ - "\u0006\u008b\u0019\u0000\u04bb\u0126\u0001\u0000\u0000\u0000\u04bc\u04bd"+ - "\u0003\u00adO\u0000\u04bd\u04be\u0001\u0000\u0000\u0000\u04be\u04bf\u0006"+ - "\u008c\u001e\u0000\u04bf\u0128\u0001\u0000\u0000\u0000\u04c0\u04c1\u0003"+ - "\u007f8\u0000\u04c1\u04c2\u0001\u0000\u0000\u0000\u04c2\u04c3\u0006\u008d"+ - "\u0017\u0000\u04c3\u012a\u0001\u0000\u0000\u0000\u04c4\u04c5\u0003\u00a3"+ - "J\u0000\u04c5\u04c6\u0001\u0000\u0000\u0000\u04c6\u04c7\u0006\u008e\u0018"+ - "\u0000\u04c7\u012c\u0001\u0000\u0000\u0000\u04c8\u04c9\u00037\u0014\u0000"+ - "\u04c9\u04ca\u0001\u0000\u0000\u0000\u04ca\u04cb\u0006\u008f\n\u0000\u04cb"+ - "\u012e\u0001\u0000\u0000\u0000\u04cc\u04cd\u00039\u0015\u0000\u04cd\u04ce"+ - "\u0001\u0000\u0000\u0000\u04ce\u04cf\u0006\u0090\n\u0000\u04cf\u0130\u0001"+ - "\u0000\u0000\u0000\u04d0\u04d1\u0003;\u0016\u0000\u04d1\u04d2\u0001\u0000"+ - "\u0000\u0000\u04d2\u04d3\u0006\u0091\n\u0000\u04d3\u0132\u0001\u0000\u0000"+ - "\u0000\u04d4\u04d5\u0003=\u0017\u0000\u04d5\u04d6\u0001\u0000\u0000\u0000"+ - "\u04d6\u04d7\u0006\u0092\u000f\u0000\u04d7\u04d8\u0006\u0092\u000b\u0000"+ - "\u04d8\u0134\u0001\u0000\u0000\u0000\u04d9\u04da\u0003g,\u0000\u04da\u04db"+ - "\u0001\u0000\u0000\u0000\u04db\u04dc\u0006\u0093\u0016\u0000\u04dc\u0136"+ - "\u0001\u0000\u0000\u0000\u04dd\u04de\u0003\u007f8\u0000\u04de\u04df\u0001"+ - "\u0000\u0000\u0000\u04df\u04e0\u0006\u0094\u0017\u0000\u04e0\u0138\u0001"+ - "\u0000\u0000\u0000\u04e1\u04e2\u0003\u00a3J\u0000\u04e2\u04e3\u0001\u0000"+ - "\u0000\u0000\u04e3\u04e4\u0006\u0095\u0018\u0000\u04e4\u013a\u0001\u0000"+ - "\u0000\u0000\u04e5\u04e6\u0003\u00adO\u0000\u04e6\u04e7\u0001\u0000\u0000"+ - "\u0000\u04e7\u04e8\u0006\u0096\u001e\u0000\u04e8\u013c\u0001\u0000\u0000"+ - "\u0000\u04e9\u04ea\u0003\u00a9M\u0000\u04ea\u04eb\u0001\u0000\u0000\u0000"+ - "\u04eb\u04ec\u0006\u0097\u001f\u0000\u04ec\u013e\u0001\u0000\u0000\u0000"+ - "\u04ed\u04ee\u00037\u0014\u0000\u04ee\u04ef\u0001\u0000\u0000\u0000\u04ef"+ - "\u04f0\u0006\u0098\n\u0000\u04f0\u0140\u0001\u0000\u0000\u0000\u04f1\u04f2"+ - "\u00039\u0015\u0000\u04f2\u04f3\u0001\u0000\u0000\u0000\u04f3\u04f4\u0006"+ - "\u0099\n\u0000\u04f4\u0142\u0001\u0000\u0000\u0000\u04f5\u04f6\u0003;"+ - "\u0016\u0000\u04f6\u04f7\u0001\u0000\u0000\u0000\u04f7\u04f8\u0006\u009a"+ - "\n\u0000\u04f8\u0144\u0001\u0000\u0000\u0000\u04f9\u04fa\u0003=\u0017"+ - "\u0000\u04fa\u04fb\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006\u009b\u000f"+ - "\u0000\u04fc\u04fd\u0006\u009b\u000b\u0000\u04fd\u0146\u0001\u0000\u0000"+ - "\u0000\u04fe\u04ff\u0007\u0001\u0000\u0000\u04ff\u0500\u0007\t\u0000\u0000"+ - "\u0500\u0501\u0007\u000f\u0000\u0000\u0501\u0502\u0007\u0007\u0000\u0000"+ - "\u0502\u0148\u0001\u0000\u0000\u0000\u0503\u0504\u00037\u0014\u0000\u0504"+ - "\u0505\u0001\u0000\u0000\u0000\u0505\u0506\u0006\u009d\n\u0000\u0506\u014a"+ - "\u0001\u0000\u0000\u0000\u0507\u0508\u00039\u0015\u0000\u0508\u0509\u0001"+ - "\u0000\u0000\u0000\u0509\u050a\u0006\u009e\n\u0000\u050a\u014c\u0001\u0000"+ - "\u0000\u0000\u050b\u050c\u0003;\u0016\u0000\u050c\u050d\u0001\u0000\u0000"+ - "\u0000\u050d\u050e\u0006\u009f\n\u0000\u050e\u014e\u0001\u0000\u0000\u0000"+ - "\u050f\u0510\u0003\u00a7L\u0000\u0510\u0511\u0001\u0000\u0000\u0000\u0511"+ - "\u0512\u0006\u00a0\u0010\u0000\u0512\u0513\u0006\u00a0\u000b\u0000\u0513"+ - "\u0150\u0001\u0000\u0000\u0000\u0514\u0515\u0005:\u0000\u0000\u0515\u0152"+ - "\u0001\u0000\u0000\u0000\u0516\u051c\u0003I\u001d\u0000\u0517\u051c\u0003"+ - "?\u0018\u0000\u0518\u051c\u0003g,\u0000\u0519\u051c\u0003A\u0019\u0000"+ - "\u051a\u051c\u0003O \u0000\u051b\u0516\u0001\u0000\u0000\u0000\u051b\u0517"+ - "\u0001\u0000\u0000\u0000\u051b\u0518\u0001\u0000\u0000\u0000\u051b\u0519"+ - "\u0001\u0000\u0000\u0000\u051b\u051a\u0001\u0000\u0000\u0000\u051c\u051d"+ - "\u0001\u0000\u0000\u0000\u051d\u051b\u0001\u0000\u0000\u0000\u051d\u051e"+ - "\u0001\u0000\u0000\u0000\u051e\u0154\u0001\u0000\u0000\u0000\u051f\u0520"+ - "\u00037\u0014\u0000\u0520\u0521\u0001\u0000\u0000\u0000\u0521\u0522\u0006"+ - "\u00a3\n\u0000\u0522\u0156\u0001\u0000\u0000\u0000\u0523\u0524\u00039"+ - "\u0015\u0000\u0524\u0525\u0001\u0000\u0000\u0000\u0525\u0526\u0006\u00a4"+ - "\n\u0000\u0526\u0158\u0001\u0000\u0000\u0000\u0527\u0528\u0003;\u0016"+ - "\u0000\u0528\u0529\u0001\u0000\u0000\u0000\u0529\u052a\u0006\u00a5\n\u0000"+ - "\u052a\u015a\u0001\u0000\u0000\u0000\u052b\u052c\u0003=\u0017\u0000\u052c"+ - "\u052d\u0001\u0000\u0000\u0000\u052d\u052e\u0006\u00a6\u000f\u0000\u052e"+ - "\u052f\u0006\u00a6\u000b\u0000\u052f\u015c\u0001\u0000\u0000\u0000\u0530"+ - "\u0531\u0003\u0151\u00a1\u0000\u0531\u0532\u0001\u0000\u0000\u0000\u0532"+ - "\u0533\u0006\u00a7\u0011\u0000\u0533\u015e\u0001\u0000\u0000\u0000\u0534"+ - "\u0535\u0003c*\u0000\u0535\u0536\u0001\u0000\u0000\u0000\u0536\u0537\u0006"+ - "\u00a8\u0012\u0000\u0537\u0160\u0001\u0000\u0000\u0000\u0538\u0539\u0003"+ - "g,\u0000\u0539\u053a\u0001\u0000\u0000\u0000\u053a\u053b\u0006\u00a9\u0016"+ - "\u0000\u053b\u0162\u0001\u0000\u0000\u0000\u053c\u053d\u0003\u010b~\u0000"+ - "\u053d\u053e\u0001\u0000\u0000\u0000\u053e\u053f\u0006\u00aa \u0000\u053f"+ - "\u0540\u0006\u00aa!\u0000\u0540\u0164\u0001\u0000\u0000\u0000\u0541\u0542"+ - "\u0003\u00cf`\u0000\u0542\u0543\u0001\u0000\u0000\u0000\u0543\u0544\u0006"+ - "\u00ab\u0014\u0000\u0544\u0166\u0001\u0000\u0000\u0000\u0545\u0546\u0003"+ - "S\"\u0000\u0546\u0547\u0001\u0000\u0000\u0000\u0547\u0548\u0006\u00ac"+ - "\u0015\u0000\u0548\u0168\u0001\u0000\u0000\u0000\u0549\u054a\u00037\u0014"+ - "\u0000\u054a\u054b\u0001\u0000\u0000\u0000\u054b\u054c\u0006\u00ad\n\u0000"+ - "\u054c\u016a\u0001\u0000\u0000\u0000\u054d\u054e\u00039\u0015\u0000\u054e"+ - "\u054f\u0001\u0000\u0000\u0000\u054f\u0550\u0006\u00ae\n\u0000\u0550\u016c"+ - "\u0001\u0000\u0000\u0000\u0551\u0552\u0003;\u0016\u0000\u0552\u0553\u0001"+ - "\u0000\u0000\u0000\u0553\u0554\u0006\u00af\n\u0000\u0554\u016e\u0001\u0000"+ - "\u0000\u0000\u0555\u0556\u0003=\u0017\u0000\u0556\u0557\u0001\u0000\u0000"+ - "\u0000\u0557\u0558\u0006\u00b0\u000f\u0000\u0558\u0559\u0006\u00b0\u000b"+ - "\u0000\u0559\u055a\u0006\u00b0\u000b\u0000\u055a\u0170\u0001\u0000\u0000"+ - "\u0000\u055b\u055c\u0003c*\u0000\u055c\u055d\u0001\u0000\u0000\u0000\u055d"+ - "\u055e\u0006\u00b1\u0012\u0000\u055e\u0172\u0001\u0000\u0000\u0000\u055f"+ - "\u0560\u0003g,\u0000\u0560\u0561\u0001\u0000\u0000\u0000\u0561\u0562\u0006"+ - "\u00b2\u0016\u0000\u0562\u0174\u0001\u0000\u0000\u0000\u0563\u0564\u0003"+ - "\u00e9m\u0000\u0564\u0565\u0001\u0000\u0000\u0000\u0565\u0566\u0006\u00b3"+ - "\u0019\u0000\u0566\u0176\u0001\u0000\u0000\u0000\u0567\u0568\u00037\u0014"+ - "\u0000\u0568\u0569\u0001\u0000\u0000\u0000\u0569\u056a\u0006\u00b4\n\u0000"+ - "\u056a\u0178\u0001\u0000\u0000\u0000\u056b\u056c\u00039\u0015\u0000\u056c"+ - "\u056d\u0001\u0000\u0000\u0000\u056d\u056e\u0006\u00b5\n\u0000\u056e\u017a"+ - "\u0001\u0000\u0000\u0000\u056f\u0570\u0003;\u0016\u0000\u0570\u0571\u0001"+ - "\u0000\u0000\u0000\u0571\u0572\u0006\u00b6\n\u0000\u0572\u017c\u0001\u0000"+ - "\u0000\u0000\u0573\u0574\u0003=\u0017\u0000\u0574\u0575\u0001\u0000\u0000"+ - "\u0000\u0575\u0576\u0006\u00b7\u000f\u0000\u0576\u0577\u0006\u00b7\u000b"+ - "\u0000\u0577\u017e\u0001\u0000\u0000\u0000\u0578\u0579\u0003\u00cf`\u0000"+ - "\u0579\u057a\u0001\u0000\u0000\u0000\u057a\u057b\u0006\u00b8\u0014\u0000"+ - "\u057b\u057c\u0006\u00b8\u000b\u0000\u057c\u057d\u0006\u00b8\"\u0000\u057d"+ - "\u0180\u0001\u0000\u0000\u0000\u057e\u057f\u0003S\"\u0000\u057f\u0580"+ - "\u0001\u0000\u0000\u0000\u0580\u0581\u0006\u00b9\u0015\u0000\u0581\u0582"+ - "\u0006\u00b9\u000b\u0000\u0582\u0583\u0006\u00b9\"\u0000\u0583\u0182\u0001"+ - "\u0000\u0000\u0000\u0584\u0585\u00037\u0014\u0000\u0585\u0586\u0001\u0000"+ - "\u0000\u0000\u0586\u0587\u0006\u00ba\n\u0000\u0587\u0184\u0001\u0000\u0000"+ - "\u0000\u0588\u0589\u00039\u0015\u0000\u0589\u058a\u0001\u0000\u0000\u0000"+ - "\u058a\u058b\u0006\u00bb\n\u0000\u058b\u0186\u0001\u0000\u0000\u0000\u058c"+ - "\u058d\u0003;\u0016\u0000\u058d\u058e\u0001\u0000\u0000\u0000\u058e\u058f"+ - "\u0006\u00bc\n\u0000\u058f\u0188\u0001\u0000\u0000\u0000\u0590\u0591\u0003"+ - "\u0151\u00a1\u0000\u0591\u0592\u0001\u0000\u0000\u0000\u0592\u0593\u0006"+ - "\u00bd\u0011\u0000\u0593\u0594\u0006\u00bd\u000b\u0000\u0594\u0595\u0006"+ - "\u00bd\t\u0000\u0595\u018a\u0001\u0000\u0000\u0000\u0596\u0597\u0003c"+ - "*\u0000\u0597\u0598\u0001\u0000\u0000\u0000\u0598\u0599\u0006\u00be\u0012"+ - "\u0000\u0599\u059a\u0006\u00be\u000b\u0000\u059a\u059b\u0006\u00be\t\u0000"+ - "\u059b\u018c\u0001\u0000\u0000\u0000\u059c\u059d\u00037\u0014\u0000\u059d"+ - "\u059e\u0001\u0000\u0000\u0000\u059e\u059f\u0006\u00bf\n\u0000\u059f\u018e"+ - "\u0001\u0000\u0000\u0000\u05a0\u05a1\u00039\u0015\u0000\u05a1\u05a2\u0001"+ - "\u0000\u0000\u0000\u05a2\u05a3\u0006\u00c0\n\u0000\u05a3\u0190\u0001\u0000"+ - "\u0000\u0000\u05a4\u05a5\u0003;\u0016\u0000\u05a5\u05a6\u0001\u0000\u0000"+ - "\u0000\u05a6\u05a7\u0006\u00c1\n\u0000\u05a7\u0192\u0001\u0000\u0000\u0000"+ - "\u05a8\u05a9\u0003\u00adO\u0000\u05a9\u05aa\u0001\u0000\u0000\u0000\u05aa"+ - "\u05ab\u0006\u00c2\u000b\u0000\u05ab\u05ac\u0006\u00c2\u0000\u0000\u05ac"+ - "\u05ad\u0006\u00c2\u001e\u0000\u05ad\u0194\u0001\u0000\u0000\u0000\u05ae"+ - "\u05af\u0003\u00a9M\u0000\u05af\u05b0\u0001\u0000\u0000\u0000\u05b0\u05b1"+ - "\u0006\u00c3\u000b\u0000\u05b1\u05b2\u0006\u00c3\u0000\u0000\u05b2\u05b3"+ - "\u0006\u00c3\u001f\u0000\u05b3\u0196\u0001\u0000\u0000\u0000\u05b4\u05b5"+ - "\u0003Y%\u0000\u05b5\u05b6\u0001\u0000\u0000\u0000\u05b6\u05b7\u0006\u00c4"+ - "\u000b\u0000\u05b7\u05b8\u0006\u00c4\u0000\u0000\u05b8\u05b9\u0006\u00c4"+ - "#\u0000\u05b9\u0198\u0001\u0000\u0000\u0000\u05ba\u05bb\u0003=\u0017\u0000"+ - "\u05bb\u05bc\u0001\u0000\u0000\u0000\u05bc\u05bd\u0006\u00c5\u000f\u0000"+ - "\u05bd\u05be\u0006\u00c5\u000b\u0000\u05be\u019a\u0001\u0000\u0000\u0000"+ - "A\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+ - "\u0243\u024d\u0251\u0254\u025d\u025f\u026a\u027d\u0282\u028b\u0292\u0297"+ - "\u0299\u02a4\u02ac\u02af\u02b1\u02b6\u02bb\u02c1\u02c8\u02cd\u02d3\u02d6"+ - "\u02de\u02e2\u0366\u036b\u0372\u0374\u0384\u0389\u038e\u0390\u0396\u03e3"+ - "\u03e8\u0417\u041b\u0420\u0425\u042a\u042c\u0430\u0432\u0487\u048b\u0490"+ - "\u051b\u051d$\u0005\u0001\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005"+ - "\u0002\u0000\u0005\u0003\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t"+ - "\u0000\u0005\u000b\u0000\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000"+ - "\u0007\u0010\u0000\u0007A\u0000\u0005\u0000\u0000\u0007\u0018\u0000\u0007"+ - "B\u0000\u0007h\u0000\u0007!\u0000\u0007\u001f\u0000\u0007L\u0000\u0007"+ - "\u0019\u0000\u0007#\u0000\u0007/\u0000\u0007@\u0000\u0007P\u0000\u0005"+ - "\n\u0000\u0005\u0007\u0000\u0007Z\u0000\u0007Y\u0000\u0007D\u0000\u0007"+ - "C\u0000\u0007X\u0000\u0005\f\u0000\u0005\u000e\u0000\u0007\u001c\u0000"; + "h\u0012\u0000\u040a\u00e0\u0001\u0000\u0000\u0000\u040b\u040c\u0004i\u0003"+ + "\u0000\u040c\u040d\u0003\u007f8\u0000\u040d\u040e\u0001\u0000\u0000\u0000"+ + "\u040e\u040f\u0006i\u0017\u0000\u040f\u00e2\u0001\u0000\u0000\u0000\u0410"+ + "\u0411\u0004j\u0004\u0000\u0411\u0412\u0003\u00a3J\u0000\u0412\u0413\u0001"+ + "\u0000\u0000\u0000\u0413\u0414\u0006j\u0018\u0000\u0414\u00e4\u0001\u0000"+ + "\u0000\u0000\u0415\u041a\u0003A\u0019\u0000\u0416\u041a\u0003?\u0018\u0000"+ + "\u0417\u041a\u0003O \u0000\u0418\u041a\u0003\u0099E\u0000\u0419\u0415"+ + "\u0001\u0000\u0000\u0000\u0419\u0416\u0001\u0000\u0000\u0000\u0419\u0417"+ + "\u0001\u0000\u0000\u0000\u0419\u0418\u0001\u0000\u0000\u0000\u041a\u00e6"+ + "\u0001\u0000\u0000\u0000\u041b\u041e\u0003A\u0019\u0000\u041c\u041e\u0003"+ + "\u0099E\u0000\u041d\u041b\u0001\u0000\u0000\u0000\u041d\u041c\u0001\u0000"+ + "\u0000\u0000\u041e\u0422\u0001\u0000\u0000\u0000\u041f\u0421\u0003\u00e5"+ + "k\u0000\u0420\u041f\u0001\u0000\u0000\u0000\u0421\u0424\u0001\u0000\u0000"+ + "\u0000\u0422\u0420\u0001\u0000\u0000\u0000\u0422\u0423\u0001\u0000\u0000"+ + "\u0000\u0423\u042f\u0001\u0000\u0000\u0000\u0424\u0422\u0001\u0000\u0000"+ + "\u0000\u0425\u0428\u0003O \u0000\u0426\u0428\u0003I\u001d\u0000\u0427"+ + "\u0425\u0001\u0000\u0000\u0000\u0427\u0426\u0001\u0000\u0000\u0000\u0428"+ + "\u042a\u0001\u0000\u0000\u0000\u0429\u042b\u0003\u00e5k\u0000\u042a\u0429"+ + "\u0001\u0000\u0000\u0000\u042b\u042c\u0001\u0000\u0000\u0000\u042c\u042a"+ + "\u0001\u0000\u0000\u0000\u042c\u042d\u0001\u0000\u0000\u0000\u042d\u042f"+ + "\u0001\u0000\u0000\u0000\u042e\u041d\u0001\u0000\u0000\u0000\u042e\u0427"+ + "\u0001\u0000\u0000\u0000\u042f\u00e8\u0001\u0000\u0000\u0000\u0430\u0433"+ + "\u0003\u00e7l\u0000\u0431\u0433\u0003\u00abN\u0000\u0432\u0430\u0001\u0000"+ + "\u0000\u0000\u0432\u0431\u0001\u0000\u0000\u0000\u0433\u0434\u0001\u0000"+ + "\u0000\u0000\u0434\u0432\u0001\u0000\u0000\u0000\u0434\u0435\u0001\u0000"+ + "\u0000\u0000\u0435\u00ea\u0001\u0000\u0000\u0000\u0436\u0437\u00037\u0014"+ + "\u0000\u0437\u0438\u0001\u0000\u0000\u0000\u0438\u0439\u0006n\n\u0000"+ + "\u0439\u00ec\u0001\u0000\u0000\u0000\u043a\u043b\u00039\u0015\u0000\u043b"+ + "\u043c\u0001\u0000\u0000\u0000\u043c\u043d\u0006o\n\u0000\u043d\u00ee"+ + "\u0001\u0000\u0000\u0000\u043e\u043f\u0003;\u0016\u0000\u043f\u0440\u0001"+ + "\u0000\u0000\u0000\u0440\u0441\u0006p\n\u0000\u0441\u00f0\u0001\u0000"+ + "\u0000\u0000\u0442\u0443\u0003=\u0017\u0000\u0443\u0444\u0001\u0000\u0000"+ + "\u0000\u0444\u0445\u0006q\u000f\u0000\u0445\u0446\u0006q\u000b\u0000\u0446"+ + "\u00f2\u0001\u0000\u0000\u0000\u0447\u0448\u0003_(\u0000\u0448\u0449\u0001"+ + "\u0000\u0000\u0000\u0449\u044a\u0006r\u0013\u0000\u044a\u00f4\u0001\u0000"+ + "\u0000\u0000\u044b\u044c\u0003c*\u0000\u044c\u044d\u0001\u0000\u0000\u0000"+ + "\u044d\u044e\u0006s\u0012\u0000\u044e\u00f6\u0001\u0000\u0000\u0000\u044f"+ + "\u0450\u0003g,\u0000\u0450\u0451\u0001\u0000\u0000\u0000\u0451\u0452\u0006"+ + "t\u0016\u0000\u0452\u00f8\u0001\u0000\u0000\u0000\u0453\u0454\u0004u\u0005"+ + "\u0000\u0454\u0455\u0003\u007f8\u0000\u0455\u0456\u0001\u0000\u0000\u0000"+ + "\u0456\u0457\u0006u\u0017\u0000\u0457\u00fa\u0001\u0000\u0000\u0000\u0458"+ + "\u0459\u0004v\u0006\u0000\u0459\u045a\u0003\u00a3J\u0000\u045a\u045b\u0001"+ + "\u0000\u0000\u0000\u045b\u045c\u0006v\u0018\u0000\u045c\u00fc\u0001\u0000"+ + "\u0000\u0000\u045d\u045e\u0007\f\u0000\u0000\u045e\u045f\u0007\u0002\u0000"+ + "\u0000\u045f\u00fe\u0001\u0000\u0000\u0000\u0460\u0461\u0003\u00e9m\u0000"+ + "\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463\u0006x\u0019\u0000\u0463"+ + "\u0100\u0001\u0000\u0000\u0000\u0464\u0465\u00037\u0014\u0000\u0465\u0466"+ + "\u0001\u0000\u0000\u0000\u0466\u0467\u0006y\n\u0000\u0467\u0102\u0001"+ + "\u0000\u0000\u0000\u0468\u0469\u00039\u0015\u0000\u0469\u046a\u0001\u0000"+ + "\u0000\u0000\u046a\u046b\u0006z\n\u0000\u046b\u0104\u0001\u0000\u0000"+ + "\u0000\u046c\u046d\u0003;\u0016\u0000\u046d\u046e\u0001\u0000\u0000\u0000"+ + "\u046e\u046f\u0006{\n\u0000\u046f\u0106\u0001\u0000\u0000\u0000\u0470"+ + "\u0471\u0003=\u0017\u0000\u0471\u0472\u0001\u0000\u0000\u0000\u0472\u0473"+ + "\u0006|\u000f\u0000\u0473\u0474\u0006|\u000b\u0000\u0474\u0108\u0001\u0000"+ + "\u0000\u0000\u0475\u0476\u0003\u00a5K\u0000\u0476\u0477\u0001\u0000\u0000"+ + "\u0000\u0477\u0478\u0006}\r\u0000\u0478\u0479\u0006}\u001a\u0000\u0479"+ + "\u010a\u0001\u0000\u0000\u0000\u047a\u047b\u0007\u0007\u0000\u0000\u047b"+ + "\u047c\u0007\t\u0000\u0000\u047c\u047d\u0001\u0000\u0000\u0000\u047d\u047e"+ + "\u0006~\u001b\u0000\u047e\u010c\u0001\u0000\u0000\u0000\u047f\u0480\u0007"+ + "\u0013\u0000\u0000\u0480\u0481\u0007\u0001\u0000\u0000\u0481\u0482\u0007"+ + "\u0005\u0000\u0000\u0482\u0483\u0007\n\u0000\u0000\u0483\u0484\u0001\u0000"+ + "\u0000\u0000\u0484\u0485\u0006\u007f\u001b\u0000\u0485\u010e\u0001\u0000"+ + "\u0000\u0000\u0486\u0487\b\"\u0000\u0000\u0487\u0110\u0001\u0000\u0000"+ + "\u0000\u0488\u048a\u0003\u010f\u0080\u0000\u0489\u0488\u0001\u0000\u0000"+ + "\u0000\u048a\u048b\u0001\u0000\u0000\u0000\u048b\u0489\u0001\u0000\u0000"+ + "\u0000\u048b\u048c\u0001\u0000\u0000\u0000\u048c\u048d\u0001\u0000\u0000"+ + "\u0000\u048d\u048e\u0003\u0151\u00a1\u0000\u048e\u0490\u0001\u0000\u0000"+ + "\u0000\u048f\u0489\u0001\u0000\u0000\u0000\u048f\u0490\u0001\u0000\u0000"+ + "\u0000\u0490\u0492\u0001\u0000\u0000\u0000\u0491\u0493\u0003\u010f\u0080"+ + "\u0000\u0492\u0491\u0001\u0000\u0000\u0000\u0493\u0494\u0001\u0000\u0000"+ + "\u0000\u0494\u0492\u0001\u0000\u0000\u0000\u0494\u0495\u0001\u0000\u0000"+ + "\u0000\u0495\u0112\u0001\u0000\u0000\u0000\u0496\u0497\u0003\u0111\u0081"+ + "\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498\u0499\u0006\u0082\u001c"+ + "\u0000\u0499\u0114\u0001\u0000\u0000\u0000\u049a\u049b\u00037\u0014\u0000"+ + "\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d\u0006\u0083\n\u0000\u049d"+ + "\u0116\u0001\u0000\u0000\u0000\u049e\u049f\u00039\u0015\u0000\u049f\u04a0"+ + "\u0001\u0000\u0000\u0000\u04a0\u04a1\u0006\u0084\n\u0000\u04a1\u0118\u0001"+ + "\u0000\u0000\u0000\u04a2\u04a3\u0003;\u0016\u0000\u04a3\u04a4\u0001\u0000"+ + "\u0000\u0000\u04a4\u04a5\u0006\u0085\n\u0000\u04a5\u011a\u0001\u0000\u0000"+ + "\u0000\u04a6\u04a7\u0003=\u0017\u0000\u04a7\u04a8\u0001\u0000\u0000\u0000"+ + "\u04a8\u04a9\u0006\u0086\u000f\u0000\u04a9\u04aa\u0006\u0086\u000b\u0000"+ + "\u04aa\u04ab\u0006\u0086\u000b\u0000\u04ab\u011c\u0001\u0000\u0000\u0000"+ + "\u04ac\u04ad\u0003_(\u0000\u04ad\u04ae\u0001\u0000\u0000\u0000\u04ae\u04af"+ + "\u0006\u0087\u0013\u0000\u04af\u011e\u0001\u0000\u0000\u0000\u04b0\u04b1"+ + "\u0003c*\u0000\u04b1\u04b2\u0001\u0000\u0000\u0000\u04b2\u04b3\u0006\u0088"+ + "\u0012\u0000\u04b3\u0120\u0001\u0000\u0000\u0000\u04b4\u04b5\u0003g,\u0000"+ + "\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6\u04b7\u0006\u0089\u0016\u0000"+ + "\u04b7\u0122\u0001\u0000\u0000\u0000\u04b8\u04b9\u0003\u010d\u007f\u0000"+ + "\u04b9\u04ba\u0001\u0000\u0000\u0000\u04ba\u04bb\u0006\u008a\u001d\u0000"+ + "\u04bb\u0124\u0001\u0000\u0000\u0000\u04bc\u04bd\u0003\u00e9m\u0000\u04bd"+ + "\u04be\u0001\u0000\u0000\u0000\u04be\u04bf\u0006\u008b\u0019\u0000\u04bf"+ + "\u0126\u0001\u0000\u0000\u0000\u04c0\u04c1\u0003\u00adO\u0000\u04c1\u04c2"+ + "\u0001\u0000\u0000\u0000\u04c2\u04c3\u0006\u008c\u001e\u0000\u04c3\u0128"+ + "\u0001\u0000\u0000\u0000\u04c4\u04c5\u0004\u008d\u0007\u0000\u04c5\u04c6"+ + "\u0003\u007f8\u0000\u04c6\u04c7\u0001\u0000\u0000\u0000\u04c7\u04c8\u0006"+ + "\u008d\u0017\u0000\u04c8\u012a\u0001\u0000\u0000\u0000\u04c9\u04ca\u0004"+ + "\u008e\b\u0000\u04ca\u04cb\u0003\u00a3J\u0000\u04cb\u04cc\u0001\u0000"+ + "\u0000\u0000\u04cc\u04cd\u0006\u008e\u0018\u0000\u04cd\u012c\u0001\u0000"+ + "\u0000\u0000\u04ce\u04cf\u00037\u0014\u0000\u04cf\u04d0\u0001\u0000\u0000"+ + "\u0000\u04d0\u04d1\u0006\u008f\n\u0000\u04d1\u012e\u0001\u0000\u0000\u0000"+ + "\u04d2\u04d3\u00039\u0015\u0000\u04d3\u04d4\u0001\u0000\u0000\u0000\u04d4"+ + "\u04d5\u0006\u0090\n\u0000\u04d5\u0130\u0001\u0000\u0000\u0000\u04d6\u04d7"+ + "\u0003;\u0016\u0000\u04d7\u04d8\u0001\u0000\u0000\u0000\u04d8\u04d9\u0006"+ + "\u0091\n\u0000\u04d9\u0132\u0001\u0000\u0000\u0000\u04da\u04db\u0003="+ + "\u0017\u0000\u04db\u04dc\u0001\u0000\u0000\u0000\u04dc\u04dd\u0006\u0092"+ + "\u000f\u0000\u04dd\u04de\u0006\u0092\u000b\u0000\u04de\u0134\u0001\u0000"+ + "\u0000\u0000\u04df\u04e0\u0003g,\u0000\u04e0\u04e1\u0001\u0000\u0000\u0000"+ + "\u04e1\u04e2\u0006\u0093\u0016\u0000\u04e2\u0136\u0001\u0000\u0000\u0000"+ + "\u04e3\u04e4\u0004\u0094\t\u0000\u04e4\u04e5\u0003\u007f8\u0000\u04e5"+ + "\u04e6\u0001\u0000\u0000\u0000\u04e6\u04e7\u0006\u0094\u0017\u0000\u04e7"+ + "\u0138\u0001\u0000\u0000\u0000\u04e8\u04e9\u0004\u0095\n\u0000\u04e9\u04ea"+ + "\u0003\u00a3J\u0000\u04ea\u04eb\u0001\u0000\u0000\u0000\u04eb\u04ec\u0006"+ + "\u0095\u0018\u0000\u04ec\u013a\u0001\u0000\u0000\u0000\u04ed\u04ee\u0003"+ + "\u00adO\u0000\u04ee\u04ef\u0001\u0000\u0000\u0000\u04ef\u04f0\u0006\u0096"+ + "\u001e\u0000\u04f0\u013c\u0001\u0000\u0000\u0000\u04f1\u04f2\u0003\u00a9"+ + "M\u0000\u04f2\u04f3\u0001\u0000\u0000\u0000\u04f3\u04f4\u0006\u0097\u001f"+ + "\u0000\u04f4\u013e\u0001\u0000\u0000\u0000\u04f5\u04f6\u00037\u0014\u0000"+ + "\u04f6\u04f7\u0001\u0000\u0000\u0000\u04f7\u04f8\u0006\u0098\n\u0000\u04f8"+ + "\u0140\u0001\u0000\u0000\u0000\u04f9\u04fa\u00039\u0015\u0000\u04fa\u04fb"+ + "\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006\u0099\n\u0000\u04fc\u0142\u0001"+ + "\u0000\u0000\u0000\u04fd\u04fe\u0003;\u0016\u0000\u04fe\u04ff\u0001\u0000"+ + "\u0000\u0000\u04ff\u0500\u0006\u009a\n\u0000\u0500\u0144\u0001\u0000\u0000"+ + "\u0000\u0501\u0502\u0003=\u0017\u0000\u0502\u0503\u0001\u0000\u0000\u0000"+ + "\u0503\u0504\u0006\u009b\u000f\u0000\u0504\u0505\u0006\u009b\u000b\u0000"+ + "\u0505\u0146\u0001\u0000\u0000\u0000\u0506\u0507\u0007\u0001\u0000\u0000"+ + "\u0507\u0508\u0007\t\u0000\u0000\u0508\u0509\u0007\u000f\u0000\u0000\u0509"+ + "\u050a\u0007\u0007\u0000\u0000\u050a\u0148\u0001\u0000\u0000\u0000\u050b"+ + "\u050c\u00037\u0014\u0000\u050c\u050d\u0001\u0000\u0000\u0000\u050d\u050e"+ + "\u0006\u009d\n\u0000\u050e\u014a\u0001\u0000\u0000\u0000\u050f\u0510\u0003"+ + "9\u0015\u0000\u0510\u0511\u0001\u0000\u0000\u0000\u0511\u0512\u0006\u009e"+ + "\n\u0000\u0512\u014c\u0001\u0000\u0000\u0000\u0513\u0514\u0003;\u0016"+ + "\u0000\u0514\u0515\u0001\u0000\u0000\u0000\u0515\u0516\u0006\u009f\n\u0000"+ + "\u0516\u014e\u0001\u0000\u0000\u0000\u0517\u0518\u0003\u00a7L\u0000\u0518"+ + "\u0519\u0001\u0000\u0000\u0000\u0519\u051a\u0006\u00a0\u0010\u0000\u051a"+ + "\u051b\u0006\u00a0\u000b\u0000\u051b\u0150\u0001\u0000\u0000\u0000\u051c"+ + "\u051d\u0005:\u0000\u0000\u051d\u0152\u0001\u0000\u0000\u0000\u051e\u0524"+ + "\u0003I\u001d\u0000\u051f\u0524\u0003?\u0018\u0000\u0520\u0524\u0003g"+ + ",\u0000\u0521\u0524\u0003A\u0019\u0000\u0522\u0524\u0003O \u0000\u0523"+ + "\u051e\u0001\u0000\u0000\u0000\u0523\u051f\u0001\u0000\u0000\u0000\u0523"+ + "\u0520\u0001\u0000\u0000\u0000\u0523\u0521\u0001\u0000\u0000\u0000\u0523"+ + "\u0522\u0001\u0000\u0000\u0000\u0524\u0525\u0001\u0000\u0000\u0000\u0525"+ + "\u0523\u0001\u0000\u0000\u0000\u0525\u0526\u0001\u0000\u0000\u0000\u0526"+ + "\u0154\u0001\u0000\u0000\u0000\u0527\u0528\u00037\u0014\u0000\u0528\u0529"+ + "\u0001\u0000\u0000\u0000\u0529\u052a\u0006\u00a3\n\u0000\u052a\u0156\u0001"+ + "\u0000\u0000\u0000\u052b\u052c\u00039\u0015\u0000\u052c\u052d\u0001\u0000"+ + "\u0000\u0000\u052d\u052e\u0006\u00a4\n\u0000\u052e\u0158\u0001\u0000\u0000"+ + "\u0000\u052f\u0530\u0003;\u0016\u0000\u0530\u0531\u0001\u0000\u0000\u0000"+ + "\u0531\u0532\u0006\u00a5\n\u0000\u0532\u015a\u0001\u0000\u0000\u0000\u0533"+ + "\u0534\u0003=\u0017\u0000\u0534\u0535\u0001\u0000\u0000\u0000\u0535\u0536"+ + "\u0006\u00a6\u000f\u0000\u0536\u0537\u0006\u00a6\u000b\u0000\u0537\u015c"+ + "\u0001\u0000\u0000\u0000\u0538\u0539\u0003\u0151\u00a1\u0000\u0539\u053a"+ + "\u0001\u0000\u0000\u0000\u053a\u053b\u0006\u00a7\u0011\u0000\u053b\u015e"+ + "\u0001\u0000\u0000\u0000\u053c\u053d\u0003c*\u0000\u053d\u053e\u0001\u0000"+ + "\u0000\u0000\u053e\u053f\u0006\u00a8\u0012\u0000\u053f\u0160\u0001\u0000"+ + "\u0000\u0000\u0540\u0541\u0003g,\u0000\u0541\u0542\u0001\u0000\u0000\u0000"+ + "\u0542\u0543\u0006\u00a9\u0016\u0000\u0543\u0162\u0001\u0000\u0000\u0000"+ + "\u0544\u0545\u0003\u010b~\u0000\u0545\u0546\u0001\u0000\u0000\u0000\u0546"+ + "\u0547\u0006\u00aa \u0000\u0547\u0548\u0006\u00aa!\u0000\u0548\u0164\u0001"+ + "\u0000\u0000\u0000\u0549\u054a\u0003\u00cf`\u0000\u054a\u054b\u0001\u0000"+ + "\u0000\u0000\u054b\u054c\u0006\u00ab\u0014\u0000\u054c\u0166\u0001\u0000"+ + "\u0000\u0000\u054d\u054e\u0003S\"\u0000\u054e\u054f\u0001\u0000\u0000"+ + "\u0000\u054f\u0550\u0006\u00ac\u0015\u0000\u0550\u0168\u0001\u0000\u0000"+ + "\u0000\u0551\u0552\u00037\u0014\u0000\u0552\u0553\u0001\u0000\u0000\u0000"+ + "\u0553\u0554\u0006\u00ad\n\u0000\u0554\u016a\u0001\u0000\u0000\u0000\u0555"+ + "\u0556\u00039\u0015\u0000\u0556\u0557\u0001\u0000\u0000\u0000\u0557\u0558"+ + "\u0006\u00ae\n\u0000\u0558\u016c\u0001\u0000\u0000\u0000\u0559\u055a\u0003"+ + ";\u0016\u0000\u055a\u055b\u0001\u0000\u0000\u0000\u055b\u055c\u0006\u00af"+ + "\n\u0000\u055c\u016e\u0001\u0000\u0000\u0000\u055d\u055e\u0003=\u0017"+ + "\u0000\u055e\u055f\u0001\u0000\u0000\u0000\u055f\u0560\u0006\u00b0\u000f"+ + "\u0000\u0560\u0561\u0006\u00b0\u000b\u0000\u0561\u0562\u0006\u00b0\u000b"+ + "\u0000\u0562\u0170\u0001\u0000\u0000\u0000\u0563\u0564\u0003c*\u0000\u0564"+ + "\u0565\u0001\u0000\u0000\u0000\u0565\u0566\u0006\u00b1\u0012\u0000\u0566"+ + "\u0172\u0001\u0000\u0000\u0000\u0567\u0568\u0003g,\u0000\u0568\u0569\u0001"+ + "\u0000\u0000\u0000\u0569\u056a\u0006\u00b2\u0016\u0000\u056a\u0174\u0001"+ + "\u0000\u0000\u0000\u056b\u056c\u0003\u00e9m\u0000\u056c\u056d\u0001\u0000"+ + "\u0000\u0000\u056d\u056e\u0006\u00b3\u0019\u0000\u056e\u0176\u0001\u0000"+ + "\u0000\u0000\u056f\u0570\u00037\u0014\u0000\u0570\u0571\u0001\u0000\u0000"+ + "\u0000\u0571\u0572\u0006\u00b4\n\u0000\u0572\u0178\u0001\u0000\u0000\u0000"+ + "\u0573\u0574\u00039\u0015\u0000\u0574\u0575\u0001\u0000\u0000\u0000\u0575"+ + "\u0576\u0006\u00b5\n\u0000\u0576\u017a\u0001\u0000\u0000\u0000\u0577\u0578"+ + "\u0003;\u0016\u0000\u0578\u0579\u0001\u0000\u0000\u0000\u0579\u057a\u0006"+ + "\u00b6\n\u0000\u057a\u017c\u0001\u0000\u0000\u0000\u057b\u057c\u0003="+ + "\u0017\u0000\u057c\u057d\u0001\u0000\u0000\u0000\u057d\u057e\u0006\u00b7"+ + "\u000f\u0000\u057e\u057f\u0006\u00b7\u000b\u0000\u057f\u017e\u0001\u0000"+ + "\u0000\u0000\u0580\u0581\u0003\u00cf`\u0000\u0581\u0582\u0001\u0000\u0000"+ + "\u0000\u0582\u0583\u0006\u00b8\u0014\u0000\u0583\u0584\u0006\u00b8\u000b"+ + "\u0000\u0584\u0585\u0006\u00b8\"\u0000\u0585\u0180\u0001\u0000\u0000\u0000"+ + "\u0586\u0587\u0003S\"\u0000\u0587\u0588\u0001\u0000\u0000\u0000\u0588"+ + "\u0589\u0006\u00b9\u0015\u0000\u0589\u058a\u0006\u00b9\u000b\u0000\u058a"+ + "\u058b\u0006\u00b9\"\u0000\u058b\u0182\u0001\u0000\u0000\u0000\u058c\u058d"+ + "\u00037\u0014\u0000\u058d\u058e\u0001\u0000\u0000\u0000\u058e\u058f\u0006"+ + "\u00ba\n\u0000\u058f\u0184\u0001\u0000\u0000\u0000\u0590\u0591\u00039"+ + "\u0015\u0000\u0591\u0592\u0001\u0000\u0000\u0000\u0592\u0593\u0006\u00bb"+ + "\n\u0000\u0593\u0186\u0001\u0000\u0000\u0000\u0594\u0595\u0003;\u0016"+ + "\u0000\u0595\u0596\u0001\u0000\u0000\u0000\u0596\u0597\u0006\u00bc\n\u0000"+ + "\u0597\u0188\u0001\u0000\u0000\u0000\u0598\u0599\u0003\u0151\u00a1\u0000"+ + "\u0599\u059a\u0001\u0000\u0000\u0000\u059a\u059b\u0006\u00bd\u0011\u0000"+ + "\u059b\u059c\u0006\u00bd\u000b\u0000\u059c\u059d\u0006\u00bd\t\u0000\u059d"+ + "\u018a\u0001\u0000\u0000\u0000\u059e\u059f\u0003c*\u0000\u059f\u05a0\u0001"+ + "\u0000\u0000\u0000\u05a0\u05a1\u0006\u00be\u0012\u0000\u05a1\u05a2\u0006"+ + "\u00be\u000b\u0000\u05a2\u05a3\u0006\u00be\t\u0000\u05a3\u018c\u0001\u0000"+ + "\u0000\u0000\u05a4\u05a5\u00037\u0014\u0000\u05a5\u05a6\u0001\u0000\u0000"+ + "\u0000\u05a6\u05a7\u0006\u00bf\n\u0000\u05a7\u018e\u0001\u0000\u0000\u0000"+ + "\u05a8\u05a9\u00039\u0015\u0000\u05a9\u05aa\u0001\u0000\u0000\u0000\u05aa"+ + "\u05ab\u0006\u00c0\n\u0000\u05ab\u0190\u0001\u0000\u0000\u0000\u05ac\u05ad"+ + "\u0003;\u0016\u0000\u05ad\u05ae\u0001\u0000\u0000\u0000\u05ae\u05af\u0006"+ + "\u00c1\n\u0000\u05af\u0192\u0001\u0000\u0000\u0000\u05b0\u05b1\u0003\u00ad"+ + "O\u0000\u05b1\u05b2\u0001\u0000\u0000\u0000\u05b2\u05b3\u0006\u00c2\u000b"+ + "\u0000\u05b3\u05b4\u0006\u00c2\u0000\u0000\u05b4\u05b5\u0006\u00c2\u001e"+ + "\u0000\u05b5\u0194\u0001\u0000\u0000\u0000\u05b6\u05b7\u0003\u00a9M\u0000"+ + "\u05b7\u05b8\u0001\u0000\u0000\u0000\u05b8\u05b9\u0006\u00c3\u000b\u0000"+ + "\u05b9\u05ba\u0006\u00c3\u0000\u0000\u05ba\u05bb\u0006\u00c3\u001f\u0000"+ + "\u05bb\u0196\u0001\u0000\u0000\u0000\u05bc\u05bd\u0003Y%\u0000\u05bd\u05be"+ + "\u0001\u0000\u0000\u0000\u05be\u05bf\u0006\u00c4\u000b\u0000\u05bf\u05c0"+ + "\u0006\u00c4\u0000\u0000\u05c0\u05c1\u0006\u00c4#\u0000\u05c1\u0198\u0001"+ + "\u0000\u0000\u0000\u05c2\u05c3\u0003=\u0017\u0000\u05c3\u05c4\u0001\u0000"+ + "\u0000\u0000\u05c4\u05c5\u0006\u00c5\u000f\u0000\u05c5\u05c6\u0006\u00c5"+ + "\u000b\u0000\u05c6\u019a\u0001\u0000\u0000\u0000A\u0000\u0001\u0002\u0003"+ + "\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u0243\u024d\u0251\u0254"+ + "\u025d\u025f\u026a\u027d\u0282\u028b\u0292\u0297\u0299\u02a4\u02ac\u02af"+ + "\u02b1\u02b6\u02bb\u02c1\u02c8\u02cd\u02d3\u02d6\u02de\u02e2\u0366\u036b"+ + "\u0372\u0374\u0384\u0389\u038e\u0390\u0396\u03e3\u03e8\u0419\u041d\u0422"+ + "\u0427\u042c\u042e\u0432\u0434\u048b\u048f\u0494\u0523\u0525$\u0005\u0001"+ + "\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005\u0003"+ + "\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0005\u000b\u0000"+ + "\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007\u0010\u0000\u0007"+ + "A\u0000\u0005\u0000\u0000\u0007\u0018\u0000\u0007B\u0000\u0007h\u0000"+ + "\u0007!\u0000\u0007\u001f\u0000\u0007L\u0000\u0007\u0019\u0000\u0007#"+ + "\u0000\u0007/\u0000\u0007@\u0000\u0007P\u0000\u0005\n\u0000\u0005\u0007"+ + "\u0000\u0007Z\u0000\u0007Y\u0000\u0007D\u0000\u0007C\u0000\u0007X\u0000"+ + "\u0005\f\u0000\u0005\u000e\u0000\u0007\u001c\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index 22d8ec32f3d92..b52d842e79fb2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -310,4 +310,4 @@ inlinestatsCommand atn: -[4, 1, 120, 603, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 134, 8, 1, 10, 1, 12, 1, 137, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 145, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 163, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 175, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 182, 8, 5, 10, 5, 12, 5, 185, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 192, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 198, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 206, 8, 5, 10, 5, 12, 5, 209, 9, 5, 1, 6, 1, 6, 3, 6, 213, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 220, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 236, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 242, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 250, 8, 9, 10, 9, 12, 9, 253, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 263, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 268, 8, 10, 10, 10, 12, 10, 271, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 279, 8, 11, 10, 11, 12, 11, 282, 9, 11, 3, 11, 284, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 3, 12, 290, 8, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 300, 8, 15, 10, 15, 12, 15, 303, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 308, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 316, 8, 17, 10, 17, 12, 17, 319, 9, 17, 1, 17, 3, 17, 322, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 327, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 337, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 343, 8, 22, 10, 22, 12, 22, 346, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 356, 8, 24, 10, 24, 12, 24, 359, 9, 24, 1, 24, 3, 24, 362, 8, 24, 1, 24, 1, 24, 3, 24, 366, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 373, 8, 26, 1, 26, 1, 26, 3, 26, 377, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 382, 8, 27, 10, 27, 12, 27, 385, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 390, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 395, 8, 29, 10, 29, 12, 29, 398, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 403, 8, 30, 10, 30, 12, 30, 406, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 411, 8, 31, 10, 31, 12, 31, 414, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 3, 33, 420, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 435, 8, 34, 10, 34, 12, 34, 438, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 446, 8, 34, 10, 34, 12, 34, 449, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 457, 8, 34, 10, 34, 12, 34, 460, 9, 34, 1, 34, 1, 34, 3, 34, 464, 8, 34, 1, 35, 1, 35, 3, 35, 468, 8, 35, 1, 36, 1, 36, 3, 36, 472, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 481, 8, 38, 10, 38, 12, 38, 484, 9, 38, 1, 39, 1, 39, 3, 39, 488, 8, 39, 1, 39, 1, 39, 3, 39, 492, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 504, 8, 42, 10, 42, 12, 42, 507, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 517, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 529, 8, 47, 10, 47, 12, 47, 532, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 542, 8, 50, 1, 51, 3, 51, 545, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 550, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 572, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 578, 8, 58, 10, 58, 12, 58, 581, 9, 58, 3, 58, 583, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 588, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 601, 8, 61, 1, 61, 0, 4, 2, 10, 18, 20, 62, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 0, 8, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 25, 25, 76, 76, 1, 0, 67, 68, 2, 0, 30, 30, 34, 34, 2, 0, 37, 37, 40, 40, 2, 0, 36, 36, 50, 50, 2, 0, 51, 51, 53, 57, 629, 0, 124, 1, 0, 0, 0, 2, 127, 1, 0, 0, 0, 4, 144, 1, 0, 0, 0, 6, 162, 1, 0, 0, 0, 8, 164, 1, 0, 0, 0, 10, 197, 1, 0, 0, 0, 12, 224, 1, 0, 0, 0, 14, 226, 1, 0, 0, 0, 16, 235, 1, 0, 0, 0, 18, 241, 1, 0, 0, 0, 20, 262, 1, 0, 0, 0, 22, 272, 1, 0, 0, 0, 24, 289, 1, 0, 0, 0, 26, 291, 1, 0, 0, 0, 28, 293, 1, 0, 0, 0, 30, 296, 1, 0, 0, 0, 32, 307, 1, 0, 0, 0, 34, 311, 1, 0, 0, 0, 36, 326, 1, 0, 0, 0, 38, 330, 1, 0, 0, 0, 40, 332, 1, 0, 0, 0, 42, 336, 1, 0, 0, 0, 44, 338, 1, 0, 0, 0, 46, 347, 1, 0, 0, 0, 48, 351, 1, 0, 0, 0, 50, 367, 1, 0, 0, 0, 52, 370, 1, 0, 0, 0, 54, 378, 1, 0, 0, 0, 56, 386, 1, 0, 0, 0, 58, 391, 1, 0, 0, 0, 60, 399, 1, 0, 0, 0, 62, 407, 1, 0, 0, 0, 64, 415, 1, 0, 0, 0, 66, 419, 1, 0, 0, 0, 68, 463, 1, 0, 0, 0, 70, 467, 1, 0, 0, 0, 72, 471, 1, 0, 0, 0, 74, 473, 1, 0, 0, 0, 76, 476, 1, 0, 0, 0, 78, 485, 1, 0, 0, 0, 80, 493, 1, 0, 0, 0, 82, 496, 1, 0, 0, 0, 84, 499, 1, 0, 0, 0, 86, 508, 1, 0, 0, 0, 88, 512, 1, 0, 0, 0, 90, 518, 1, 0, 0, 0, 92, 522, 1, 0, 0, 0, 94, 525, 1, 0, 0, 0, 96, 533, 1, 0, 0, 0, 98, 537, 1, 0, 0, 0, 100, 541, 1, 0, 0, 0, 102, 544, 1, 0, 0, 0, 104, 549, 1, 0, 0, 0, 106, 553, 1, 0, 0, 0, 108, 555, 1, 0, 0, 0, 110, 557, 1, 0, 0, 0, 112, 560, 1, 0, 0, 0, 114, 564, 1, 0, 0, 0, 116, 567, 1, 0, 0, 0, 118, 587, 1, 0, 0, 0, 120, 591, 1, 0, 0, 0, 122, 596, 1, 0, 0, 0, 124, 125, 3, 2, 1, 0, 125, 126, 5, 0, 0, 1, 126, 1, 1, 0, 0, 0, 127, 128, 6, 1, -1, 0, 128, 129, 3, 4, 2, 0, 129, 135, 1, 0, 0, 0, 130, 131, 10, 1, 0, 0, 131, 132, 5, 24, 0, 0, 132, 134, 3, 6, 3, 0, 133, 130, 1, 0, 0, 0, 134, 137, 1, 0, 0, 0, 135, 133, 1, 0, 0, 0, 135, 136, 1, 0, 0, 0, 136, 3, 1, 0, 0, 0, 137, 135, 1, 0, 0, 0, 138, 145, 3, 110, 55, 0, 139, 145, 3, 34, 17, 0, 140, 145, 3, 28, 14, 0, 141, 145, 3, 114, 57, 0, 142, 143, 4, 2, 1, 0, 143, 145, 3, 48, 24, 0, 144, 138, 1, 0, 0, 0, 144, 139, 1, 0, 0, 0, 144, 140, 1, 0, 0, 0, 144, 141, 1, 0, 0, 0, 144, 142, 1, 0, 0, 0, 145, 5, 1, 0, 0, 0, 146, 163, 3, 50, 25, 0, 147, 163, 3, 8, 4, 0, 148, 163, 3, 80, 40, 0, 149, 163, 3, 74, 37, 0, 150, 163, 3, 52, 26, 0, 151, 163, 3, 76, 38, 0, 152, 163, 3, 82, 41, 0, 153, 163, 3, 84, 42, 0, 154, 163, 3, 88, 44, 0, 155, 163, 3, 90, 45, 0, 156, 163, 3, 116, 58, 0, 157, 163, 3, 92, 46, 0, 158, 159, 4, 3, 2, 0, 159, 163, 3, 122, 61, 0, 160, 161, 4, 3, 3, 0, 161, 163, 3, 120, 60, 0, 162, 146, 1, 0, 0, 0, 162, 147, 1, 0, 0, 0, 162, 148, 1, 0, 0, 0, 162, 149, 1, 0, 0, 0, 162, 150, 1, 0, 0, 0, 162, 151, 1, 0, 0, 0, 162, 152, 1, 0, 0, 0, 162, 153, 1, 0, 0, 0, 162, 154, 1, 0, 0, 0, 162, 155, 1, 0, 0, 0, 162, 156, 1, 0, 0, 0, 162, 157, 1, 0, 0, 0, 162, 158, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 163, 7, 1, 0, 0, 0, 164, 165, 5, 16, 0, 0, 165, 166, 3, 10, 5, 0, 166, 9, 1, 0, 0, 0, 167, 168, 6, 5, -1, 0, 168, 169, 5, 43, 0, 0, 169, 198, 3, 10, 5, 8, 170, 198, 3, 16, 8, 0, 171, 198, 3, 12, 6, 0, 172, 174, 3, 16, 8, 0, 173, 175, 5, 43, 0, 0, 174, 173, 1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 177, 5, 38, 0, 0, 177, 178, 5, 42, 0, 0, 178, 183, 3, 16, 8, 0, 179, 180, 5, 33, 0, 0, 180, 182, 3, 16, 8, 0, 181, 179, 1, 0, 0, 0, 182, 185, 1, 0, 0, 0, 183, 181, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 186, 1, 0, 0, 0, 185, 183, 1, 0, 0, 0, 186, 187, 5, 49, 0, 0, 187, 198, 1, 0, 0, 0, 188, 189, 3, 16, 8, 0, 189, 191, 5, 39, 0, 0, 190, 192, 5, 43, 0, 0, 191, 190, 1, 0, 0, 0, 191, 192, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193, 194, 5, 44, 0, 0, 194, 198, 1, 0, 0, 0, 195, 196, 4, 5, 4, 0, 196, 198, 3, 14, 7, 0, 197, 167, 1, 0, 0, 0, 197, 170, 1, 0, 0, 0, 197, 171, 1, 0, 0, 0, 197, 172, 1, 0, 0, 0, 197, 188, 1, 0, 0, 0, 197, 195, 1, 0, 0, 0, 198, 207, 1, 0, 0, 0, 199, 200, 10, 5, 0, 0, 200, 201, 5, 29, 0, 0, 201, 206, 3, 10, 5, 6, 202, 203, 10, 4, 0, 0, 203, 204, 5, 46, 0, 0, 204, 206, 3, 10, 5, 5, 205, 199, 1, 0, 0, 0, 205, 202, 1, 0, 0, 0, 206, 209, 1, 0, 0, 0, 207, 205, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 11, 1, 0, 0, 0, 209, 207, 1, 0, 0, 0, 210, 212, 3, 16, 8, 0, 211, 213, 5, 43, 0, 0, 212, 211, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 5, 41, 0, 0, 215, 216, 3, 106, 53, 0, 216, 225, 1, 0, 0, 0, 217, 219, 3, 16, 8, 0, 218, 220, 5, 43, 0, 0, 219, 218, 1, 0, 0, 0, 219, 220, 1, 0, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 5, 48, 0, 0, 222, 223, 3, 106, 53, 0, 223, 225, 1, 0, 0, 0, 224, 210, 1, 0, 0, 0, 224, 217, 1, 0, 0, 0, 225, 13, 1, 0, 0, 0, 226, 227, 3, 16, 8, 0, 227, 228, 5, 63, 0, 0, 228, 229, 3, 106, 53, 0, 229, 15, 1, 0, 0, 0, 230, 236, 3, 18, 9, 0, 231, 232, 3, 18, 9, 0, 232, 233, 3, 108, 54, 0, 233, 234, 3, 18, 9, 0, 234, 236, 1, 0, 0, 0, 235, 230, 1, 0, 0, 0, 235, 231, 1, 0, 0, 0, 236, 17, 1, 0, 0, 0, 237, 238, 6, 9, -1, 0, 238, 242, 3, 20, 10, 0, 239, 240, 7, 0, 0, 0, 240, 242, 3, 18, 9, 3, 241, 237, 1, 0, 0, 0, 241, 239, 1, 0, 0, 0, 242, 251, 1, 0, 0, 0, 243, 244, 10, 2, 0, 0, 244, 245, 7, 1, 0, 0, 245, 250, 3, 18, 9, 3, 246, 247, 10, 1, 0, 0, 247, 248, 7, 0, 0, 0, 248, 250, 3, 18, 9, 2, 249, 243, 1, 0, 0, 0, 249, 246, 1, 0, 0, 0, 250, 253, 1, 0, 0, 0, 251, 249, 1, 0, 0, 0, 251, 252, 1, 0, 0, 0, 252, 19, 1, 0, 0, 0, 253, 251, 1, 0, 0, 0, 254, 255, 6, 10, -1, 0, 255, 263, 3, 68, 34, 0, 256, 263, 3, 58, 29, 0, 257, 263, 3, 22, 11, 0, 258, 259, 5, 42, 0, 0, 259, 260, 3, 10, 5, 0, 260, 261, 5, 49, 0, 0, 261, 263, 1, 0, 0, 0, 262, 254, 1, 0, 0, 0, 262, 256, 1, 0, 0, 0, 262, 257, 1, 0, 0, 0, 262, 258, 1, 0, 0, 0, 263, 269, 1, 0, 0, 0, 264, 265, 10, 1, 0, 0, 265, 266, 5, 32, 0, 0, 266, 268, 3, 26, 13, 0, 267, 264, 1, 0, 0, 0, 268, 271, 1, 0, 0, 0, 269, 267, 1, 0, 0, 0, 269, 270, 1, 0, 0, 0, 270, 21, 1, 0, 0, 0, 271, 269, 1, 0, 0, 0, 272, 273, 3, 24, 12, 0, 273, 283, 5, 42, 0, 0, 274, 284, 5, 60, 0, 0, 275, 280, 3, 10, 5, 0, 276, 277, 5, 33, 0, 0, 277, 279, 3, 10, 5, 0, 278, 276, 1, 0, 0, 0, 279, 282, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 274, 1, 0, 0, 0, 283, 275, 1, 0, 0, 0, 283, 284, 1, 0, 0, 0, 284, 285, 1, 0, 0, 0, 285, 286, 5, 49, 0, 0, 286, 23, 1, 0, 0, 0, 287, 290, 5, 63, 0, 0, 288, 290, 3, 72, 36, 0, 289, 287, 1, 0, 0, 0, 289, 288, 1, 0, 0, 0, 290, 25, 1, 0, 0, 0, 291, 292, 3, 64, 32, 0, 292, 27, 1, 0, 0, 0, 293, 294, 5, 12, 0, 0, 294, 295, 3, 30, 15, 0, 295, 29, 1, 0, 0, 0, 296, 301, 3, 32, 16, 0, 297, 298, 5, 33, 0, 0, 298, 300, 3, 32, 16, 0, 299, 297, 1, 0, 0, 0, 300, 303, 1, 0, 0, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 31, 1, 0, 0, 0, 303, 301, 1, 0, 0, 0, 304, 305, 3, 58, 29, 0, 305, 306, 5, 31, 0, 0, 306, 308, 1, 0, 0, 0, 307, 304, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 310, 3, 10, 5, 0, 310, 33, 1, 0, 0, 0, 311, 312, 5, 6, 0, 0, 312, 317, 3, 36, 18, 0, 313, 314, 5, 33, 0, 0, 314, 316, 3, 36, 18, 0, 315, 313, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 321, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 320, 322, 3, 42, 21, 0, 321, 320, 1, 0, 0, 0, 321, 322, 1, 0, 0, 0, 322, 35, 1, 0, 0, 0, 323, 324, 3, 38, 19, 0, 324, 325, 5, 104, 0, 0, 325, 327, 1, 0, 0, 0, 326, 323, 1, 0, 0, 0, 326, 327, 1, 0, 0, 0, 327, 328, 1, 0, 0, 0, 328, 329, 3, 40, 20, 0, 329, 37, 1, 0, 0, 0, 330, 331, 5, 76, 0, 0, 331, 39, 1, 0, 0, 0, 332, 333, 7, 2, 0, 0, 333, 41, 1, 0, 0, 0, 334, 337, 3, 44, 22, 0, 335, 337, 3, 46, 23, 0, 336, 334, 1, 0, 0, 0, 336, 335, 1, 0, 0, 0, 337, 43, 1, 0, 0, 0, 338, 339, 5, 75, 0, 0, 339, 344, 5, 76, 0, 0, 340, 341, 5, 33, 0, 0, 341, 343, 5, 76, 0, 0, 342, 340, 1, 0, 0, 0, 343, 346, 1, 0, 0, 0, 344, 342, 1, 0, 0, 0, 344, 345, 1, 0, 0, 0, 345, 45, 1, 0, 0, 0, 346, 344, 1, 0, 0, 0, 347, 348, 5, 65, 0, 0, 348, 349, 3, 44, 22, 0, 349, 350, 5, 66, 0, 0, 350, 47, 1, 0, 0, 0, 351, 352, 5, 19, 0, 0, 352, 357, 3, 36, 18, 0, 353, 354, 5, 33, 0, 0, 354, 356, 3, 36, 18, 0, 355, 353, 1, 0, 0, 0, 356, 359, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 357, 358, 1, 0, 0, 0, 358, 361, 1, 0, 0, 0, 359, 357, 1, 0, 0, 0, 360, 362, 3, 54, 27, 0, 361, 360, 1, 0, 0, 0, 361, 362, 1, 0, 0, 0, 362, 365, 1, 0, 0, 0, 363, 364, 5, 28, 0, 0, 364, 366, 3, 30, 15, 0, 365, 363, 1, 0, 0, 0, 365, 366, 1, 0, 0, 0, 366, 49, 1, 0, 0, 0, 367, 368, 5, 4, 0, 0, 368, 369, 3, 30, 15, 0, 369, 51, 1, 0, 0, 0, 370, 372, 5, 15, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 28, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 53, 1, 0, 0, 0, 378, 383, 3, 56, 28, 0, 379, 380, 5, 33, 0, 0, 380, 382, 3, 56, 28, 0, 381, 379, 1, 0, 0, 0, 382, 385, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 55, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 386, 389, 3, 32, 16, 0, 387, 388, 5, 16, 0, 0, 388, 390, 3, 10, 5, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 57, 1, 0, 0, 0, 391, 396, 3, 72, 36, 0, 392, 393, 5, 35, 0, 0, 393, 395, 3, 72, 36, 0, 394, 392, 1, 0, 0, 0, 395, 398, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 396, 397, 1, 0, 0, 0, 397, 59, 1, 0, 0, 0, 398, 396, 1, 0, 0, 0, 399, 404, 3, 66, 33, 0, 400, 401, 5, 35, 0, 0, 401, 403, 3, 66, 33, 0, 402, 400, 1, 0, 0, 0, 403, 406, 1, 0, 0, 0, 404, 402, 1, 0, 0, 0, 404, 405, 1, 0, 0, 0, 405, 61, 1, 0, 0, 0, 406, 404, 1, 0, 0, 0, 407, 412, 3, 60, 30, 0, 408, 409, 5, 33, 0, 0, 409, 411, 3, 60, 30, 0, 410, 408, 1, 0, 0, 0, 411, 414, 1, 0, 0, 0, 412, 410, 1, 0, 0, 0, 412, 413, 1, 0, 0, 0, 413, 63, 1, 0, 0, 0, 414, 412, 1, 0, 0, 0, 415, 416, 7, 3, 0, 0, 416, 65, 1, 0, 0, 0, 417, 420, 5, 80, 0, 0, 418, 420, 3, 70, 35, 0, 419, 417, 1, 0, 0, 0, 419, 418, 1, 0, 0, 0, 420, 67, 1, 0, 0, 0, 421, 464, 5, 44, 0, 0, 422, 423, 3, 104, 52, 0, 423, 424, 5, 67, 0, 0, 424, 464, 1, 0, 0, 0, 425, 464, 3, 102, 51, 0, 426, 464, 3, 104, 52, 0, 427, 464, 3, 98, 49, 0, 428, 464, 3, 70, 35, 0, 429, 464, 3, 106, 53, 0, 430, 431, 5, 65, 0, 0, 431, 436, 3, 100, 50, 0, 432, 433, 5, 33, 0, 0, 433, 435, 3, 100, 50, 0, 434, 432, 1, 0, 0, 0, 435, 438, 1, 0, 0, 0, 436, 434, 1, 0, 0, 0, 436, 437, 1, 0, 0, 0, 437, 439, 1, 0, 0, 0, 438, 436, 1, 0, 0, 0, 439, 440, 5, 66, 0, 0, 440, 464, 1, 0, 0, 0, 441, 442, 5, 65, 0, 0, 442, 447, 3, 98, 49, 0, 443, 444, 5, 33, 0, 0, 444, 446, 3, 98, 49, 0, 445, 443, 1, 0, 0, 0, 446, 449, 1, 0, 0, 0, 447, 445, 1, 0, 0, 0, 447, 448, 1, 0, 0, 0, 448, 450, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 450, 451, 5, 66, 0, 0, 451, 464, 1, 0, 0, 0, 452, 453, 5, 65, 0, 0, 453, 458, 3, 106, 53, 0, 454, 455, 5, 33, 0, 0, 455, 457, 3, 106, 53, 0, 456, 454, 1, 0, 0, 0, 457, 460, 1, 0, 0, 0, 458, 456, 1, 0, 0, 0, 458, 459, 1, 0, 0, 0, 459, 461, 1, 0, 0, 0, 460, 458, 1, 0, 0, 0, 461, 462, 5, 66, 0, 0, 462, 464, 1, 0, 0, 0, 463, 421, 1, 0, 0, 0, 463, 422, 1, 0, 0, 0, 463, 425, 1, 0, 0, 0, 463, 426, 1, 0, 0, 0, 463, 427, 1, 0, 0, 0, 463, 428, 1, 0, 0, 0, 463, 429, 1, 0, 0, 0, 463, 430, 1, 0, 0, 0, 463, 441, 1, 0, 0, 0, 463, 452, 1, 0, 0, 0, 464, 69, 1, 0, 0, 0, 465, 468, 5, 47, 0, 0, 466, 468, 5, 64, 0, 0, 467, 465, 1, 0, 0, 0, 467, 466, 1, 0, 0, 0, 468, 71, 1, 0, 0, 0, 469, 472, 3, 64, 32, 0, 470, 472, 3, 70, 35, 0, 471, 469, 1, 0, 0, 0, 471, 470, 1, 0, 0, 0, 472, 73, 1, 0, 0, 0, 473, 474, 5, 9, 0, 0, 474, 475, 5, 26, 0, 0, 475, 75, 1, 0, 0, 0, 476, 477, 5, 14, 0, 0, 477, 482, 3, 78, 39, 0, 478, 479, 5, 33, 0, 0, 479, 481, 3, 78, 39, 0, 480, 478, 1, 0, 0, 0, 481, 484, 1, 0, 0, 0, 482, 480, 1, 0, 0, 0, 482, 483, 1, 0, 0, 0, 483, 77, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 487, 3, 10, 5, 0, 486, 488, 7, 4, 0, 0, 487, 486, 1, 0, 0, 0, 487, 488, 1, 0, 0, 0, 488, 491, 1, 0, 0, 0, 489, 490, 5, 45, 0, 0, 490, 492, 7, 5, 0, 0, 491, 489, 1, 0, 0, 0, 491, 492, 1, 0, 0, 0, 492, 79, 1, 0, 0, 0, 493, 494, 5, 8, 0, 0, 494, 495, 3, 62, 31, 0, 495, 81, 1, 0, 0, 0, 496, 497, 5, 2, 0, 0, 497, 498, 3, 62, 31, 0, 498, 83, 1, 0, 0, 0, 499, 500, 5, 11, 0, 0, 500, 505, 3, 86, 43, 0, 501, 502, 5, 33, 0, 0, 502, 504, 3, 86, 43, 0, 503, 501, 1, 0, 0, 0, 504, 507, 1, 0, 0, 0, 505, 503, 1, 0, 0, 0, 505, 506, 1, 0, 0, 0, 506, 85, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 508, 509, 3, 60, 30, 0, 509, 510, 5, 84, 0, 0, 510, 511, 3, 60, 30, 0, 511, 87, 1, 0, 0, 0, 512, 513, 5, 1, 0, 0, 513, 514, 3, 20, 10, 0, 514, 516, 3, 106, 53, 0, 515, 517, 3, 94, 47, 0, 516, 515, 1, 0, 0, 0, 516, 517, 1, 0, 0, 0, 517, 89, 1, 0, 0, 0, 518, 519, 5, 7, 0, 0, 519, 520, 3, 20, 10, 0, 520, 521, 3, 106, 53, 0, 521, 91, 1, 0, 0, 0, 522, 523, 5, 10, 0, 0, 523, 524, 3, 58, 29, 0, 524, 93, 1, 0, 0, 0, 525, 530, 3, 96, 48, 0, 526, 527, 5, 33, 0, 0, 527, 529, 3, 96, 48, 0, 528, 526, 1, 0, 0, 0, 529, 532, 1, 0, 0, 0, 530, 528, 1, 0, 0, 0, 530, 531, 1, 0, 0, 0, 531, 95, 1, 0, 0, 0, 532, 530, 1, 0, 0, 0, 533, 534, 3, 64, 32, 0, 534, 535, 5, 31, 0, 0, 535, 536, 3, 68, 34, 0, 536, 97, 1, 0, 0, 0, 537, 538, 7, 6, 0, 0, 538, 99, 1, 0, 0, 0, 539, 542, 3, 102, 51, 0, 540, 542, 3, 104, 52, 0, 541, 539, 1, 0, 0, 0, 541, 540, 1, 0, 0, 0, 542, 101, 1, 0, 0, 0, 543, 545, 7, 0, 0, 0, 544, 543, 1, 0, 0, 0, 544, 545, 1, 0, 0, 0, 545, 546, 1, 0, 0, 0, 546, 547, 5, 27, 0, 0, 547, 103, 1, 0, 0, 0, 548, 550, 7, 0, 0, 0, 549, 548, 1, 0, 0, 0, 549, 550, 1, 0, 0, 0, 550, 551, 1, 0, 0, 0, 551, 552, 5, 26, 0, 0, 552, 105, 1, 0, 0, 0, 553, 554, 5, 25, 0, 0, 554, 107, 1, 0, 0, 0, 555, 556, 7, 7, 0, 0, 556, 109, 1, 0, 0, 0, 557, 558, 5, 5, 0, 0, 558, 559, 3, 112, 56, 0, 559, 111, 1, 0, 0, 0, 560, 561, 5, 65, 0, 0, 561, 562, 3, 2, 1, 0, 562, 563, 5, 66, 0, 0, 563, 113, 1, 0, 0, 0, 564, 565, 5, 13, 0, 0, 565, 566, 5, 100, 0, 0, 566, 115, 1, 0, 0, 0, 567, 568, 5, 3, 0, 0, 568, 571, 5, 90, 0, 0, 569, 570, 5, 88, 0, 0, 570, 572, 3, 60, 30, 0, 571, 569, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 572, 582, 1, 0, 0, 0, 573, 574, 5, 89, 0, 0, 574, 579, 3, 118, 59, 0, 575, 576, 5, 33, 0, 0, 576, 578, 3, 118, 59, 0, 577, 575, 1, 0, 0, 0, 578, 581, 1, 0, 0, 0, 579, 577, 1, 0, 0, 0, 579, 580, 1, 0, 0, 0, 580, 583, 1, 0, 0, 0, 581, 579, 1, 0, 0, 0, 582, 573, 1, 0, 0, 0, 582, 583, 1, 0, 0, 0, 583, 117, 1, 0, 0, 0, 584, 585, 3, 60, 30, 0, 585, 586, 5, 31, 0, 0, 586, 588, 1, 0, 0, 0, 587, 584, 1, 0, 0, 0, 587, 588, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 3, 60, 30, 0, 590, 119, 1, 0, 0, 0, 591, 592, 5, 18, 0, 0, 592, 593, 3, 36, 18, 0, 593, 594, 5, 88, 0, 0, 594, 595, 3, 62, 31, 0, 595, 121, 1, 0, 0, 0, 596, 597, 5, 17, 0, 0, 597, 600, 3, 54, 27, 0, 598, 599, 5, 28, 0, 0, 599, 601, 3, 30, 15, 0, 600, 598, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 123, 1, 0, 0, 0, 59, 135, 144, 162, 174, 183, 191, 197, 205, 207, 212, 219, 224, 235, 241, 249, 251, 262, 269, 280, 283, 289, 301, 307, 317, 321, 326, 336, 344, 357, 361, 365, 372, 376, 383, 389, 396, 404, 412, 419, 436, 447, 458, 463, 467, 471, 482, 487, 491, 505, 516, 530, 541, 544, 549, 571, 579, 582, 587, 600] \ No newline at end of file +[4, 1, 120, 605, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 134, 8, 1, 10, 1, 12, 1, 137, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 145, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 163, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 175, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 182, 8, 5, 10, 5, 12, 5, 185, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 192, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 198, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 206, 8, 5, 10, 5, 12, 5, 209, 9, 5, 1, 6, 1, 6, 3, 6, 213, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 220, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 236, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 242, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 250, 8, 9, 10, 9, 12, 9, 253, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 263, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 268, 8, 10, 10, 10, 12, 10, 271, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 279, 8, 11, 10, 11, 12, 11, 282, 9, 11, 3, 11, 284, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 3, 12, 290, 8, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 300, 8, 15, 10, 15, 12, 15, 303, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 308, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 316, 8, 17, 10, 17, 12, 17, 319, 9, 17, 1, 17, 3, 17, 322, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 327, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 337, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 343, 8, 22, 10, 22, 12, 22, 346, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 356, 8, 24, 10, 24, 12, 24, 359, 9, 24, 1, 24, 3, 24, 362, 8, 24, 1, 24, 1, 24, 3, 24, 366, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 373, 8, 26, 1, 26, 1, 26, 3, 26, 377, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 382, 8, 27, 10, 27, 12, 27, 385, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 390, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 395, 8, 29, 10, 29, 12, 29, 398, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 403, 8, 30, 10, 30, 12, 30, 406, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 411, 8, 31, 10, 31, 12, 31, 414, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 421, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 436, 8, 34, 10, 34, 12, 34, 439, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 447, 8, 34, 10, 34, 12, 34, 450, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 458, 8, 34, 10, 34, 12, 34, 461, 9, 34, 1, 34, 1, 34, 3, 34, 465, 8, 34, 1, 35, 1, 35, 3, 35, 469, 8, 35, 1, 36, 1, 36, 1, 36, 3, 36, 474, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 483, 8, 38, 10, 38, 12, 38, 486, 9, 38, 1, 39, 1, 39, 3, 39, 490, 8, 39, 1, 39, 1, 39, 3, 39, 494, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 506, 8, 42, 10, 42, 12, 42, 509, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 519, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 531, 8, 47, 10, 47, 12, 47, 534, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 544, 8, 50, 1, 51, 3, 51, 547, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 552, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 574, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 580, 8, 58, 10, 58, 12, 58, 583, 9, 58, 3, 58, 585, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 590, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 603, 8, 61, 1, 61, 0, 4, 2, 10, 18, 20, 62, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 0, 8, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 25, 25, 76, 76, 1, 0, 67, 68, 2, 0, 30, 30, 34, 34, 2, 0, 37, 37, 40, 40, 2, 0, 36, 36, 50, 50, 2, 0, 51, 51, 53, 57, 631, 0, 124, 1, 0, 0, 0, 2, 127, 1, 0, 0, 0, 4, 144, 1, 0, 0, 0, 6, 162, 1, 0, 0, 0, 8, 164, 1, 0, 0, 0, 10, 197, 1, 0, 0, 0, 12, 224, 1, 0, 0, 0, 14, 226, 1, 0, 0, 0, 16, 235, 1, 0, 0, 0, 18, 241, 1, 0, 0, 0, 20, 262, 1, 0, 0, 0, 22, 272, 1, 0, 0, 0, 24, 289, 1, 0, 0, 0, 26, 291, 1, 0, 0, 0, 28, 293, 1, 0, 0, 0, 30, 296, 1, 0, 0, 0, 32, 307, 1, 0, 0, 0, 34, 311, 1, 0, 0, 0, 36, 326, 1, 0, 0, 0, 38, 330, 1, 0, 0, 0, 40, 332, 1, 0, 0, 0, 42, 336, 1, 0, 0, 0, 44, 338, 1, 0, 0, 0, 46, 347, 1, 0, 0, 0, 48, 351, 1, 0, 0, 0, 50, 367, 1, 0, 0, 0, 52, 370, 1, 0, 0, 0, 54, 378, 1, 0, 0, 0, 56, 386, 1, 0, 0, 0, 58, 391, 1, 0, 0, 0, 60, 399, 1, 0, 0, 0, 62, 407, 1, 0, 0, 0, 64, 415, 1, 0, 0, 0, 66, 420, 1, 0, 0, 0, 68, 464, 1, 0, 0, 0, 70, 468, 1, 0, 0, 0, 72, 473, 1, 0, 0, 0, 74, 475, 1, 0, 0, 0, 76, 478, 1, 0, 0, 0, 78, 487, 1, 0, 0, 0, 80, 495, 1, 0, 0, 0, 82, 498, 1, 0, 0, 0, 84, 501, 1, 0, 0, 0, 86, 510, 1, 0, 0, 0, 88, 514, 1, 0, 0, 0, 90, 520, 1, 0, 0, 0, 92, 524, 1, 0, 0, 0, 94, 527, 1, 0, 0, 0, 96, 535, 1, 0, 0, 0, 98, 539, 1, 0, 0, 0, 100, 543, 1, 0, 0, 0, 102, 546, 1, 0, 0, 0, 104, 551, 1, 0, 0, 0, 106, 555, 1, 0, 0, 0, 108, 557, 1, 0, 0, 0, 110, 559, 1, 0, 0, 0, 112, 562, 1, 0, 0, 0, 114, 566, 1, 0, 0, 0, 116, 569, 1, 0, 0, 0, 118, 589, 1, 0, 0, 0, 120, 593, 1, 0, 0, 0, 122, 598, 1, 0, 0, 0, 124, 125, 3, 2, 1, 0, 125, 126, 5, 0, 0, 1, 126, 1, 1, 0, 0, 0, 127, 128, 6, 1, -1, 0, 128, 129, 3, 4, 2, 0, 129, 135, 1, 0, 0, 0, 130, 131, 10, 1, 0, 0, 131, 132, 5, 24, 0, 0, 132, 134, 3, 6, 3, 0, 133, 130, 1, 0, 0, 0, 134, 137, 1, 0, 0, 0, 135, 133, 1, 0, 0, 0, 135, 136, 1, 0, 0, 0, 136, 3, 1, 0, 0, 0, 137, 135, 1, 0, 0, 0, 138, 145, 3, 110, 55, 0, 139, 145, 3, 34, 17, 0, 140, 145, 3, 28, 14, 0, 141, 145, 3, 114, 57, 0, 142, 143, 4, 2, 1, 0, 143, 145, 3, 48, 24, 0, 144, 138, 1, 0, 0, 0, 144, 139, 1, 0, 0, 0, 144, 140, 1, 0, 0, 0, 144, 141, 1, 0, 0, 0, 144, 142, 1, 0, 0, 0, 145, 5, 1, 0, 0, 0, 146, 163, 3, 50, 25, 0, 147, 163, 3, 8, 4, 0, 148, 163, 3, 80, 40, 0, 149, 163, 3, 74, 37, 0, 150, 163, 3, 52, 26, 0, 151, 163, 3, 76, 38, 0, 152, 163, 3, 82, 41, 0, 153, 163, 3, 84, 42, 0, 154, 163, 3, 88, 44, 0, 155, 163, 3, 90, 45, 0, 156, 163, 3, 116, 58, 0, 157, 163, 3, 92, 46, 0, 158, 159, 4, 3, 2, 0, 159, 163, 3, 122, 61, 0, 160, 161, 4, 3, 3, 0, 161, 163, 3, 120, 60, 0, 162, 146, 1, 0, 0, 0, 162, 147, 1, 0, 0, 0, 162, 148, 1, 0, 0, 0, 162, 149, 1, 0, 0, 0, 162, 150, 1, 0, 0, 0, 162, 151, 1, 0, 0, 0, 162, 152, 1, 0, 0, 0, 162, 153, 1, 0, 0, 0, 162, 154, 1, 0, 0, 0, 162, 155, 1, 0, 0, 0, 162, 156, 1, 0, 0, 0, 162, 157, 1, 0, 0, 0, 162, 158, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 163, 7, 1, 0, 0, 0, 164, 165, 5, 16, 0, 0, 165, 166, 3, 10, 5, 0, 166, 9, 1, 0, 0, 0, 167, 168, 6, 5, -1, 0, 168, 169, 5, 43, 0, 0, 169, 198, 3, 10, 5, 8, 170, 198, 3, 16, 8, 0, 171, 198, 3, 12, 6, 0, 172, 174, 3, 16, 8, 0, 173, 175, 5, 43, 0, 0, 174, 173, 1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 177, 5, 38, 0, 0, 177, 178, 5, 42, 0, 0, 178, 183, 3, 16, 8, 0, 179, 180, 5, 33, 0, 0, 180, 182, 3, 16, 8, 0, 181, 179, 1, 0, 0, 0, 182, 185, 1, 0, 0, 0, 183, 181, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 186, 1, 0, 0, 0, 185, 183, 1, 0, 0, 0, 186, 187, 5, 49, 0, 0, 187, 198, 1, 0, 0, 0, 188, 189, 3, 16, 8, 0, 189, 191, 5, 39, 0, 0, 190, 192, 5, 43, 0, 0, 191, 190, 1, 0, 0, 0, 191, 192, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193, 194, 5, 44, 0, 0, 194, 198, 1, 0, 0, 0, 195, 196, 4, 5, 4, 0, 196, 198, 3, 14, 7, 0, 197, 167, 1, 0, 0, 0, 197, 170, 1, 0, 0, 0, 197, 171, 1, 0, 0, 0, 197, 172, 1, 0, 0, 0, 197, 188, 1, 0, 0, 0, 197, 195, 1, 0, 0, 0, 198, 207, 1, 0, 0, 0, 199, 200, 10, 5, 0, 0, 200, 201, 5, 29, 0, 0, 201, 206, 3, 10, 5, 6, 202, 203, 10, 4, 0, 0, 203, 204, 5, 46, 0, 0, 204, 206, 3, 10, 5, 5, 205, 199, 1, 0, 0, 0, 205, 202, 1, 0, 0, 0, 206, 209, 1, 0, 0, 0, 207, 205, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 11, 1, 0, 0, 0, 209, 207, 1, 0, 0, 0, 210, 212, 3, 16, 8, 0, 211, 213, 5, 43, 0, 0, 212, 211, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 5, 41, 0, 0, 215, 216, 3, 106, 53, 0, 216, 225, 1, 0, 0, 0, 217, 219, 3, 16, 8, 0, 218, 220, 5, 43, 0, 0, 219, 218, 1, 0, 0, 0, 219, 220, 1, 0, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 5, 48, 0, 0, 222, 223, 3, 106, 53, 0, 223, 225, 1, 0, 0, 0, 224, 210, 1, 0, 0, 0, 224, 217, 1, 0, 0, 0, 225, 13, 1, 0, 0, 0, 226, 227, 3, 16, 8, 0, 227, 228, 5, 63, 0, 0, 228, 229, 3, 106, 53, 0, 229, 15, 1, 0, 0, 0, 230, 236, 3, 18, 9, 0, 231, 232, 3, 18, 9, 0, 232, 233, 3, 108, 54, 0, 233, 234, 3, 18, 9, 0, 234, 236, 1, 0, 0, 0, 235, 230, 1, 0, 0, 0, 235, 231, 1, 0, 0, 0, 236, 17, 1, 0, 0, 0, 237, 238, 6, 9, -1, 0, 238, 242, 3, 20, 10, 0, 239, 240, 7, 0, 0, 0, 240, 242, 3, 18, 9, 3, 241, 237, 1, 0, 0, 0, 241, 239, 1, 0, 0, 0, 242, 251, 1, 0, 0, 0, 243, 244, 10, 2, 0, 0, 244, 245, 7, 1, 0, 0, 245, 250, 3, 18, 9, 3, 246, 247, 10, 1, 0, 0, 247, 248, 7, 0, 0, 0, 248, 250, 3, 18, 9, 2, 249, 243, 1, 0, 0, 0, 249, 246, 1, 0, 0, 0, 250, 253, 1, 0, 0, 0, 251, 249, 1, 0, 0, 0, 251, 252, 1, 0, 0, 0, 252, 19, 1, 0, 0, 0, 253, 251, 1, 0, 0, 0, 254, 255, 6, 10, -1, 0, 255, 263, 3, 68, 34, 0, 256, 263, 3, 58, 29, 0, 257, 263, 3, 22, 11, 0, 258, 259, 5, 42, 0, 0, 259, 260, 3, 10, 5, 0, 260, 261, 5, 49, 0, 0, 261, 263, 1, 0, 0, 0, 262, 254, 1, 0, 0, 0, 262, 256, 1, 0, 0, 0, 262, 257, 1, 0, 0, 0, 262, 258, 1, 0, 0, 0, 263, 269, 1, 0, 0, 0, 264, 265, 10, 1, 0, 0, 265, 266, 5, 32, 0, 0, 266, 268, 3, 26, 13, 0, 267, 264, 1, 0, 0, 0, 268, 271, 1, 0, 0, 0, 269, 267, 1, 0, 0, 0, 269, 270, 1, 0, 0, 0, 270, 21, 1, 0, 0, 0, 271, 269, 1, 0, 0, 0, 272, 273, 3, 24, 12, 0, 273, 283, 5, 42, 0, 0, 274, 284, 5, 60, 0, 0, 275, 280, 3, 10, 5, 0, 276, 277, 5, 33, 0, 0, 277, 279, 3, 10, 5, 0, 278, 276, 1, 0, 0, 0, 279, 282, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 274, 1, 0, 0, 0, 283, 275, 1, 0, 0, 0, 283, 284, 1, 0, 0, 0, 284, 285, 1, 0, 0, 0, 285, 286, 5, 49, 0, 0, 286, 23, 1, 0, 0, 0, 287, 290, 5, 63, 0, 0, 288, 290, 3, 72, 36, 0, 289, 287, 1, 0, 0, 0, 289, 288, 1, 0, 0, 0, 290, 25, 1, 0, 0, 0, 291, 292, 3, 64, 32, 0, 292, 27, 1, 0, 0, 0, 293, 294, 5, 12, 0, 0, 294, 295, 3, 30, 15, 0, 295, 29, 1, 0, 0, 0, 296, 301, 3, 32, 16, 0, 297, 298, 5, 33, 0, 0, 298, 300, 3, 32, 16, 0, 299, 297, 1, 0, 0, 0, 300, 303, 1, 0, 0, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 31, 1, 0, 0, 0, 303, 301, 1, 0, 0, 0, 304, 305, 3, 58, 29, 0, 305, 306, 5, 31, 0, 0, 306, 308, 1, 0, 0, 0, 307, 304, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 310, 3, 10, 5, 0, 310, 33, 1, 0, 0, 0, 311, 312, 5, 6, 0, 0, 312, 317, 3, 36, 18, 0, 313, 314, 5, 33, 0, 0, 314, 316, 3, 36, 18, 0, 315, 313, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 321, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 320, 322, 3, 42, 21, 0, 321, 320, 1, 0, 0, 0, 321, 322, 1, 0, 0, 0, 322, 35, 1, 0, 0, 0, 323, 324, 3, 38, 19, 0, 324, 325, 5, 104, 0, 0, 325, 327, 1, 0, 0, 0, 326, 323, 1, 0, 0, 0, 326, 327, 1, 0, 0, 0, 327, 328, 1, 0, 0, 0, 328, 329, 3, 40, 20, 0, 329, 37, 1, 0, 0, 0, 330, 331, 5, 76, 0, 0, 331, 39, 1, 0, 0, 0, 332, 333, 7, 2, 0, 0, 333, 41, 1, 0, 0, 0, 334, 337, 3, 44, 22, 0, 335, 337, 3, 46, 23, 0, 336, 334, 1, 0, 0, 0, 336, 335, 1, 0, 0, 0, 337, 43, 1, 0, 0, 0, 338, 339, 5, 75, 0, 0, 339, 344, 5, 76, 0, 0, 340, 341, 5, 33, 0, 0, 341, 343, 5, 76, 0, 0, 342, 340, 1, 0, 0, 0, 343, 346, 1, 0, 0, 0, 344, 342, 1, 0, 0, 0, 344, 345, 1, 0, 0, 0, 345, 45, 1, 0, 0, 0, 346, 344, 1, 0, 0, 0, 347, 348, 5, 65, 0, 0, 348, 349, 3, 44, 22, 0, 349, 350, 5, 66, 0, 0, 350, 47, 1, 0, 0, 0, 351, 352, 5, 19, 0, 0, 352, 357, 3, 36, 18, 0, 353, 354, 5, 33, 0, 0, 354, 356, 3, 36, 18, 0, 355, 353, 1, 0, 0, 0, 356, 359, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 357, 358, 1, 0, 0, 0, 358, 361, 1, 0, 0, 0, 359, 357, 1, 0, 0, 0, 360, 362, 3, 54, 27, 0, 361, 360, 1, 0, 0, 0, 361, 362, 1, 0, 0, 0, 362, 365, 1, 0, 0, 0, 363, 364, 5, 28, 0, 0, 364, 366, 3, 30, 15, 0, 365, 363, 1, 0, 0, 0, 365, 366, 1, 0, 0, 0, 366, 49, 1, 0, 0, 0, 367, 368, 5, 4, 0, 0, 368, 369, 3, 30, 15, 0, 369, 51, 1, 0, 0, 0, 370, 372, 5, 15, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 28, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 53, 1, 0, 0, 0, 378, 383, 3, 56, 28, 0, 379, 380, 5, 33, 0, 0, 380, 382, 3, 56, 28, 0, 381, 379, 1, 0, 0, 0, 382, 385, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 55, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 386, 389, 3, 32, 16, 0, 387, 388, 5, 16, 0, 0, 388, 390, 3, 10, 5, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 57, 1, 0, 0, 0, 391, 396, 3, 72, 36, 0, 392, 393, 5, 35, 0, 0, 393, 395, 3, 72, 36, 0, 394, 392, 1, 0, 0, 0, 395, 398, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 396, 397, 1, 0, 0, 0, 397, 59, 1, 0, 0, 0, 398, 396, 1, 0, 0, 0, 399, 404, 3, 66, 33, 0, 400, 401, 5, 35, 0, 0, 401, 403, 3, 66, 33, 0, 402, 400, 1, 0, 0, 0, 403, 406, 1, 0, 0, 0, 404, 402, 1, 0, 0, 0, 404, 405, 1, 0, 0, 0, 405, 61, 1, 0, 0, 0, 406, 404, 1, 0, 0, 0, 407, 412, 3, 60, 30, 0, 408, 409, 5, 33, 0, 0, 409, 411, 3, 60, 30, 0, 410, 408, 1, 0, 0, 0, 411, 414, 1, 0, 0, 0, 412, 410, 1, 0, 0, 0, 412, 413, 1, 0, 0, 0, 413, 63, 1, 0, 0, 0, 414, 412, 1, 0, 0, 0, 415, 416, 7, 3, 0, 0, 416, 65, 1, 0, 0, 0, 417, 421, 5, 80, 0, 0, 418, 419, 4, 33, 10, 0, 419, 421, 3, 70, 35, 0, 420, 417, 1, 0, 0, 0, 420, 418, 1, 0, 0, 0, 421, 67, 1, 0, 0, 0, 422, 465, 5, 44, 0, 0, 423, 424, 3, 104, 52, 0, 424, 425, 5, 67, 0, 0, 425, 465, 1, 0, 0, 0, 426, 465, 3, 102, 51, 0, 427, 465, 3, 104, 52, 0, 428, 465, 3, 98, 49, 0, 429, 465, 3, 70, 35, 0, 430, 465, 3, 106, 53, 0, 431, 432, 5, 65, 0, 0, 432, 437, 3, 100, 50, 0, 433, 434, 5, 33, 0, 0, 434, 436, 3, 100, 50, 0, 435, 433, 1, 0, 0, 0, 436, 439, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 437, 438, 1, 0, 0, 0, 438, 440, 1, 0, 0, 0, 439, 437, 1, 0, 0, 0, 440, 441, 5, 66, 0, 0, 441, 465, 1, 0, 0, 0, 442, 443, 5, 65, 0, 0, 443, 448, 3, 98, 49, 0, 444, 445, 5, 33, 0, 0, 445, 447, 3, 98, 49, 0, 446, 444, 1, 0, 0, 0, 447, 450, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 448, 449, 1, 0, 0, 0, 449, 451, 1, 0, 0, 0, 450, 448, 1, 0, 0, 0, 451, 452, 5, 66, 0, 0, 452, 465, 1, 0, 0, 0, 453, 454, 5, 65, 0, 0, 454, 459, 3, 106, 53, 0, 455, 456, 5, 33, 0, 0, 456, 458, 3, 106, 53, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 462, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 463, 5, 66, 0, 0, 463, 465, 1, 0, 0, 0, 464, 422, 1, 0, 0, 0, 464, 423, 1, 0, 0, 0, 464, 426, 1, 0, 0, 0, 464, 427, 1, 0, 0, 0, 464, 428, 1, 0, 0, 0, 464, 429, 1, 0, 0, 0, 464, 430, 1, 0, 0, 0, 464, 431, 1, 0, 0, 0, 464, 442, 1, 0, 0, 0, 464, 453, 1, 0, 0, 0, 465, 69, 1, 0, 0, 0, 466, 469, 5, 47, 0, 0, 467, 469, 5, 64, 0, 0, 468, 466, 1, 0, 0, 0, 468, 467, 1, 0, 0, 0, 469, 71, 1, 0, 0, 0, 470, 474, 3, 64, 32, 0, 471, 472, 4, 36, 11, 0, 472, 474, 3, 70, 35, 0, 473, 470, 1, 0, 0, 0, 473, 471, 1, 0, 0, 0, 474, 73, 1, 0, 0, 0, 475, 476, 5, 9, 0, 0, 476, 477, 5, 26, 0, 0, 477, 75, 1, 0, 0, 0, 478, 479, 5, 14, 0, 0, 479, 484, 3, 78, 39, 0, 480, 481, 5, 33, 0, 0, 481, 483, 3, 78, 39, 0, 482, 480, 1, 0, 0, 0, 483, 486, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 484, 485, 1, 0, 0, 0, 485, 77, 1, 0, 0, 0, 486, 484, 1, 0, 0, 0, 487, 489, 3, 10, 5, 0, 488, 490, 7, 4, 0, 0, 489, 488, 1, 0, 0, 0, 489, 490, 1, 0, 0, 0, 490, 493, 1, 0, 0, 0, 491, 492, 5, 45, 0, 0, 492, 494, 7, 5, 0, 0, 493, 491, 1, 0, 0, 0, 493, 494, 1, 0, 0, 0, 494, 79, 1, 0, 0, 0, 495, 496, 5, 8, 0, 0, 496, 497, 3, 62, 31, 0, 497, 81, 1, 0, 0, 0, 498, 499, 5, 2, 0, 0, 499, 500, 3, 62, 31, 0, 500, 83, 1, 0, 0, 0, 501, 502, 5, 11, 0, 0, 502, 507, 3, 86, 43, 0, 503, 504, 5, 33, 0, 0, 504, 506, 3, 86, 43, 0, 505, 503, 1, 0, 0, 0, 506, 509, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 85, 1, 0, 0, 0, 509, 507, 1, 0, 0, 0, 510, 511, 3, 60, 30, 0, 511, 512, 5, 84, 0, 0, 512, 513, 3, 60, 30, 0, 513, 87, 1, 0, 0, 0, 514, 515, 5, 1, 0, 0, 515, 516, 3, 20, 10, 0, 516, 518, 3, 106, 53, 0, 517, 519, 3, 94, 47, 0, 518, 517, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 89, 1, 0, 0, 0, 520, 521, 5, 7, 0, 0, 521, 522, 3, 20, 10, 0, 522, 523, 3, 106, 53, 0, 523, 91, 1, 0, 0, 0, 524, 525, 5, 10, 0, 0, 525, 526, 3, 58, 29, 0, 526, 93, 1, 0, 0, 0, 527, 532, 3, 96, 48, 0, 528, 529, 5, 33, 0, 0, 529, 531, 3, 96, 48, 0, 530, 528, 1, 0, 0, 0, 531, 534, 1, 0, 0, 0, 532, 530, 1, 0, 0, 0, 532, 533, 1, 0, 0, 0, 533, 95, 1, 0, 0, 0, 534, 532, 1, 0, 0, 0, 535, 536, 3, 64, 32, 0, 536, 537, 5, 31, 0, 0, 537, 538, 3, 68, 34, 0, 538, 97, 1, 0, 0, 0, 539, 540, 7, 6, 0, 0, 540, 99, 1, 0, 0, 0, 541, 544, 3, 102, 51, 0, 542, 544, 3, 104, 52, 0, 543, 541, 1, 0, 0, 0, 543, 542, 1, 0, 0, 0, 544, 101, 1, 0, 0, 0, 545, 547, 7, 0, 0, 0, 546, 545, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 1, 0, 0, 0, 548, 549, 5, 27, 0, 0, 549, 103, 1, 0, 0, 0, 550, 552, 7, 0, 0, 0, 551, 550, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 5, 26, 0, 0, 554, 105, 1, 0, 0, 0, 555, 556, 5, 25, 0, 0, 556, 107, 1, 0, 0, 0, 557, 558, 7, 7, 0, 0, 558, 109, 1, 0, 0, 0, 559, 560, 5, 5, 0, 0, 560, 561, 3, 112, 56, 0, 561, 111, 1, 0, 0, 0, 562, 563, 5, 65, 0, 0, 563, 564, 3, 2, 1, 0, 564, 565, 5, 66, 0, 0, 565, 113, 1, 0, 0, 0, 566, 567, 5, 13, 0, 0, 567, 568, 5, 100, 0, 0, 568, 115, 1, 0, 0, 0, 569, 570, 5, 3, 0, 0, 570, 573, 5, 90, 0, 0, 571, 572, 5, 88, 0, 0, 572, 574, 3, 60, 30, 0, 573, 571, 1, 0, 0, 0, 573, 574, 1, 0, 0, 0, 574, 584, 1, 0, 0, 0, 575, 576, 5, 89, 0, 0, 576, 581, 3, 118, 59, 0, 577, 578, 5, 33, 0, 0, 578, 580, 3, 118, 59, 0, 579, 577, 1, 0, 0, 0, 580, 583, 1, 0, 0, 0, 581, 579, 1, 0, 0, 0, 581, 582, 1, 0, 0, 0, 582, 585, 1, 0, 0, 0, 583, 581, 1, 0, 0, 0, 584, 575, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 117, 1, 0, 0, 0, 586, 587, 3, 60, 30, 0, 587, 588, 5, 31, 0, 0, 588, 590, 1, 0, 0, 0, 589, 586, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 1, 0, 0, 0, 591, 592, 3, 60, 30, 0, 592, 119, 1, 0, 0, 0, 593, 594, 5, 18, 0, 0, 594, 595, 3, 36, 18, 0, 595, 596, 5, 88, 0, 0, 596, 597, 3, 62, 31, 0, 597, 121, 1, 0, 0, 0, 598, 599, 5, 17, 0, 0, 599, 602, 3, 54, 27, 0, 600, 601, 5, 28, 0, 0, 601, 603, 3, 30, 15, 0, 602, 600, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 123, 1, 0, 0, 0, 59, 135, 144, 162, 174, 183, 191, 197, 205, 207, 212, 219, 224, 235, 241, 249, 251, 262, 269, 280, 283, 289, 301, 307, 317, 321, 326, 336, 344, 357, 361, 365, 372, 376, 383, 389, 396, 404, 412, 420, 437, 448, 459, 464, 468, 473, 484, 489, 493, 507, 518, 532, 543, 546, 551, 573, 581, 584, 589, 602] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index af5c03a27592b..9f8197830640c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -1886,26 +1886,21 @@ public final FunctionNameContext functionName() throws RecognitionException { try { setState(289); _errHandler.sync(this); - switch (_input.LA(1)) { - case MATCH: + switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { + case 1: enterOuterAlt(_localctx, 1); { setState(287); match(MATCH); } break; - case PARAM: - case NAMED_OR_POSITIONAL_PARAM: - case UNQUOTED_IDENTIFIER: - case QUOTED_IDENTIFIER: + case 2: enterOuterAlt(_localctx, 2); { setState(288); identifierOrParameter(); } break; - default: - throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -3260,26 +3255,25 @@ public final IdentifierPatternContext identifierPattern() throws RecognitionExce IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); enterRule(_localctx, 66, RULE_identifierPattern); try { - setState(419); + setState(420); _errHandler.sync(this); - switch (_input.LA(1)) { - case ID_PATTERN: + switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) { + case 1: enterOuterAlt(_localctx, 1); { setState(417); match(ID_PATTERN); } break; - case PARAM: - case NAMED_OR_POSITIONAL_PARAM: + case 2: enterOuterAlt(_localctx, 2); { setState(418); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); + setState(419); parameter(); } break; - default: - throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -3549,14 +3543,14 @@ public final ConstantContext constant() throws RecognitionException { enterRule(_localctx, 68, RULE_constant); int _la; try { - setState(463); + setState(464); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(421); + setState(422); match(NULL); } break; @@ -3564,9 +3558,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(422); - integerValue(); setState(423); + integerValue(); + setState(424); match(UNQUOTED_IDENTIFIER); } break; @@ -3574,7 +3568,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(425); + setState(426); decimalValue(); } break; @@ -3582,7 +3576,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(426); + setState(427); integerValue(); } break; @@ -3590,7 +3584,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(427); + setState(428); booleanValue(); } break; @@ -3598,7 +3592,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(428); + setState(429); parameter(); } break; @@ -3606,7 +3600,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(429); + setState(430); string(); } break; @@ -3614,27 +3608,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(430); - match(OPENING_BRACKET); setState(431); + match(OPENING_BRACKET); + setState(432); numericValue(); - setState(436); + setState(437); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(432); - match(COMMA); setState(433); + match(COMMA); + setState(434); numericValue(); } } - setState(438); + setState(439); _errHandler.sync(this); _la = _input.LA(1); } - setState(439); + setState(440); match(CLOSING_BRACKET); } break; @@ -3642,27 +3636,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(441); - match(OPENING_BRACKET); setState(442); + match(OPENING_BRACKET); + setState(443); booleanValue(); - setState(447); + setState(448); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(443); - match(COMMA); setState(444); + match(COMMA); + setState(445); booleanValue(); } } - setState(449); + setState(450); _errHandler.sync(this); _la = _input.LA(1); } - setState(450); + setState(451); match(CLOSING_BRACKET); } break; @@ -3670,27 +3664,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(452); - match(OPENING_BRACKET); setState(453); + match(OPENING_BRACKET); + setState(454); string(); - setState(458); + setState(459); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(454); - match(COMMA); setState(455); + match(COMMA); + setState(456); string(); } } - setState(460); + setState(461); _errHandler.sync(this); _la = _input.LA(1); } - setState(461); + setState(462); match(CLOSING_BRACKET); } break; @@ -3764,14 +3758,14 @@ public final ParameterContext parameter() throws RecognitionException { ParameterContext _localctx = new ParameterContext(_ctx, getState()); enterRule(_localctx, 70, RULE_parameter); try { - setState(467); + setState(468); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(465); + setState(466); match(PARAM); } break; @@ -3779,7 +3773,7 @@ public final ParameterContext parameter() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(466); + setState(467); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3830,27 +3824,25 @@ public final IdentifierOrParameterContext identifierOrParameter() throws Recogni IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); enterRule(_localctx, 72, RULE_identifierOrParameter); try { - setState(471); + setState(473); _errHandler.sync(this); - switch (_input.LA(1)) { - case UNQUOTED_IDENTIFIER: - case QUOTED_IDENTIFIER: + switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { + case 1: enterOuterAlt(_localctx, 1); { - setState(469); + setState(470); identifier(); } break; - case PARAM: - case NAMED_OR_POSITIONAL_PARAM: + case 2: enterOuterAlt(_localctx, 2); { - setState(470); + setState(471); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); + setState(472); parameter(); } break; - default: - throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -3894,9 +3886,9 @@ public final LimitCommandContext limitCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(473); + setState(475); match(LIMIT); - setState(474); + setState(476); match(INTEGER_LITERAL); } } @@ -3951,25 +3943,25 @@ public final SortCommandContext sortCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(476); + setState(478); match(SORT); - setState(477); + setState(479); orderExpression(); - setState(482); + setState(484); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,45,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(478); + setState(480); match(COMMA); - setState(479); + setState(481); orderExpression(); } } } - setState(484); + setState(486); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,45,_ctx); } @@ -4025,14 +4017,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(485); - booleanExpression(0); setState(487); + booleanExpression(0); + setState(489); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { case 1: { - setState(486); + setState(488); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -4046,14 +4038,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(491); + setState(493); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { case 1: { - setState(489); + setState(491); match(NULLS); - setState(490); + setState(492); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -4112,9 +4104,9 @@ public final KeepCommandContext keepCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(493); + setState(495); match(KEEP); - setState(494); + setState(496); qualifiedNamePatterns(); } } @@ -4161,9 +4153,9 @@ public final DropCommandContext dropCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(496); + setState(498); match(DROP); - setState(497); + setState(499); qualifiedNamePatterns(); } } @@ -4218,25 +4210,25 @@ public final RenameCommandContext renameCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(499); + setState(501); match(RENAME); - setState(500); + setState(502); renameClause(); - setState(505); + setState(507); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,48,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(501); + setState(503); match(COMMA); - setState(502); + setState(504); renameClause(); } } } - setState(507); + setState(509); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,48,_ctx); } @@ -4290,11 +4282,11 @@ public final RenameClauseContext renameClause() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(508); + setState(510); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(509); + setState(511); match(AS); - setState(510); + setState(512); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4347,18 +4339,18 @@ public final DissectCommandContext dissectCommand() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(512); + setState(514); match(DISSECT); - setState(513); + setState(515); primaryExpression(0); - setState(514); - string(); setState(516); + string(); + setState(518); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { case 1: { - setState(515); + setState(517); commandOptions(); } break; @@ -4411,11 +4403,11 @@ public final GrokCommandContext grokCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(518); + setState(520); match(GROK); - setState(519); + setState(521); primaryExpression(0); - setState(520); + setState(522); string(); } } @@ -4462,9 +4454,9 @@ public final MvExpandCommandContext mvExpandCommand() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(522); + setState(524); match(MV_EXPAND); - setState(523); + setState(525); qualifiedName(); } } @@ -4518,23 +4510,23 @@ public final CommandOptionsContext commandOptions() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(525); + setState(527); commandOption(); - setState(530); + setState(532); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,50,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(526); + setState(528); match(COMMA); - setState(527); + setState(529); commandOption(); } } } - setState(532); + setState(534); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,50,_ctx); } @@ -4586,11 +4578,11 @@ public final CommandOptionContext commandOption() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(533); + setState(535); identifier(); - setState(534); + setState(536); match(ASSIGN); - setState(535); + setState(537); constant(); } } @@ -4636,7 +4628,7 @@ public final BooleanValueContext booleanValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(537); + setState(539); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4691,20 +4683,20 @@ public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); enterRule(_localctx, 100, RULE_numericValue); try { - setState(541); + setState(543); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(539); + setState(541); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(540); + setState(542); integerValue(); } break; @@ -4753,12 +4745,12 @@ public final DecimalValueContext decimalValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(544); + setState(546); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(543); + setState(545); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4771,7 +4763,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(546); + setState(548); match(DECIMAL_LITERAL); } } @@ -4818,12 +4810,12 @@ public final IntegerValueContext integerValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(549); + setState(551); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(548); + setState(550); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4836,7 +4828,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(551); + setState(553); match(INTEGER_LITERAL); } } @@ -4880,7 +4872,7 @@ public final StringContext string() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(553); + setState(555); match(QUOTED_STRING); } } @@ -4930,7 +4922,7 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(555); + setState(557); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 281474976710656000L) != 0)) ) { _errHandler.recoverInline(this); @@ -4985,9 +4977,9 @@ public final ExplainCommandContext explainCommand() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(557); + setState(559); match(EXPLAIN); - setState(558); + setState(560); subqueryExpression(); } } @@ -5035,11 +5027,11 @@ public final SubqueryExpressionContext subqueryExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(560); + setState(562); match(OPENING_BRACKET); - setState(561); + setState(563); query(0); - setState(562); + setState(564); match(CLOSING_BRACKET); } } @@ -5096,9 +5088,9 @@ public final ShowCommandContext showCommand() throws RecognitionException { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(564); + setState(566); match(SHOW); - setState(565); + setState(567); match(INFO); } } @@ -5161,46 +5153,46 @@ public final EnrichCommandContext enrichCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(567); + setState(569); match(ENRICH); - setState(568); + setState(570); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(571); + setState(573); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { case 1: { - setState(569); + setState(571); match(ON); - setState(570); + setState(572); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(582); + setState(584); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { case 1: { - setState(573); + setState(575); match(WITH); - setState(574); + setState(576); enrichWithClause(); - setState(579); + setState(581); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,55,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(575); + setState(577); match(COMMA); - setState(576); + setState(578); enrichWithClause(); } } } - setState(581); + setState(583); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,55,_ctx); } @@ -5257,19 +5249,19 @@ public final EnrichWithClauseContext enrichWithClause() throws RecognitionExcept try { enterOuterAlt(_localctx, 1); { - setState(587); + setState(589); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { case 1: { - setState(584); + setState(586); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(585); + setState(587); match(ASSIGN); } break; } - setState(589); + setState(591); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5322,13 +5314,13 @@ public final LookupCommandContext lookupCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(591); + setState(593); match(DEV_LOOKUP); - setState(592); + setState(594); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(593); + setState(595); match(ON); - setState(594); + setState(596); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5381,18 +5373,18 @@ public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(596); + setState(598); match(DEV_INLINESTATS); - setState(597); + setState(599); ((InlinestatsCommandContext)_localctx).stats = aggFields(); - setState(600); + setState(602); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) { case 1: { - setState(598); + setState(600); match(BY); - setState(599); + setState(601); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5424,6 +5416,10 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex); case 10: return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex); + case 33: + return identifierPattern_sempred((IdentifierPatternContext)_localctx, predIndex); + case 36: + return identifierOrParameter_sempred((IdentifierOrParameterContext)_localctx, predIndex); } return true; } @@ -5477,9 +5473,23 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in } return true; } + private boolean identifierPattern_sempred(IdentifierPatternContext _localctx, int predIndex) { + switch (predIndex) { + case 10: + return this.isDevVersion(); + } + return true; + } + private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _localctx, int predIndex) { + switch (predIndex) { + case 11: + return this.isDevVersion(); + } + return true; + } public static final String _serializedATN = - "\u0004\u0001x\u025b\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ + "\u0004\u0001x\u025d\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ @@ -5540,326 +5550,327 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in "\u0001\u001d\u0005\u001d\u018b\b\u001d\n\u001d\f\u001d\u018e\t\u001d\u0001"+ "\u001e\u0001\u001e\u0001\u001e\u0005\u001e\u0193\b\u001e\n\u001e\f\u001e"+ "\u0196\t\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u019b\b"+ - "\u001f\n\u001f\f\u001f\u019e\t\u001f\u0001 \u0001 \u0001!\u0001!\u0003"+ - "!\u01a4\b!\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01b3\b\"\n\"\f\"\u01b6"+ - "\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01be\b\""+ - "\n\"\f\"\u01c1\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005"+ - "\"\u01c9\b\"\n\"\f\"\u01cc\t\"\u0001\"\u0001\"\u0003\"\u01d0\b\"\u0001"+ - "#\u0001#\u0003#\u01d4\b#\u0001$\u0001$\u0003$\u01d8\b$\u0001%\u0001%\u0001"+ - "%\u0001&\u0001&\u0001&\u0001&\u0005&\u01e1\b&\n&\f&\u01e4\t&\u0001\'\u0001"+ - "\'\u0003\'\u01e8\b\'\u0001\'\u0001\'\u0003\'\u01ec\b\'\u0001(\u0001(\u0001"+ - "(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001*\u0005*\u01f8\b*\n*"+ - "\f*\u01fb\t*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0003"+ - ",\u0205\b,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001"+ - "/\u0001/\u0005/\u0211\b/\n/\f/\u0214\t/\u00010\u00010\u00010\u00010\u0001"+ - "1\u00011\u00012\u00012\u00032\u021e\b2\u00013\u00033\u0221\b3\u00013\u0001"+ - "3\u00014\u00034\u0226\b4\u00014\u00014\u00015\u00015\u00016\u00016\u0001"+ - "7\u00017\u00017\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u0001"+ - ":\u0001:\u0001:\u0001:\u0003:\u023c\b:\u0001:\u0001:\u0001:\u0001:\u0005"+ - ":\u0242\b:\n:\f:\u0245\t:\u0003:\u0247\b:\u0001;\u0001;\u0001;\u0003;"+ - "\u024c\b;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001"+ - "=\u0001=\u0001=\u0003=\u0259\b=\u0001=\u0000\u0004\u0002\n\u0012\u0014"+ - ">\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a"+ - "\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvxz\u0000\b\u0001"+ - "\u0000:;\u0001\u0000<>\u0002\u0000\u0019\u0019LL\u0001\u0000CD\u0002\u0000"+ - "\u001e\u001e\"\"\u0002\u0000%%((\u0002\u0000$$22\u0002\u00003359\u0275"+ - "\u0000|\u0001\u0000\u0000\u0000\u0002\u007f\u0001\u0000\u0000\u0000\u0004"+ - "\u0090\u0001\u0000\u0000\u0000\u0006\u00a2\u0001\u0000\u0000\u0000\b\u00a4"+ - "\u0001\u0000\u0000\u0000\n\u00c5\u0001\u0000\u0000\u0000\f\u00e0\u0001"+ - "\u0000\u0000\u0000\u000e\u00e2\u0001\u0000\u0000\u0000\u0010\u00eb\u0001"+ - "\u0000\u0000\u0000\u0012\u00f1\u0001\u0000\u0000\u0000\u0014\u0106\u0001"+ - "\u0000\u0000\u0000\u0016\u0110\u0001\u0000\u0000\u0000\u0018\u0121\u0001"+ - "\u0000\u0000\u0000\u001a\u0123\u0001\u0000\u0000\u0000\u001c\u0125\u0001"+ - "\u0000\u0000\u0000\u001e\u0128\u0001\u0000\u0000\u0000 \u0133\u0001\u0000"+ - "\u0000\u0000\"\u0137\u0001\u0000\u0000\u0000$\u0146\u0001\u0000\u0000"+ - "\u0000&\u014a\u0001\u0000\u0000\u0000(\u014c\u0001\u0000\u0000\u0000*"+ - "\u0150\u0001\u0000\u0000\u0000,\u0152\u0001\u0000\u0000\u0000.\u015b\u0001"+ - "\u0000\u0000\u00000\u015f\u0001\u0000\u0000\u00002\u016f\u0001\u0000\u0000"+ - "\u00004\u0172\u0001\u0000\u0000\u00006\u017a\u0001\u0000\u0000\u00008"+ - "\u0182\u0001\u0000\u0000\u0000:\u0187\u0001\u0000\u0000\u0000<\u018f\u0001"+ - "\u0000\u0000\u0000>\u0197\u0001\u0000\u0000\u0000@\u019f\u0001\u0000\u0000"+ - "\u0000B\u01a3\u0001\u0000\u0000\u0000D\u01cf\u0001\u0000\u0000\u0000F"+ - "\u01d3\u0001\u0000\u0000\u0000H\u01d7\u0001\u0000\u0000\u0000J\u01d9\u0001"+ - "\u0000\u0000\u0000L\u01dc\u0001\u0000\u0000\u0000N\u01e5\u0001\u0000\u0000"+ - "\u0000P\u01ed\u0001\u0000\u0000\u0000R\u01f0\u0001\u0000\u0000\u0000T"+ - "\u01f3\u0001\u0000\u0000\u0000V\u01fc\u0001\u0000\u0000\u0000X\u0200\u0001"+ - "\u0000\u0000\u0000Z\u0206\u0001\u0000\u0000\u0000\\\u020a\u0001\u0000"+ - "\u0000\u0000^\u020d\u0001\u0000\u0000\u0000`\u0215\u0001\u0000\u0000\u0000"+ - "b\u0219\u0001\u0000\u0000\u0000d\u021d\u0001\u0000\u0000\u0000f\u0220"+ - "\u0001\u0000\u0000\u0000h\u0225\u0001\u0000\u0000\u0000j\u0229\u0001\u0000"+ - "\u0000\u0000l\u022b\u0001\u0000\u0000\u0000n\u022d\u0001\u0000\u0000\u0000"+ - "p\u0230\u0001\u0000\u0000\u0000r\u0234\u0001\u0000\u0000\u0000t\u0237"+ - "\u0001\u0000\u0000\u0000v\u024b\u0001\u0000\u0000\u0000x\u024f\u0001\u0000"+ - "\u0000\u0000z\u0254\u0001\u0000\u0000\u0000|}\u0003\u0002\u0001\u0000"+ - "}~\u0005\u0000\u0000\u0001~\u0001\u0001\u0000\u0000\u0000\u007f\u0080"+ - "\u0006\u0001\uffff\uffff\u0000\u0080\u0081\u0003\u0004\u0002\u0000\u0081"+ - "\u0087\u0001\u0000\u0000\u0000\u0082\u0083\n\u0001\u0000\u0000\u0083\u0084"+ - "\u0005\u0018\u0000\u0000\u0084\u0086\u0003\u0006\u0003\u0000\u0085\u0082"+ - "\u0001\u0000\u0000\u0000\u0086\u0089\u0001\u0000\u0000\u0000\u0087\u0085"+ - "\u0001\u0000\u0000\u0000\u0087\u0088\u0001\u0000\u0000\u0000\u0088\u0003"+ - "\u0001\u0000\u0000\u0000\u0089\u0087\u0001\u0000\u0000\u0000\u008a\u0091"+ - "\u0003n7\u0000\u008b\u0091\u0003\"\u0011\u0000\u008c\u0091\u0003\u001c"+ - "\u000e\u0000\u008d\u0091\u0003r9\u0000\u008e\u008f\u0004\u0002\u0001\u0000"+ - "\u008f\u0091\u00030\u0018\u0000\u0090\u008a\u0001\u0000\u0000\u0000\u0090"+ - "\u008b\u0001\u0000\u0000\u0000\u0090\u008c\u0001\u0000\u0000\u0000\u0090"+ - "\u008d\u0001\u0000\u0000\u0000\u0090\u008e\u0001\u0000\u0000\u0000\u0091"+ - "\u0005\u0001\u0000\u0000\u0000\u0092\u00a3\u00032\u0019\u0000\u0093\u00a3"+ - "\u0003\b\u0004\u0000\u0094\u00a3\u0003P(\u0000\u0095\u00a3\u0003J%\u0000"+ - "\u0096\u00a3\u00034\u001a\u0000\u0097\u00a3\u0003L&\u0000\u0098\u00a3"+ - "\u0003R)\u0000\u0099\u00a3\u0003T*\u0000\u009a\u00a3\u0003X,\u0000\u009b"+ - "\u00a3\u0003Z-\u0000\u009c\u00a3\u0003t:\u0000\u009d\u00a3\u0003\\.\u0000"+ - "\u009e\u009f\u0004\u0003\u0002\u0000\u009f\u00a3\u0003z=\u0000\u00a0\u00a1"+ - "\u0004\u0003\u0003\u0000\u00a1\u00a3\u0003x<\u0000\u00a2\u0092\u0001\u0000"+ - "\u0000\u0000\u00a2\u0093\u0001\u0000\u0000\u0000\u00a2\u0094\u0001\u0000"+ - "\u0000\u0000\u00a2\u0095\u0001\u0000\u0000\u0000\u00a2\u0096\u0001\u0000"+ - "\u0000\u0000\u00a2\u0097\u0001\u0000\u0000\u0000\u00a2\u0098\u0001\u0000"+ - "\u0000\u0000\u00a2\u0099\u0001\u0000\u0000\u0000\u00a2\u009a\u0001\u0000"+ - "\u0000\u0000\u00a2\u009b\u0001\u0000\u0000\u0000\u00a2\u009c\u0001\u0000"+ - "\u0000\u0000\u00a2\u009d\u0001\u0000\u0000\u0000\u00a2\u009e\u0001\u0000"+ - "\u0000\u0000\u00a2\u00a0\u0001\u0000\u0000\u0000\u00a3\u0007\u0001\u0000"+ - "\u0000\u0000\u00a4\u00a5\u0005\u0010\u0000\u0000\u00a5\u00a6\u0003\n\u0005"+ - "\u0000\u00a6\t\u0001\u0000\u0000\u0000\u00a7\u00a8\u0006\u0005\uffff\uffff"+ - "\u0000\u00a8\u00a9\u0005+\u0000\u0000\u00a9\u00c6\u0003\n\u0005\b\u00aa"+ - "\u00c6\u0003\u0010\b\u0000\u00ab\u00c6\u0003\f\u0006\u0000\u00ac\u00ae"+ - "\u0003\u0010\b\u0000\u00ad\u00af\u0005+\u0000\u0000\u00ae\u00ad\u0001"+ - "\u0000\u0000\u0000\u00ae\u00af\u0001\u0000\u0000\u0000\u00af\u00b0\u0001"+ - "\u0000\u0000\u0000\u00b0\u00b1\u0005&\u0000\u0000\u00b1\u00b2\u0005*\u0000"+ - "\u0000\u00b2\u00b7\u0003\u0010\b\u0000\u00b3\u00b4\u0005!\u0000\u0000"+ - "\u00b4\u00b6\u0003\u0010\b\u0000\u00b5\u00b3\u0001\u0000\u0000\u0000\u00b6"+ - "\u00b9\u0001\u0000\u0000\u0000\u00b7\u00b5\u0001\u0000\u0000\u0000\u00b7"+ - "\u00b8\u0001\u0000\u0000\u0000\u00b8\u00ba\u0001\u0000\u0000\u0000\u00b9"+ - "\u00b7\u0001\u0000\u0000\u0000\u00ba\u00bb\u00051\u0000\u0000\u00bb\u00c6"+ - "\u0001\u0000\u0000\u0000\u00bc\u00bd\u0003\u0010\b\u0000\u00bd\u00bf\u0005"+ - "\'\u0000\u0000\u00be\u00c0\u0005+\u0000\u0000\u00bf\u00be\u0001\u0000"+ - "\u0000\u0000\u00bf\u00c0\u0001\u0000\u0000\u0000\u00c0\u00c1\u0001\u0000"+ - "\u0000\u0000\u00c1\u00c2\u0005,\u0000\u0000\u00c2\u00c6\u0001\u0000\u0000"+ - "\u0000\u00c3\u00c4\u0004\u0005\u0004\u0000\u00c4\u00c6\u0003\u000e\u0007"+ - "\u0000\u00c5\u00a7\u0001\u0000\u0000\u0000\u00c5\u00aa\u0001\u0000\u0000"+ - "\u0000\u00c5\u00ab\u0001\u0000\u0000\u0000\u00c5\u00ac\u0001\u0000\u0000"+ - "\u0000\u00c5\u00bc\u0001\u0000\u0000\u0000\u00c5\u00c3\u0001\u0000\u0000"+ - "\u0000\u00c6\u00cf\u0001\u0000\u0000\u0000\u00c7\u00c8\n\u0005\u0000\u0000"+ - "\u00c8\u00c9\u0005\u001d\u0000\u0000\u00c9\u00ce\u0003\n\u0005\u0006\u00ca"+ - "\u00cb\n\u0004\u0000\u0000\u00cb\u00cc\u0005.\u0000\u0000\u00cc\u00ce"+ - "\u0003\n\u0005\u0005\u00cd\u00c7\u0001\u0000\u0000\u0000\u00cd\u00ca\u0001"+ - "\u0000\u0000\u0000\u00ce\u00d1\u0001\u0000\u0000\u0000\u00cf\u00cd\u0001"+ - "\u0000\u0000\u0000\u00cf\u00d0\u0001\u0000\u0000\u0000\u00d0\u000b\u0001"+ - "\u0000\u0000\u0000\u00d1\u00cf\u0001\u0000\u0000\u0000\u00d2\u00d4\u0003"+ - "\u0010\b\u0000\u00d3\u00d5\u0005+\u0000\u0000\u00d4\u00d3\u0001\u0000"+ - "\u0000\u0000\u00d4\u00d5\u0001\u0000\u0000\u0000\u00d5\u00d6\u0001\u0000"+ - "\u0000\u0000\u00d6\u00d7\u0005)\u0000\u0000\u00d7\u00d8\u0003j5\u0000"+ - "\u00d8\u00e1\u0001\u0000\u0000\u0000\u00d9\u00db\u0003\u0010\b\u0000\u00da"+ - "\u00dc\u0005+\u0000\u0000\u00db\u00da\u0001\u0000\u0000\u0000\u00db\u00dc"+ - "\u0001\u0000\u0000\u0000\u00dc\u00dd\u0001\u0000\u0000\u0000\u00dd\u00de"+ - "\u00050\u0000\u0000\u00de\u00df\u0003j5\u0000\u00df\u00e1\u0001\u0000"+ - "\u0000\u0000\u00e0\u00d2\u0001\u0000\u0000\u0000\u00e0\u00d9\u0001\u0000"+ - "\u0000\u0000\u00e1\r\u0001\u0000\u0000\u0000\u00e2\u00e3\u0003\u0010\b"+ - "\u0000\u00e3\u00e4\u0005?\u0000\u0000\u00e4\u00e5\u0003j5\u0000\u00e5"+ - "\u000f\u0001\u0000\u0000\u0000\u00e6\u00ec\u0003\u0012\t\u0000\u00e7\u00e8"+ - "\u0003\u0012\t\u0000\u00e8\u00e9\u0003l6\u0000\u00e9\u00ea\u0003\u0012"+ - "\t\u0000\u00ea\u00ec\u0001\u0000\u0000\u0000\u00eb\u00e6\u0001\u0000\u0000"+ - "\u0000\u00eb\u00e7\u0001\u0000\u0000\u0000\u00ec\u0011\u0001\u0000\u0000"+ - "\u0000\u00ed\u00ee\u0006\t\uffff\uffff\u0000\u00ee\u00f2\u0003\u0014\n"+ - "\u0000\u00ef\u00f0\u0007\u0000\u0000\u0000\u00f0\u00f2\u0003\u0012\t\u0003"+ - "\u00f1\u00ed\u0001\u0000\u0000\u0000\u00f1\u00ef\u0001\u0000\u0000\u0000"+ - "\u00f2\u00fb\u0001\u0000\u0000\u0000\u00f3\u00f4\n\u0002\u0000\u0000\u00f4"+ - "\u00f5\u0007\u0001\u0000\u0000\u00f5\u00fa\u0003\u0012\t\u0003\u00f6\u00f7"+ - "\n\u0001\u0000\u0000\u00f7\u00f8\u0007\u0000\u0000\u0000\u00f8\u00fa\u0003"+ - "\u0012\t\u0002\u00f9\u00f3\u0001\u0000\u0000\u0000\u00f9\u00f6\u0001\u0000"+ - "\u0000\u0000\u00fa\u00fd\u0001\u0000\u0000\u0000\u00fb\u00f9\u0001\u0000"+ - "\u0000\u0000\u00fb\u00fc\u0001\u0000\u0000\u0000\u00fc\u0013\u0001\u0000"+ - "\u0000\u0000\u00fd\u00fb\u0001\u0000\u0000\u0000\u00fe\u00ff\u0006\n\uffff"+ - "\uffff\u0000\u00ff\u0107\u0003D\"\u0000\u0100\u0107\u0003:\u001d\u0000"+ - "\u0101\u0107\u0003\u0016\u000b\u0000\u0102\u0103\u0005*\u0000\u0000\u0103"+ - "\u0104\u0003\n\u0005\u0000\u0104\u0105\u00051\u0000\u0000\u0105\u0107"+ - "\u0001\u0000\u0000\u0000\u0106\u00fe\u0001\u0000\u0000\u0000\u0106\u0100"+ - "\u0001\u0000\u0000\u0000\u0106\u0101\u0001\u0000\u0000\u0000\u0106\u0102"+ - "\u0001\u0000\u0000\u0000\u0107\u010d\u0001\u0000\u0000\u0000\u0108\u0109"+ - "\n\u0001\u0000\u0000\u0109\u010a\u0005 \u0000\u0000\u010a\u010c\u0003"+ - "\u001a\r\u0000\u010b\u0108\u0001\u0000\u0000\u0000\u010c\u010f\u0001\u0000"+ - "\u0000\u0000\u010d\u010b\u0001\u0000\u0000\u0000\u010d\u010e\u0001\u0000"+ - "\u0000\u0000\u010e\u0015\u0001\u0000\u0000\u0000\u010f\u010d\u0001\u0000"+ - "\u0000\u0000\u0110\u0111\u0003\u0018\f\u0000\u0111\u011b\u0005*\u0000"+ - "\u0000\u0112\u011c\u0005<\u0000\u0000\u0113\u0118\u0003\n\u0005\u0000"+ - "\u0114\u0115\u0005!\u0000\u0000\u0115\u0117\u0003\n\u0005\u0000\u0116"+ - "\u0114\u0001\u0000\u0000\u0000\u0117\u011a\u0001\u0000\u0000\u0000\u0118"+ - "\u0116\u0001\u0000\u0000\u0000\u0118\u0119\u0001\u0000\u0000\u0000\u0119"+ - "\u011c\u0001\u0000\u0000\u0000\u011a\u0118\u0001\u0000\u0000\u0000\u011b"+ - "\u0112\u0001\u0000\u0000\u0000\u011b\u0113\u0001\u0000\u0000\u0000\u011b"+ - "\u011c\u0001\u0000\u0000\u0000\u011c\u011d\u0001\u0000\u0000\u0000\u011d"+ - "\u011e\u00051\u0000\u0000\u011e\u0017\u0001\u0000\u0000\u0000\u011f\u0122"+ - "\u0005?\u0000\u0000\u0120\u0122\u0003H$\u0000\u0121\u011f\u0001\u0000"+ - "\u0000\u0000\u0121\u0120\u0001\u0000\u0000\u0000\u0122\u0019\u0001\u0000"+ - "\u0000\u0000\u0123\u0124\u0003@ \u0000\u0124\u001b\u0001\u0000\u0000\u0000"+ - "\u0125\u0126\u0005\f\u0000\u0000\u0126\u0127\u0003\u001e\u000f\u0000\u0127"+ - "\u001d\u0001\u0000\u0000\u0000\u0128\u012d\u0003 \u0010\u0000\u0129\u012a"+ - "\u0005!\u0000\u0000\u012a\u012c\u0003 \u0010\u0000\u012b\u0129\u0001\u0000"+ - "\u0000\u0000\u012c\u012f\u0001\u0000\u0000\u0000\u012d\u012b\u0001\u0000"+ - "\u0000\u0000\u012d\u012e\u0001\u0000\u0000\u0000\u012e\u001f\u0001\u0000"+ - "\u0000\u0000\u012f\u012d\u0001\u0000\u0000\u0000\u0130\u0131\u0003:\u001d"+ - "\u0000\u0131\u0132\u0005\u001f\u0000\u0000\u0132\u0134\u0001\u0000\u0000"+ - "\u0000\u0133\u0130\u0001\u0000\u0000\u0000\u0133\u0134\u0001\u0000\u0000"+ - "\u0000\u0134\u0135\u0001\u0000\u0000\u0000\u0135\u0136\u0003\n\u0005\u0000"+ - "\u0136!\u0001\u0000\u0000\u0000\u0137\u0138\u0005\u0006\u0000\u0000\u0138"+ - "\u013d\u0003$\u0012\u0000\u0139\u013a\u0005!\u0000\u0000\u013a\u013c\u0003"+ - "$\u0012\u0000\u013b\u0139\u0001\u0000\u0000\u0000\u013c\u013f\u0001\u0000"+ - "\u0000\u0000\u013d\u013b\u0001\u0000\u0000\u0000\u013d\u013e\u0001\u0000"+ - "\u0000\u0000\u013e\u0141\u0001\u0000\u0000\u0000\u013f\u013d\u0001\u0000"+ - "\u0000\u0000\u0140\u0142\u0003*\u0015\u0000\u0141\u0140\u0001\u0000\u0000"+ - "\u0000\u0141\u0142\u0001\u0000\u0000\u0000\u0142#\u0001\u0000\u0000\u0000"+ - "\u0143\u0144\u0003&\u0013\u0000\u0144\u0145\u0005h\u0000\u0000\u0145\u0147"+ - "\u0001\u0000\u0000\u0000\u0146\u0143\u0001\u0000\u0000\u0000\u0146\u0147"+ - "\u0001\u0000\u0000\u0000\u0147\u0148\u0001\u0000\u0000\u0000\u0148\u0149"+ - "\u0003(\u0014\u0000\u0149%\u0001\u0000\u0000\u0000\u014a\u014b\u0005L"+ - "\u0000\u0000\u014b\'\u0001\u0000\u0000\u0000\u014c\u014d\u0007\u0002\u0000"+ - "\u0000\u014d)\u0001\u0000\u0000\u0000\u014e\u0151\u0003,\u0016\u0000\u014f"+ - "\u0151\u0003.\u0017\u0000\u0150\u014e\u0001\u0000\u0000\u0000\u0150\u014f"+ - "\u0001\u0000\u0000\u0000\u0151+\u0001\u0000\u0000\u0000\u0152\u0153\u0005"+ - "K\u0000\u0000\u0153\u0158\u0005L\u0000\u0000\u0154\u0155\u0005!\u0000"+ - "\u0000\u0155\u0157\u0005L\u0000\u0000\u0156\u0154\u0001\u0000\u0000\u0000"+ - "\u0157\u015a\u0001\u0000\u0000\u0000\u0158\u0156\u0001\u0000\u0000\u0000"+ - "\u0158\u0159\u0001\u0000\u0000\u0000\u0159-\u0001\u0000\u0000\u0000\u015a"+ - "\u0158\u0001\u0000\u0000\u0000\u015b\u015c\u0005A\u0000\u0000\u015c\u015d"+ - "\u0003,\u0016\u0000\u015d\u015e\u0005B\u0000\u0000\u015e/\u0001\u0000"+ - "\u0000\u0000\u015f\u0160\u0005\u0013\u0000\u0000\u0160\u0165\u0003$\u0012"+ - "\u0000\u0161\u0162\u0005!\u0000\u0000\u0162\u0164\u0003$\u0012\u0000\u0163"+ - "\u0161\u0001\u0000\u0000\u0000\u0164\u0167\u0001\u0000\u0000\u0000\u0165"+ - "\u0163\u0001\u0000\u0000\u0000\u0165\u0166\u0001\u0000\u0000\u0000\u0166"+ - "\u0169\u0001\u0000\u0000\u0000\u0167\u0165\u0001\u0000\u0000\u0000\u0168"+ - "\u016a\u00036\u001b\u0000\u0169\u0168\u0001\u0000\u0000\u0000\u0169\u016a"+ - "\u0001\u0000\u0000\u0000\u016a\u016d\u0001\u0000\u0000\u0000\u016b\u016c"+ - "\u0005\u001c\u0000\u0000\u016c\u016e\u0003\u001e\u000f\u0000\u016d\u016b"+ - "\u0001\u0000\u0000\u0000\u016d\u016e\u0001\u0000\u0000\u0000\u016e1\u0001"+ - "\u0000\u0000\u0000\u016f\u0170\u0005\u0004\u0000\u0000\u0170\u0171\u0003"+ - "\u001e\u000f\u0000\u01713\u0001\u0000\u0000\u0000\u0172\u0174\u0005\u000f"+ - "\u0000\u0000\u0173\u0175\u00036\u001b\u0000\u0174\u0173\u0001\u0000\u0000"+ - "\u0000\u0174\u0175\u0001\u0000\u0000\u0000\u0175\u0178\u0001\u0000\u0000"+ - "\u0000\u0176\u0177\u0005\u001c\u0000\u0000\u0177\u0179\u0003\u001e\u000f"+ - "\u0000\u0178\u0176\u0001\u0000\u0000\u0000\u0178\u0179\u0001\u0000\u0000"+ - "\u0000\u01795\u0001\u0000\u0000\u0000\u017a\u017f\u00038\u001c\u0000\u017b"+ - "\u017c\u0005!\u0000\u0000\u017c\u017e\u00038\u001c\u0000\u017d\u017b\u0001"+ - "\u0000\u0000\u0000\u017e\u0181\u0001\u0000\u0000\u0000\u017f\u017d\u0001"+ - "\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000\u0000\u01807\u0001\u0000"+ - "\u0000\u0000\u0181\u017f\u0001\u0000\u0000\u0000\u0182\u0185\u0003 \u0010"+ - "\u0000\u0183\u0184\u0005\u0010\u0000\u0000\u0184\u0186\u0003\n\u0005\u0000"+ - "\u0185\u0183\u0001\u0000\u0000\u0000\u0185\u0186\u0001\u0000\u0000\u0000"+ - "\u01869\u0001\u0000\u0000\u0000\u0187\u018c\u0003H$\u0000\u0188\u0189"+ - "\u0005#\u0000\u0000\u0189\u018b\u0003H$\u0000\u018a\u0188\u0001\u0000"+ - "\u0000\u0000\u018b\u018e\u0001\u0000\u0000\u0000\u018c\u018a\u0001\u0000"+ - "\u0000\u0000\u018c\u018d\u0001\u0000\u0000\u0000\u018d;\u0001\u0000\u0000"+ - "\u0000\u018e\u018c\u0001\u0000\u0000\u0000\u018f\u0194\u0003B!\u0000\u0190"+ - "\u0191\u0005#\u0000\u0000\u0191\u0193\u0003B!\u0000\u0192\u0190\u0001"+ - "\u0000\u0000\u0000\u0193\u0196\u0001\u0000\u0000\u0000\u0194\u0192\u0001"+ - "\u0000\u0000\u0000\u0194\u0195\u0001\u0000\u0000\u0000\u0195=\u0001\u0000"+ - "\u0000\u0000\u0196\u0194\u0001\u0000\u0000\u0000\u0197\u019c\u0003<\u001e"+ - "\u0000\u0198\u0199\u0005!\u0000\u0000\u0199\u019b\u0003<\u001e\u0000\u019a"+ - "\u0198\u0001\u0000\u0000\u0000\u019b\u019e\u0001\u0000\u0000\u0000\u019c"+ - "\u019a\u0001\u0000\u0000\u0000\u019c\u019d\u0001\u0000\u0000\u0000\u019d"+ - "?\u0001\u0000\u0000\u0000\u019e\u019c\u0001\u0000\u0000\u0000\u019f\u01a0"+ - "\u0007\u0003\u0000\u0000\u01a0A\u0001\u0000\u0000\u0000\u01a1\u01a4\u0005"+ - "P\u0000\u0000\u01a2\u01a4\u0003F#\u0000\u01a3\u01a1\u0001\u0000\u0000"+ - "\u0000\u01a3\u01a2\u0001\u0000\u0000\u0000\u01a4C\u0001\u0000\u0000\u0000"+ - "\u01a5\u01d0\u0005,\u0000\u0000\u01a6\u01a7\u0003h4\u0000\u01a7\u01a8"+ - "\u0005C\u0000\u0000\u01a8\u01d0\u0001\u0000\u0000\u0000\u01a9\u01d0\u0003"+ - "f3\u0000\u01aa\u01d0\u0003h4\u0000\u01ab\u01d0\u0003b1\u0000\u01ac\u01d0"+ - "\u0003F#\u0000\u01ad\u01d0\u0003j5\u0000\u01ae\u01af\u0005A\u0000\u0000"+ - "\u01af\u01b4\u0003d2\u0000\u01b0\u01b1\u0005!\u0000\u0000\u01b1\u01b3"+ - "\u0003d2\u0000\u01b2\u01b0\u0001\u0000\u0000\u0000\u01b3\u01b6\u0001\u0000"+ - "\u0000\u0000\u01b4\u01b2\u0001\u0000\u0000\u0000\u01b4\u01b5\u0001\u0000"+ - "\u0000\u0000\u01b5\u01b7\u0001\u0000\u0000\u0000\u01b6\u01b4\u0001\u0000"+ - "\u0000\u0000\u01b7\u01b8\u0005B\u0000\u0000\u01b8\u01d0\u0001\u0000\u0000"+ - "\u0000\u01b9\u01ba\u0005A\u0000\u0000\u01ba\u01bf\u0003b1\u0000\u01bb"+ - "\u01bc\u0005!\u0000\u0000\u01bc\u01be\u0003b1\u0000\u01bd\u01bb\u0001"+ - "\u0000\u0000\u0000\u01be\u01c1\u0001\u0000\u0000\u0000\u01bf\u01bd\u0001"+ - "\u0000\u0000\u0000\u01bf\u01c0\u0001\u0000\u0000\u0000\u01c0\u01c2\u0001"+ - "\u0000\u0000\u0000\u01c1\u01bf\u0001\u0000\u0000\u0000\u01c2\u01c3\u0005"+ - "B\u0000\u0000\u01c3\u01d0\u0001\u0000\u0000\u0000\u01c4\u01c5\u0005A\u0000"+ - "\u0000\u01c5\u01ca\u0003j5\u0000\u01c6\u01c7\u0005!\u0000\u0000\u01c7"+ - "\u01c9\u0003j5\u0000\u01c8\u01c6\u0001\u0000\u0000\u0000\u01c9\u01cc\u0001"+ - "\u0000\u0000\u0000\u01ca\u01c8\u0001\u0000\u0000\u0000\u01ca\u01cb\u0001"+ - "\u0000\u0000\u0000\u01cb\u01cd\u0001\u0000\u0000\u0000\u01cc\u01ca\u0001"+ - "\u0000\u0000\u0000\u01cd\u01ce\u0005B\u0000\u0000\u01ce\u01d0\u0001\u0000"+ - "\u0000\u0000\u01cf\u01a5\u0001\u0000\u0000\u0000\u01cf\u01a6\u0001\u0000"+ - "\u0000\u0000\u01cf\u01a9\u0001\u0000\u0000\u0000\u01cf\u01aa\u0001\u0000"+ - "\u0000\u0000\u01cf\u01ab\u0001\u0000\u0000\u0000\u01cf\u01ac\u0001\u0000"+ - "\u0000\u0000\u01cf\u01ad\u0001\u0000\u0000\u0000\u01cf\u01ae\u0001\u0000"+ - "\u0000\u0000\u01cf\u01b9\u0001\u0000\u0000\u0000\u01cf\u01c4\u0001\u0000"+ - "\u0000\u0000\u01d0E\u0001\u0000\u0000\u0000\u01d1\u01d4\u0005/\u0000\u0000"+ - "\u01d2\u01d4\u0005@\u0000\u0000\u01d3\u01d1\u0001\u0000\u0000\u0000\u01d3"+ - "\u01d2\u0001\u0000\u0000\u0000\u01d4G\u0001\u0000\u0000\u0000\u01d5\u01d8"+ - "\u0003@ \u0000\u01d6\u01d8\u0003F#\u0000\u01d7\u01d5\u0001\u0000\u0000"+ - "\u0000\u01d7\u01d6\u0001\u0000\u0000\u0000\u01d8I\u0001\u0000\u0000\u0000"+ - "\u01d9\u01da\u0005\t\u0000\u0000\u01da\u01db\u0005\u001a\u0000\u0000\u01db"+ - "K\u0001\u0000\u0000\u0000\u01dc\u01dd\u0005\u000e\u0000\u0000\u01dd\u01e2"+ - "\u0003N\'\u0000\u01de\u01df\u0005!\u0000\u0000\u01df\u01e1\u0003N\'\u0000"+ - "\u01e0\u01de\u0001\u0000\u0000\u0000\u01e1\u01e4\u0001\u0000\u0000\u0000"+ - "\u01e2\u01e0\u0001\u0000\u0000\u0000\u01e2\u01e3\u0001\u0000\u0000\u0000"+ - "\u01e3M\u0001\u0000\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5"+ - "\u01e7\u0003\n\u0005\u0000\u01e6\u01e8\u0007\u0004\u0000\u0000\u01e7\u01e6"+ - "\u0001\u0000\u0000\u0000\u01e7\u01e8\u0001\u0000\u0000\u0000\u01e8\u01eb"+ - "\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005-\u0000\u0000\u01ea\u01ec\u0007"+ - "\u0005\u0000\u0000\u01eb\u01e9\u0001\u0000\u0000\u0000\u01eb\u01ec\u0001"+ - "\u0000\u0000\u0000\u01ecO\u0001\u0000\u0000\u0000\u01ed\u01ee\u0005\b"+ - "\u0000\u0000\u01ee\u01ef\u0003>\u001f\u0000\u01efQ\u0001\u0000\u0000\u0000"+ - "\u01f0\u01f1\u0005\u0002\u0000\u0000\u01f1\u01f2\u0003>\u001f\u0000\u01f2"+ - "S\u0001\u0000\u0000\u0000\u01f3\u01f4\u0005\u000b\u0000\u0000\u01f4\u01f9"+ - "\u0003V+\u0000\u01f5\u01f6\u0005!\u0000\u0000\u01f6\u01f8\u0003V+\u0000"+ - "\u01f7\u01f5\u0001\u0000\u0000\u0000\u01f8\u01fb\u0001\u0000\u0000\u0000"+ - "\u01f9\u01f7\u0001\u0000\u0000\u0000\u01f9\u01fa\u0001\u0000\u0000\u0000"+ - "\u01faU\u0001\u0000\u0000\u0000\u01fb\u01f9\u0001\u0000\u0000\u0000\u01fc"+ - "\u01fd\u0003<\u001e\u0000\u01fd\u01fe\u0005T\u0000\u0000\u01fe\u01ff\u0003"+ - "<\u001e\u0000\u01ffW\u0001\u0000\u0000\u0000\u0200\u0201\u0005\u0001\u0000"+ - "\u0000\u0201\u0202\u0003\u0014\n\u0000\u0202\u0204\u0003j5\u0000\u0203"+ - "\u0205\u0003^/\u0000\u0204\u0203\u0001\u0000\u0000\u0000\u0204\u0205\u0001"+ - "\u0000\u0000\u0000\u0205Y\u0001\u0000\u0000\u0000\u0206\u0207\u0005\u0007"+ - "\u0000\u0000\u0207\u0208\u0003\u0014\n\u0000\u0208\u0209\u0003j5\u0000"+ - "\u0209[\u0001\u0000\u0000\u0000\u020a\u020b\u0005\n\u0000\u0000\u020b"+ - "\u020c\u0003:\u001d\u0000\u020c]\u0001\u0000\u0000\u0000\u020d\u0212\u0003"+ - "`0\u0000\u020e\u020f\u0005!\u0000\u0000\u020f\u0211\u0003`0\u0000\u0210"+ - "\u020e\u0001\u0000\u0000\u0000\u0211\u0214\u0001\u0000\u0000\u0000\u0212"+ - "\u0210\u0001\u0000\u0000\u0000\u0212\u0213\u0001\u0000\u0000\u0000\u0213"+ - "_\u0001\u0000\u0000\u0000\u0214\u0212\u0001\u0000\u0000\u0000\u0215\u0216"+ - "\u0003@ \u0000\u0216\u0217\u0005\u001f\u0000\u0000\u0217\u0218\u0003D"+ - "\"\u0000\u0218a\u0001\u0000\u0000\u0000\u0219\u021a\u0007\u0006\u0000"+ - "\u0000\u021ac\u0001\u0000\u0000\u0000\u021b\u021e\u0003f3\u0000\u021c"+ - "\u021e\u0003h4\u0000\u021d\u021b\u0001\u0000\u0000\u0000\u021d\u021c\u0001"+ - "\u0000\u0000\u0000\u021ee\u0001\u0000\u0000\u0000\u021f\u0221\u0007\u0000"+ - "\u0000\u0000\u0220\u021f\u0001\u0000\u0000\u0000\u0220\u0221\u0001\u0000"+ - "\u0000\u0000\u0221\u0222\u0001\u0000\u0000\u0000\u0222\u0223\u0005\u001b"+ - "\u0000\u0000\u0223g\u0001\u0000\u0000\u0000\u0224\u0226\u0007\u0000\u0000"+ - "\u0000\u0225\u0224\u0001\u0000\u0000\u0000\u0225\u0226\u0001\u0000\u0000"+ - "\u0000\u0226\u0227\u0001\u0000\u0000\u0000\u0227\u0228\u0005\u001a\u0000"+ - "\u0000\u0228i\u0001\u0000\u0000\u0000\u0229\u022a\u0005\u0019\u0000\u0000"+ - "\u022ak\u0001\u0000\u0000\u0000\u022b\u022c\u0007\u0007\u0000\u0000\u022c"+ - "m\u0001\u0000\u0000\u0000\u022d\u022e\u0005\u0005\u0000\u0000\u022e\u022f"+ - "\u0003p8\u0000\u022fo\u0001\u0000\u0000\u0000\u0230\u0231\u0005A\u0000"+ - "\u0000\u0231\u0232\u0003\u0002\u0001\u0000\u0232\u0233\u0005B\u0000\u0000"+ - "\u0233q\u0001\u0000\u0000\u0000\u0234\u0235\u0005\r\u0000\u0000\u0235"+ - "\u0236\u0005d\u0000\u0000\u0236s\u0001\u0000\u0000\u0000\u0237\u0238\u0005"+ - "\u0003\u0000\u0000\u0238\u023b\u0005Z\u0000\u0000\u0239\u023a\u0005X\u0000"+ - "\u0000\u023a\u023c\u0003<\u001e\u0000\u023b\u0239\u0001\u0000\u0000\u0000"+ - "\u023b\u023c\u0001\u0000\u0000\u0000\u023c\u0246\u0001\u0000\u0000\u0000"+ - "\u023d\u023e\u0005Y\u0000\u0000\u023e\u0243\u0003v;\u0000\u023f\u0240"+ - "\u0005!\u0000\u0000\u0240\u0242\u0003v;\u0000\u0241\u023f\u0001\u0000"+ - "\u0000\u0000\u0242\u0245\u0001\u0000\u0000\u0000\u0243\u0241\u0001\u0000"+ - "\u0000\u0000\u0243\u0244\u0001\u0000\u0000\u0000\u0244\u0247\u0001\u0000"+ - "\u0000\u0000\u0245\u0243\u0001\u0000\u0000\u0000\u0246\u023d\u0001\u0000"+ - "\u0000\u0000\u0246\u0247\u0001\u0000\u0000\u0000\u0247u\u0001\u0000\u0000"+ - "\u0000\u0248\u0249\u0003<\u001e\u0000\u0249\u024a\u0005\u001f\u0000\u0000"+ - "\u024a\u024c\u0001\u0000\u0000\u0000\u024b\u0248\u0001\u0000\u0000\u0000"+ - "\u024b\u024c\u0001\u0000\u0000\u0000\u024c\u024d\u0001\u0000\u0000\u0000"+ - "\u024d\u024e\u0003<\u001e\u0000\u024ew\u0001\u0000\u0000\u0000\u024f\u0250"+ - "\u0005\u0012\u0000\u0000\u0250\u0251\u0003$\u0012\u0000\u0251\u0252\u0005"+ - "X\u0000\u0000\u0252\u0253\u0003>\u001f\u0000\u0253y\u0001\u0000\u0000"+ - "\u0000\u0254\u0255\u0005\u0011\u0000\u0000\u0255\u0258\u00036\u001b\u0000"+ - "\u0256\u0257\u0005\u001c\u0000\u0000\u0257\u0259\u0003\u001e\u000f\u0000"+ - "\u0258\u0256\u0001\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000\u0000"+ - "\u0259{\u0001\u0000\u0000\u0000;\u0087\u0090\u00a2\u00ae\u00b7\u00bf\u00c5"+ - "\u00cd\u00cf\u00d4\u00db\u00e0\u00eb\u00f1\u00f9\u00fb\u0106\u010d\u0118"+ - "\u011b\u0121\u012d\u0133\u013d\u0141\u0146\u0150\u0158\u0165\u0169\u016d"+ - "\u0174\u0178\u017f\u0185\u018c\u0194\u019c\u01a3\u01b4\u01bf\u01ca\u01cf"+ - "\u01d3\u01d7\u01e2\u01e7\u01eb\u01f9\u0204\u0212\u021d\u0220\u0225\u023b"+ - "\u0243\u0246\u024b\u0258"; + "\u001f\n\u001f\f\u001f\u019e\t\u001f\u0001 \u0001 \u0001!\u0001!\u0001"+ + "!\u0003!\u01a5\b!\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ + "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01b4\b\"\n"+ + "\"\f\"\u01b7\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005"+ + "\"\u01bf\b\"\n\"\f\"\u01c2\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\""+ + "\u0001\"\u0005\"\u01ca\b\"\n\"\f\"\u01cd\t\"\u0001\"\u0001\"\u0003\"\u01d1"+ + "\b\"\u0001#\u0001#\u0003#\u01d5\b#\u0001$\u0001$\u0001$\u0003$\u01da\b"+ + "$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0005&\u01e3\b&\n&"+ + "\f&\u01e6\t&\u0001\'\u0001\'\u0003\'\u01ea\b\'\u0001\'\u0001\'\u0003\'"+ + "\u01ee\b\'\u0001(\u0001(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001"+ + "*\u0001*\u0005*\u01fa\b*\n*\f*\u01fd\t*\u0001+\u0001+\u0001+\u0001+\u0001"+ + ",\u0001,\u0001,\u0001,\u0003,\u0207\b,\u0001-\u0001-\u0001-\u0001-\u0001"+ + ".\u0001.\u0001.\u0001/\u0001/\u0001/\u0005/\u0213\b/\n/\f/\u0216\t/\u0001"+ + "0\u00010\u00010\u00010\u00011\u00011\u00012\u00012\u00032\u0220\b2\u0001"+ + "3\u00033\u0223\b3\u00013\u00013\u00014\u00034\u0228\b4\u00014\u00014\u0001"+ + "5\u00015\u00016\u00016\u00017\u00017\u00017\u00018\u00018\u00018\u0001"+ + "8\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u023e\b:\u0001"+ + ":\u0001:\u0001:\u0001:\u0005:\u0244\b:\n:\f:\u0247\t:\u0003:\u0249\b:"+ + "\u0001;\u0001;\u0001;\u0003;\u024e\b;\u0001;\u0001;\u0001<\u0001<\u0001"+ + "<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001=\u0003=\u025b\b=\u0001=\u0000"+ + "\u0004\u0002\n\u0012\u0014>\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ + "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ + "TVXZ\\^`bdfhjlnprtvxz\u0000\b\u0001\u0000:;\u0001\u0000<>\u0002\u0000"+ + "\u0019\u0019LL\u0001\u0000CD\u0002\u0000\u001e\u001e\"\"\u0002\u0000%"+ + "%((\u0002\u0000$$22\u0002\u00003359\u0277\u0000|\u0001\u0000\u0000\u0000"+ + "\u0002\u007f\u0001\u0000\u0000\u0000\u0004\u0090\u0001\u0000\u0000\u0000"+ + "\u0006\u00a2\u0001\u0000\u0000\u0000\b\u00a4\u0001\u0000\u0000\u0000\n"+ + "\u00c5\u0001\u0000\u0000\u0000\f\u00e0\u0001\u0000\u0000\u0000\u000e\u00e2"+ + "\u0001\u0000\u0000\u0000\u0010\u00eb\u0001\u0000\u0000\u0000\u0012\u00f1"+ + "\u0001\u0000\u0000\u0000\u0014\u0106\u0001\u0000\u0000\u0000\u0016\u0110"+ + "\u0001\u0000\u0000\u0000\u0018\u0121\u0001\u0000\u0000\u0000\u001a\u0123"+ + "\u0001\u0000\u0000\u0000\u001c\u0125\u0001\u0000\u0000\u0000\u001e\u0128"+ + "\u0001\u0000\u0000\u0000 \u0133\u0001\u0000\u0000\u0000\"\u0137\u0001"+ + "\u0000\u0000\u0000$\u0146\u0001\u0000\u0000\u0000&\u014a\u0001\u0000\u0000"+ + "\u0000(\u014c\u0001\u0000\u0000\u0000*\u0150\u0001\u0000\u0000\u0000,"+ + "\u0152\u0001\u0000\u0000\u0000.\u015b\u0001\u0000\u0000\u00000\u015f\u0001"+ + "\u0000\u0000\u00002\u016f\u0001\u0000\u0000\u00004\u0172\u0001\u0000\u0000"+ + "\u00006\u017a\u0001\u0000\u0000\u00008\u0182\u0001\u0000\u0000\u0000:"+ + "\u0187\u0001\u0000\u0000\u0000<\u018f\u0001\u0000\u0000\u0000>\u0197\u0001"+ + "\u0000\u0000\u0000@\u019f\u0001\u0000\u0000\u0000B\u01a4\u0001\u0000\u0000"+ + "\u0000D\u01d0\u0001\u0000\u0000\u0000F\u01d4\u0001\u0000\u0000\u0000H"+ + "\u01d9\u0001\u0000\u0000\u0000J\u01db\u0001\u0000\u0000\u0000L\u01de\u0001"+ + "\u0000\u0000\u0000N\u01e7\u0001\u0000\u0000\u0000P\u01ef\u0001\u0000\u0000"+ + "\u0000R\u01f2\u0001\u0000\u0000\u0000T\u01f5\u0001\u0000\u0000\u0000V"+ + "\u01fe\u0001\u0000\u0000\u0000X\u0202\u0001\u0000\u0000\u0000Z\u0208\u0001"+ + "\u0000\u0000\u0000\\\u020c\u0001\u0000\u0000\u0000^\u020f\u0001\u0000"+ + "\u0000\u0000`\u0217\u0001\u0000\u0000\u0000b\u021b\u0001\u0000\u0000\u0000"+ + "d\u021f\u0001\u0000\u0000\u0000f\u0222\u0001\u0000\u0000\u0000h\u0227"+ + "\u0001\u0000\u0000\u0000j\u022b\u0001\u0000\u0000\u0000l\u022d\u0001\u0000"+ + "\u0000\u0000n\u022f\u0001\u0000\u0000\u0000p\u0232\u0001\u0000\u0000\u0000"+ + "r\u0236\u0001\u0000\u0000\u0000t\u0239\u0001\u0000\u0000\u0000v\u024d"+ + "\u0001\u0000\u0000\u0000x\u0251\u0001\u0000\u0000\u0000z\u0256\u0001\u0000"+ + "\u0000\u0000|}\u0003\u0002\u0001\u0000}~\u0005\u0000\u0000\u0001~\u0001"+ + "\u0001\u0000\u0000\u0000\u007f\u0080\u0006\u0001\uffff\uffff\u0000\u0080"+ + "\u0081\u0003\u0004\u0002\u0000\u0081\u0087\u0001\u0000\u0000\u0000\u0082"+ + "\u0083\n\u0001\u0000\u0000\u0083\u0084\u0005\u0018\u0000\u0000\u0084\u0086"+ + "\u0003\u0006\u0003\u0000\u0085\u0082\u0001\u0000\u0000\u0000\u0086\u0089"+ + "\u0001\u0000\u0000\u0000\u0087\u0085\u0001\u0000\u0000\u0000\u0087\u0088"+ + "\u0001\u0000\u0000\u0000\u0088\u0003\u0001\u0000\u0000\u0000\u0089\u0087"+ + "\u0001\u0000\u0000\u0000\u008a\u0091\u0003n7\u0000\u008b\u0091\u0003\""+ + "\u0011\u0000\u008c\u0091\u0003\u001c\u000e\u0000\u008d\u0091\u0003r9\u0000"+ + "\u008e\u008f\u0004\u0002\u0001\u0000\u008f\u0091\u00030\u0018\u0000\u0090"+ + "\u008a\u0001\u0000\u0000\u0000\u0090\u008b\u0001\u0000\u0000\u0000\u0090"+ + "\u008c\u0001\u0000\u0000\u0000\u0090\u008d\u0001\u0000\u0000\u0000\u0090"+ + "\u008e\u0001\u0000\u0000\u0000\u0091\u0005\u0001\u0000\u0000\u0000\u0092"+ + "\u00a3\u00032\u0019\u0000\u0093\u00a3\u0003\b\u0004\u0000\u0094\u00a3"+ + "\u0003P(\u0000\u0095\u00a3\u0003J%\u0000\u0096\u00a3\u00034\u001a\u0000"+ + "\u0097\u00a3\u0003L&\u0000\u0098\u00a3\u0003R)\u0000\u0099\u00a3\u0003"+ + "T*\u0000\u009a\u00a3\u0003X,\u0000\u009b\u00a3\u0003Z-\u0000\u009c\u00a3"+ + "\u0003t:\u0000\u009d\u00a3\u0003\\.\u0000\u009e\u009f\u0004\u0003\u0002"+ + "\u0000\u009f\u00a3\u0003z=\u0000\u00a0\u00a1\u0004\u0003\u0003\u0000\u00a1"+ + "\u00a3\u0003x<\u0000\u00a2\u0092\u0001\u0000\u0000\u0000\u00a2\u0093\u0001"+ + "\u0000\u0000\u0000\u00a2\u0094\u0001\u0000\u0000\u0000\u00a2\u0095\u0001"+ + "\u0000\u0000\u0000\u00a2\u0096\u0001\u0000\u0000\u0000\u00a2\u0097\u0001"+ + "\u0000\u0000\u0000\u00a2\u0098\u0001\u0000\u0000\u0000\u00a2\u0099\u0001"+ + "\u0000\u0000\u0000\u00a2\u009a\u0001\u0000\u0000\u0000\u00a2\u009b\u0001"+ + "\u0000\u0000\u0000\u00a2\u009c\u0001\u0000\u0000\u0000\u00a2\u009d\u0001"+ + "\u0000\u0000\u0000\u00a2\u009e\u0001\u0000\u0000\u0000\u00a2\u00a0\u0001"+ + "\u0000\u0000\u0000\u00a3\u0007\u0001\u0000\u0000\u0000\u00a4\u00a5\u0005"+ + "\u0010\u0000\u0000\u00a5\u00a6\u0003\n\u0005\u0000\u00a6\t\u0001\u0000"+ + "\u0000\u0000\u00a7\u00a8\u0006\u0005\uffff\uffff\u0000\u00a8\u00a9\u0005"+ + "+\u0000\u0000\u00a9\u00c6\u0003\n\u0005\b\u00aa\u00c6\u0003\u0010\b\u0000"+ + "\u00ab\u00c6\u0003\f\u0006\u0000\u00ac\u00ae\u0003\u0010\b\u0000\u00ad"+ + "\u00af\u0005+\u0000\u0000\u00ae\u00ad\u0001\u0000\u0000\u0000\u00ae\u00af"+ + "\u0001\u0000\u0000\u0000\u00af\u00b0\u0001\u0000\u0000\u0000\u00b0\u00b1"+ + "\u0005&\u0000\u0000\u00b1\u00b2\u0005*\u0000\u0000\u00b2\u00b7\u0003\u0010"+ + "\b\u0000\u00b3\u00b4\u0005!\u0000\u0000\u00b4\u00b6\u0003\u0010\b\u0000"+ + "\u00b5\u00b3\u0001\u0000\u0000\u0000\u00b6\u00b9\u0001\u0000\u0000\u0000"+ + "\u00b7\u00b5\u0001\u0000\u0000\u0000\u00b7\u00b8\u0001\u0000\u0000\u0000"+ + "\u00b8\u00ba\u0001\u0000\u0000\u0000\u00b9\u00b7\u0001\u0000\u0000\u0000"+ + "\u00ba\u00bb\u00051\u0000\u0000\u00bb\u00c6\u0001\u0000\u0000\u0000\u00bc"+ + "\u00bd\u0003\u0010\b\u0000\u00bd\u00bf\u0005\'\u0000\u0000\u00be\u00c0"+ + "\u0005+\u0000\u0000\u00bf\u00be\u0001\u0000\u0000\u0000\u00bf\u00c0\u0001"+ + "\u0000\u0000\u0000\u00c0\u00c1\u0001\u0000\u0000\u0000\u00c1\u00c2\u0005"+ + ",\u0000\u0000\u00c2\u00c6\u0001\u0000\u0000\u0000\u00c3\u00c4\u0004\u0005"+ + "\u0004\u0000\u00c4\u00c6\u0003\u000e\u0007\u0000\u00c5\u00a7\u0001\u0000"+ + "\u0000\u0000\u00c5\u00aa\u0001\u0000\u0000\u0000\u00c5\u00ab\u0001\u0000"+ + "\u0000\u0000\u00c5\u00ac\u0001\u0000\u0000\u0000\u00c5\u00bc\u0001\u0000"+ + "\u0000\u0000\u00c5\u00c3\u0001\u0000\u0000\u0000\u00c6\u00cf\u0001\u0000"+ + "\u0000\u0000\u00c7\u00c8\n\u0005\u0000\u0000\u00c8\u00c9\u0005\u001d\u0000"+ + "\u0000\u00c9\u00ce\u0003\n\u0005\u0006\u00ca\u00cb\n\u0004\u0000\u0000"+ + "\u00cb\u00cc\u0005.\u0000\u0000\u00cc\u00ce\u0003\n\u0005\u0005\u00cd"+ + "\u00c7\u0001\u0000\u0000\u0000\u00cd\u00ca\u0001\u0000\u0000\u0000\u00ce"+ + "\u00d1\u0001\u0000\u0000\u0000\u00cf\u00cd\u0001\u0000\u0000\u0000\u00cf"+ + "\u00d0\u0001\u0000\u0000\u0000\u00d0\u000b\u0001\u0000\u0000\u0000\u00d1"+ + "\u00cf\u0001\u0000\u0000\u0000\u00d2\u00d4\u0003\u0010\b\u0000\u00d3\u00d5"+ + "\u0005+\u0000\u0000\u00d4\u00d3\u0001\u0000\u0000\u0000\u00d4\u00d5\u0001"+ + "\u0000\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7\u0005"+ + ")\u0000\u0000\u00d7\u00d8\u0003j5\u0000\u00d8\u00e1\u0001\u0000\u0000"+ + "\u0000\u00d9\u00db\u0003\u0010\b\u0000\u00da\u00dc\u0005+\u0000\u0000"+ + "\u00db\u00da\u0001\u0000\u0000\u0000\u00db\u00dc\u0001\u0000\u0000\u0000"+ + "\u00dc\u00dd\u0001\u0000\u0000\u0000\u00dd\u00de\u00050\u0000\u0000\u00de"+ + "\u00df\u0003j5\u0000\u00df\u00e1\u0001\u0000\u0000\u0000\u00e0\u00d2\u0001"+ + "\u0000\u0000\u0000\u00e0\u00d9\u0001\u0000\u0000\u0000\u00e1\r\u0001\u0000"+ + "\u0000\u0000\u00e2\u00e3\u0003\u0010\b\u0000\u00e3\u00e4\u0005?\u0000"+ + "\u0000\u00e4\u00e5\u0003j5\u0000\u00e5\u000f\u0001\u0000\u0000\u0000\u00e6"+ + "\u00ec\u0003\u0012\t\u0000\u00e7\u00e8\u0003\u0012\t\u0000\u00e8\u00e9"+ + "\u0003l6\u0000\u00e9\u00ea\u0003\u0012\t\u0000\u00ea\u00ec\u0001\u0000"+ + "\u0000\u0000\u00eb\u00e6\u0001\u0000\u0000\u0000\u00eb\u00e7\u0001\u0000"+ + "\u0000\u0000\u00ec\u0011\u0001\u0000\u0000\u0000\u00ed\u00ee\u0006\t\uffff"+ + "\uffff\u0000\u00ee\u00f2\u0003\u0014\n\u0000\u00ef\u00f0\u0007\u0000\u0000"+ + "\u0000\u00f0\u00f2\u0003\u0012\t\u0003\u00f1\u00ed\u0001\u0000\u0000\u0000"+ + "\u00f1\u00ef\u0001\u0000\u0000\u0000\u00f2\u00fb\u0001\u0000\u0000\u0000"+ + "\u00f3\u00f4\n\u0002\u0000\u0000\u00f4\u00f5\u0007\u0001\u0000\u0000\u00f5"+ + "\u00fa\u0003\u0012\t\u0003\u00f6\u00f7\n\u0001\u0000\u0000\u00f7\u00f8"+ + "\u0007\u0000\u0000\u0000\u00f8\u00fa\u0003\u0012\t\u0002\u00f9\u00f3\u0001"+ + "\u0000\u0000\u0000\u00f9\u00f6\u0001\u0000\u0000\u0000\u00fa\u00fd\u0001"+ + "\u0000\u0000\u0000\u00fb\u00f9\u0001\u0000\u0000\u0000\u00fb\u00fc\u0001"+ + "\u0000\u0000\u0000\u00fc\u0013\u0001\u0000\u0000\u0000\u00fd\u00fb\u0001"+ + "\u0000\u0000\u0000\u00fe\u00ff\u0006\n\uffff\uffff\u0000\u00ff\u0107\u0003"+ + "D\"\u0000\u0100\u0107\u0003:\u001d\u0000\u0101\u0107\u0003\u0016\u000b"+ + "\u0000\u0102\u0103\u0005*\u0000\u0000\u0103\u0104\u0003\n\u0005\u0000"+ + "\u0104\u0105\u00051\u0000\u0000\u0105\u0107\u0001\u0000\u0000\u0000\u0106"+ + "\u00fe\u0001\u0000\u0000\u0000\u0106\u0100\u0001\u0000\u0000\u0000\u0106"+ + "\u0101\u0001\u0000\u0000\u0000\u0106\u0102\u0001\u0000\u0000\u0000\u0107"+ + "\u010d\u0001\u0000\u0000\u0000\u0108\u0109\n\u0001\u0000\u0000\u0109\u010a"+ + "\u0005 \u0000\u0000\u010a\u010c\u0003\u001a\r\u0000\u010b\u0108\u0001"+ + "\u0000\u0000\u0000\u010c\u010f\u0001\u0000\u0000\u0000\u010d\u010b\u0001"+ + "\u0000\u0000\u0000\u010d\u010e\u0001\u0000\u0000\u0000\u010e\u0015\u0001"+ + "\u0000\u0000\u0000\u010f\u010d\u0001\u0000\u0000\u0000\u0110\u0111\u0003"+ + "\u0018\f\u0000\u0111\u011b\u0005*\u0000\u0000\u0112\u011c\u0005<\u0000"+ + "\u0000\u0113\u0118\u0003\n\u0005\u0000\u0114\u0115\u0005!\u0000\u0000"+ + "\u0115\u0117\u0003\n\u0005\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0117"+ + "\u011a\u0001\u0000\u0000\u0000\u0118\u0116\u0001\u0000\u0000\u0000\u0118"+ + "\u0119\u0001\u0000\u0000\u0000\u0119\u011c\u0001\u0000\u0000\u0000\u011a"+ + "\u0118\u0001\u0000\u0000\u0000\u011b\u0112\u0001\u0000\u0000\u0000\u011b"+ + "\u0113\u0001\u0000\u0000\u0000\u011b\u011c\u0001\u0000\u0000\u0000\u011c"+ + "\u011d\u0001\u0000\u0000\u0000\u011d\u011e\u00051\u0000\u0000\u011e\u0017"+ + "\u0001\u0000\u0000\u0000\u011f\u0122\u0005?\u0000\u0000\u0120\u0122\u0003"+ + "H$\u0000\u0121\u011f\u0001\u0000\u0000\u0000\u0121\u0120\u0001\u0000\u0000"+ + "\u0000\u0122\u0019\u0001\u0000\u0000\u0000\u0123\u0124\u0003@ \u0000\u0124"+ + "\u001b\u0001\u0000\u0000\u0000\u0125\u0126\u0005\f\u0000\u0000\u0126\u0127"+ + "\u0003\u001e\u000f\u0000\u0127\u001d\u0001\u0000\u0000\u0000\u0128\u012d"+ + "\u0003 \u0010\u0000\u0129\u012a\u0005!\u0000\u0000\u012a\u012c\u0003 "+ + "\u0010\u0000\u012b\u0129\u0001\u0000\u0000\u0000\u012c\u012f\u0001\u0000"+ + "\u0000\u0000\u012d\u012b\u0001\u0000\u0000\u0000\u012d\u012e\u0001\u0000"+ + "\u0000\u0000\u012e\u001f\u0001\u0000\u0000\u0000\u012f\u012d\u0001\u0000"+ + "\u0000\u0000\u0130\u0131\u0003:\u001d\u0000\u0131\u0132\u0005\u001f\u0000"+ + "\u0000\u0132\u0134\u0001\u0000\u0000\u0000\u0133\u0130\u0001\u0000\u0000"+ + "\u0000\u0133\u0134\u0001\u0000\u0000\u0000\u0134\u0135\u0001\u0000\u0000"+ + "\u0000\u0135\u0136\u0003\n\u0005\u0000\u0136!\u0001\u0000\u0000\u0000"+ + "\u0137\u0138\u0005\u0006\u0000\u0000\u0138\u013d\u0003$\u0012\u0000\u0139"+ + "\u013a\u0005!\u0000\u0000\u013a\u013c\u0003$\u0012\u0000\u013b\u0139\u0001"+ + "\u0000\u0000\u0000\u013c\u013f\u0001\u0000\u0000\u0000\u013d\u013b\u0001"+ + "\u0000\u0000\u0000\u013d\u013e\u0001\u0000\u0000\u0000\u013e\u0141\u0001"+ + "\u0000\u0000\u0000\u013f\u013d\u0001\u0000\u0000\u0000\u0140\u0142\u0003"+ + "*\u0015\u0000\u0141\u0140\u0001\u0000\u0000\u0000\u0141\u0142\u0001\u0000"+ + "\u0000\u0000\u0142#\u0001\u0000\u0000\u0000\u0143\u0144\u0003&\u0013\u0000"+ + "\u0144\u0145\u0005h\u0000\u0000\u0145\u0147\u0001\u0000\u0000\u0000\u0146"+ + "\u0143\u0001\u0000\u0000\u0000\u0146\u0147\u0001\u0000\u0000\u0000\u0147"+ + "\u0148\u0001\u0000\u0000\u0000\u0148\u0149\u0003(\u0014\u0000\u0149%\u0001"+ + "\u0000\u0000\u0000\u014a\u014b\u0005L\u0000\u0000\u014b\'\u0001\u0000"+ + "\u0000\u0000\u014c\u014d\u0007\u0002\u0000\u0000\u014d)\u0001\u0000\u0000"+ + "\u0000\u014e\u0151\u0003,\u0016\u0000\u014f\u0151\u0003.\u0017\u0000\u0150"+ + "\u014e\u0001\u0000\u0000\u0000\u0150\u014f\u0001\u0000\u0000\u0000\u0151"+ + "+\u0001\u0000\u0000\u0000\u0152\u0153\u0005K\u0000\u0000\u0153\u0158\u0005"+ + "L\u0000\u0000\u0154\u0155\u0005!\u0000\u0000\u0155\u0157\u0005L\u0000"+ + "\u0000\u0156\u0154\u0001\u0000\u0000\u0000\u0157\u015a\u0001\u0000\u0000"+ + "\u0000\u0158\u0156\u0001\u0000\u0000\u0000\u0158\u0159\u0001\u0000\u0000"+ + "\u0000\u0159-\u0001\u0000\u0000\u0000\u015a\u0158\u0001\u0000\u0000\u0000"+ + "\u015b\u015c\u0005A\u0000\u0000\u015c\u015d\u0003,\u0016\u0000\u015d\u015e"+ + "\u0005B\u0000\u0000\u015e/\u0001\u0000\u0000\u0000\u015f\u0160\u0005\u0013"+ + "\u0000\u0000\u0160\u0165\u0003$\u0012\u0000\u0161\u0162\u0005!\u0000\u0000"+ + "\u0162\u0164\u0003$\u0012\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0164"+ + "\u0167\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000\u0165"+ + "\u0166\u0001\u0000\u0000\u0000\u0166\u0169\u0001\u0000\u0000\u0000\u0167"+ + "\u0165\u0001\u0000\u0000\u0000\u0168\u016a\u00036\u001b\u0000\u0169\u0168"+ + "\u0001\u0000\u0000\u0000\u0169\u016a\u0001\u0000\u0000\u0000\u016a\u016d"+ + "\u0001\u0000\u0000\u0000\u016b\u016c\u0005\u001c\u0000\u0000\u016c\u016e"+ + "\u0003\u001e\u000f\u0000\u016d\u016b\u0001\u0000\u0000\u0000\u016d\u016e"+ + "\u0001\u0000\u0000\u0000\u016e1\u0001\u0000\u0000\u0000\u016f\u0170\u0005"+ + "\u0004\u0000\u0000\u0170\u0171\u0003\u001e\u000f\u0000\u01713\u0001\u0000"+ + "\u0000\u0000\u0172\u0174\u0005\u000f\u0000\u0000\u0173\u0175\u00036\u001b"+ + "\u0000\u0174\u0173\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000"+ + "\u0000\u0175\u0178\u0001\u0000\u0000\u0000\u0176\u0177\u0005\u001c\u0000"+ + "\u0000\u0177\u0179\u0003\u001e\u000f\u0000\u0178\u0176\u0001\u0000\u0000"+ + "\u0000\u0178\u0179\u0001\u0000\u0000\u0000\u01795\u0001\u0000\u0000\u0000"+ + "\u017a\u017f\u00038\u001c\u0000\u017b\u017c\u0005!\u0000\u0000\u017c\u017e"+ + "\u00038\u001c\u0000\u017d\u017b\u0001\u0000\u0000\u0000\u017e\u0181\u0001"+ + "\u0000\u0000\u0000\u017f\u017d\u0001\u0000\u0000\u0000\u017f\u0180\u0001"+ + "\u0000\u0000\u0000\u01807\u0001\u0000\u0000\u0000\u0181\u017f\u0001\u0000"+ + "\u0000\u0000\u0182\u0185\u0003 \u0010\u0000\u0183\u0184\u0005\u0010\u0000"+ + "\u0000\u0184\u0186\u0003\n\u0005\u0000\u0185\u0183\u0001\u0000\u0000\u0000"+ + "\u0185\u0186\u0001\u0000\u0000\u0000\u01869\u0001\u0000\u0000\u0000\u0187"+ + "\u018c\u0003H$\u0000\u0188\u0189\u0005#\u0000\u0000\u0189\u018b\u0003"+ + "H$\u0000\u018a\u0188\u0001\u0000\u0000\u0000\u018b\u018e\u0001\u0000\u0000"+ + "\u0000\u018c\u018a\u0001\u0000\u0000\u0000\u018c\u018d\u0001\u0000\u0000"+ + "\u0000\u018d;\u0001\u0000\u0000\u0000\u018e\u018c\u0001\u0000\u0000\u0000"+ + "\u018f\u0194\u0003B!\u0000\u0190\u0191\u0005#\u0000\u0000\u0191\u0193"+ + "\u0003B!\u0000\u0192\u0190\u0001\u0000\u0000\u0000\u0193\u0196\u0001\u0000"+ + "\u0000\u0000\u0194\u0192\u0001\u0000\u0000\u0000\u0194\u0195\u0001\u0000"+ + "\u0000\u0000\u0195=\u0001\u0000\u0000\u0000\u0196\u0194\u0001\u0000\u0000"+ + "\u0000\u0197\u019c\u0003<\u001e\u0000\u0198\u0199\u0005!\u0000\u0000\u0199"+ + "\u019b\u0003<\u001e\u0000\u019a\u0198\u0001\u0000\u0000\u0000\u019b\u019e"+ + "\u0001\u0000\u0000\u0000\u019c\u019a\u0001\u0000\u0000\u0000\u019c\u019d"+ + "\u0001\u0000\u0000\u0000\u019d?\u0001\u0000\u0000\u0000\u019e\u019c\u0001"+ + "\u0000\u0000\u0000\u019f\u01a0\u0007\u0003\u0000\u0000\u01a0A\u0001\u0000"+ + "\u0000\u0000\u01a1\u01a5\u0005P\u0000\u0000\u01a2\u01a3\u0004!\n\u0000"+ + "\u01a3\u01a5\u0003F#\u0000\u01a4\u01a1\u0001\u0000\u0000\u0000\u01a4\u01a2"+ + "\u0001\u0000\u0000\u0000\u01a5C\u0001\u0000\u0000\u0000\u01a6\u01d1\u0005"+ + ",\u0000\u0000\u01a7\u01a8\u0003h4\u0000\u01a8\u01a9\u0005C\u0000\u0000"+ + "\u01a9\u01d1\u0001\u0000\u0000\u0000\u01aa\u01d1\u0003f3\u0000\u01ab\u01d1"+ + "\u0003h4\u0000\u01ac\u01d1\u0003b1\u0000\u01ad\u01d1\u0003F#\u0000\u01ae"+ + "\u01d1\u0003j5\u0000\u01af\u01b0\u0005A\u0000\u0000\u01b0\u01b5\u0003"+ + "d2\u0000\u01b1\u01b2\u0005!\u0000\u0000\u01b2\u01b4\u0003d2\u0000\u01b3"+ + "\u01b1\u0001\u0000\u0000\u0000\u01b4\u01b7\u0001\u0000\u0000\u0000\u01b5"+ + "\u01b3\u0001\u0000\u0000\u0000\u01b5\u01b6\u0001\u0000\u0000\u0000\u01b6"+ + "\u01b8\u0001\u0000\u0000\u0000\u01b7\u01b5\u0001\u0000\u0000\u0000\u01b8"+ + "\u01b9\u0005B\u0000\u0000\u01b9\u01d1\u0001\u0000\u0000\u0000\u01ba\u01bb"+ + "\u0005A\u0000\u0000\u01bb\u01c0\u0003b1\u0000\u01bc\u01bd\u0005!\u0000"+ + "\u0000\u01bd\u01bf\u0003b1\u0000\u01be\u01bc\u0001\u0000\u0000\u0000\u01bf"+ + "\u01c2\u0001\u0000\u0000\u0000\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0"+ + "\u01c1\u0001\u0000\u0000\u0000\u01c1\u01c3\u0001\u0000\u0000\u0000\u01c2"+ + "\u01c0\u0001\u0000\u0000\u0000\u01c3\u01c4\u0005B\u0000\u0000\u01c4\u01d1"+ + "\u0001\u0000\u0000\u0000\u01c5\u01c6\u0005A\u0000\u0000\u01c6\u01cb\u0003"+ + "j5\u0000\u01c7\u01c8\u0005!\u0000\u0000\u01c8\u01ca\u0003j5\u0000\u01c9"+ + "\u01c7\u0001\u0000\u0000\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb"+ + "\u01c9\u0001\u0000\u0000\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc"+ + "\u01ce\u0001\u0000\u0000\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce"+ + "\u01cf\u0005B\u0000\u0000\u01cf\u01d1\u0001\u0000\u0000\u0000\u01d0\u01a6"+ + "\u0001\u0000\u0000\u0000\u01d0\u01a7\u0001\u0000\u0000\u0000\u01d0\u01aa"+ + "\u0001\u0000\u0000\u0000\u01d0\u01ab\u0001\u0000\u0000\u0000\u01d0\u01ac"+ + "\u0001\u0000\u0000\u0000\u01d0\u01ad\u0001\u0000\u0000\u0000\u01d0\u01ae"+ + "\u0001\u0000\u0000\u0000\u01d0\u01af\u0001\u0000\u0000\u0000\u01d0\u01ba"+ + "\u0001\u0000\u0000\u0000\u01d0\u01c5\u0001\u0000\u0000\u0000\u01d1E\u0001"+ + "\u0000\u0000\u0000\u01d2\u01d5\u0005/\u0000\u0000\u01d3\u01d5\u0005@\u0000"+ + "\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d4\u01d3\u0001\u0000\u0000"+ + "\u0000\u01d5G\u0001\u0000\u0000\u0000\u01d6\u01da\u0003@ \u0000\u01d7"+ + "\u01d8\u0004$\u000b\u0000\u01d8\u01da\u0003F#\u0000\u01d9\u01d6\u0001"+ + "\u0000\u0000\u0000\u01d9\u01d7\u0001\u0000\u0000\u0000\u01daI\u0001\u0000"+ + "\u0000\u0000\u01db\u01dc\u0005\t\u0000\u0000\u01dc\u01dd\u0005\u001a\u0000"+ + "\u0000\u01ddK\u0001\u0000\u0000\u0000\u01de\u01df\u0005\u000e\u0000\u0000"+ + "\u01df\u01e4\u0003N\'\u0000\u01e0\u01e1\u0005!\u0000\u0000\u01e1\u01e3"+ + "\u0003N\'\u0000\u01e2\u01e0\u0001\u0000\u0000\u0000\u01e3\u01e6\u0001"+ + "\u0000\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e4\u01e5\u0001"+ + "\u0000\u0000\u0000\u01e5M\u0001\u0000\u0000\u0000\u01e6\u01e4\u0001\u0000"+ + "\u0000\u0000\u01e7\u01e9\u0003\n\u0005\u0000\u01e8\u01ea\u0007\u0004\u0000"+ + "\u0000\u01e9\u01e8\u0001\u0000\u0000\u0000\u01e9\u01ea\u0001\u0000\u0000"+ + "\u0000\u01ea\u01ed\u0001\u0000\u0000\u0000\u01eb\u01ec\u0005-\u0000\u0000"+ + "\u01ec\u01ee\u0007\u0005\u0000\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000"+ + "\u01ed\u01ee\u0001\u0000\u0000\u0000\u01eeO\u0001\u0000\u0000\u0000\u01ef"+ + "\u01f0\u0005\b\u0000\u0000\u01f0\u01f1\u0003>\u001f\u0000\u01f1Q\u0001"+ + "\u0000\u0000\u0000\u01f2\u01f3\u0005\u0002\u0000\u0000\u01f3\u01f4\u0003"+ + ">\u001f\u0000\u01f4S\u0001\u0000\u0000\u0000\u01f5\u01f6\u0005\u000b\u0000"+ + "\u0000\u01f6\u01fb\u0003V+\u0000\u01f7\u01f8\u0005!\u0000\u0000\u01f8"+ + "\u01fa\u0003V+\u0000\u01f9\u01f7\u0001\u0000\u0000\u0000\u01fa\u01fd\u0001"+ + "\u0000\u0000\u0000\u01fb\u01f9\u0001\u0000\u0000\u0000\u01fb\u01fc\u0001"+ + "\u0000\u0000\u0000\u01fcU\u0001\u0000\u0000\u0000\u01fd\u01fb\u0001\u0000"+ + "\u0000\u0000\u01fe\u01ff\u0003<\u001e\u0000\u01ff\u0200\u0005T\u0000\u0000"+ + "\u0200\u0201\u0003<\u001e\u0000\u0201W\u0001\u0000\u0000\u0000\u0202\u0203"+ + "\u0005\u0001\u0000\u0000\u0203\u0204\u0003\u0014\n\u0000\u0204\u0206\u0003"+ + "j5\u0000\u0205\u0207\u0003^/\u0000\u0206\u0205\u0001\u0000\u0000\u0000"+ + "\u0206\u0207\u0001\u0000\u0000\u0000\u0207Y\u0001\u0000\u0000\u0000\u0208"+ + "\u0209\u0005\u0007\u0000\u0000\u0209\u020a\u0003\u0014\n\u0000\u020a\u020b"+ + "\u0003j5\u0000\u020b[\u0001\u0000\u0000\u0000\u020c\u020d\u0005\n\u0000"+ + "\u0000\u020d\u020e\u0003:\u001d\u0000\u020e]\u0001\u0000\u0000\u0000\u020f"+ + "\u0214\u0003`0\u0000\u0210\u0211\u0005!\u0000\u0000\u0211\u0213\u0003"+ + "`0\u0000\u0212\u0210\u0001\u0000\u0000\u0000\u0213\u0216\u0001\u0000\u0000"+ + "\u0000\u0214\u0212\u0001\u0000\u0000\u0000\u0214\u0215\u0001\u0000\u0000"+ + "\u0000\u0215_\u0001\u0000\u0000\u0000\u0216\u0214\u0001\u0000\u0000\u0000"+ + "\u0217\u0218\u0003@ \u0000\u0218\u0219\u0005\u001f\u0000\u0000\u0219\u021a"+ + "\u0003D\"\u0000\u021aa\u0001\u0000\u0000\u0000\u021b\u021c\u0007\u0006"+ + "\u0000\u0000\u021cc\u0001\u0000\u0000\u0000\u021d\u0220\u0003f3\u0000"+ + "\u021e\u0220\u0003h4\u0000\u021f\u021d\u0001\u0000\u0000\u0000\u021f\u021e"+ + "\u0001\u0000\u0000\u0000\u0220e\u0001\u0000\u0000\u0000\u0221\u0223\u0007"+ + "\u0000\u0000\u0000\u0222\u0221\u0001\u0000\u0000\u0000\u0222\u0223\u0001"+ + "\u0000\u0000\u0000\u0223\u0224\u0001\u0000\u0000\u0000\u0224\u0225\u0005"+ + "\u001b\u0000\u0000\u0225g\u0001\u0000\u0000\u0000\u0226\u0228\u0007\u0000"+ + "\u0000\u0000\u0227\u0226\u0001\u0000\u0000\u0000\u0227\u0228\u0001\u0000"+ + "\u0000\u0000\u0228\u0229\u0001\u0000\u0000\u0000\u0229\u022a\u0005\u001a"+ + "\u0000\u0000\u022ai\u0001\u0000\u0000\u0000\u022b\u022c\u0005\u0019\u0000"+ + "\u0000\u022ck\u0001\u0000\u0000\u0000\u022d\u022e\u0007\u0007\u0000\u0000"+ + "\u022em\u0001\u0000\u0000\u0000\u022f\u0230\u0005\u0005\u0000\u0000\u0230"+ + "\u0231\u0003p8\u0000\u0231o\u0001\u0000\u0000\u0000\u0232\u0233\u0005"+ + "A\u0000\u0000\u0233\u0234\u0003\u0002\u0001\u0000\u0234\u0235\u0005B\u0000"+ + "\u0000\u0235q\u0001\u0000\u0000\u0000\u0236\u0237\u0005\r\u0000\u0000"+ + "\u0237\u0238\u0005d\u0000\u0000\u0238s\u0001\u0000\u0000\u0000\u0239\u023a"+ + "\u0005\u0003\u0000\u0000\u023a\u023d\u0005Z\u0000\u0000\u023b\u023c\u0005"+ + "X\u0000\u0000\u023c\u023e\u0003<\u001e\u0000\u023d\u023b\u0001\u0000\u0000"+ + "\u0000\u023d\u023e\u0001\u0000\u0000\u0000\u023e\u0248\u0001\u0000\u0000"+ + "\u0000\u023f\u0240\u0005Y\u0000\u0000\u0240\u0245\u0003v;\u0000\u0241"+ + "\u0242\u0005!\u0000\u0000\u0242\u0244\u0003v;\u0000\u0243\u0241\u0001"+ + "\u0000\u0000\u0000\u0244\u0247\u0001\u0000\u0000\u0000\u0245\u0243\u0001"+ + "\u0000\u0000\u0000\u0245\u0246\u0001\u0000\u0000\u0000\u0246\u0249\u0001"+ + "\u0000\u0000\u0000\u0247\u0245\u0001\u0000\u0000\u0000\u0248\u023f\u0001"+ + "\u0000\u0000\u0000\u0248\u0249\u0001\u0000\u0000\u0000\u0249u\u0001\u0000"+ + "\u0000\u0000\u024a\u024b\u0003<\u001e\u0000\u024b\u024c\u0005\u001f\u0000"+ + "\u0000\u024c\u024e\u0001\u0000\u0000\u0000\u024d\u024a\u0001\u0000\u0000"+ + "\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u024f\u0001\u0000\u0000"+ + "\u0000\u024f\u0250\u0003<\u001e\u0000\u0250w\u0001\u0000\u0000\u0000\u0251"+ + "\u0252\u0005\u0012\u0000\u0000\u0252\u0253\u0003$\u0012\u0000\u0253\u0254"+ + "\u0005X\u0000\u0000\u0254\u0255\u0003>\u001f\u0000\u0255y\u0001\u0000"+ + "\u0000\u0000\u0256\u0257\u0005\u0011\u0000\u0000\u0257\u025a\u00036\u001b"+ + "\u0000\u0258\u0259\u0005\u001c\u0000\u0000\u0259\u025b\u0003\u001e\u000f"+ + "\u0000\u025a\u0258\u0001\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000"+ + "\u0000\u025b{\u0001\u0000\u0000\u0000;\u0087\u0090\u00a2\u00ae\u00b7\u00bf"+ + "\u00c5\u00cd\u00cf\u00d4\u00db\u00e0\u00eb\u00f1\u00f9\u00fb\u0106\u010d"+ + "\u0118\u011b\u0121\u012d\u0133\u013d\u0141\u0146\u0150\u0158\u0165\u0169"+ + "\u016d\u0174\u0178\u017f\u0185\u018c\u0194\u019c\u01a4\u01b5\u01c0\u01cb"+ + "\u01d0\u01d4\u01d9\u01e4\u01e9\u01ed\u01fb\u0206\u0214\u021f\u0222\u0227"+ + "\u023d\u0245\u0248\u024d\u025a"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java index 1872fa6e8f1f0..ae2379318474b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java @@ -21,6 +21,7 @@ import java.util.List; import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; +import static org.elasticsearch.transport.RemoteClusterAware.isRemoteIndexName; import static org.elasticsearch.xpack.esql.core.util.StringUtils.EXCLUSION; import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD; import static org.elasticsearch.xpack.esql.parser.ParserUtils.source; @@ -61,11 +62,14 @@ public String visitIndexPattern(List ctx) { Holder hasSeenStar = new Holder<>(false); ctx.forEach(c -> { String indexPattern = visitIndexString(c.indexString()); - hasSeenStar.set(indexPattern.contains(WILDCARD) || hasSeenStar.get()); - validateIndexPattern(indexPattern, c, hasSeenStar.get()); - patterns.add( - c.clusterString() != null ? c.clusterString().getText() + REMOTE_CLUSTER_INDEX_SEPARATOR + indexPattern : indexPattern - ); + String clusterString = c.clusterString() != null ? c.clusterString().getText() : null; + // skip validating index on remote cluster, because the behavior of remote cluster is not consistent with local cluster + // For example, invalid#index is an invalid index name, however FROM *:invalid#index does not return an error + if (clusterString == null) { + hasSeenStar.set(indexPattern.contains(WILDCARD) || hasSeenStar.get()); + validateIndexPattern(indexPattern, c, hasSeenStar.get()); + } + patterns.add(clusterString != null ? clusterString + REMOTE_CLUSTER_INDEX_SEPARATOR + indexPattern : indexPattern); }); return Strings.collectionToDelimitedString(patterns, ","); } @@ -75,6 +79,9 @@ private static void validateIndexPattern(String indexPattern, EsqlBaseParser.Ind String[] indices = indexPattern.split(","); boolean hasExclusion = false; for (String index : indices) { + if (isRemoteIndexName(index)) { // skip the validation if there is remote cluster + continue; + } hasSeenStar = index.contains(WILDCARD) || hasSeenStar; index = index.replace(WILDCARD, "").strip(); if (index.isBlank()) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java index 0123186ced1aa..7deaff6ebe6bb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java @@ -99,6 +99,60 @@ public void testNamedParams() throws IOException { Locale locale = randomLocale(random()); QueryBuilder filter = randomQueryBuilder(); + String paramsString = """ + ,"params":[ {"n1" : "8.15.0"}, { "n2" : 0.05}, {"n3" : -799810013}, + {"n4" : "127.0.0.1"}, {"n5" : "esql"}, {"n_6" : null}, {"n7_" : false}, + {"_n1" : "8.15.0"}, { "__n2" : 0.05}, {"__3" : -799810013}, + {"__4n" : "127.0.0.1"}, {"_n5" : "esql"}, {"_n6" : null}, {"_n7" : false}] }"""; + + List params = List.of( + paramAsConstant("n1", "8.15.0"), + paramAsConstant("n2", 0.05), + paramAsConstant("n3", -799810013), + paramAsConstant("n4", "127.0.0.1"), + paramAsConstant("n5", "esql"), + paramAsConstant("n_6", null), + paramAsConstant("n7_", false), + paramAsConstant("_n1", "8.15.0"), + paramAsConstant("__n2", 0.05), + paramAsConstant("__3", -799810013), + paramAsConstant("__4n", "127.0.0.1"), + paramAsConstant("_n5", "esql"), + paramAsConstant("_n6", null), + paramAsConstant("_n7", false) + ); + String json = String.format(Locale.ROOT, """ + { + "query": "%s", + "columnar": %s, + "locale": "%s", + "filter": %s + %s""", query, columnar, locale.toLanguageTag(), filter, paramsString); + + EsqlQueryRequest request = parseEsqlQueryRequestSync(json); + + assertEquals(query, request.query()); + assertEquals(columnar, request.columnar()); + assertEquals(locale.toLanguageTag(), request.locale().toLanguageTag()); + assertEquals(locale, request.locale()); + assertEquals(filter, request.filter()); + assertEquals(params.size(), request.params().size()); + + for (int i = 0; i < request.params().size(); i++) { + assertEquals(params.get(i), request.params().get(i + 1)); + } + } + + public void testNamedParamsForIdentifiersPatterns() throws IOException { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); + String query = randomAlphaOfLengthBetween(1, 100); + boolean columnar = randomBoolean(); + Locale locale = randomLocale(random()); + QueryBuilder filter = randomQueryBuilder(); + String paramsString = """ ,"params":[ {"n1" : {"value" : "f1", "kind" : "Identifier"}}, {"n2" : {"value" : "f1*", "Kind" : "identifier"}}, @@ -106,14 +160,7 @@ public void testNamedParams() throws IOException { {"n4" : {"value" : "*", "kind" : "pattern"}}, {"n5" : {"value" : "esql", "kind" : "Value"}}, {"n_6" : {"value" : "null", "kind" : "identifier"}}, - {"n7_" : {"value" : "f.1.1"}}, - {"_n1" : "8.15.0"}, - { "__n2" : 0.05}, - {"__3" : -799810013}, - {"__4n" : "127.0.0.1"}, - {"_n5" : "esql"}, - {"_n6" : null}, - {"_n7" : false}] }"""; + {"n7_" : {"value" : "f.1.1"}}] }"""; List params = List.of( paramAsIdentifier("n1", "f1"), @@ -122,14 +169,7 @@ public void testNamedParams() throws IOException { paramAsPattern("n4", "*"), paramAsConstant("n5", "esql"), paramAsIdentifier("n_6", "null"), - paramAsConstant("n7_", "f.1.1"), - paramAsConstant("_n1", "8.15.0"), - paramAsConstant("__n2", 0.05), - paramAsConstant("__3", -799810013), - paramAsConstant("__4n", "127.0.0.1"), - paramAsConstant("_n5", "esql"), - paramAsConstant("_n6", null), - paramAsConstant("_n7", false) + paramAsConstant("n7_", "f.1.1") ); String json = String.format(Locale.ROOT, """ { @@ -217,9 +257,20 @@ public void testInvalidParams() throws IOException { e2.getCause().getMessage(), containsString("Params cannot contain both named and unnamed parameters; got [{1:v1}, {1x:v1}] and [{1}, {2}]") ); + } + + public void testInvalidParamsForIdentifiersPatterns() throws IOException { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); + String query = randomAlphaOfLengthBetween(1, 100); + boolean columnar = randomBoolean(); + Locale locale = randomLocale(random()); + QueryBuilder filter = randomQueryBuilder(); // invalid named parameter for identifier and identifier pattern - String paramsString3 = """ + String paramsString1 = """ "params":[ {"n1" : {"v" : "v1"}}, {"n2" : {"value" : "v2", "type" : "identifier"}}, {"n3" : {"value" : "v3", "kind" : "id" }}, {"n4" : {"value" : "v4", "kind" : true}}, {"n5" : {"value" : "v5", "kind" : ["identifier", "pattern"]}}, {"n6" : {"value" : "v6", "kind" : 0}}, @@ -227,18 +278,18 @@ public void testInvalidParams() throws IOException { {"n9" : {"kind" : "identifier"}}, {"n10" : {"v" : "v10", "kind" : "identifier"}}, {"n11" : {"value" : "v11", "kind" : "pattern"}}, {"n12" : {"value" : ["x", "y"], "kind" : "identifier"}}, {"n13" : {"value" : "v13", "kind" : "identifier", "type" : "pattern"}}, {"n14" : {"v" : "v14", "kind" : "value"}}]"""; - String json3 = String.format(Locale.ROOT, """ + String json1 = String.format(Locale.ROOT, """ { %s "query": "%s", "columnar": %s, "locale": "%s", "filter": %s - }""", paramsString3, query, columnar, locale.toLanguageTag(), filter); + }""", paramsString1, query, columnar, locale.toLanguageTag(), filter); - Exception e3 = expectThrows(XContentParseException.class, () -> parseEsqlQueryRequestSync(json3)); + Exception e1 = expectThrows(XContentParseException.class, () -> parseEsqlQueryRequestSync(json1)); assertThat( - e3.getCause().getMessage(), + e1.getCause().getMessage(), containsString( "Failed to parse params: [2:16] [v] is not a valid param attribute, a valid attribute is any of VALUE, KIND; " + "[2:39] [type] is not a valid param attribute, a valid attribute is any of VALUE, KIND; " @@ -252,10 +303,12 @@ public void testInvalidParams() throws IOException { + "[6:1] [null] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; " + "[6:35] [v] is not a valid param attribute, a valid attribute is any of VALUE, KIND; " + "[6:35] [n10={v=v10, kind=identifier}] does not have a value specified; " - + "[6:35] [null] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; " + + "[6:35] [null] is not a valid value for IDENTIFIER parameter, " + + "a valid value for IDENTIFIER parameter is a string; " + "[7:1] [v11] is not a valid value for PATTERN parameter, " + "a valid value for PATTERN parameter is a string and contains *; " - + "[7:50] [[x, y]] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; " + + "[7:50] [[x, y]] is not a valid value for IDENTIFIER parameter," + + " a valid value for IDENTIFIER parameter is a string; " + "[7:50] n12={kind=identifier, value=[x, y]} is not supported as a parameter; " + "[8:1] [type] is not a valid param attribute, a valid attribute is any of VALUE, KIND; " + "[8:73] [v] is not a valid param attribute, a valid attribute is any of VALUE, KIND; " diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index d365ee3bb2e51..3048686efbe44 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.LoadMapping; import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expressions; @@ -2084,6 +2085,10 @@ public void testCoalesceWithMixedNumericTypes() { } public void testNamedParamsForIdentifiers() { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); assertProjectionWithMapping( """ from test @@ -2174,6 +2179,10 @@ public void testNamedParamsForIdentifiers() { } public void testInvalidNamedParamsForIdentifiers() { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); // missing field assertError( """ @@ -2243,6 +2252,10 @@ public void testInvalidNamedParamsForIdentifiers() { } public void testNamedParamsForIdentifierPatterns() { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); assertProjectionWithMapping( """ from test @@ -2273,6 +2286,10 @@ public void testNamedParamsForIdentifierPatterns() { } public void testInvalidNamedParamsForIdentifierPatterns() { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); // missing pattern assertError( """ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java index ebad8e6e13b8c..39170f1a305df 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java @@ -446,9 +446,9 @@ EnrichResolution resolvePolicies(Collection clusters, Collection listener) { assertThat("Must only called on the local cluster", cluster, equalTo(LOCAL_CLUSTER_GROUP_KEY)); - return transports.get("").getConnection(transports.get(remoteCluster).getLocalDiscoNode()); + listener.onResponse(transports.get("").getConnection(transports.get(remoteCluster).getLocalNode())); } static ClusterService mockClusterService(Map policies) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index ca9950a4bfe77..84a41ef040c8e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -1283,13 +1283,6 @@ public void allMemoryReleased() { } } - /** - * All string types (keyword, text, match_only_text, etc). - */ - protected static DataType[] strings() { - return DataType.types().stream().filter(DataType::isString).toArray(DataType[]::new); - } - /** * Validate that we know the types for all the test cases already created * @param suppliers - list of suppliers before adding in the illegal type combinations @@ -1316,10 +1309,9 @@ private static boolean isAggregation() { */ private static boolean shouldHideSignature(List argTypes, DataType returnType) { for (DataType dt : DataType.UNDER_CONSTRUCTION.keySet()) { - if (returnType == dt) { + if (returnType == dt || argTypes.contains(dt)) { return true; } - return argTypes.contains(dt); } return false; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index b3942a71edadb..2ba175657b6c2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -21,7 +21,6 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.versionfield.Version; import org.hamcrest.Matcher; @@ -91,7 +90,7 @@ public static List stringCases( List lhsSuppliers = new ArrayList<>(); List rhsSuppliers = new ArrayList<>(); List suppliers = new ArrayList<>(); - for (DataType type : AbstractConvertFunction.STRING_TYPES) { + for (DataType type : DataType.stringTypes()) { lhsSuppliers.addAll(stringCases(type)); rhsSuppliers.addAll(stringCases(type)); casesCrossProduct( @@ -760,7 +759,7 @@ public static void forUnaryStrings( Function expectedValue, Function> expectedWarnings ) { - for (DataType type : AbstractConvertFunction.STRING_TYPES) { + for (DataType type : DataType.stringTypes()) { unary( suppliers, expectedEvaluatorToString, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java index b79252c694084..15ea029a05554 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java @@ -22,13 +22,15 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matcher; -import java.nio.ByteOrder; import java.util.List; import java.util.function.Supplier; import java.util.stream.Stream; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.closeTo; @FunctionName("st_centroid_agg") public class SpatialCentroidTests extends AbstractAggregationTestCase { @@ -74,16 +76,58 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier count++; } - var expected = new BytesRef( - WellKnownBinary.toWKB(new Point(xSum.value() / count, ySum.value() / count), ByteOrder.LITTLE_ENDIAN) - ); + var expectedX = xSum.value() / count; + var expectedY = ySum.value() / count; return new TestCaseSupplier.TestCase( List.of(fieldTypedData), "SpatialCentroid[field=Attribute[channel=0]]", fieldTypedData.type(), - equalTo(expected) + centroidMatches(expectedX, expectedY, 1e-14) ); }); } + + @SuppressWarnings("SameParameterValue") + private static Matcher centroidMatches(double x, double y, double error) { + return new TestCentroidMatcher(x, y, error); + } + + private static class TestCentroidMatcher extends BaseMatcher { + private final double x; + private final double y; + private final Matcher mx; + private final Matcher my; + + private TestCentroidMatcher(double x, double y, double error) { + this.x = x; + this.y = y; + this.mx = closeTo(x, error); + this.my = closeTo(y, error); + } + + @Override + public boolean matches(Object item) { + if (item instanceof BytesRef wkb) { + var point = (Point) WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length); + return mx.matches(point.getX()) && my.matches(point.getY()); + } + return false; + } + + @Override + public void describeMismatch(Object item, Description description) { + if (item instanceof BytesRef wkb) { + var point = (Point) WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length); + description.appendText("was ").appendValue(point); + } else { + description.appendText("was ").appendValue(item); + } + } + + @Override + public void describeTo(Description description) { + description.appendValue(" POINT (" + x + " " + y + ")"); + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java index d37bc89635c1d..967b4d854c325 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.hamcrest.Matcher; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -40,8 +39,8 @@ public static Iterable parameters() { Set supported = Set.of(DataType.KEYWORD, DataType.TEXT); List> supportedPerPosition = List.of(supported, supported); List suppliers = new LinkedList<>(); - for (DataType fieldType : validStringDataTypes()) { - for (DataType queryType : validStringDataTypes()) { + for (DataType fieldType : DataType.stringTypes()) { + for (DataType queryType : DataType.stringTypes()) { suppliers.add( new TestCaseSupplier( "<" + fieldType + "-ES field, " + queryType + ">", @@ -67,10 +66,6 @@ private static String matchTypeErrorSupplier(boolean includeOrdinal, List validStringDataTypes() { - return Arrays.stream(DataType.values()).filter(DataType::isString).toList(); - } - private static TestCaseSupplier.TestCase testCase( DataType fieldType, String field, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java index 2dfdb05ec8ecc..b4b4ebcaacde6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.hamcrest.Matcher; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.function.Supplier; @@ -36,7 +35,7 @@ public QueryStringTests(@Name("TestCase") Supplier te @ParametersFactory public static Iterable parameters() { List suppliers = new LinkedList<>(); - for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) { + for (DataType strType : DataType.stringTypes()) { suppliers.add( new TestCaseSupplier( "<" + strType + ">", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java index 7799c3c756f23..2852b92ba156e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java @@ -134,9 +134,9 @@ public static Iterable parameters() { "ToDatetimeFromStringEvaluator[field=" + read + "]", List.of( new TestCaseSupplier.TypedDataSupplier( - "", - // millis before "0001-01-01T00:00:00.000Z" - () -> new BytesRef(randomDateString(Long.MIN_VALUE, -62135596800001L)), + "", + // millis before "-9999-12-31T23:59:59.999Z" + () -> new BytesRef(randomDateString(Long.MIN_VALUE, -377736739200000L)), DataType.KEYWORD ) ), @@ -154,8 +154,8 @@ public static Iterable parameters() { "ToDatetimeFromStringEvaluator[field=" + read + "]", List.of( new TestCaseSupplier.TypedDataSupplier( - "", - // millis before "0001-01-01T00:00:00.000Z" + "", + // millis after "9999-12-31T23:59:59.999Z" () -> new BytesRef(randomDateString(253402300800000L, Long.MAX_VALUE)), DataType.KEYWORD ) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java index 46a8086f9479c..57f11331818dc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java @@ -49,7 +49,7 @@ public static Iterable parameters() { ); // But strings that are shaped like versions do parse to valid versions - for (DataType inputType : AbstractConvertFunction.STRING_TYPES) { + for (DataType inputType : DataType.stringTypes()) { TestCaseSupplier.unary( suppliers, read, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java index 29cc959e6a943..0a419d44e3448 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.MultivalueTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; import java.math.BigDecimal; import java.util.ArrayList; @@ -28,6 +29,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -375,27 +377,25 @@ private static TestCaseSupplier makeSupplier( var values = (List) fieldTypedData.data(); var percentile = ((Number) percentileTypedData.data()).doubleValue(); - var expected = calculatePercentile(values, percentile); + var expectedMatcher = makePercentileMatcher(values, percentile); return new TestCaseSupplier.TestCase( List.of(fieldTypedData, percentileTypedData), evaluatorString(fieldSupplier.type(), percentileSupplier.type()), fieldSupplier.type(), - expected instanceof Double expectedDouble - ? closeTo(expectedDouble, Math.abs(expectedDouble * 0.0000001)) - : equalTo(expected) + expectedMatcher ); } ); } - private static Number calculatePercentile(List rawValues, double percentile) { + private static Matcher makePercentileMatcher(List rawValues, double percentile) { if (rawValues.isEmpty() || percentile < 0 || percentile > 100) { - return null; + return nullValue(); } if (rawValues.size() == 1) { - return rawValues.get(0); + return equalTo(rawValues.get(0)); } int valueCount = rawValues.size(); @@ -407,49 +407,62 @@ private static Number calculatePercentile(List rawValues, double percent if (rawValues.get(0) instanceof Integer) { var values = rawValues.stream().mapToInt(Number::intValue).sorted().toArray(); + int expected; if (percentile == 0) { - return values[0]; + expected = values[0]; } else if (percentile == 100) { - return values[valueCount - 1]; + expected = values[valueCount - 1]; } else { assert lowerIndex >= 0 && upperIndex < valueCount; var difference = (long) values[upperIndex] - values[lowerIndex]; - return values[lowerIndex] + (int) (fraction * difference); + expected = values[lowerIndex] + (int) (fraction * difference); } + + return equalTo(expected); } if (rawValues.get(0) instanceof Long) { var values = rawValues.stream().mapToLong(Number::longValue).sorted().toArray(); + long expected; if (percentile == 0) { - return values[0]; + expected = values[0]; } else if (percentile == 100) { - return values[valueCount - 1]; + expected = values[valueCount - 1]; } else { assert lowerIndex >= 0 && upperIndex < valueCount; - return calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])).longValue(); + expected = calculatePercentile(fraction, BigDecimal.valueOf(values[lowerIndex]), BigDecimal.valueOf(values[upperIndex])) + .longValue(); } + + // Double*bigLong may lose precision, we allow a small range + return anyOf(equalTo(Math.min(expected, expected - 1)), equalTo(expected), equalTo(Math.max(expected, expected + 1))); } if (rawValues.get(0) instanceof Double) { var values = rawValues.stream().mapToDouble(Number::doubleValue).sorted().toArray(); + double expected; if (percentile == 0) { - return values[0]; + expected = values[0]; } else if (percentile == 100) { - return values[valueCount - 1]; + expected = values[valueCount - 1]; } else { assert lowerIndex >= 0 && upperIndex < valueCount; - return calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])).doubleValue(); + expected = calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])) + .doubleValue(); } + + return closeTo(expected, Math.abs(expected * 0.0000001)); } throw new IllegalArgumentException("Unsupported type: " + rawValues.get(0).getClass()); } private static BigDecimal calculatePercentile(double fraction, BigDecimal lowerValue, BigDecimal upperValue) { - return lowerValue.add(new BigDecimal(fraction).multiply(upperValue.subtract(lowerValue))); + var difference = upperValue.subtract(lowerValue); + return lowerValue.add(new BigDecimal(fraction).multiply(difference)); } private static TestCaseSupplier.TypedData percentileWithType(Number value, DataType type) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java index 6794de80f7433..c93d871ca2b8c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java @@ -267,19 +267,10 @@ private static DataType pickSpatialType(DataType leftType, DataType rightType) { private static Matcher spatialEvaluatorString(DataType leftType, DataType rightType) { String crsType = isSpatialGeo(pickSpatialType(leftType, rightType)) ? "Geo" : "Cartesian"; - String paramSuffix = paramSuffix(); - String channels = channelsText("left" + paramSuffix, "right" + paramSuffix); + String channels = channelsText("left", "right"); return equalTo(getFunctionClassName() + crsType + "SourceAndSourceEvaluator[" + channels + "]"); } - private static String paramSuffix() { - try { - return getSpatialRelatesFunctionClass().getSimpleName().contains("Distance") ? "Value" : ""; - } catch (ClassNotFoundException e) { - return ""; - } - } - private static String channelsText(String... args) { return IntStream.range(0, args.length).mapToObj(i -> args[i] + "=Attribute[channel=" + i + "]").collect(Collectors.joining(", ")); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java index f77a892d8682e..d069f7ffe2298 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java @@ -21,7 +21,7 @@ public abstract class AbstractTrimTests extends AbstractScalarFunctionTestCase { static Iterable parameters(String name, boolean trimLeading, boolean trimTrailing) { List suppliers = new ArrayList<>(); - for (DataType type : strings()) { + for (DataType type : DataType.stringTypes()) { suppliers.add(new TestCaseSupplier("no whitespace/" + type, List.of(type), () -> { String text = randomAlphaOfLength(8); return testCase(name, type, text, text); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java index bbe92ae4a6618..2ad953c9296b7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java @@ -58,9 +58,7 @@ public static Iterable parameters() { if (rhs == DataType.NULL || DataType.isRepresentable(rhs) == false) { continue; } - boolean lhsIsString = lhs == DataType.KEYWORD || lhs == DataType.TEXT; - boolean rhsIsString = rhs == DataType.KEYWORD || rhs == DataType.TEXT; - if (lhsIsString && rhsIsString) { + if (DataType.isString(lhs) && DataType.isString(rhs)) { continue; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java index 6d086e2626cb6..1b2e9c41cb25c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.hamcrest.Matcher; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.function.Supplier; @@ -33,8 +32,8 @@ public EndsWithTests(@Name("TestCase") Supplier testC @ParametersFactory public static Iterable parameters() { List suppliers = new LinkedList<>(); - for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) { - for (DataType suffixType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) { + for (DataType strType : DataType.stringTypes()) { + for (DataType suffixType : DataType.stringTypes()) { suppliers.add( new TestCaseSupplier( "<" + strType + ">, empty <" + suffixType + ">", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java index 207125bed2a19..a10f97c45aa04 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java @@ -35,13 +35,11 @@ public LocateTests(@Name("TestCase") Supplier testCas this.testCase = testCaseSupplier.get(); } - private static final DataType[] STRING_TYPES = new DataType[] { DataType.KEYWORD, DataType.TEXT }; - @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - for (DataType strType : STRING_TYPES) { - for (DataType substrType : STRING_TYPES) { + for (DataType strType : DataType.stringTypes()) { + for (DataType substrType : DataType.stringTypes()) { suppliers.add( supplier( "", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java index 5a34d850cffe3..dab2fca212ff4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java @@ -69,7 +69,7 @@ static Iterable parameters(Function escapeString, Supp casesForString(cases, "6 bytes, 2 code points", () -> "❗️", false, escapeString, optionalPattern); casesForString(cases, "100 random code points", () -> randomUnicodeOfCodepointLength(100), true, escapeString, optionalPattern); for (DataType type : DataType.types()) { - if (type == DataType.KEYWORD || type == DataType.TEXT || type == DataType.NULL) { + if (DataType.isString(type) || type == DataType.NULL) { continue; } if (DataType.isRepresentable(type) == false) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java index 8bc8cf3184a75..60ed3b05ad642 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.function.Supplier; @@ -32,8 +31,8 @@ public StartsWithTests(@Name("TestCase") Supplier tes @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) { - for (DataType prefixType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) { + for (DataType strType : DataType.stringTypes()) { + for (DataType prefixType : DataType.stringTypes()) { suppliers.add(new TestCaseSupplier(List.of(strType, prefixType), () -> { String str = randomAlphaOfLength(5); String prefix = randomAlphaOfLength(5); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java index a1bcdec2b7c5c..e6fef186721a0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java @@ -144,8 +144,15 @@ void expectError(String query, List params, String errorMessage) { assertThat(e.getMessage(), containsString(errorMessage)); } - void expectInvalidIndexNameErrorWithLineNumber(String query, String arg, String lineNumber, String indexString) { - expectError(LoggerMessageFormat.format(null, query, arg), lineNumber + "Invalid index name [" + indexString); + void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber) { + if ((indexString.contains("|") || indexString.contains(" ")) == false) { + expectInvalidIndexNameErrorWithLineNumber(query, indexString, lineNumber, indexString); + } + expectInvalidIndexNameErrorWithLineNumber(query, "\"" + indexString + "\"", lineNumber, indexString); + } + + void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String error) { + expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + error); } void expectDateMathErrorWithLineNumber(String query, String arg, String lineNumber, String error) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index c797f426d2ae5..094d301875d8e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.Build; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -58,6 +59,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -424,7 +426,12 @@ public void testInlineStatsWithoutGroups() { } public void testStringAsIndexPattern() { - for (String command : List.of("FROM", "METRICS")) { + List commands = new ArrayList<>(); + commands.add("FROM"); + if (Build.current().isSnapshot()) { + commands.add("METRICS"); + } + for (String command : commands) { assertStringAsIndexPattern("foo", command + " \"foo\""); assertStringAsIndexPattern("foo,test-*", command + """ "foo","test-*" @@ -456,15 +463,23 @@ public void testStringAsIndexPattern() { assertStringAsIndexPattern("`backtick`,``multiple`back``ticks```", command + " `backtick`, ``multiple`back``ticks```"); assertStringAsIndexPattern("test,metadata,metaata,.metadata", command + " test,\"metadata\", metaata, .metadata"); assertStringAsIndexPattern(".dot", command + " .dot"); - assertStringAsIndexPattern("cluster:index", command + " cluster:index"); - assertStringAsIndexPattern("cluster:.index", command + " cluster:.index"); - assertStringAsIndexPattern("cluster*:index*", command + " cluster*:index*"); - assertStringAsIndexPattern("cluster*:*", command + " cluster*:*"); - assertStringAsIndexPattern("*:index*", command + " *:index*"); - assertStringAsIndexPattern("*:*", command + " *:*"); + assertStringAsIndexPattern("cluster:index|pattern", command + " cluster:\"index|pattern\""); + assertStringAsIndexPattern("*:index|pattern", command + " \"*:index|pattern\""); + clusterAndIndexAsIndexPattern(command, "cluster:index"); + clusterAndIndexAsIndexPattern(command, "cluster:.index"); + clusterAndIndexAsIndexPattern(command, "cluster*:index*"); + clusterAndIndexAsIndexPattern(command, "cluster*:*"); + clusterAndIndexAsIndexPattern(command, "cluster*:*"); + clusterAndIndexAsIndexPattern(command, "*:index*"); + clusterAndIndexAsIndexPattern(command, "*:*"); } } + private void clusterAndIndexAsIndexPattern(String command, String clusterAndIndex) { + assertStringAsIndexPattern(clusterAndIndex, command + " " + clusterAndIndex); + assertStringAsIndexPattern(clusterAndIndex, command + " \"" + clusterAndIndex + "\""); + } + public void testStringAsLookupIndexPattern() { assumeTrue("requires snapshot build", Build.current().isSnapshot()); assertStringAsLookupIndexPattern("foo", "ROW x = 1 | LOOKUP \"foo\" ON j"); @@ -486,221 +501,104 @@ public void testStringAsLookupIndexPattern() { assertStringAsLookupIndexPattern("`backtick`", "ROW x = 1 | LOOKUP `backtick` ON j"); assertStringAsLookupIndexPattern("``multiple`back``ticks```", "ROW x = 1 | LOOKUP ``multiple`back``ticks``` ON j"); assertStringAsLookupIndexPattern(".dot", "ROW x = 1 | LOOKUP .dot ON j"); - assertStringAsLookupIndexPattern("cluster:index", "ROW x = 1 | LOOKUP cluster:index ON j"); - assertStringAsLookupIndexPattern("cluster:.index", "ROW x = 1 | LOOKUP cluster:.index ON j"); - assertStringAsLookupIndexPattern("cluster*:index*", "ROW x = 1 | LOOKUP cluster*:index* ON j"); - assertStringAsLookupIndexPattern("cluster*:*", "ROW x = 1 | LOOKUP cluster*:* ON j"); - assertStringAsLookupIndexPattern("*:index*", "ROW x = 1 | LOOKUP *:index* ON j"); - assertStringAsLookupIndexPattern("*:*", "ROW x = 1 | LOOKUP *:* ON j"); + clusterAndIndexAsLookupIndexPattern("cluster:index"); + clusterAndIndexAsLookupIndexPattern("cluster:.index"); + clusterAndIndexAsLookupIndexPattern("cluster*:index*"); + clusterAndIndexAsLookupIndexPattern("cluster*:*"); + clusterAndIndexAsLookupIndexPattern("*:index*"); + clusterAndIndexAsLookupIndexPattern("*:*"); + } + + private void clusterAndIndexAsLookupIndexPattern(String clusterAndIndex) { + assertStringAsLookupIndexPattern(clusterAndIndex, "ROW x = 1 | LOOKUP " + clusterAndIndex + " ON j"); + assertStringAsLookupIndexPattern(clusterAndIndex, "ROW x = 1 | LOOKUP \"" + clusterAndIndex + "\"" + " ON j"); } public void testInvalidCharacterInIndexPattern() { Map commands = new HashMap<>(); - commands.put("FROM {}", "line 1:8: "); + commands.put("FROM {}", "line 1:7: "); if (Build.current().isSnapshot()) { - commands.put("METRICS {}", "line 1:11: "); - commands.put("ROW x = 1 | LOOKUP {} ON j", "line 1:22: "); + commands.put("METRICS {}", "line 1:10: "); + commands.put("ROW x = 1 | LOOKUP {} ON j", "line 1:21: "); } - List clusterStrings = List.of(" ", " *:", " cluster:"); String lineNumber; for (String command : commands.keySet()) { lineNumber = commands.get(command); - for (String clusterString : clusterStrings) { - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index|pattern\"", lineNumber, "index|pattern"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index pattern\"", lineNumber, "index pattern"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index#pattern\"", lineNumber, "index#pattern"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index#pattern", lineNumber, "index#pattern"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index?pattern\"", lineNumber, "index?pattern"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index?pattern", lineNumber, "index?pattern"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index>pattern\"", lineNumber, "index>pattern"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index>pattern", lineNumber, "index>pattern"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index\"", - lineNumber, - "-logstash-" - ); - expectInvalidIndexNameErrorWithLineNumber( - command, - clusterString + "--", - lineNumber, - "-" - ); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"\"", lineNumber, "logstash#"); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "", lineNumber, "logstash#"); - expectInvalidIndexNameErrorWithLineNumber( - command, - clusterString + "\"+\"", - lineNumber, - "+" - ); - expectInvalidIndexNameErrorWithLineNumber( - command, - clusterString + "+", - lineNumber, - "+" - ); - expectInvalidIndexNameErrorWithLineNumber( - command, - clusterString + "\"_\"", - lineNumber, - "_" - ); - expectInvalidIndexNameErrorWithLineNumber( - command, - clusterString + "_", - lineNumber, - "_" - ); - expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"<>\"", lineNumber, ">", lineNumber, ">>\"", lineNumber, ">>", lineNumber, "\"", - lineNumber, - "logstash- " - ); - } + expectInvalidIndexNameErrorWithLineNumber(command, "index|pattern", lineNumber); + expectInvalidIndexNameErrorWithLineNumber(command, "index pattern", lineNumber); + expectInvalidIndexNameErrorWithLineNumber(command, "index#pattern", lineNumber); + expectInvalidIndexNameErrorWithLineNumber(command, "index?pattern", lineNumber); + expectInvalidIndexNameErrorWithLineNumber(command, "index>pattern", lineNumber); + expectInvalidIndexNameErrorWithLineNumber(command, "index", lineNumber); + expectInvalidIndexNameErrorWithLineNumber(command, "_", lineNumber); + expectInvalidIndexNameErrorWithLineNumber(command, "index\\pattern", lineNumber, "index\\pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"index\\\\pattern\"", lineNumber, "index\\pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"--indexpattern\"", lineNumber, "-indexpattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "--indexpattern", lineNumber, "-indexpattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "<--logstash-{now/M{yyyy.MM}}>", lineNumber, "-logstash-"); + expectInvalidIndexNameErrorWithLineNumber( + command, + "\"--\"", + lineNumber, + "-" + ); + expectInvalidIndexNameErrorWithLineNumber(command, "", lineNumber, "logstash#"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"\"", lineNumber, "logstash#"); + expectInvalidIndexNameErrorWithLineNumber(command, "<>", lineNumber, ">\"", lineNumber, ">>", lineNumber, ">>\"", lineNumber, "\"", lineNumber, "logstash- "); } - // comma separated indices + // comma separated indices, with exclusions // Invalid index names after removing exclusion fail, when there is no index name with wildcard before it for (String command : commands.keySet()) { if (command.contains("LOOKUP")) { continue; } - for (String clusterString : clusterStrings) { - lineNumber = command.contains("FROM") - ? "line 1:" + (22 + clusterString.length() - 1) + ": " - : "line 1:" + (25 + clusterString.length() - 1) + ": "; - expectInvalidIndexNameErrorWithLineNumber( - command, - clusterString + "indexpattern, --indexpattern", - lineNumber, - "-indexpattern" - ); - expectInvalidIndexNameErrorWithLineNumber( - command, - clusterString + "indexpattern, \"--indexpattern\"", - lineNumber, - "-indexpattern" - ); - expectInvalidIndexNameErrorWithLineNumber( - command, - clusterString + "\"indexpattern, --indexpattern\"", - commands.get(command), - "-indexpattern" - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "indexpattern,-indexpattern"), - statement(command, clusterString + "indexpattern, -indexpattern") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "indexpattern,-indexpattern"), - statement(command, clusterString + "indexpattern, \"-indexpattern\"") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "indexpattern, -indexpattern"), - statement(command, clusterString + "\"indexpattern, -indexpattern\"") - ); - } + + lineNumber = command.contains("FROM") ? "line 1:21: " : "line 1:24: "; + expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, --indexpattern", lineNumber, "-indexpattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, \"--indexpattern\"", lineNumber, "-indexpattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern, --indexpattern\"", commands.get(command), "-indexpattern"); + clustersAndIndices(command, "indexpattern", "-indexpattern"); } // Invalid index names, except invalid DateMath, are ignored if there is an index name with wildcard before it + String dateMathError = "unit [D] not supported for date math [/D]"; for (String command : commands.keySet()) { if (command.contains("LOOKUP")) { continue; } - for (String clusterString : clusterStrings) { - lineNumber = command.contains("FROM") - ? "line 1:" + (11 + clusterString.length() - 1) + ": " - : "line 1:" + (14 + clusterString.length() - 1) + ": "; - assertEquals( - unresolvedRelation(clusterString.strip() + "*,-index#pattern"), - statement(command, clusterString + "*, \"-index#pattern\"") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "*,-index#pattern"), - statement(command, clusterString + "*, -index#pattern") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "*, -index#pattern"), - statement(command, clusterString + "\"*, -index#pattern\"") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "index*,-index#pattern"), - statement(command, clusterString + "index*, \"-index#pattern\"") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "index*,-index#pattern"), - statement(command, clusterString + "index*, -index#pattern") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "index*, -index#pattern"), - statement(command, clusterString + "\"index*, -index#pattern\"") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "*,-<--logstash-{now/M{yyyy.MM}}>"), - statement(command, clusterString + "*, \"-<--logstash-{now/M{yyyy.MM}}>\"") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "*,-<--logstash-{now/M{yyyy.MM}}>"), - statement(command, clusterString + "*, -<--logstash-{now/M{yyyy.MM}}>") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "*, -<--logstash-{now/M{yyyy.MM}}>"), - statement(command, clusterString + "\"*, -<--logstash-{now/M{yyyy.MM}}>\"") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "index*,-<--logstash#-{now/M{yyyy.MM}}>"), - statement(command, clusterString + "index*, \"-<--logstash#-{now/M{yyyy.MM}}>\"") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "index*,-<--logstash#-{now/M{yyyy.MM}}>"), - statement(command, clusterString + "index*, -<--logstash#-{now/M{yyyy.MM}}>") - ); - assertEquals( - unresolvedRelation(clusterString.strip() + "index*, -<--logstash#-{now/M{yyyy.MM}}>"), - statement(command, clusterString + "\"index*, -<--logstash#-{now/M{yyyy.MM}}>\"") - ); - expectDateMathErrorWithLineNumber( - command, - clusterString + "*, \"-<-logstash-{now/D}>\"", - lineNumber, - "unit [D] not supported for date math [/D]" - ); - expectDateMathErrorWithLineNumber( - command, - clusterString + "*, -<-logstash-{now/D}>", - lineNumber, - "unit [D] not supported for date math [/D]" - ); - expectDateMathErrorWithLineNumber( - command, - clusterString + "\"*, -<-logstash-{now/D}>\"", - commands.get(command), - "unit [D] not supported for date math [/D]" - ); - } + lineNumber = command.contains("FROM") ? "line 1:10: " : "line 1:13: "; + clustersAndIndices(command, "*", "-index#pattern"); + clustersAndIndices(command, "index*", "-index#pattern"); + clustersAndIndices(command, "*", "-<--logstash-{now/M{yyyy.MM}}>"); + clustersAndIndices(command, "index*", "-<--logstash#-{now/M{yyyy.MM}}>"); + expectDateMathErrorWithLineNumber(command, "*, \"-<-logstash-{now/D}>\"", lineNumber, dateMathError); + expectDateMathErrorWithLineNumber(command, "*, -<-logstash-{now/D}>", lineNumber, dateMathError); + expectDateMathErrorWithLineNumber(command, "\"*, -<-logstash-{now/D}>\"", commands.get(command), dateMathError); } } + private void clustersAndIndices(String command, String indexString1, String indexString2) { + assertEquals(unresolvedRelation(indexString1 + "," + indexString2), statement(command, indexString1 + ", " + indexString2)); + assertEquals( + unresolvedRelation(indexString1 + "," + indexString2), + statement(command, indexString1 + ", \"" + indexString2 + "\"") + ); + assertEquals( + unresolvedRelation(indexString1 + ", " + indexString2), + statement(command, "\"" + indexString1 + ", " + indexString2 + "\"") + ); + } + public void testInvalidQuotingAsFromIndexPattern() { expectError("FROM \"foo", ": token recognition error at: '\"foo'"); expectError("FROM \"foo | LIMIT 1", ": token recognition error at: '\"foo | LIMIT 1'"); @@ -984,11 +882,14 @@ public void testDeprecatedIsNullFunction() { "row x = is_null(f)", "line 1:10: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" ); - expectError( - "from test | eval x = ?fn1(f)", - List.of(paramAsIdentifier("fn1", "IS_NULL")), - "line 1:23: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" - ); + + if (Build.current().isSnapshot()) { + expectError( + "from test | eval x = ?fn1(f)", + List.of(paramAsIdentifier("fn1", "IS_NULL")), + "line 1:23: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" + ); + } } public void testMetadataFieldOnOtherSources() { @@ -1661,6 +1562,10 @@ public void testIntervalParam() { } public void testParamForIdentifier() { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); // field names can appear in eval/where/stats/sort/keep/drop/rename/dissect/grok/enrich/mvexpand // eval, where assertEquals( @@ -1918,6 +1823,10 @@ public void testParamForIdentifier() { } public void testParamForIdentifierPattern() { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); // name patterns can appear in keep and drop // all patterns LogicalPlan plan = statement( @@ -2007,6 +1916,10 @@ public void testParamForIdentifierPattern() { } public void testParamInInvalidPosition() { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); // param for pattern is not supported in eval/where/stats/sort/rename/dissect/grok/enrich/mvexpand // where/stats/sort/dissect/grok are covered in RestEsqlTestCase List invalidParamPositions = List.of("eval ?f1 = 1", "stats x = ?f1(*)", "mv_expand ?f1", "rename ?f1 as ?f2"); @@ -2058,6 +1971,10 @@ public void testParamInInvalidPosition() { } public void testMissingParam() { + assumeTrue( + "named parameters for identifiers and patterns require snapshot build", + EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES.isEnabled() + ); // cover all processing commands eval/where/stats/sort/rename/dissect/grok/enrich/mvexpand/keep/drop String error = "Unknown query parameter [f1], did you mean [f4]?"; String errorMvExpandFunctionNameCommandOption = "Query parameter [?f1] is null or undefined, cannot be used as an identifier"; @@ -2139,13 +2056,11 @@ public void testIdPatternUnquoted() throws Exception { public void testIdPatternQuoted() throws Exception { var string = "`escaped string`"; - List temp = breakIntoFragments(string); assertThat(breakIntoFragments(string), contains(string)); } public void testIdPatternQuotedWithDoubleBackticks() throws Exception { var string = "`escaped``string`"; - List temp = breakIntoFragments(string); assertThat(breakIntoFragments(string), contains(string)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java index 8ad083683f696..babb9fc8c0bd1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java @@ -62,8 +62,7 @@ public void testCommonTypeNull() { } public void testCommonTypeStrings() { - List STRINGS = Arrays.stream(DataType.values()).filter(DataType::isString).toList(); - for (DataType dataType1 : STRINGS) { + for (DataType dataType1 : DataType.stringTypes()) { for (DataType dataType2 : DataType.values()) { if (dataType2 == NULL) { assertEqualsCommonType(dataType1, NULL, dataType1); diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java index a6c369734f0e3..a79424b8b7d59 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/rest/RestFleetSearchAction.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.fleet.rest; -import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.client.internal.node.NodeClient; @@ -57,12 +56,11 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - SearchRequest searchRequest; - if (request.hasParam("min_compatible_shard_node")) { - searchRequest = new SearchRequest(Version.fromString(request.param("min_compatible_shard_node"))); - } else { - searchRequest = new SearchRequest(); - } + SearchRequest searchRequest = new SearchRequest(); + // access the BwC param, but just drop it + // this might be set by old clients + request.param("min_compatible_shard_node"); + String[] indices = searchRequest.indices(); if (indices.length > 1) { throw new IllegalArgumentException( diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java index 369752ea5ed75..f07d17fee87f5 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java @@ -6,27 +6,20 @@ */ package org.elasticsearch.xpack.frozen.rest.action; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.action.RestBuilderListener; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.rest.RestStatus.GONE; import static org.elasticsearch.rest.RestUtils.getAckTimeout; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; @@ -53,22 +46,6 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.path().endsWith("/_freeze")) { - // translate to a get indices request, so that we'll 404 on non-existent indices - final GetIndexRequest getIndexRequest = new GetIndexRequest(); - getIndexRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); - getIndexRequest.masterNodeTimeout(getMasterNodeTimeout(request)); - getIndexRequest.indicesOptions(IndicesOptions.fromRequest(request, getIndexRequest.indicesOptions())); - return channel -> client.admin().indices().getIndex(getIndexRequest, new RestBuilderListener<>(channel) { - @Override - public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBuilder builder) throws Exception { - builder.close(); - // but if the index *does* exist, we still just respond with 410 -- there's no such thing as _freeze anymore - return new RestResponse(channel, GONE, new UnsupportedOperationException(FREEZE_REMOVED)); - } - }); - } - final var freezeRequest = new FreezeRequest( getMasterNodeTimeout(request), getAckTimeout(request), diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java index 7dd360f8a2d82..973151e73d8dc 100644 --- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java +++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest; import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest.TermBoost; @@ -78,11 +77,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { - deprecationLogger.compatibleCritical("graph_with_types", TYPES_DEPRECATION_MESSAGE); - request.param("type"); - } - GraphExploreRequest graphRequest = new GraphExploreRequest(Strings.splitStringByCommaToArray(request.param("index"))); graphRequest.indicesOptions(IndicesOptions.fromRequest(request, graphRequest.indicesOptions())); graphRequest.routing(request.param("routing")); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java index 55aad5c55a2ac..5ee1e40869dbc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java @@ -9,13 +9,13 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.inference.InferenceServiceRegistry; -import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; @@ -29,11 +29,8 @@ import org.elasticsearch.xpack.inference.registry.ModelRegistry; import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.concurrent.Executor; -import java.util.stream.Collectors; public class TransportGetInferenceModelAction extends HandledTransportAction< GetInferenceModelAction.Request, @@ -99,69 +96,38 @@ private void getSingleModel( var model = service.get() .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings()); - - service.get() - .updateModelsWithDynamicFields( - List.of(model), - delegate.delegateFailureAndWrap( - (l2, updatedModels) -> l2.onResponse( - new GetInferenceModelAction.Response( - updatedModels.stream().map(Model::getConfigurations).collect(Collectors.toList()) - ) - ) - ) - ); + delegate.onResponse(new GetInferenceModelAction.Response(List.of(model.getConfigurations()))); })); } private void getAllModels(ActionListener listener) { - modelRegistry.getAllModels(listener.delegateFailureAndWrap((l, models) -> executor.execute(() -> parseModels(models, listener)))); + modelRegistry.getAllModels( + listener.delegateFailureAndWrap((l, models) -> executor.execute(ActionRunnable.supply(l, () -> parseModels(models)))) + ); } private void getModelsByTaskType(TaskType taskType, ActionListener listener) { modelRegistry.getModelsByTaskType( taskType, - listener.delegateFailureAndWrap((l, models) -> executor.execute(() -> parseModels(models, listener))) + listener.delegateFailureAndWrap((l, models) -> executor.execute(ActionRunnable.supply(l, () -> parseModels(models)))) ); } - private void parseModels(List unparsedModels, ActionListener listener) { - var parsedModelsByService = new HashMap>(); - try { - for (var unparsedModel : unparsedModels) { - var service = serviceRegistry.getService(unparsedModel.service()); - if (service.isEmpty()) { - throw serviceNotFoundException(unparsedModel.service(), unparsedModel.inferenceEntityId()); - } - var list = parsedModelsByService.computeIfAbsent(service.get().name(), s -> new ArrayList<>()); - list.add( - service.get() - .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings()) - ); - } - - var groupedListener = new GroupedActionListener>( - parsedModelsByService.entrySet().size(), - listener.delegateFailureAndWrap((delegate, listOfListOfModels) -> { - var modifiable = new ArrayList(); - for (var l : listOfListOfModels) { - modifiable.addAll(l); - } - modifiable.sort(Comparator.comparing(Model::getInferenceEntityId)); - delegate.onResponse( - new GetInferenceModelAction.Response(modifiable.stream().map(Model::getConfigurations).collect(Collectors.toList())) - ); - }) - ); + private GetInferenceModelAction.Response parseModels(List unparsedModels) { + var parsedModels = new ArrayList(); - for (var entry : parsedModelsByService.entrySet()) { - serviceRegistry.getService(entry.getKey()) - .get() // must be non-null to get this far - .updateModelsWithDynamicFields(entry.getValue(), groupedListener); + for (var unparsedModel : unparsedModels) { + var service = serviceRegistry.getService(unparsedModel.service()); + if (service.isEmpty()) { + throw serviceNotFoundException(unparsedModel.service(), unparsedModel.inferenceEntityId()); } - } catch (Exception e) { - listener.onFailure(e); + parsedModels.add( + service.get() + .parsePersistedConfig(unparsedModel.inferenceEntityId(), unparsedModel.taskType(), unparsedModel.settings()) + .getConfigurations() + ); } + return new GetInferenceModelAction.Response(parsedModels); } private ElasticsearchStatusException serviceNotFoundException(String service, String inferenceId) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java index 3ae8dc0550391..c5897f32d6eb8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java @@ -324,7 +324,7 @@ private ElasticsearchStatusException unexpectedResultTypeException(String got, S public void onFailure(Exception e) { var errorResult = new ErrorChunkedInferenceResults(e); for (var pos : positions) { - errors.setOnce(pos.inputIndex(), errorResult); + errors.set(pos.inputIndex(), errorResult); } if (resultCount.incrementAndGet() == totalNumberOfRequests) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java index 170b39e0bf76c..8525710c6cf23 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java @@ -7,65 +7,14 @@ package org.elasticsearch.xpack.inference.services.elastic; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import java.net.URI; -import java.net.URISyntaxException; import java.util.List; -import java.util.Objects; -import java.util.Set; -/** - * Class encapsulating any global setting for the EIS integration. - */ public class ElasticInferenceServiceSettings { - public static final Setting EIS_GATEWAY_URL = Setting.simpleString( - "xpack.inference.eis.gateway.url", - new EisGatewayURLValidator(), - Setting.Property.NodeScope - ); - - private static final Logger log = LogManager.getLogger(ElasticInferenceServiceSettings.class); - - /** - * Class to validate the EIS Gateway url set via `xpack.inference.eis.gateway.url`. - */ - public static class EisGatewayURLValidator implements Setting.Validator { - - private static final Set VALID_EIS_GATEWAY_SCHEMES = Set.of("http", "https"); - - @Override - public void validate(String value) { - if (Objects.isNull(value) || value.isEmpty()) { - // No validation needed, if eis-gateway URL is not set - log.debug("eis-gateway url not set. Skipping validation"); - return; - } - - try { - var uri = new URI(value); - var scheme = uri.getScheme(); - - if (scheme == null || VALID_EIS_GATEWAY_SCHEMES.contains(scheme) == false) { - throw new IllegalArgumentException( - "[" - + scheme - + "] is not a valid URI scheme for the setting [" - + ElasticInferenceServiceSettings.EIS_GATEWAY_URL.getKey() - + "]. Use one of [" - + String.join(",", VALID_EIS_GATEWAY_SCHEMES) - + "]" - ); - } - } catch (URISyntaxException e) { - throw new IllegalArgumentException("[" + e.getInput() + "] is not a valid URI", e); - } - } - } + static final Setting EIS_GATEWAY_URL = Setting.simpleString("xpack.inference.eis.gateway.url", Setting.Property.NodeScope); // Adjust this variable to be volatile, if the setting can be updated at some point in time private final String eisGatewayUrl; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java index b18b362dd099e..bbbae736dbeb9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java @@ -108,12 +108,6 @@ private URI createUri() throws URISyntaxException { default -> throw new IllegalArgumentException("Unsupported model for EIS [" + modelId + "]"); } - var uriString = elasticInferenceServiceComponents().eisGatewayUrl() + "/sparse-text-embedding/" + modelIdUriPath; - - // We perform the same validation here as when reading the setting to make sure that our extended URI is still valid - // This method throws, if the URI is invalid - new ElasticInferenceServiceSettings.EisGatewayURLValidator().validate(uriString); - - return new URI(uriString); + return new URI(elasticInferenceServiceComponents().eisGatewayUrl() + "/sparse-text-embedding/" + modelIdUriPath); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java index 642f6f144abc0..d38def8dca47f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java @@ -21,7 +21,7 @@ public abstract class ElasticsearchInternalModel extends Model { - protected ElasticsearchInternalServiceSettings internalServiceSettings; + protected final ElasticsearchInternalServiceSettings internalServiceSettings; public ElasticsearchInternalModel( String inferenceEntityId, @@ -91,15 +91,6 @@ public ElasticsearchInternalServiceSettings getServiceSettings() { return (ElasticsearchInternalServiceSettings) super.getServiceSettings(); } - public void updateNumAllocation(Integer numAllocations) { - this.internalServiceSettings = new ElasticsearchInternalServiceSettings( - numAllocations, - this.internalServiceSettings.getNumThreads(), - this.internalServiceSettings.modelId(), - this.internalServiceSettings.getAdaptiveAllocationsSettings() - ); - } - @Override public String toString() { return Strings.toString(this.getConfigurations()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 4546280b39fe2..389a9fa369c21 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -32,7 +32,6 @@ import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; -import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; @@ -57,7 +56,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -788,47 +786,11 @@ public List defaultConfigIds() { ); } + /** + * Default configurations that can be out of the box without creating an endpoint first. + * @param defaultsListener Config listener + */ @Override - public void updateModelsWithDynamicFields(List models, ActionListener> listener) { - var modelsByDeploymentIds = new HashMap(); - for (var model : models) { - if (model instanceof ElasticsearchInternalModel esModel) { - modelsByDeploymentIds.put(esModel.internalServiceSettings.deloymentId(), esModel); - } else { - listener.onFailure( - new ElasticsearchStatusException( - "Cannot update model [{}] as it is not an Elasticsearch service model", - RestStatus.INTERNAL_SERVER_ERROR, - model.getInferenceEntityId() - ) - ); - return; - } - } - - if (modelsByDeploymentIds.isEmpty()) { - listener.onResponse(models); - return; - } - - String deploymentIds = String.join(",", modelsByDeploymentIds.keySet()); - client.execute( - GetDeploymentStatsAction.INSTANCE, - new GetDeploymentStatsAction.Request(deploymentIds), - ActionListener.wrap(stats -> { - for (var deploymentStats : stats.getStats().results()) { - var model = modelsByDeploymentIds.get(deploymentStats.getDeploymentId()); - model.updateNumAllocation(deploymentStats.getNumberOfAllocations()); - } - listener.onResponse(new ArrayList<>(modelsByDeploymentIds.values())); - }, e -> { - logger.warn("Get deployment stats failed, cannot update the endpoint's number of allocations", e); - // continue with the original response - listener.onResponse(models); - }) - ); - } - public void defaultConfigs(ActionListener> defaultsListener) { preferredModelVariantFn.accept(defaultsListener.delegateFailureAndWrap((delegate, preferredModelVariant) -> { if (PreferredModelVariant.LINUX_X86_OPTIMIZED.equals(preferredModelVariant)) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index 5bd8d8cfc5c13..fedf48fb583a3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -194,10 +194,6 @@ public String modelId() { return modelId; } - public String deloymentId() { - return modelId; - } - public Integer getNumAllocations() { return numAllocations; } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettingsTests.java deleted file mode 100644 index a053a5eb33cfe..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettingsTests.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.services.elastic; - -import org.elasticsearch.test.ESTestCase; - -public class ElasticInferenceServiceSettingsTests extends ESTestCase { - - public void testEisGatewayURLValidator_Validate_ThrowError_OnMissingURIScheme() { - expectThrows( - IllegalArgumentException.class, - () -> new ElasticInferenceServiceSettings.EisGatewayURLValidator().validate("www.missing-scheme-gateway-url.com") - ); - } - - public void testEisGatewayURLValidator_Validate_ThrowError_OnWrongURIScheme() { - expectThrows( - IllegalArgumentException.class, - () -> new ElasticInferenceServiceSettings.EisGatewayURLValidator().validate("file://www.missing-scheme-gateway-url.com") - ); - } - - public void testEisGatewayURLValidator_Validate_DoesNotThrowError_ForHTTP() { - var scheme = "http"; - - try { - new ElasticInferenceServiceSettings.EisGatewayURLValidator().validate(scheme + "://www.valid-gateway-url.com"); - } catch (Exception e) { - fail(e, "Should not throw exception for " + "[" + scheme + "]"); - } - } - - public void testEisGatewayURLValidator_Validate_DoesNotThrowError_ForHTTPS() { - var scheme = "https"; - - try { - new ElasticInferenceServiceSettings.EisGatewayURLValidator().validate(scheme + "://www.valid-gateway-url.com"); - } catch (Exception e) { - fail(e, "Should not throw exception for " + "[" + scheme + "]"); - } - } - - public void testEisGatewayURLValidator_Validate_DoesNotThrowError_IfURLNull() { - try { - new ElasticInferenceServiceSettings.EisGatewayURLValidator().validate(null); - } catch (Exception e) { - fail(e, "Should not throw exception for, if eis-gateway URL is null"); - } - } - - public void testEisGatewayURLValidator_Validate_DoesNotThrowError_IfURLEmpty() { - try { - new ElasticInferenceServiceSettings.EisGatewayURLValidator().validate(""); - } catch (Exception e) { - fail(e, "Should not throw exception for, if eis-gateway URL is empty"); - } - } - -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java index 27d86e3d59461..c9f4234331221 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java @@ -15,34 +15,6 @@ public class ElasticInferenceServiceSparseEmbeddingsModelTests extends ESTestCase { - public void testCreateURI_ThrowError_OnMissingURIScheme() { - expectThrows(IllegalArgumentException.class, () -> createModel("www.missing-scheme-gateway-url.com")); - } - - public void testCreateURI_ThrowError_OnWrongURIScheme() { - expectThrows(IllegalArgumentException.class, () -> createModel("file://www.missing-scheme-gateway-url.com")); - } - - public void testCreateURI_DoesNotThrowError_ForHTTP() { - var scheme = "http"; - - try { - createModel(scheme + "://www.valid-gateway-url.com"); - } catch (Exception e) { - fail(e, "Should not throw exception for " + "[" + scheme + "]"); - } - } - - public void testCreateURI_DoesNotThrowError_ForHTTPS() { - var scheme = "https"; - - try { - createModel(scheme + "://www.valid-gateway-url.com"); - } catch (Exception e) { - fail(e, "Should not throw exception for " + "[" + scheme + "]"); - } - } - public static ElasticInferenceServiceSparseEmbeddingsModel createModel(String url) { return createModel(url, null); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index 5a5eae9f51670..d10c70c6f0f5e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -492,7 +492,7 @@ private ElasticInferenceService createServiceWithMockSender() { return new ElasticInferenceService( mock(HttpRequestSender.Factory.class), createWithEmptySettings(threadPool), - new ElasticInferenceServiceComponents("http://valid-eis-gateway-url.com") + new ElasticInferenceServiceComponents(null) ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java deleted file mode 100644 index 74cdab79fe79b..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModelTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.services.elasticsearch; - -import org.elasticsearch.inference.TaskType; -import org.elasticsearch.test.ESTestCase; - -public class ElserInternalModelTests extends ESTestCase { - public void testUpdateNumAllocation() { - var model = new ElserInternalModel( - "foo", - TaskType.SPARSE_EMBEDDING, - ElasticsearchInternalService.NAME, - new ElserInternalServiceSettings(null, 1, "elser", null), - new ElserMlNodeTaskSettings(), - null - ); - - model.updateNumAllocation(1); - assertEquals(1, model.internalServiceSettings.getNumAllocations().intValue()); - - model.updateNumAllocation(null); - assertNull(model.internalServiceSettings.getNumAllocations()); - } -} diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml index d33f57f763db8..47d87583fffb2 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml @@ -8,6 +8,8 @@ setup: indices.create: index: test-index body: + settings: + number_of_shards: 1 mappings: properties: text: diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml index 52c500c102cee..3f2bca2e4bcd9 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml @@ -280,3 +280,784 @@ create logsdb data stream with timestamp object mapping: - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "composable template [logsdb-index-template] template after composition with component templates [logsdb-mappings] is invalid" } + +--- +create logsdb data stream with custom sorting without host.name: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-http-prod ] + priority: 10000 + template: + settings: + index: + sort.field: [ agent.id ] + sort.order: [ desc ] + mode: logsdb + mappings: + properties: + agent.id: + type: keyword + host.hostname: + type: keyword + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-http-prod + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logs-http-prod + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.agent.properties.id.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.type: null } + +--- +create logsdb data stream with custom sorting and host object: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-nginx-prod ] + priority: 10000 + template: + settings: + index: + sort.field: [ host.hostname, host.region ] + sort.order: [ desc, desc ] + mode: logsdb + mappings: + properties: + host: + type: object + properties: + ip: + type: ip + hostname: + type: keyword + region: + type: keyword + name: + type: integer + + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-nginx-prod + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logs-nginx-prod + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.ip.type: ip } + - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.region.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.type: integer } # Overrides LogsDB injected + +--- +create logsdb data stream with custom sorting and dynamically mapped host.name: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-kafka-qa ] + priority: 10000 + template: + settings: + index: + sort.field: [ "agent.id", "@timestamp" ] + sort.order: [ desc, asc ] + mode: logsdb + mappings: + properties: + agent: + type: object + properties: + name: + type: keyword + id: + type: keyword + + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-kafka-qa + - is_true: acknowledged + + - do: + bulk: + index: logs-kafka-qa + refresh: true + body: + - { "create": { } } + - { "@timestamp": "2022-01-01T00:00:00", agent.name: "foo", agent.id: "foo-568", host: { id: "db8fdcf1-b1e2-444b-8c6a-0466c61dcce4" } } + - { "create": { } } + - { "@timestamp": "2022-01-01T00:01:00", agent.name: "bar", agent.id: "foo-309", host: { id: "35e1ed10-961e-46c7-83ea-4109c913a1d6" } } + + - do: + indices.get_data_stream: + name: logs-kafka-qa + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.agent.properties.name.type: keyword } + - match: { .$backing_index.mappings.properties.agent.properties.id.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name: null } + - match: { .$backing_index.mappings.properties.host.properties.id.type: text } + +--- +create logsdb data stream with custom sorting and host.name object: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-nginx-qa ] + priority: 10000 + template: + settings: + index: + sort.field: [ "host.name.value", "@timestamp" ] + sort.order: [ desc, desc ] + mode: logsdb + mappings: + properties: + host: + type: object + properties: + name: + type: object + properties: + value: + type: keyword + alias: + type: keyword + + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-nginx-qa + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logs-nginx-qa + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.name.properties.value.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.properties.alias.type: keyword } + +--- +create logsdb data stream with default sorting on malformed host.name: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-win-prod ] + priority: 10000 + template: + settings: + index: + mode: logsdb + mappings: + properties: + agent: + type: object + properties: + name: + type: keyword + id: + type: keyword + + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-win-prod + - is_true: acknowledged + + - do: + bulk: + index: logs-win-prod + refresh: true + body: + - { "create": { } } + - { "@timestamp": "2022-01-01T00:00:00", agent.name: "foo", agent.id: "foo-568", host: { name: 192.168.10.12, id: "e70e91cd-bb3f-43f0-909c-2748e7fdfd54" } } + - { "create": { } } + - { "@timestamp": "2022-01-01T00:01:00", agent.name: "bar", agent.id: "foo-309", host: { name: 192.168.15.17, id: "ad2e3edb-2c4b-4f12-83dd-255691ed614c" } } + + - do: + indices.get_data_stream: + name: logs-win-prod + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.agent.properties.name.type: keyword } + - match: { .$backing_index.mappings.properties.agent.properties.id.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword } # LogsDB injected + - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 } # LogsDB injected + - match: { .$backing_index.mappings.properties.host.properties.id.type: text } + +--- +create logsdb data stream with custom sorting and host.name date field: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-http-prod ] + priority: 10000 + template: + settings: + index: + sort.field: [ host.name, host.hostname ] + sort.order: [ desc, desc ] + mode: logsdb + mappings: + properties: + host: + type: object + properties: + hostname: + type: keyword + name: + type: date + + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-http-prod + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logs-http-prod + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.type: date } + +--- +create logsdb data stream with custom sorting and missing host.name field mapping: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-http-qa ] + priority: 10000 + template: + settings: + index: + sort.field: [ host.name, host.hostname ] + sort.order: [ desc, desc ] + mode: logsdb + mappings: + properties: + host: + type: object + properties: + hostname: + type: keyword + + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-http-qa + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logs-http-qa + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.hostname.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 } + +--- +create logsdb data stream with custom sorting and host.name field without doc values: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-http-dev ] + priority: 10000 + template: + settings: + index: + sort.field: [ "host.name", "@timestamp" ] + sort.order: [ desc, desc ] + mode: logsdb + mappings: + properties: + host: + type: object + properties: + name: + type: keyword + doc_values: false + + data_stream: { } + - is_true: acknowledged + + - do: + catch: bad_request + indices.create_data_stream: + name: logs-http-dev + + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "docvalues not found for index sort field:[host.name]" } + +--- +create logsdb data stream with incompatible ignore_above on host.name: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logsdb-index-template-ignore-above] has index patterns [logsdb-ignore-above] matching patterns from existing older templates [global]" + indices.put_index_template: + name: logsdb-index-template-ignore-above + body: + index_patterns: [ logsdb-ignore-above ] + priority: 10000 + template: + settings: + index: + sort.field: [ host.name ] + sort.order: [ desc ] + mode: logsdb + mappings: + properties: + host.name: + type: keyword + ignore_above: 128 + data_stream: {} + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logsdb-ignore-above + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logsdb-ignore-above + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 128 } + +--- +create logsdb data stream with no sorting and host.name as text: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logsdb-index-template-non-keyword] has index patterns [logsdb-non-keyword] matching patterns from existing older templates [global]" + indices.put_index_template: + name: logsdb-index-template-non-keyword + body: + index_patterns: [ logsdb-non-keyword ] + priority: 10000 + template: + settings: + mode: logsdb + mappings: + properties: + host.name: + type: text + data_stream: {} + - is_true: acknowledged + + - do: + catch: bad_request + indices.create_data_stream: + name: logsdb-non-keyword + + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "docvalues not found for index sort field:[host.name]" } + +--- +create logsdb data stream without index sorting and ignore_above on host.name: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logsdb-index-template-ignore-above-override] has index patterns [logsdb-ignore-above-override] matching patterns from existing older templates [global]" + indices.put_index_template: + name: logsdb-index-template-ignore-above-override + body: + index_patterns: [ logsdb-ignore-above-override ] + priority: 10000 + template: + settings: + index: + mode: logsdb + mappings: + properties: + host.name: + type: keyword + ignore_above: 128 + data_stream: {} + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logsdb-ignore-above-override + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logsdb-ignore-above-override + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 128 } + +--- +create logsdb data stream with host.name as alias and sorting on it: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logsdb-index-template-alias] has index patterns [logsdb-alias] matching patterns from existing older templates [global]" + indices.put_index_template: + name: logsdb-index-template-alias + body: + index_patterns: [ logsdb-alias ] + template: + settings: + index: + sort.field: [ host.name ] + sort.order: [ desc ] + mode: logsdb + mappings: + properties: + host.name: + type: alias + path: host.hostname + host.hostname: + type: + keyword + data_stream: {} + - do: + catch: bad_request + indices.create_data_stream: + name: logsdb-alias + + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "Cannot use alias [host.name] as an index sort field" } + +--- +create logsdb data stream with multi-fields on host.name: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logsdb-index-template-multi-fields] has index patterns [logsdb-multi-fields] matching patterns from existing older templates [global]" + indices.put_index_template: + name: logsdb-index-template-multi-fields + body: + index_patterns: [ logsdb-multi-fields ] + template: + settings: + index: + sort.field: [ host.name.keyword ] + sort.order: [ asc ] + mode: logsdb + mappings: + properties: + host.name: + type: "text" + fields: + keyword: + type: "keyword" + data_stream: {} + + - do: + indices.create_data_stream: + name: logsdb-multi-fields + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logsdb-multi-fields + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.name.fields.keyword.type: keyword } + +--- +create logsdb data stream with multi-fields on host.name and no sorting: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [ logsdb-no-sort-multi-fields-template ] has index patterns [logsdb-no-sort-multi-fields] matching patterns from existing older templates [global]" + indices.put_index_template: + name: logsdb-no-sort-multi-fields-template + body: + index_patterns: [ logsdb-no-sort-multi-fields ] + template: + settings: + mode: logsdb + mappings: + properties: + host.name: + type: text + fields: + keyword: + type: keyword + data_stream: {} + + - do: + catch: bad_request + indices.create_data_stream: + name: logsdb-no-sort-multi-fields + + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "docvalues not found for index sort field:[host.name]" } + +--- +create logsdb data stream with custom empty sorting: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-http-empty ] + priority: 10000 + template: + settings: + index: + sort.field: [ ] + sort.order: [ ] + mode: logsdb + mappings: + properties: + hostname: + type: keyword + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-http-empty + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logs-http-empty + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } + - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword } + - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 } + +--- +create logsdb data stream with custom sorting on timestamp: + - skip: + features: [ "allowed_warnings" ] + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + + - do: + allowed_warnings: + - "index template [logs-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-template] will take precedence during new index creation" + indices.put_index_template: + name: logs-template + body: + index_patterns: [ logs-http-dev ] + priority: 10000 + template: + settings: + index: + sort.field: [ "@timestamp" ] + sort.order: [ "asc" ] + mode: logsdb + mappings: + properties: + hostname: + type: keyword + data_stream: { } + - is_true: acknowledged + + - do: + indices.create_data_stream: + name: logs-http-dev + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: logs-http-dev + + - set: { data_streams.0.indices.0.index_name: backing_index } + - do: + indices.get_mapping: + index: $backing_index + + - match: { .$backing_index.mappings.properties.@timestamp.type: date } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index bb322cf79adc0..d981a60adbdb5 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -747,7 +747,15 @@ public void testMultiIndexDelete() throws Exception { } } }""", AnomalyDetectorsIndex.jobResultsAliasedName(jobId), Job.ID, jobId)); - client().performRequest(extraIndex1); + + // Creating an index with a leading dot (".") is now deprecated. + // Ensure the ensuing warning exception doesn't cause a test case failure + try { + client().performRequest(extraIndex1); + } catch (org.elasticsearch.client.WarningFailureException e) { + logger.warn(e.getMessage()); + } + Request extraIndex2 = new Request("PUT", indexName + "-002"); extraIndex2.setJsonEntity(Strings.format(""" { @@ -762,7 +770,14 @@ public void testMultiIndexDelete() throws Exception { } } }""", AnomalyDetectorsIndex.jobResultsAliasedName(jobId), Job.ID, jobId)); - client().performRequest(extraIndex2); + + // Creating an index with a leading dot (".") is now deprecated. + // Ensure the ensuing warning exception doesn't cause a test case failure + try { + client().performRequest(extraIndex2); + } catch (org.elasticsearch.client.WarningFailureException e) { + logger.warn(e.getMessage()); + } // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 String indicesBeforeDelete = EntityUtils.toString( @@ -983,10 +998,10 @@ private Response openJob(String jobId, Optional timeout) throws IOExc } private void closeJob(String jobId) throws IOException { - Response openResponse = client().performRequest( + Response closeResponse = client().performRequest( new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_close") ); - assertThat(entityAsMap(openResponse), hasEntry("closed", true)); + assertThat(entityAsMap(closeResponse), hasEntry("closed", true)); } private Response putJob(String jobId, String jsonBody) throws IOException { @@ -1000,5 +1015,8 @@ public void clearMlState() throws Exception { new MlRestTestStateCleaner(logger, adminClient()).resetFeatures(); // Don't check analytics jobs as they are independent of anomaly detection jobs and should not be created by this test. waitForPendingTasks(adminClient(), taskName -> taskName.contains(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME)); + // Finally, clean up any lingering persistent tasks (such as "_close", "_close[n]" etc.) that may negatively + // impact subsequent tests. + client().performRequest(new Request("POST", "/_tasks/_cancel")); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java index 1c3a73a409dd1..9624d619ff20a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java @@ -433,7 +433,8 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo public boolean maybeStartAllocation(TrainedModelAssignment assignment) { if (assignment.getAdaptiveAllocationsSettings() != null && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE - && assignment.getAdaptiveAllocationsSettings().getMinNumberOfAllocations() == 0) { + && (assignment.getAdaptiveAllocationsSettings().getMinNumberOfAllocations() == null + || assignment.getAdaptiveAllocationsSettings().getMinNumberOfAllocations() == 0)) { // Prevent against a flurry of scale up requests. if (deploymentIdsWithInFlightScaleFromZeroRequests.contains(assignment.getDeploymentId()) == false) { diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index 37dd93b7f16d9..d2df9b5629704 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -3,6 +3,8 @@ _meta: description: Default mappings for the OpenTelemetry metrics index template installed by x-pack managed: true template: + settings: + index.mapping.ignore_malformed: true mappings: properties: start_timestamp: @@ -19,27 +21,22 @@ template: - histogram: mapping: type: histogram - ignore_malformed: true - counter_long: mapping: type: long time_series_metric: counter - ignore_malformed: true - gauge_long: mapping: type: long time_series_metric: gauge - ignore_malformed: true - counter_double: mapping: type: double time_series_metric: counter - ignore_malformed: true - gauge_double: mapping: type: double time_series_metric: gauge - ignore_malformed: true - summary: mapping: type: aggregate_metric_double diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index 3b4c3127bb71c..f8489605ad1bf 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -27,14 +27,3 @@ template: data_stream.type: type: constant_keyword value: metrics - dynamic_templates: - - ecs_ip: - mapping: - type: ip - path_match: [ "ip", "*.ip", "*_ip" ] - match_mapping_type: string - - all_strings_to_keywords: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index 52873287696ab..b2d30c7f85cc4 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin otel-data. This must be increased whenever an existing template is # changed, in order for it to be updated on Elasticsearch upgrade. -version: 5 +version: 6 component-templates: - otel@mappings diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml index be4de6dca6c76..6bc0cee78be4f 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -163,3 +163,27 @@ Structured log body: fields: ["event.dataset"] - length: { hits.hits: 1 } - match: { hits.hits.0.fields.event\.dataset: ["generic.otel"] } +--- +host.name pass-through: + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - "@timestamp": 2024-07-18T14:48:33.467654000Z + resource: + attributes: + host.name: localhost + - is_false: errors + - do: + search: + index: logs-generic.otel-default + body: + query: + term: + host.name: localhost + fields: [ "*" ] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.resource\.attributes\.host\.name: [ "localhost" ] } + - match: { hits.hits.0.fields.host\.name: [ "localhost" ] } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml index 81aded9cef3be..1aafe3765813b 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml @@ -245,3 +245,37 @@ IP dimensions: - match: { .$idx0name.mappings.properties.metrics.properties.summary.type: 'aggregate_metric_double' } - match: { .$idx0name.mappings.properties.metrics.properties.summary_minmax.type: 'aggregate_metric_double' } - match: { .$idx0name.mappings.properties.metrics.properties.histogram.type: 'histogram' } +--- +Empty IP field: + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {"dynamic_templates":{"metrics.foo.bar":"counter_long"}} + - "@timestamp": 2024-07-18T14:48:33.467654000Z + resource: + attributes: + host.name: localhost + host.ip: "" + metrics: + foo.bar: 42 + - is_false: errors + + - do: + indices.get_data_stream: + name: metrics-generic.otel-default + - set: { data_streams.0.indices.0.index_name: idx0name } + + - do: + indices.get_mapping: + index: $idx0name + expand_wildcards: hidden + - match: { .$idx0name.mappings.properties.resource.properties.attributes.properties.host\.ip.type: 'ip' } + - do: + search: + index: metrics-generic.otel-default + body: + fields: ["*"] + - length: { hits.hits: 1 } + - match: { hits.hits.0._ignored: ["resource.attributes.host.ip"] } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java deleted file mode 100644 index 6431c83ee1c2e..0000000000000 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.ql.plugin; - -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.search.VersionMismatchException; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.xpack.ql.util.Holder; - -import java.util.function.Consumer; - -public final class TransportActionUtils { - - /** - * Execute a *QL request and re-try it in case the first request failed with a {@code VersionMismatchException} - * - * @param clusterService The cluster service instance - * @param onFailure On-failure handler in case the request doesn't fail with a {@code VersionMismatchException} - * @param queryRunner *QL query execution code, typically a Plan Executor running the query - * @param retryRequest Re-trial logic - * @param log Log4j logger - */ - public static void executeRequestWithRetryAttempt( - ClusterService clusterService, - Consumer onFailure, - Consumer> queryRunner, - Consumer retryRequest, - Logger log - ) { - - Holder retrySecondTime = new Holder(false); - queryRunner.accept(e -> { - // the search request likely ran on nodes with different versions of ES - // we will retry on a node with an older version that should generate a backwards compatible _search request - if (e instanceof SearchPhaseExecutionException - && ((SearchPhaseExecutionException) e).getCause() instanceof VersionMismatchException) { - if (log.isDebugEnabled()) { - log.debug("Caught exception type [{}] with cause [{}].", e.getClass().getName(), e.getCause()); - } - DiscoveryNode localNode = clusterService.state().nodes().getLocalNode(); - DiscoveryNode candidateNode = null; - for (DiscoveryNode node : clusterService.state().nodes()) { - // find the first node that's older than the current node - if (node != localNode && node.getVersion().before(localNode.getVersion())) { - candidateNode = node; - break; - } - } - if (candidateNode != null) { - if (log.isDebugEnabled()) { - log.debug( - "Candidate node to resend the request to: address [{}], id [{}], name [{}], version [{}]", - candidateNode.getAddress(), - candidateNode.getId(), - candidateNode.getName(), - candidateNode.getVersion() - ); - } - // re-send the request to the older node - retryRequest.accept(candidateNode); - } else { - retrySecondTime.set(true); - } - } else { - onFailure.accept(e); - } - }); - if (retrySecondTime.get()) { - if (log.isDebugEnabled()) { - log.debug("No candidate node found, likely all were upgraded in the meantime. Re-trying the original request."); - } - queryRunner.accept(onFailure); - } - } -} diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java index def26a613775a..edd5e557aadf0 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -174,7 +174,10 @@ public void testRRFPagination() { ); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 @@ -221,7 +224,10 @@ public void testRRFWithAggs() { ); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 @@ -273,7 +279,10 @@ public void testRRFWithCollapse() { ); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 @@ -327,7 +336,10 @@ public void testRRFRetrieverWithCollapseAndAggs() { ); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 @@ -390,7 +402,10 @@ public void testMultipleRRFRetrievers() { ); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 @@ -453,7 +468,10 @@ public void testRRFExplainWithNamedRetrievers() { standard0.retrieverName("my_custom_retriever"); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 @@ -509,7 +527,10 @@ public void testRRFExplainWithAnotherNestedRRF() { standard0.retrieverName("my_custom_retriever"); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 2, 3, 6, and 7 @@ -577,7 +598,10 @@ public void testRRFInnerRetrieverSearchError() { QueryBuilders.constantScoreQuery(QueryBuilders.rangeQuery(VECTOR_FIELD).gte(10)) ); StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); source.retriever( @@ -613,7 +637,10 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder } }; StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); source.retriever( @@ -646,7 +673,10 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder } }; StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); source.retriever( diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java index ea251917cfae2..69c61fe3bca1f 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java @@ -142,7 +142,10 @@ public void testRRFRetrieverWithNestedQuery() { ); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( - QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L) + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(20L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_3")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(5L)) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); // this one retrieves docs 6 diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java index 10aff2f4d68cd..fb20f834937d3 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java @@ -36,16 +36,17 @@ /** * The builder to support RRF. Adds user-defined parameters for window size and rank constant. + * + * @deprecated RRF support is provided through the retriever framework. Please use {@link RRFRetrieverBuilder instead} */ +@Deprecated public class RRFRankBuilder extends RankBuilder { - public static final int DEFAULT_RANK_CONSTANT = 60; - public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant"); static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(RRFRankPlugin.NAME, args -> { int windowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0]; - int rankConstant = args[1] == null ? DEFAULT_RANK_CONSTANT : (int) args[1]; + int rankConstant = args[1] == null ? RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT : (int) args[1]; return new RRFRankBuilder(windowSize, rankConstant); }); diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java index 272df248e53e9..4cd10801b298c 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java @@ -19,7 +19,7 @@ import java.util.Arrays; import java.util.Objects; -import static org.elasticsearch.xpack.rank.rrf.RRFRankBuilder.DEFAULT_RANK_CONSTANT; +import static org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT; /** * {@code RRFRankDoc} supports additional ranking information diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java index 12c43a2f169f8..c3c9f19cde6ef 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.search.rank.RankBuilder; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; @@ -31,7 +32,6 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; -import static org.elasticsearch.xpack.rank.rrf.RRFRankPlugin.NAME; /** * An rrf retriever is used to represent an rrf rank element, but @@ -50,6 +50,7 @@ public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder PARSER = new ConstructingObjectParser<>( NAME, @@ -57,8 +58,8 @@ public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder { List childRetrievers = (List) args[0]; List innerRetrievers = childRetrievers.stream().map(r -> new RetrieverSource(r, null)).toList(); - int rankWindowSize = args[1] == null ? RRFRankBuilder.DEFAULT_RANK_WINDOW_SIZE : (int) args[1]; - int rankConstant = args[2] == null ? RRFRankBuilder.DEFAULT_RANK_CONSTANT : (int) args[2]; + int rankWindowSize = args[1] == null ? RankBuilder.DEFAULT_RANK_WINDOW_SIZE : (int) args[1]; + int rankConstant = args[2] == null ? DEFAULT_RANK_CONSTANT : (int) args[2]; return new RRFRetrieverBuilder(innerRetrievers, rankWindowSize, rankConstant); } ); diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java index d324effe41c22..cae758457a2ac 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java @@ -41,7 +41,7 @@ public static RRFRetrieverBuilder createRandomRRFRetrieverBuilder() { if (randomBoolean()) { rankWindowSize = randomIntBetween(1, 10000); } - int rankConstant = RRFRankBuilder.DEFAULT_RANK_CONSTANT; + int rankConstant = RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT; if (randomBoolean()) { rankConstant = randomIntBetween(1, 1000000); } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml index 647540644ce9e..a5c346b386999 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml @@ -2,6 +2,8 @@ setup: - requires: cluster_features: "gte_v8.8.0" reason: 'rank added in 8.8' + - skip: + features: "warnings" - do: indices.create: @@ -59,7 +61,14 @@ setup: --- "Simple rank with bm25 search and kNN search": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" search: index: test body: @@ -94,7 +103,15 @@ setup: --- "Simple rank with multiple bm25 sub searches": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -135,7 +152,15 @@ setup: --- "Simple rank with multiple bm25 sub_searches and a knn search": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml index b4893bfec0849..94a1457a7acc8 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml @@ -63,7 +63,15 @@ setup: --- "Standard pagination within rank_window_size": # this test retrieves the same results from two queries, and applies a simple pagination skipping the first result + - requires: + cluster_features: [ "gte_v8.16.0" ] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -170,7 +178,15 @@ setup: --- "Standard pagination outside rank_window_size": # in this example, from starts *after* rank_window_size so, we expect 0 results to be returned + - requires: + cluster_features: [ "gte_v8.16.0" ] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -274,7 +290,15 @@ setup: --- "Standard pagination partially outside rank_window_size": # in this example we have that from starts *within* rank_window_size, but "from + size" goes over + - requires: + cluster_features: [ "gte_v8.16.0" ] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -384,7 +408,15 @@ setup: # queryA has a result set of [1, 2, 3, 4] and # queryB has a result set of [4, 3, 1, 2] # so for rank_constant=10, the expected order is [1, 4, 3, 2] + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -488,6 +520,9 @@ setup: - match: { hits.hits.1._id: "4" } - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -597,7 +632,15 @@ setup: # queryA has a result set of [5, 1] and # queryB has a result set of [4, 3, 1, 2] # so for rank_constant=10, the expected order is [1, 5, 4, 3, 2] + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -685,6 +728,9 @@ setup: - match: { hits.hits.1._id: "5" } - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -772,6 +818,9 @@ setup: - match: { hits.hits.1._id: "3" } - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -867,7 +916,15 @@ setup: # queryB has a result set of [4, 3] # so for rank_constant=10, the expected order is [5, 4, 1, 3], # and the rank_window_size-sized result set that we'd paginate over is [5, 4] + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -955,6 +1012,10 @@ setup: - match: { hits.hits.1._id: "4" } - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] + search: index: test body: diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml index bca39dea4ae57..36e70581f39f2 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml @@ -67,7 +67,14 @@ setup: --- "RRF using single knn and single BM25 with a scripted metric aggregation": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" search: index: test body: @@ -140,7 +147,14 @@ setup: --- "RRF using multi-knn only with a scripted metric aggregation": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" search: index: test body: @@ -195,7 +209,14 @@ setup: --- "RRF using multi-knn and single BM25 with a scripted metric aggregation": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" search: index: test body: diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml index 5718cd3455526..1b74ffee62a11 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml @@ -75,7 +75,14 @@ setup: --- "using a top level knn and query": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" search: index: test body: @@ -129,7 +136,15 @@ setup: --- "using sub_searches": + - requires: + cluster_features: [ "gte_v8.16.0" ] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -194,7 +209,14 @@ setup: --- "using named top level knn and query": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" search: index: test body: @@ -251,7 +273,15 @@ setup: --- "using named sub_searches": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: @@ -320,7 +350,15 @@ setup: --- "using a mix of named and unnamed queries": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" + - Deprecated field [sub_searches] used, replaced by [retriever] search: index: test body: diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml index e34885419c7f7..a9ddb4f902929 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml @@ -172,7 +172,14 @@ setup: --- "using query and dfs knn search": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "deprecation added in 8.16" + test_runner_features: warnings + - do: + warnings: + - "Deprecated field [rank] used, replaced by [retriever]" search: index: test body: diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml index 517c162c33e95..f3914843b80ec 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml @@ -11,6 +11,8 @@ setup: body: mappings: properties: + id: + type: integer text: type: text text_to_highlight: @@ -39,6 +41,7 @@ setup: index: test id: "1" body: + id: 1 text: "term1" vector: [1.0] @@ -47,6 +50,7 @@ setup: index: test id: "2" body: + id: 2 text: "term2" text_to_highlight: "search for the truth" keyword: "biology" @@ -57,6 +61,7 @@ setup: index: test id: "3" body: + id: 3 text: "term3" text_to_highlight: "nothing related" keyword: "technology" @@ -67,6 +72,7 @@ setup: index: test id: "4" body: + id: 4 text: "term4" vector: [4.0] - do: @@ -74,6 +80,7 @@ setup: index: test id: "5" body: + id: 5 text: "term5" text_to_highlight: "You know, for Search!" keyword: "technology" @@ -81,9 +88,10 @@ setup: vector: [5.0] - do: index: + id: 6 index: test - id: "6" body: + id: 6 text: "term6" keyword: "biology" integer: 6 @@ -93,6 +101,7 @@ setup: index: test id: "7" body: + id: 7 text: "term7" keyword: "astronomy" vector: [77.0] @@ -102,6 +111,7 @@ setup: index: test id: "8" body: + id: 8 text: "term8" keyword: "technology" nested: { views: 100} @@ -110,6 +120,7 @@ setup: index: test id: "9" body: + id: 9 text: "term9" integer: 2 keyword: "technology" @@ -439,7 +450,19 @@ setup: rank_window_size: 5 rank_constant: 10 size: 3 - collapse: { field: keyword, inner_hits: { name: sub_hits, size: 2 } } + collapse: { + field: keyword, + inner_hits: { + name: sub_hits, + size: 2, + sort: + { + id: { + order: desc + } + } + } + } - match: { hits.total : 9 } @@ -456,8 +479,8 @@ setup: - match: { hits.hits.1.inner_hits.sub_hits.hits.total : 4 } - length: { hits.hits.1.inner_hits.sub_hits.hits.hits : 2 } - - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "5" } - - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "3" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "9" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "8" } - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 } - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "6" } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyAction.java index 5615e2bc7915c..ec32b19f7e666 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyAction.java @@ -8,9 +8,7 @@ package org.elasticsearch.xpack.security.rest.action.apikey; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -66,7 +64,7 @@ public List routes() { @Override protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { try (XContentParser parser = request.contentParser()) { - final InvalidateApiKeyRequest invalidateApiKeyRequest = getObjectParser(request).parse(parser, null); + final InvalidateApiKeyRequest invalidateApiKeyRequest = PARSER.parse(parser, null); return channel -> client.execute( InvalidateApiKeyAction.INSTANCE, invalidateApiKeyRequest, @@ -86,41 +84,6 @@ public String getName() { return "xpack_security_invalidate_api_key"; } - private static ConstructingObjectParser getObjectParser(RestRequest request) { - if (request.getRestApiVersion() == RestApiVersion.V_7) { - final ConstructingObjectParser objectParser = new ConstructingObjectParser<>( - "invalidate_api_key_v7", - a -> { - final String id = (String) a[5]; - @SuppressWarnings("unchecked") - final List ids = (List) a[4]; - if (id != null && ids != null) { - throw new IllegalArgumentException("Must use either [id] or [ids], not both at the same time"); - } - final String[] idsArray; - if (Strings.hasText(id)) { - idsArray = new String[] { id }; - } else if (ids != null) { - idsArray = ids.toArray(String[]::new); - } else { - idsArray = null; - } - return new InvalidateApiKeyRequest( - (String) a[0], - (String) a[1], - (String) a[2], - (a[3] == null) ? false : (Boolean) a[3], - idsArray - ); - } - ); - initObjectParser(objectParser, true); - return objectParser; - } else { - return PARSER; - } - } - private static void initObjectParser(ConstructingObjectParser objectParser, boolean restCompatMode) { objectParser.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("realm_name")); objectParser.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("username")); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java index 3d3f96b98d5e5..d294fb50046d6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java @@ -158,7 +158,7 @@ public TransportRequestHandler interceptHandler( } } ); - DiscoveryNode remoteNode = remoteTransportService.getLocalDiscoNode(); + DiscoveryNode remoteNode = remoteTransportService.getLocalNode(); remoteTransportService.registerRequestHandler( RemoteClusterNodesAction.TYPE.name(), EsExecutors.DIRECT_EXECUTOR_SERVICE, diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java index b2ce91140de76..06293df4f4559 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.PriorityQueue; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.search.ClosePointInTimeRequest; @@ -101,7 +100,6 @@ import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.LENIENT; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.NONE; -import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_UNSIGNED_LONG; // TODO: add retry/back-off public class Querier { @@ -202,7 +200,7 @@ public static void closePointInTime(Client client, BytesReference pointInTimeId, public static SearchRequest prepareRequest(SearchSourceBuilder source, SqlConfiguration cfg, boolean includeFrozen, String... indices) { source.timeout(cfg.requestTimeout()); - SearchRequest searchRequest = new SearchRequest(Version.fromId(INTRODUCING_UNSIGNED_LONG.id)); + SearchRequest searchRequest = new SearchRequest(); if (source.pointInTimeBuilder() == null) { searchRequest.indices(indices); searchRequest.indicesOptions( diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java index 7a76ffe8eb109..41fa66ae36aeb 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.cluster.service.ClusterService; @@ -57,7 +56,6 @@ import static java.util.Collections.unmodifiableList; import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; -import static org.elasticsearch.xpack.ql.plugin.TransportActionUtils.executeRequestWithRetryAttempt; import static org.elasticsearch.xpack.sql.plugin.Transports.clusterName; import static org.elasticsearch.xpack.sql.plugin.Transports.username; import static org.elasticsearch.xpack.sql.proto.Mode.CLI; @@ -161,22 +159,11 @@ public static void operation( ); if (Strings.hasText(request.cursor()) == false) { - executeRequestWithRetryAttempt( - clusterService, - listener::onFailure, - onFailure -> planExecutor.sql( - cfg, - request.query(), - request.params(), - wrap(p -> listener.onResponse(createResponseWithSchema(request, p, task)), onFailure) - ), - node -> transportService.sendRequest( - node, - SqlQueryAction.NAME, - request, - new ActionListenerResponseHandler<>(listener, SqlQueryResponse::new, EsExecutors.DIRECT_EXECUTOR_SERVICE) - ), - log + planExecutor.sql( + cfg, + request.query(), + request.params(), + wrap(p -> listener.onResponse(createResponseWithSchema(request, p, task)), listener::onFailure) ); } else { Tuple decoded = Cursors.decodeFromStringWithZone(request.cursor(), planExecutor.writeableRegistry());